def setUp(self): unittest.TestCase.setUp(self) self.hpc = HPC.by_name("alvin") self.rdir = "/home/metno_op/work/emep/metno_hpc_test" self.testFiles = ["script.job", "status"]
def setUp(self): unittest.TestCase.setUp(self) self.hpc = HPC.by_name("frost") self.rdir = "/home/metno_op/work/emep/metno_hpc_test" self.strangeFiles = ["file with spaces", "file with wildcards*"] self.testFiles = ["script.job", "status"] + self.strangeFiles
def setUp(self): unittest.TestCase.setUp(self) self.hpc = HPC.by_name("ppi_centos7_direct") self.rdir = "/lustre/storeB/users/heikok/tmp/metno_hpc_test" self.testFiles = ["script.job", "status"]
def __init__(self, directory, hpc, directory2, remote, remoteUser, remoteDir, dryrun=False): self.dryrun = dryrun self.hpc = HPC.by_name(hpc) self.remote = remote self.remote_user = remoteUser self.remote_dir = remoteDir self.ssh = SSHConnection(remoteUser, remote) self.scpdestination = "{remote}:{remoteDir}".format( remote=self.remote, remoteDir=self.remote_dir) if self.remote_user: self.scpdestination = self.remote_user + '@' + self.scpdestination if dirIsWritable(directory): self.directory = directory if dirIsWritable(directory2): self.directory2 = directory2 else: if (self.dryrun): print("directory2: '{}' not writable and disabled".format( directory2), file=sys.stderr) self.directory2 = "" elif dirIsWritable(directory2): if (self.dryrun): print( "directory: '{}' not writable and disabled, using '{}' as default " .format(directory, directory2), file=sys.stderr) self.directory = directory2 self.directory2 = "" else: raise Exception("{dir1} and {dir2} not writable".format( dir1=directory, dir2=directory2)) workdir = os.path.join(self.directory, self.WORK_DIR) if not os.path.isdir(workdir): os.mkdir(workdir) self.statusfile = os.path.join(self.directory, "snapRemoteRunner_working") # make sure only one instance is running, not failsafe (no flock on lustre, eventually in different directories, but good enough) if (os.path.exists(self.statusfile)): file_modified = datetime.datetime.fromtimestamp( os.lstat(self.statusfile).st_mtime) if (self.dryrun): with open(self.statusfile, 'rt') as fh: msg = fh.read() print("status-file exists at '{}' with:".format( self.statusfile), file=sys.stderr) print(msg, file=sys.stderr) else: if datetime.datetime.now( ) - file_modified > datetime.timedelta(hours=3): # return statusfile if hanging for more than 3 hours print("cleaning up {} after 3 hours".format( self.statusfile), file=sys.stderr) _cleanupFileCallable(self.statusfile)() return else: if not self.dryrun: with open(self.statusfile, 'wt') as fh: atexit.register(_cleanupFileCallable(self.statusfile)) fh.write("working pid: {} on node: {}\n".format( os.getpid(), os.uname().nodename)) if DEBUG: print("working pid: {} on node: {}\n".format( os.getpid(), os.uname().nodename)) self._check_and_unpack_new_files()
def setUp(self): unittest.TestCase.setUp(self) self.vilje = HPC.by_name("vilje") self.rdir = "/work/forecast/metno_hpc_test" self.testFiles = ["script.job", "status"]