def test_e_MultipleFiles(self): """Test that the wildcards work""" MassStorageFile = self.fileClass from Ganga.GPI import LocalFile, Job, ArgSplitter _ext = '.root' _ext2 = '.txt' file_1 = generate_unique_temp_file(_ext) file_2 = generate_unique_temp_file(_ext) file_3 = generate_unique_temp_file(_ext2) TestMassStorageWN._managed_files.append(file_1) TestMassStorageWN._managed_files.append(file_2) TestMassStorageWN._managed_files.append(file_3) j = Job() j.inputfiles = [LocalFile(file_1), LocalFile(file_2), LocalFile(file_3)] j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ]) j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat='{jid}/{sjid}/{fname}'), MassStorageFile(namePattern='*'+_ext2)] j.submit()
def test_a_Submit(self): """Test the ability to submit a job with some LocalFiles""" MassStorageFile = self.fileClass from Ganga.GPI import jobs, Job, LocalFile _ext = '.txt' file_1 = generate_unique_temp_file(_ext) TestMassStorageWN._managed_files.append(file_1) j = Job() j.inputfiles = [LocalFile(file_1)] j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.standardFormat)] j.submit() for f in j.outputfiles: assert f.outputfilenameformat == self.standardFormat
def test_c_SplitJob(self): """Test submitting subjobs""" MassStorageFile = self.fileClass from Ganga.GPI import Job, LocalFile, ArgSplitter _ext = '.txt2' file_1 = generate_unique_temp_file(_ext) TestMassStorageWN._managed_files.append(file_1) j = Job() j.inputfiles = [LocalFile(file_1)] j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ]) j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.extendedFormat)] j.submit() for f in j.outputfiles: assert f.outputfilenameformat == self.extendedFormat
def test_c_testCopyJob(self): """ Test copying a completed job with a wildcard in the outputfiles """ from Ganga.GPI import jobs j = jobs[-1] j2 = j.copy() assert len(j2.outputfiles) == 1 MassStorageFile = self.fileClass assert j2.outputfiles == [MassStorageFile(namePattern='*'+TestMassStorageClient._ext)] assert len(j2.inputfiles) == 2 self.cleanUp()
def test_a_testClientInputSubmit(self): """Test that a job can be submitted with inputfiles in the input""" MassStorageFile = self.fileClass from Ganga.GPI import LocalFile, Job, ArgSplitter _ext = '.root' file_1 = generate_unique_temp_file(_ext) file_2 = generate_unique_temp_file(_ext) self._managed_files.append(file_1) self._managed_files.append(file_2) msf_1 = MassStorageFile(file_1) msf_2 = MassStorageFile(file_2) msf_1.put() msf_2.put() j = Job() j.inputfiles = [msf_1, msf_2] j.splitter = ArgSplitter(args=[[_] for _ in range(self.sj_len)]) j.outputfiles = [LocalFile(namePattern='*' + _ext)] j.submit()
def test_a_test_put(self): """Test that a job can be submitted with inputfiles in the input""" MassStorageFile = self.fileClass _ext = '.root' file_1 = generate_unique_temp_file(_ext) file_2 = generate_unique_temp_file(_ext) self._temp_files.append(file_1) self._temp_files.append(file_2) msf_1 = MassStorageFile(file_1) msf_2 = MassStorageFile(file_2) self._managed_files.append(msf_1) self._managed_files.append(msf_2) msf_1.put() msf_2.put() for file_ in [msf for msf in (msf_1, msf_2)]: assert os.path.isfile( os.path.join(self.outputFilePath, file_.namePattern)) file_.localDir = '' assert file_.localDir == ''
def test_a_testClientSideSubmit(self): """Test the client side code whilst stil using the Local backend""" MassStorageFile = self.fileClass from Ganga.GPI import LocalFile, Job, ArgSplitter TestMassStorageClient.cleanUp() assert getConfig('Output')[_getName(self.fileClass)]['backendPostprocess']['Local'] == 'client' file_1 = generate_unique_temp_file(TestMassStorageClient._ext) file_2 = generate_unique_temp_file(TestMassStorageClient._ext) TestMassStorageClient._managed_files.append(file_1) TestMassStorageClient._managed_files.append(file_2) j = Job() j.inputfiles = [LocalFile(file_1), LocalFile(file_2)] j.splitter = ArgSplitter(args = [[_] for _ in range(TestMassStorageClient.sj_len)]) j.outputfiles = [MassStorageFile(namePattern='*'+TestMassStorageClient._ext)] j.submit()
def string_file_shortcut(v, item): if isinstance(v, str): # use proxy class to enable all user conversions on the value itself # but return the implementation object (not proxy) key = findOutputFileTypeByFileName(v) if key is not None: if key == 'MassStorageFile': from .MassStorageFile import MassStorageFile return stripProxy(MassStorageFile._proxyClass(v)) elif key == 'LCGSEFile': from .LCGSEFile import LCGSEFile return stripProxy(LCGSEFile._proxyClass(v)) elif key == 'DiracFile': try: from GangaDirac.Lib.Files.DiracFile import DiracFile return stripProxy(DiracFile._proxyClass(v)) except: Ganga.Utility.logging.log_unknown_exception() pass return stripProxy(LocalFile._proxyClass(v)) return None
def test_g_MultipleFiles(self): """Test that the wildcards work""" MassStorageFile = self.fileClass from Ganga.GPI import LocalFile, Job, ArgSplitter _ext = '.root' file_1 = generate_unique_temp_file(_ext) file_2 = generate_unique_temp_file(_ext) TestMassStorageWN._managed_files.append(file_1) TestMassStorageWN._managed_files.append(file_2) j = Job() j.inputfiles = [LocalFile(file_1), LocalFile(file_2)] j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ]) j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.customOutputFormat)] for f in j.outputfiles: assert f.outputfilenameformat == self.customOutputFormat j.submit() for f in j.outputfiles: assert f.outputfilenameformat == self.customOutputFormat
def test_a_testClientInputSubmit(self): """Test that a job can be submitted with inputfiles in the input""" MassStorageFile = self.fileClass from Ganga.GPI import LocalFile, Job, ArgSplitter _ext = '.root' file_1 = generate_unique_temp_file(_ext) file_2 = generate_unique_temp_file(_ext) self._managed_files.append(file_1) self._managed_files.append(file_2) msf_1 = MassStorageFile(file_1) msf_2 = MassStorageFile(file_2) msf_1.put() msf_2.put() j = Job() j.inputfiles = [msf_1, msf_2] j.splitter = ArgSplitter(args = [[_] for _ in range(self.sj_len)]) j.outputfiles = [LocalFile(namePattern='*'+_ext)] j.submit()
def test_a_test_put(self): """Test that a job can be submitted with inputfiles in the input""" MassStorageFile = self.fileClass _ext = '.root' file_1 = generate_unique_temp_file(_ext) file_2 = generate_unique_temp_file(_ext) self._temp_files.append(file_1) self._temp_files.append(file_2) msf_1 = MassStorageFile(file_1) msf_2 = MassStorageFile(file_2) self._managed_files.append(msf_1) self._managed_files.append(msf_2) msf_1.put() msf_2.put() for file_ in [msf for msf in (msf_1, msf_2)]: assert os.path.isfile(os.path.join(self.outputFilePath, file_.namePattern)) file_.localDir = '' assert file_.localDir == ''