def test_h_testXMLIndex(self): # Check index of job from Ganga.Core.GangaRepository.PickleStreamer import to_file, from_file from Ganga.GPI import jobs j = jobs(0) assert path.isfile(getIndexFile(j)) with open(getIndexFile(j)) as handler: obj, errs = from_file(handler) assert isinstance(obj, tuple) from Ganga.GPIDev.Base.Proxy import stripProxy, getName raw_j = stripProxy(j) index_cache = raw_j._getRegistry().getIndexCache(raw_j) assert isinstance(index_cache, dict) index_cls = getName(raw_j) index_cat = raw_j._category this_index_cache = (index_cat, index_cls, index_cache) print("just-built index: %s" % str(this_index_cache)) print("from disk: %s" % str(obj)) assert this_index_cache == obj
def test_i_testSJXMLIndex(self): # Check index of all sj from Ganga.Core.GangaRepository.PickleStreamer import to_file, from_file from Ganga.GPI import jobs assert len(jobs) == 2 j=jobs(0) with open(getSJXMLIndex(j)) as handler: obj, errs = from_file(handler) assert isinstance(obj, dict) from Ganga.GPIDev.Base.Proxy import stripProxy, getName raw_j = stripProxy(j) new_dict = {} for sj in j.subjobs: raw_sj = stripProxy(sj) temp_index = raw_sj._getRegistry().getIndexCache(raw_sj) new_dict[sj.id] = temp_index assert raw_sj._category == raw_j._category for k, v in new_dict.iteritems(): for k1, v1 in v.iteritems(): if k1 != 'modified': assert obj[k][k1] == new_dict[k][k1]
def load_subJobIndex(self): import os.path index_file = os.path.join(self._jobDirectory, self._subjob_master_index_name ) if os.path.isfile( index_file ): index_file_obj = None try: from Ganga.Core.GangaRepository.PickleStreamer import from_file try: index_file_obj = open( index_file, "r" ) self._subjobIndexData = from_file( index_file_obj )[0] except IOError, err: self._subjobIndexData = None if self._subjobIndexData is None: self._subjobIndexData = {} else: for subjob in self._subjobIndexData.keys(): index_data = self._subjobIndexData.get(subjob) if index_data is not None and 'modified' in index_data: mod_time = index_data['modified'] disk_location = self.__get_dataFile(str(subjob)) import os disk_time = os.stat(disk_location).st_ctime if mod_time != disk_time: self._subjobIndexData = {} break else: self._subjobIndexData = {} except Exception, err: logger.error( "Subjob Index file open, error: %s" % str(err) ) self._subjobIndexData = {}
def test_f_testXMLIndex(self): # Check XML Index content from Ganga.Core.GangaRepository.PickleStreamer import to_file, from_file from Ganga.GPI import jobs j = jobs(0) assert path.isfile(getIndexFile(j)) with open(getIndexFile(j)) as handler: obj, errs = from_file(handler) assert isinstance(obj, tuple) from Ganga.GPIDev.Base.Proxy import stripProxy, getName raw_j = stripProxy(j) index_cache = raw_j._getRegistry().getIndexCache(raw_j) assert isinstance(index_cache, dict) index_cls = getName(raw_j) index_cat = raw_j._category this_index_cache = (index_cat, index_cls, index_cache) assert this_index_cache == obj
def getAssociatedFiles(self): """ Load the list of associated files from the saved XML. This is necessary to keep the list consistent when copying jobs. """ if os.path.isfile(os.path.join(self.path(), 'associated_files.xml')): with open(os.path.join(self.path(), 'associated_files.xml'), "r") as fobj: tmpobj, errs = from_file(fobj) self.associated_files = tmpobj
def test_i_testSJXMLIndex(self): # Check index of all sj from Ganga.Core.GangaRepository.PickleStreamer import to_file, from_file from Ganga.GPI import jobs assert len(jobs) == 2 j = jobs(0) with open(getSJXMLIndex(j)) as handler: obj, errs = from_file(handler) assert isinstance(obj, dict) from Ganga.GPIDev.Base.Proxy import stripProxy, getName raw_j = stripProxy(j) new_dict = {} for sj in j.subjobs: raw_sj = stripProxy(sj) temp_index = raw_sj._getRegistry().getIndexCache(raw_sj) new_dict[sj.id] = temp_index assert raw_sj._category == raw_j._category for k, v in new_dict.iteritems(): for k1, v1 in v.iteritems(): if k1 != 'modified': assert obj[k][k1] == new_dict[k][k1]
def test_e_testXMLContent(self): # Check content of XML is as expected from Ganga.Core.GangaRepository.VStreamer import to_file, from_file from Ganga.GPI import jobs, Job from Ganga.GPIDev.Base.Proxy import stripProxy from tempfile import NamedTemporaryFile j = jobs(0) assert path.isfile(getXMLFile(j)) with open(getXMLFile(j)) as handler: tmpobj, errs = from_file(handler) assert hasattr(tmpobj, 'name') assert tmpobj.name == testStr ignore_subs = [ 'time', 'subjobs', 'info', 'application', 'backend', 'id' ] with NamedTemporaryFile(delete=False) as new_temp_file: temp_name = new_temp_file.name to_file(stripProxy(j), new_temp_file, ignore_subs) new_temp_file.flush() with NamedTemporaryFile(delete=False) as new_temp_file2: temp_name2 = new_temp_file2.name j2 = Job() j2.name = testStr j2.submit() from GangaTest.Framework.utils import sleep_until_completed sleep_until_completed(j2) to_file(stripProxy(j2), new_temp_file2, ignore_subs) new_temp_file2.flush() #import filecmp #assert filecmp.cmp(handler.name, new_temp_file.name) #assert not filecmp.cmp(new_temp_file.name, new_temp_file2.name) #assert open(getXMLFile(j)).read() == open(temp_name).read() assert open(temp_name).read() == open(temp_name2).read() unlink(temp_name) unlink(temp_name2)
def test_e_testXMLContent(self): # Check content of XML is as expected from Ganga.Core.GangaRepository.VStreamer import to_file, from_file from Ganga.GPI import jobs, Job from Ganga.GPIDev.Base.Proxy import stripProxy from tempfile import NamedTemporaryFile j = jobs(0) assert path.isfile(getXMLFile(j)) with open(getXMLFile(j)) as handler: tmpobj, errs = from_file(handler) assert hasattr(tmpobj, 'name') assert tmpobj.name == testStr ignore_subs = ['time', 'subjobs', 'info', 'application', 'backend', 'id'] with NamedTemporaryFile(delete=False) as new_temp_file: temp_name = new_temp_file.name to_file(stripProxy(j), new_temp_file, ignore_subs) new_temp_file.flush() with NamedTemporaryFile(delete=False) as new_temp_file2: temp_name2 = new_temp_file2.name j2 = Job() j2.name = testStr j2.submit() from GangaTest.Framework.utils import sleep_until_completed sleep_until_completed(j2) to_file(stripProxy(j2), new_temp_file2, ignore_subs) new_temp_file2.flush() #import filecmp #assert filecmp.cmp(handler.name, new_temp_file.name) #assert not filecmp.cmp(new_temp_file.name, new_temp_file2.name) #assert open(getXMLFile(j)).read() == open(temp_name).read() assert open(temp_name).read() == open(temp_name2).read() unlink(temp_name) unlink(temp_name2)
def test_g_testSJXMLContent(self): # Check SJ content from Ganga.Core.GangaRepository.VStreamer import to_file, from_file from Ganga.GPI import jobs from tempfile import NamedTemporaryFile from Ganga.GPIDev.Base.Proxy import stripProxy ignore_subs = [ 'subjobs', 'time', 'backend', 'id', 'splitter', 'info', 'application', 'inputdata' ] with NamedTemporaryFile(delete=False) as new_temp_file_a: temp_name_a = new_temp_file_a.name j = jobs(0) to_file(stripProxy(j), new_temp_file_a, ignore_subs) new_temp_file_a.flush() counter = 0 for sj in j.subjobs: XMLFileName = getSJXMLFile(sj) assert path.isfile(XMLFileName) with open(XMLFileName) as handler: tmpobj, errs = from_file(handler) assert hasattr(tmpobj, 'id') assert tmpobj.id == counter with NamedTemporaryFile(delete=False) as new_temp_file: temp_name = new_temp_file.name to_file(stripProxy(sj), new_temp_file, ignore_subs) new_temp_file.flush() #import filecmp #assert filecmp.cmp(XMLFileName, temp_name) assert open(temp_name_a).read() == open(temp_name).read() unlink(temp_name) counter += 1 assert counter == len(jobs(0).subjobs) unlink(temp_name_a)
def test_f_testXMLContent(self): # Check their content from Ganga.Core.GangaRepository.VStreamer import to_file, from_file from Ganga.GPI import jobs, Job from Ganga.GPIDev.Base.Proxy import stripProxy from tempfile import NamedTemporaryFile j = jobs(0) XMLFileName = getXMLFile(j) assert path.isfile(XMLFileName) with open(XMLFileName) as handler: tmpobj, errs = from_file(handler) assert hasattr(tmpobj, 'name') assert tmpobj.name == testStr ignore_subs = [ 'status', 'subjobs', 'time', 'backend', 'id', 'splitter', 'info', 'application' ] with NamedTemporaryFile(delete=False) as new_temp_file: temp_name = new_temp_file.name to_file(stripProxy(j), new_temp_file, ignore_subs) new_temp_file.flush() with NamedTemporaryFile(delete=False) as new_temp_file2: temp_name2 = new_temp_file2.name j2 = Job() j2.name = testStr to_file(stripProxy(j2), new_temp_file2, ignore_subs) new_temp_file2.flush() #assert open(XMLFileName).read() == open(temp_name).read() assert open(temp_name).read() == open(temp_name2).read() unlink(temp_name) unlink(temp_name2)
def test_g_testSJXMLContent(self): # Check SJ content from Ganga.Core.GangaRepository.VStreamer import to_file, from_file from Ganga.GPI import jobs from tempfile import NamedTemporaryFile from Ganga.GPIDev.Base.Proxy import stripProxy ignore_subs = ['subjobs', 'time', 'backend', 'id', 'splitter', 'info', 'application', 'inputdata'] with NamedTemporaryFile(delete=False) as new_temp_file_a: temp_name_a = new_temp_file_a.name j=jobs(0) to_file(stripProxy(j), new_temp_file_a, ignore_subs) new_temp_file_a.flush() counter = 0 for sj in j.subjobs: XMLFileName = getSJXMLFile(sj) assert path.isfile(XMLFileName) with open(XMLFileName) as handler: tmpobj, errs = from_file(handler) assert hasattr(tmpobj, 'id') assert tmpobj.id == counter with NamedTemporaryFile(delete=False) as new_temp_file: temp_name = new_temp_file.name to_file(stripProxy(sj), new_temp_file, ignore_subs) new_temp_file.flush() #import filecmp #assert filecmp.cmp(XMLFileName, temp_name) assert open(temp_name_a).read() == open(temp_name).read() unlink(temp_name) counter+=1 assert counter == len(jobs(0).subjobs) unlink(temp_name_a)
def test_f_testXMLContent(self): # Check their content from Ganga.Core.GangaRepository.VStreamer import to_file, from_file from Ganga.GPI import jobs, Job from Ganga.GPIDev.Base.Proxy import stripProxy from tempfile import NamedTemporaryFile j=jobs(0) XMLFileName = getXMLFile(j) assert path.isfile(XMLFileName) with open(XMLFileName) as handler: tmpobj, errs = from_file(handler) assert hasattr(tmpobj, 'name') assert tmpobj.name == testStr ignore_subs = ['status', 'subjobs', 'time', 'backend', 'id', 'splitter', 'info', 'application'] with NamedTemporaryFile(delete=False) as new_temp_file: temp_name = new_temp_file.name to_file(stripProxy(j), new_temp_file, ignore_subs) new_temp_file.flush() with NamedTemporaryFile(delete=False) as new_temp_file2: temp_name2 = new_temp_file2.name j2=Job() j2.name=testStr to_file(stripProxy(j2), new_temp_file2, ignore_subs) new_temp_file2.flush() #assert open(XMLFileName).read() == open(temp_name).read() assert open(temp_name).read() == open(temp_name2).read() unlink(temp_name) unlink(temp_name2)
def test_d_testXMLContent(self): # Check content of XML is as expected from Ganga.Core.GangaRepository.VStreamer import to_file, from_file from Ganga.GPI import jobs, Job, ArgSplitter from Ganga.GPIDev.Base.Proxy import stripProxy from tempfile import NamedTemporaryFile j = jobs(0) assert path.isfile(getXMLFile(j)) with open(getXMLFile(j)) as handler: tmpobj, errs = from_file(handler) assert tmpobj.splitter assert tmpobj.splitter.args == getNestedList() ignore_subs = '' with NamedTemporaryFile(delete=False) as new_temp_file: to_file(stripProxy(j), new_temp_file, ignore_subs) new_temp_file.flush() with NamedTemporaryFile(delete=False) as new_temp_file2: j2 = Job() j2.splitter = ArgSplitter() j2.splitter.args = getNestedList() to_file(stripProxy(j2), new_temp_file2, ignore_subs) new_temp_file2.flush() assert open(handler.name).read() == open(new_temp_file.name).read() assert open(handler.name) != open(new_temp_file2.name).read() unlink(new_temp_file.name) unlink(new_temp_file2.name)