def _getItem(self, index): """Actual meat of loading the subjob from disk is required, parsing and storing a copy in memory (_cached_subjobs) for future use""" logger.debug("Requesting: %s" % str(index)) subjob_data = None if not index in self._cachedJobs.keys(): # obtain a lock to make sure multiple loads of the same object don't happen with self._load_lock: # just make sure we haven't loaded this object already while waiting on the lock if index in self._cachedJobs: return self._cachedJobs[index] # Now try to load the subjob if len(self) < index: raise GangaException("Subjob: %s does NOT exist" % str(index)) subjob_data = self.__get_dataFile(str(index)) try: sj_file = self._loadSubJobFromDisk(subjob_data) except IOError as x: if x.errno == errno.ENOENT: raise IOError("Subobject %s not found: %s" % (index, x)) else: raise RepositoryError(self,"IOError on loading subobject %s: %s" % (index, x)) from Ganga.Core.GangaRepository.VStreamer import from_file # load the subobject into a temporary object loaded_sj = None try: loaded_sj = from_file(sj_file)[0] except Exception as err: try: subjob_data = self.__get_dataFile(str(index), True) loaded_sj = from_file(sj_file)[0] except Exception as err: logger.debug("Failed to Load XML for job: %s using: %s" % (str(index), str(subjob_data))) logger.debug("Err:\n%s" % str(err)) raise err # if load was successful then set parent and add to the _cachedJobs dict if loaded_sj: loaded_sj._setParent( self._definedParent ) self._cachedJobs[index] = loaded_sj return self._cachedJobs[index]
def load_subJobIndex(self): """Load the index from all sujobs ynto _subjobIndexData or empty it is an error occurs""" index_file = path.join(self._jobDirectory, self._subjob_master_index_name) if path.isfile(index_file): index_file_obj = None try: from Ganga.Core.GangaRepository.PickleStreamer import from_file try: index_file_obj = open(index_file, "r") self._subjobIndexData = from_file(index_file_obj)[0] except IOError as err: self._subjobIndexData = None self._setDirty() if self._subjobIndexData is None: self._subjobIndexData = {} else: for subjob_id in self._subjobIndexData: index_data = self._subjobIndexData.get(subjob_id) ## CANNOT PERFORM REASONABLE DISK CHECKING ON AFS ## SLOW FILE ACCESS WRITE AND METADATA MEANS FILE DATA DOES NOT MATCH MOD TIME #if index_data is not None and 'modified' in index_data: # mod_time = index_data['modified'] # disk_location = self.__get_dataFile(str(subjob_id)) # disk_time = stat(disk_location).st_ctime # diff = disk_time - mod_time # if disk_time > mod_time and (diff*diff < 9.): # logger.warning("objs: %s" % self._cachedJobs.keys()) # logger.warning("%s != %s" % (mod_time, disk_time)) # logger.warning("SubJob: %s has been modified, re-loading" % (subjob_id)) # new_data = self._registry.getIndexCache( self.__getitem__(subjob_id) ) # self._subjobIndexData[subjob_id] = new_data # break #else: if index_data is None: logger.warning( "Cannot find subjob index %s, rebuilding" % subjob_id) new_data = self._registry.getIndexCache( self.__getitem__(subjob_id)) self._subjobIndexData[subjob_id] = new_data continue #self._subjobIndexData = {} except Exception as err: logger.debug("Subjob Index file open, error: %s" % err) self._subjobIndexData = {} self._setDirty() finally: if index_file_obj is not None: index_file_obj.close() if self._subjobIndexData is None: self._subjobIndexData = {} else: self._setDirty() return
def _getItem(self, index): """Actual meat of loading the subjob from disk is required, parsing and storing a copy in memory (_cached_subjobs) for future use""" logger.debug("Requesting: %s" % str(index)) #if index == 0: #import traceback #traceback.print_stack() subjob_data = None if not index in self._cachedJobs.keys(): if len(self) < index: raise GangaException("Subjob: %s does NOT exist" % str(index)) subjob_data = self.__get_dataFile(str(index)) try: sj_file = self._loadSubJobFromDisk(subjob_data) except IOError as x: if x.errno == errno.ENOENT: raise IOError("Subobject %s not found: %s" % (index, x)) else: raise RepositoryError( self, "IOError on loading subobject %s: %s" % (index, x)) from Ganga.Core.GangaRepository.VStreamer import from_file try: self._cachedJobs[index] = from_file(sj_file)[0] except Exception as err: try: subjob_data = self.__get_dataFile(str(index), True) self._cachedJobs[index] = from_file(sj_file)[0] except Exception as err: logger.debug("Failed to Load XML for job: %s using: %s" % (str(index), str(subjob_data))) logger.debug("Err:\n%s" % str(err)) raise err if self._definedParent is not None: self._cachedJobs[index]._setParent(self._definedParent) return self._cachedJobs[index]
def load_subJobIndex(self): """Load the index from all sujobs ynto _subjobIndexData or empty it is an error occurs""" index_file = path.join(self._jobDirectory, self._subjob_master_index_name ) if path.isfile( index_file ): index_file_obj = None try: from Ganga.Core.GangaRepository.PickleStreamer import from_file try: index_file_obj = open(index_file, "r" ) self._subjobIndexData = from_file( index_file_obj )[0] except IOError as err: self._subjobIndexData = None self._setDirty() if self._subjobIndexData is None: self._subjobIndexData = {} else: for subjob_id in self._subjobIndexData: index_data = self._subjobIndexData.get(subjob_id) ## CANNOT PERFORM REASONABLE DISK CHECKING ON AFS ## SLOW FILE ACCESS WRITE AND METADATA MEANS FILE DATA DOES NOT MATCH MOD TIME #if index_data is not None and 'modified' in index_data: # mod_time = index_data['modified'] # disk_location = self.__get_dataFile(str(subjob_id)) # disk_time = stat(disk_location).st_ctime # diff = disk_time - mod_time # if disk_time > mod_time and (diff*diff < 9.): # logger.warning("objs: %s" % self._cachedJobs.keys()) # logger.warning("%s != %s" % (mod_time, disk_time)) # logger.warning("SubJob: %s has been modified, re-loading" % (subjob_id)) # new_data = self._registry.getIndexCache( self.__getitem__(subjob_id) ) # self._subjobIndexData[subjob_id] = new_data # break #else: if index_data is None: logger.warning("Cannot find subjob index %s, rebuilding" % subjob_id) new_data = self._registry.getIndexCache( self.__getitem__(subjob_id) ) self._subjobIndexData[subjob_id] = new_data continue #self._subjobIndexData = {} except Exception as err: logger.debug( "Subjob Index file open, error: %s" % err ) self._subjobIndexData = {} self._setDirty() finally: if index_file_obj is not None: index_file_obj.close() if self._subjobIndexData is None: self._subjobIndexData = {} else: self._setDirty() return
def _getItem(self, index): """Actual meat of loading the subjob from disk is required, parsing and storing a copy in memory (_cached_subjobs) for future use""" logger.debug("Requesting: %s" % str(index)) # if index == 0: # import traceback # traceback.print_stack() subjob_data = None if not index in self._cachedJobs.keys(): if len(self) < index: raise GangaException("Subjob: %s does NOT exist" % str(index)) subjob_data = self.__get_dataFile(str(index)) try: sj_file = self._loadSubJobFromDisk(subjob_data) except IOError as x: if x.errno == errno.ENOENT: raise IOError("Subobject %s not found: %s" % (index, x)) else: raise RepositoryError(self, "IOError on loading subobject %s: %s" % (index, x)) from Ganga.Core.GangaRepository.VStreamer import from_file try: self._cachedJobs[index] = from_file(sj_file)[0] except Exception as err: try: subjob_data = self.__get_dataFile(str(index), True) self._cachedJobs[index] = from_file(sj_file)[0] except Exception as err: logger.debug("Failed to Load XML for job: %s using: %s" % (str(index), str(subjob_data))) logger.debug("Err:\n%s" % str(err)) raise err if self._definedParent is not None: self._cachedJobs[index]._setParent(self._definedParent) return self._cachedJobs[index]
def _getItem(self, index): """Actual meat of loading the subjob from disk is required, parsing and storing a copy in memory (_cached_subjobs) for future use""" logger.debug("Requesting subjob: #%s" % index) subjob_data = None if not index in self._cachedJobs.keys(): logger.debug("Attempting to load subjob: #%s from disk" % index) # obtain a lock to make sure multiple loads of the same object don't happen with self._load_lock: # just make sure we haven't loaded this object already while waiting on the lock if index in self._cachedJobs: return self._cachedJobs[index] has_loaded_backup = False # Now try to load the subjob if len(self) < index: raise GangaException("Subjob: %s does NOT exist" % index) subjob_data = self.__get_dataFile(str(index)) try: sj_file = self._loadSubJobFromDisk(subjob_data) except (XMLFileError, IOError) as x: logger.warning("Error loading XML file: %s" % x) try: logger.debug("Loading subjob #%s for job #%s from disk, recent changes may be lost" % (index, self.getMasterID())) subjob_data = self.__get_dataFile(str(index), True) sj_file = self._loadSubJobFromDisk(subjob_data) has_loaded_backup = True except (IOError, XMLFileError) as err: logger.error("Error loading subjob XML:\n%s" % err) if isinstance(x, IOError) and x.errno == errno.ENOENT: raise IOError("Subobject %s not found: %s" % (index, x)) else: raise RepositoryError(self,"IOError on loading subobject %s: %s" % (index, x)) from Ganga.Core.GangaRepository.VStreamer import from_file # load the subobject into a temporary object loaded_sj = None try: loaded_sj = from_file(sj_file)[0] except (IOError, XMLFileError) as err: try: logger.warning("Loading subjob #%s for job #%s from backup, recent changes may be lost" % (index, self.getMasterID())) subjob_data = self.__get_dataFile(str(index), True) sj_file = self._loadSubJobFromDisk(subjob_data) loaded_sj = from_file(sj_file)[0] has_loaded_backup = True except (IOError, XMLFileError) as err: logger.debug("Failed to Load XML for job: %s using: %s" % (index, subjob_data)) logger.debug("Err:\n%s" % err) raise loaded_sj._setParent( self._definedParent ) if has_loaded_backup: loaded_sj._setDirty() else: loaded_sj._setFlushed() self._cachedJobs[index] = loaded_sj return self._cachedJobs[index]
def _getItem(self, index): """Actual meat of loading the subjob from disk is required, parsing and storing a copy in memory (_cached_subjobs) for future use Args: index (int): The index corresponding to the subjob object we want """ logger.debug("Requesting subjob: #%s" % index) if index not in self._cachedJobs: logger.debug("Attempting to load subjob: #%s from disk" % index) # obtain a lock to make sure multiple loads of the same object don't happen with self._load_lock: # just make sure we haven't loaded this object already while waiting on the lock if index in self._cachedJobs: return self._cachedJobs[index] has_loaded_backup = False # Now try to load the subjob if len(self) < index: raise GangaException("Subjob: %s does NOT exist" % index) subjob_data = self.__get_dataFile(str(index)) try: sj_file = self._loadSubJobFromDisk(subjob_data) except (XMLFileError, IOError) as x: logger.warning("Error loading XML file: %s" % x) try: logger.debug( "Loading subjob #%s for job #%s from disk, recent changes may be lost" % (index, self.getMasterID())) subjob_data = self.__get_dataFile(str(index), True) sj_file = self._loadSubJobFromDisk(subjob_data) has_loaded_backup = True except (IOError, XMLFileError) as err: logger.debug("Error loading subjob XML:\n%s" % err) if isinstance(x, IOError) and x.errno == errno.ENOENT: raise IOError("Subobject %s not found: %s" % (index, x)) else: raise RepositoryError( self, "IOError on loading subobject %s: %s" % (index, x)) from Ganga.Core.GangaRepository.VStreamer import from_file # load the subobject into a temporary object try: loaded_sj = from_file(sj_file)[0] except (IOError, XMLFileError) as err: try: logger.warning( "Loading subjob #%s for job #%s from backup, recent changes may be lost" % (index, self.getMasterID())) subjob_data = self.__get_dataFile(str(index), True) sj_file = self._loadSubJobFromDisk(subjob_data) loaded_sj = from_file(sj_file)[0] has_loaded_backup = True except (IOError, XMLFileError) as err: logger.debug( "Failed to Load XML for job: %s using: %s" % (index, subjob_data)) logger.debug("Err:\n%s" % err) raise loaded_sj._setParent(self._definedParent) if has_loaded_backup: loaded_sj._setDirty() else: loaded_sj._setFlushed() self._cachedJobs[index] = loaded_sj return self._cachedJobs[index]