def _getItem(self, index):
"""Actual meat of loading the subjob from disk is required, parsing and storing a copy in memory
(_cached_subjobs) for future use"""
logger.debug("Requesting: %s" % str(index))
subjob_data = None
if not index in self._cachedJobs.keys():
# obtain a lock to make sure multiple loads of the same object don't happen
with self._load_lock:
# just make sure we haven't loaded this object already while waiting on the lock
if index in self._cachedJobs:
return self._cachedJobs[index]
# Now try to load the subjob
if len(self) < index:
raise GangaException("Subjob: %s does NOT exist" % str(index))
subjob_data = self.__get_dataFile(str(index))
try:
sj_file = self._loadSubJobFromDisk(subjob_data)
except IOError as x:
if x.errno == errno.ENOENT:
raise IOError("Subobject %s not found: %s" % (index, x))
else:
raise RepositoryError(self,"IOError on loading subobject %s: %s" % (index, x))
from Ganga.Core.GangaRepository.VStreamer import from_file
# load the subobject into a temporary object
loaded_sj = None
try:
loaded_sj = from_file(sj_file)[0]
except Exception as err:
try:
subjob_data = self.__get_dataFile(str(index), True)
loaded_sj = from_file(sj_file)[0]
except Exception as err:
logger.debug("Failed to Load XML for job: %s using: %s" % (str(index), str(subjob_data)))
logger.debug("Err:\n%s" % str(err))
raise err
# if load was successful then set parent and add to the _cachedJobs dict
if loaded_sj:
loaded_sj._setParent( self._definedParent )
self._cachedJobs[index] = loaded_sj
return self._cachedJobs[index]
def load_subJobIndex(self):
"""Load the index from all sujobs ynto _subjobIndexData or empty it is an error occurs"""
index_file = path.join(self._jobDirectory,
self._subjob_master_index_name)
if path.isfile(index_file):
index_file_obj = None
try:
from Ganga.Core.GangaRepository.PickleStreamer import from_file
try:
index_file_obj = open(index_file, "r")
self._subjobIndexData = from_file(index_file_obj)[0]
except IOError as err:
self._subjobIndexData = None
self._setDirty()
if self._subjobIndexData is None:
self._subjobIndexData = {}
else:
for subjob_id in self._subjobIndexData:
index_data = self._subjobIndexData.get(subjob_id)
## CANNOT PERFORM REASONABLE DISK CHECKING ON AFS
## SLOW FILE ACCESS WRITE AND METADATA MEANS FILE DATA DOES NOT MATCH MOD TIME
#if index_data is not None and 'modified' in index_data:
# mod_time = index_data['modified']
# disk_location = self.__get_dataFile(str(subjob_id))
# disk_time = stat(disk_location).st_ctime
# diff = disk_time - mod_time
# if disk_time > mod_time and (diff*diff < 9.):
# logger.warning("objs: %s" % self._cachedJobs.keys())
# logger.warning("%s != %s" % (mod_time, disk_time))
# logger.warning("SubJob: %s has been modified, re-loading" % (subjob_id))
# new_data = self._registry.getIndexCache( self.__getitem__(subjob_id) )
# self._subjobIndexData[subjob_id] = new_data
# break
#else:
if index_data is None:
logger.warning(
"Cannot find subjob index %s, rebuilding" %
subjob_id)
new_data = self._registry.getIndexCache(
self.__getitem__(subjob_id))
self._subjobIndexData[subjob_id] = new_data
continue
#self._subjobIndexData = {}
except Exception as err:
logger.debug("Subjob Index file open, error: %s" % err)
self._subjobIndexData = {}
self._setDirty()
finally:
if index_file_obj is not None:
index_file_obj.close()
if self._subjobIndexData is None:
self._subjobIndexData = {}
else:
self._setDirty()
return
def _getItem(self, index):
"""Actual meat of loading the subjob from disk is required, parsing and storing a copy in memory (_cached_subjobs) for future use"""
logger.debug("Requesting: %s" % str(index))
#if index == 0:
#import traceback
#traceback.print_stack()
subjob_data = None
if not index in self._cachedJobs.keys():
if len(self) < index:
raise GangaException("Subjob: %s does NOT exist" % str(index))
subjob_data = self.__get_dataFile(str(index))
try:
sj_file = self._loadSubJobFromDisk(subjob_data)
except IOError as x:
if x.errno == errno.ENOENT:
raise IOError("Subobject %s not found: %s" % (index, x))
else:
raise RepositoryError(
self,
"IOError on loading subobject %s: %s" % (index, x))
from Ganga.Core.GangaRepository.VStreamer import from_file
try:
self._cachedJobs[index] = from_file(sj_file)[0]
except Exception as err:
try:
subjob_data = self.__get_dataFile(str(index), True)
self._cachedJobs[index] = from_file(sj_file)[0]
except Exception as err:
logger.debug("Failed to Load XML for job: %s using: %s" %
(str(index), str(subjob_data)))
logger.debug("Err:\n%s" % str(err))
raise err
if self._definedParent is not None:
self._cachedJobs[index]._setParent(self._definedParent)
return self._cachedJobs[index]
def load_subJobIndex(self):
"""Load the index from all sujobs ynto _subjobIndexData or empty it is an error occurs"""
index_file = path.join(self._jobDirectory, self._subjob_master_index_name )
if path.isfile( index_file ):
index_file_obj = None
try:
from Ganga.Core.GangaRepository.PickleStreamer import from_file
try:
index_file_obj = open(index_file, "r" )
self._subjobIndexData = from_file( index_file_obj )[0]
except IOError as err:
self._subjobIndexData = None
self._setDirty()
if self._subjobIndexData is None:
self._subjobIndexData = {}
else:
for subjob_id in self._subjobIndexData:
index_data = self._subjobIndexData.get(subjob_id)
## CANNOT PERFORM REASONABLE DISK CHECKING ON AFS
## SLOW FILE ACCESS WRITE AND METADATA MEANS FILE DATA DOES NOT MATCH MOD TIME
#if index_data is not None and 'modified' in index_data:
# mod_time = index_data['modified']
# disk_location = self.__get_dataFile(str(subjob_id))
# disk_time = stat(disk_location).st_ctime
# diff = disk_time - mod_time
# if disk_time > mod_time and (diff*diff < 9.):
# logger.warning("objs: %s" % self._cachedJobs.keys())
# logger.warning("%s != %s" % (mod_time, disk_time))
# logger.warning("SubJob: %s has been modified, re-loading" % (subjob_id))
# new_data = self._registry.getIndexCache( self.__getitem__(subjob_id) )
# self._subjobIndexData[subjob_id] = new_data
# break
#else:
if index_data is None:
logger.warning("Cannot find subjob index %s, rebuilding" % subjob_id)
new_data = self._registry.getIndexCache( self.__getitem__(subjob_id) )
self._subjobIndexData[subjob_id] = new_data
continue
#self._subjobIndexData = {}
except Exception as err:
logger.debug( "Subjob Index file open, error: %s" % err )
self._subjobIndexData = {}
self._setDirty()
finally:
if index_file_obj is not None:
index_file_obj.close()
if self._subjobIndexData is None:
self._subjobIndexData = {}
else:
self._setDirty()
return
def _getItem(self, index):
"""Actual meat of loading the subjob from disk is required, parsing and storing a copy in memory (_cached_subjobs) for future use"""
logger.debug("Requesting: %s" % str(index))
# if index == 0:
# import traceback
# traceback.print_stack()
subjob_data = None
if not index in self._cachedJobs.keys():
if len(self) < index:
raise GangaException("Subjob: %s does NOT exist" % str(index))
subjob_data = self.__get_dataFile(str(index))
try:
sj_file = self._loadSubJobFromDisk(subjob_data)
except IOError as x:
if x.errno == errno.ENOENT:
raise IOError("Subobject %s not found: %s" % (index, x))
else:
raise RepositoryError(self, "IOError on loading subobject %s: %s" % (index, x))
from Ganga.Core.GangaRepository.VStreamer import from_file
try:
self._cachedJobs[index] = from_file(sj_file)[0]
except Exception as err:
try:
subjob_data = self.__get_dataFile(str(index), True)
self._cachedJobs[index] = from_file(sj_file)[0]
except Exception as err:
logger.debug("Failed to Load XML for job: %s using: %s" % (str(index), str(subjob_data)))
logger.debug("Err:\n%s" % str(err))
raise err
if self._definedParent is not None:
self._cachedJobs[index]._setParent(self._definedParent)
return self._cachedJobs[index]
def _getItem(self, index):
"""Actual meat of loading the subjob from disk is required, parsing and storing a copy in memory
(_cached_subjobs) for future use"""
logger.debug("Requesting subjob: #%s" % index)
subjob_data = None
if not index in self._cachedJobs.keys():
logger.debug("Attempting to load subjob: #%s from disk" % index)
# obtain a lock to make sure multiple loads of the same object don't happen
with self._load_lock:
# just make sure we haven't loaded this object already while waiting on the lock
if index in self._cachedJobs:
return self._cachedJobs[index]
has_loaded_backup = False
# Now try to load the subjob
if len(self) < index:
raise GangaException("Subjob: %s does NOT exist" % index)
subjob_data = self.__get_dataFile(str(index))
try:
sj_file = self._loadSubJobFromDisk(subjob_data)
except (XMLFileError, IOError) as x:
logger.warning("Error loading XML file: %s" % x)
try:
logger.debug("Loading subjob #%s for job #%s from disk, recent changes may be lost" % (index, self.getMasterID()))
subjob_data = self.__get_dataFile(str(index), True)
sj_file = self._loadSubJobFromDisk(subjob_data)
has_loaded_backup = True
except (IOError, XMLFileError) as err:
logger.error("Error loading subjob XML:\n%s" % err)
if isinstance(x, IOError) and x.errno == errno.ENOENT:
raise IOError("Subobject %s not found: %s" % (index, x))
else:
raise RepositoryError(self,"IOError on loading subobject %s: %s" % (index, x))
from Ganga.Core.GangaRepository.VStreamer import from_file
# load the subobject into a temporary object
loaded_sj = None
try:
loaded_sj = from_file(sj_file)[0]
except (IOError, XMLFileError) as err:
try:
logger.warning("Loading subjob #%s for job #%s from backup, recent changes may be lost" % (index, self.getMasterID()))
subjob_data = self.__get_dataFile(str(index), True)
sj_file = self._loadSubJobFromDisk(subjob_data)
loaded_sj = from_file(sj_file)[0]
has_loaded_backup = True
except (IOError, XMLFileError) as err:
logger.debug("Failed to Load XML for job: %s using: %s" % (index, subjob_data))
logger.debug("Err:\n%s" % err)
raise
loaded_sj._setParent( self._definedParent )
if has_loaded_backup:
loaded_sj._setDirty()
else:
loaded_sj._setFlushed()
self._cachedJobs[index] = loaded_sj
return self._cachedJobs[index]
def _getItem(self, index):
"""Actual meat of loading the subjob from disk is required, parsing and storing a copy in memory
(_cached_subjobs) for future use
Args:
index (int): The index corresponding to the subjob object we want
"""
logger.debug("Requesting subjob: #%s" % index)
if index not in self._cachedJobs:
logger.debug("Attempting to load subjob: #%s from disk" % index)
# obtain a lock to make sure multiple loads of the same object don't happen
with self._load_lock:
# just make sure we haven't loaded this object already while waiting on the lock
if index in self._cachedJobs:
return self._cachedJobs[index]
has_loaded_backup = False
# Now try to load the subjob
if len(self) < index:
raise GangaException("Subjob: %s does NOT exist" % index)
subjob_data = self.__get_dataFile(str(index))
try:
sj_file = self._loadSubJobFromDisk(subjob_data)
except (XMLFileError, IOError) as x:
logger.warning("Error loading XML file: %s" % x)
try:
logger.debug(
"Loading subjob #%s for job #%s from disk, recent changes may be lost"
% (index, self.getMasterID()))
subjob_data = self.__get_dataFile(str(index), True)
sj_file = self._loadSubJobFromDisk(subjob_data)
has_loaded_backup = True
except (IOError, XMLFileError) as err:
logger.debug("Error loading subjob XML:\n%s" % err)
if isinstance(x, IOError) and x.errno == errno.ENOENT:
raise IOError("Subobject %s not found: %s" %
(index, x))
else:
raise RepositoryError(
self, "IOError on loading subobject %s: %s" %
(index, x))
from Ganga.Core.GangaRepository.VStreamer import from_file
# load the subobject into a temporary object
try:
loaded_sj = from_file(sj_file)[0]
except (IOError, XMLFileError) as err:
try:
logger.warning(
"Loading subjob #%s for job #%s from backup, recent changes may be lost"
% (index, self.getMasterID()))
subjob_data = self.__get_dataFile(str(index), True)
sj_file = self._loadSubJobFromDisk(subjob_data)
loaded_sj = from_file(sj_file)[0]
has_loaded_backup = True
except (IOError, XMLFileError) as err:
logger.debug(
"Failed to Load XML for job: %s using: %s" %
(index, subjob_data))
logger.debug("Err:\n%s" % err)
raise
loaded_sj._setParent(self._definedParent)
if has_loaded_backup:
loaded_sj._setDirty()
else:
loaded_sj._setFlushed()
self._cachedJobs[index] = loaded_sj
return self._cachedJobs[index]