def save(self):
'''
Saves the results in serialized form, together with the MD5 signature
of the control class and the input file.
'''
# uncomment to avoid saving results
# print ('*** save commented for debugging ***')
# return
##
if 'ext_input' in self.parameters and self.parameters['ext_input']:
return
md5_parameters = self.parameters['md5']
md5_input = utils.md5sum(self.ifile) # run md5 in self.ifile
try:
with open(os.path.join(self.dest_path, 'data.pkl'), 'wb') as fo:
pickle.dump(md5_parameters, fo)
pickle.dump(md5_input, fo)
pickle.dump(self.results, fo)
except Exception as e:
LOG.error(f"Can't serialize descriptors because of exception: {e}")
def load(self):
'''
Loads the results in serialized form, together with the MD5 signature
of the control class and the input file.
'''
if 'ext_input' in self.parameters and self.parameters['ext_input']:
return False
try:
picklfile = os.path.join(self.dest_path, 'data.pkl')
if not os.path.isfile(picklfile):
return False
with open(picklfile, 'rb') as fi:
md5_parameters = pickle.load(fi)
if md5_parameters != self.parameters['md5']:
return False
md5_input = pickle.load(fi)
if md5_input != utils.md5sum(self.ifile):
return False
self.results = pickle.load(fi)
# these values are added programatically and therefore not
# checked by the md5
self.results['meta']['endpoint'] = self.parameters['endpoint']
self.results['meta']['version'] = self.parameters['version']
except Exception as e:
LOG.error('Error loading pickle with exception: {}'.format(e))
return False
dest_path_pkl = os.path.join(self.dest_path, 'data.pkl')
LOG.info(f'Recycling data from {dest_path_pkl}')
return True
def load(self):
'''
Loads the results in serialized form, together with the MD5 signature
of the control class and the input file.
'''
if self.param.getVal('ext_input'):
return False
try:
picklfile = os.path.join(self.dest_path, 'data.pkl')
if not os.path.isfile(picklfile):
return False
with open(picklfile, 'rb') as fi:
md5_parameters = pickle.load(fi)
if md5_parameters != self.param.getVal('md5'):
return False
md5_input = pickle.load(fi)
if md5_input != utils.md5sum(self.ifile):
return False
success, message = self.conveyor.load(fi)
if not success:
LOG.error(
f'Failed to load pickle file with error: "{message}"')
return False
except Exception as e:
self.conveyor.setError(
'Error loading pickle with exception: {}'.format(e))
LOG.error('Error loading pickle with exception: {}'.format(e))
return False
LOG.info(f'Recycling data from {picklfile}')
return True
def loadYaml(self, model, version):
''' load a set of parameters from the configuration file present
at the model directory
adds some parameters identifying the model and the
hash of the configuration file
'''
# obtain the path and the default name of the model parameters
parameters_file_path = utils.model_path(model, version)
parameters_file_name = os.path.join(parameters_file_path,
'parameters.yaml')
# load the main class dictionary (p) from this yaml file
if not os.path.isfile(parameters_file_name):
return False, 'file not found'
try:
with open(parameters_file_name, 'r') as pfile:
self.p = yaml.safe_load(pfile)
except Exception as e:
return False, e
# check version of the parameter file
# no 'version' key mans version < 2.0
if 'param_format' in self.p:
self.extended = True
else:
self.extended = False
self.param_format = 1.0
# add keys for the model and a MD5 hash
self.setVal('endpoint', model)
self.setVal('version', version)
self.setVal('model_path', parameters_file_path)
self.setVal('md5', utils.md5sum(parameters_file_name))
return True, 'OK'
def __init__(self, model, version):
parameters_file_path = utils.model_path(model, version)
parameters_file_name = os.path.join(parameters_file_path,
'parameters.yaml')
if not os.path.isfile(parameters_file_name):
LOG.critical('Unable to load model parameters. Aborting...')
sys.exit()
try:
with open(parameters_file_name, 'r') as pfile:
self.p = yaml.load(pfile)
except Exception as e:
LOG.critical('Unable to load model parameters. Aborting...')
sys.exit()
self.setVal('endpoint', model)
self.setVal('version', version)
self.setVal('model_path', parameters_file_path)
self.setVal('md5', utils.md5sum(parameters_file_name))
return