def gzip_inputs(self): """ Gzipping the inputs and saving them in the datastore """ logging.info('gzipping the input files') fnames = readinput.get_input_files(self.oqparam) self.datastore.store_files(fnames)
def zip_job(job_ini, archive_zip='', risk_ini='', oq=None, log=logging.info): """ Zip the given job.ini file into the given archive, together with all related files. """ if not os.path.exists(job_ini): sys.exit('%s does not exist' % job_ini) archive_zip = archive_zip or 'job.zip' if isinstance(archive_zip, str): # actually it should be path-like if not archive_zip.endswith('.zip'): sys.exit('%s does not end with .zip' % archive_zip) if os.path.exists(archive_zip): sys.exit('%s exists already' % archive_zip) # do not validate to avoid permissions error on the export_dir oq = oq or oqvalidation.OqParam(**readinput.get_params(job_ini)) if risk_ini: risk_ini = os.path.normpath(os.path.abspath(risk_ini)) oqr = readinput.get_oqparam(risk_ini) del oqr.inputs['job_ini'] oq.inputs.update(oqr.inputs) oq.shakemap_uri.update(oqr.shakemap_uri) files = readinput.get_input_files(oq) if risk_ini: files = [risk_ini] + files return general.zipfiles(files, archive_zip, log=log)
def read_inputs(self): """ Read risk data and sources if any """ oq = self.oqparam self._read_risk_data() self.check_overflow() # check if self.sitecol is too large if getattr(self, 'sitecol', None): # can be None for the ruptures-only calculator with hdf5.File(self.datastore.tempname, 'w') as tmp: tmp['sitecol'] = self.sitecol if ('source_model_logic_tree' in oq.inputs and oq.hazard_calculation_id is None): with self.monitor('composite source model', measuremem=True): self.csm = csm = readinput.get_composite_source_model( oq, self.datastore.hdf5) self.csm_info = csm.info self.datastore['source_model_lt'] = csm.source_model_lt res = views.view('dupl_sources', self.datastore) logging.info(f'The composite source model has {res.val:_d} ' 'ruptures') if res: logging.info(res) self.init() # do this at the end of pre-execute if not oq.hazard_calculation_id: logging.info('gzipping the input files') fnames = readinput.get_input_files(oq) self.datastore.store_files(fnames)
def zip_source_model(ssmLT, archive_zip='', log=logging.info): """ Zip the source model files starting from the smmLT.xml file """ basedir = os.path.dirname(ssmLT) if os.path.basename(ssmLT) != 'ssmLT.xml': orig = ssmLT ssmLT = os.path.join(basedir, 'ssmLT.xml') with open(ssmLT, 'wb') as f: f.write(open(orig, 'rb').read()) archive_zip = archive_zip or os.path.join(basedir, 'ssmLT.zip') if os.path.exists(archive_zip): sys.exit('%s exists already' % archive_zip) info = logictree.collect_info(ssmLT) files = info.h5paths + info.smpaths oq = mock.Mock(inputs={'source_model_logic_tree': ssmLT}, random_seed=42, number_of_logic_tree_samples=0, sampling_method='early_weights') oq._input_files = readinput.get_input_files(oq) checksum = readinput.get_checksum32(oq) checkfile = os.path.join(os.path.dirname(ssmLT), 'CHECKSUM.txt') with open(checkfile, 'w') as f: f.write(str(checksum)) files.extend([os.path.abspath(ssmLT), os.path.abspath(checkfile)]) general.zipfiles(files, archive_zip, log=log, cleanup=True) return archive_zip
def get_calc(self, testfile, job_ini, **kw): """ Return the outputs of the calculation as a dictionary """ self.testdir = os.path.dirname(testfile) if os.path.isfile(testfile) \ else testfile params = readinput.get_params(os.path.join(self.testdir, job_ini), kw) oqvalidation.OqParam.calculation_mode.validator.choices = tuple( base.calculators) oq = oqvalidation.OqParam(**params) oq._input_files = readinput.get_input_files(oq) oq.validate() # change this when debugging the test log = logs.init('calc', params) return base.calculators(oq, log.calc_id)
def zip_job(job_ini, archive_zip='', risk_ini='', oq=None, log=logging.info): """ Zip the given job.ini file into the given archive, together with all related files. """ if not os.path.exists(job_ini): sys.exit('%s does not exist' % job_ini) archive_zip = archive_zip or 'job.zip' if isinstance(archive_zip, str): # actually it should be path-like if not archive_zip.endswith('.zip'): sys.exit('%s does not end with .zip' % archive_zip) if os.path.exists(archive_zip): sys.exit('%s exists already' % archive_zip) # do not validate to avoid permissions error on the export_dir oq = oq or readinput.get_oqparam(job_ini, validate=False) if risk_ini: risk_ini = os.path.normpath(os.path.abspath(risk_ini)) risk_inputs = readinput.get_params([risk_ini])['inputs'] del risk_inputs['job_ini'] oq.inputs.update(risk_inputs) files = readinput.get_input_files(oq) if risk_ini: files = [risk_ini] + files return general.zipfiles(files, archive_zip, log=log)