def _insert_input_files(params, job, force_inputs): """Create uiapi.input records for all input files :param dict params: The job config params. :param job: The :class:`openquake.db.models.OqJob` instance to use :param bool force_inputs: If `True` the model input files will be parsed and the resulting content written to the database no matter what. """ inputs_seen = [] def ln_input2job(job, path, input_type): """Link identical or newly created input to the given job.""" digest = _file_digest(path) linked_inputs = inputs4job(job.id) if any(li.digest == digest and li.input_type == input_type for li in linked_inputs): return in_model = (_identical_input(input_type, digest) if not force_inputs else None) if in_model is None: in_model = Input(path=path, input_type=input_type, owner=job.owner, size=os.path.getsize(path), digest=digest) in_model.save() # Make sure we don't link to the same input more than once. if in_model.id in inputs_seen: return else: inputs_seen.append(in_model.id) i2j = Input2job(input=in_model, oq_job=job) i2j.save() # insert input files in input table for param_key, file_type in INPUT_FILE_TYPES.items(): if param_key not in params: continue path = params[param_key] ln_input2job(job, path, file_type) # insert source models referenced in the logic tree if 'SOURCE_MODEL_LOGIC_TREE_FILE' in params: for path in _get_source_models(params['SOURCE_MODEL_LOGIC_TREE_FILE']): ln_input2job(job, path, "source")
def _insert_input_files(params, input_set): """Create uiapi.input records for all input files""" # insert input files in input table for param_key, file_type in INPUT_FILE_TYPES.items(): if param_key not in params: continue path = params[param_key] in_model = Input(input_set=input_set, path=path, input_type=file_type, size=os.path.getsize(path)) in_model.save() # insert soft-linked source models in input table if 'SOURCE_MODEL_LOGIC_TREE_FILE' in params: for path in get_source_models(params['SOURCE_MODEL_LOGIC_TREE_FILE']): in_model = Input(input_set=input_set, path=path, input_type='source', size=os.path.getsize(path)) in_model.save()
def _insert_input_files(params, job, force_inputs): """Create uiapi.input records for all input files :param dict params: The job config params. :param job: The :class:`openquake.db.models.OqJob` instance to use :param bool force_inputs: If `True` the model input files will be parsed and the resulting content written to the database no matter what. """ inputs_seen = set() def ln_input2job(path, input_type): """Link identical or newly created input to the given job.""" digest = _file_digest(path) linked_inputs = inputs4job(job.id) if any(li.digest == digest and li.input_type == input_type for li in linked_inputs): return in_model = (_identical_input(input_type, digest, job.owner.id) if not force_inputs else None) if in_model is None: # Save the raw input file contents to the DB: model_content = ModelContent() with open(path, 'rb') as fh: model_content.raw_content = fh.read() # Try to guess the content type: model_content.content_type = _get_content_type(path) model_content.save() in_model = Input(path=path, input_type=input_type, owner=job.owner, size=os.path.getsize(path), digest=digest, model_content=model_content) in_model.save() # Make sure we don't link to the same input more than once. if in_model.id not in inputs_seen: inputs_seen.add(in_model.id) i2j = Input2job(input=in_model, oq_job=job) i2j.save() return in_model # insert input files in input table for param_key, file_type in INPUT_FILE_TYPES.items(): if param_key not in params: continue path = os.path.join(params['BASE_PATH'], params[param_key]) input_obj = ln_input2job(path, file_type) if file_type == "lt_source": # collect relative paths to source model files # that are referenced by the logic tree source_models = logictree.read_logic_trees( params['BASE_PATH'], params['SOURCE_MODEL_LOGIC_TREE_FILE'], params['GMPE_LOGIC_TREE_FILE'] ) # insert source model files for srcrelpath in source_models: abspath = os.path.join(params['BASE_PATH'], srcrelpath) hzrd_src_input = ln_input2job(abspath, "source") Src2ltsrc.objects.get_or_create( hzrd_src=hzrd_src_input, lt_src=input_obj, defaults={'filename': srcrelpath} )
def _insert_input_files(params, job, force_inputs): """Create uiapi.input records for all input files :param dict params: The job config params. :param job: The :class:`openquake.db.models.OqJob` instance to use :param bool force_inputs: If `True` the model input files will be parsed and the resulting content written to the database no matter what. """ inputs_seen = [] def ln_input2job(path, input_type): """Link identical or newly created input to the given job.""" digest = _file_digest(path) linked_inputs = inputs4job(job.id) if any(li.digest == digest and li.input_type == input_type for li in linked_inputs): return in_model = (_identical_input(input_type, digest, job.owner.id) if not force_inputs else None) if in_model is None: # Save the raw input file contents to the DB: model_content = ModelContent() with open(path, 'rb') as fh: model_content.raw_content = fh.read() # Try to guess the content type: model_content.content_type = _get_content_type(path) model_content.save() in_model = Input(path=path, input_type=input_type, owner=job.owner, size=os.path.getsize(path), digest=digest, model_content=model_content) in_model.save() # Make sure we don't link to the same input more than once. if in_model.id not in inputs_seen: inputs_seen.append(in_model.id) i2j = Input2job(input=in_model, oq_job=job) i2j.save() return in_model def _insert_referenced_sources(lt_src_input, lt_src_path): """ :param lt_src_input: an `:class:openquake.db.models.Input` of type "lt_source" :param str lt_src_path: path to the current logic tree source """ # insert source models referenced in the logic tree for path in _get_source_models(lt_src_path): hzrd_src_input = ln_input2job(path, "source") one_or_none = Src2ltsrc.objects.filter(hzrd_src=hzrd_src_input, lt_src=lt_src_input) if len(one_or_none) == 0: # No link table entry found, create one. src_link = Src2ltsrc( hzrd_src=hzrd_src_input, lt_src=lt_src_input, filename=os.path.basename(path)) src_link.save() # insert input files in input table for param_key, file_type in INPUT_FILE_TYPES.items(): if param_key not in params: continue path = params[param_key] input_obj = ln_input2job(path, file_type) if file_type == "lt_source": _insert_referenced_sources(input_obj, path)
def _insert_input_files(params, job, force_inputs): """Create uiapi.input records for all input files :param dict params: The job config params. :param job: The :class:`openquake.db.models.OqJob` instance to use :param bool force_inputs: If `True` the model input files will be parsed and the resulting content written to the database no matter what. """ inputs_seen = [] def ln_input2job(path, input_type): """Link identical or newly created input to the given job.""" digest = _file_digest(path) linked_inputs = inputs4job(job.id) if any(li.digest == digest and li.input_type == input_type for li in linked_inputs): return in_model = (_identical_input(input_type, digest, job.owner.id) if not force_inputs else None) if in_model is None: # Save the raw input file contents to the DB: model_content = ModelContent() with open(path, 'rb') as fh: model_content.raw_content = fh.read() # Try to guess the content type: model_content.content_type = _get_content_type(path) model_content.save() in_model = Input(path=path, input_type=input_type, owner=job.owner, size=os.path.getsize(path), digest=digest, model_content=model_content) in_model.save() # Make sure we don't link to the same input more than once. if in_model.id not in inputs_seen: inputs_seen.append(in_model.id) i2j = Input2job(input=in_model, oq_job=job) i2j.save() return in_model def _insert_referenced_sources(lt_src_input, lt_src_path): """ :param lt_src_input: an `:class:openquake.db.models.Input` of type "lt_source" :param str lt_src_path: path to the current logic tree source """ # insert source models referenced in the logic tree for path in _get_source_models(lt_src_path): hzrd_src_input = ln_input2job(path, "source") one_or_none = Src2ltsrc.objects.filter(hzrd_src=hzrd_src_input, lt_src=lt_src_input) if len(one_or_none) == 0: # No link table entry found, create one. src_link = Src2ltsrc(hzrd_src=hzrd_src_input, lt_src=lt_src_input, filename=os.path.basename(path)) src_link.save() # insert input files in input table for param_key, file_type in INPUT_FILE_TYPES.items(): if param_key not in params: continue path = params[param_key] input_obj = ln_input2job(path, file_type) if file_type == "lt_source": _insert_referenced_sources(input_obj, path)