def _insert_input_files(params, job, force_inputs):
"""Create uiapi.input records for all input files
:param dict params: The job config params.
:param job: The :class:`openquake.db.models.OqJob` instance to use
:param bool force_inputs: If `True` the model input files will be parsed
and the resulting content written to the database no matter what.
"""
inputs_seen = []
def ln_input2job(job, path, input_type):
"""Link identical or newly created input to the given job."""
digest = _file_digest(path)
linked_inputs = inputs4job(job.id)
if any(li.digest == digest and li.input_type == input_type
for li in linked_inputs):
return
in_model = (_identical_input(input_type, digest)
if not force_inputs else None)
if in_model is None:
in_model = Input(path=path, input_type=input_type, owner=job.owner,
size=os.path.getsize(path), digest=digest)
in_model.save()
# Make sure we don't link to the same input more than once.
if in_model.id in inputs_seen:
return
else:
inputs_seen.append(in_model.id)
i2j = Input2job(input=in_model, oq_job=job)
i2j.save()
# insert input files in input table
for param_key, file_type in INPUT_FILE_TYPES.items():
if param_key not in params:
continue
path = params[param_key]
ln_input2job(job, path, file_type)
# insert source models referenced in the logic tree
if 'SOURCE_MODEL_LOGIC_TREE_FILE' in params:
for path in _get_source_models(params['SOURCE_MODEL_LOGIC_TREE_FILE']):
ln_input2job(job, path, "source")
def _insert_input_files(params, input_set):
"""Create uiapi.input records for all input files"""
# insert input files in input table
for param_key, file_type in INPUT_FILE_TYPES.items():
if param_key not in params:
continue
path = params[param_key]
in_model = Input(input_set=input_set, path=path,
input_type=file_type, size=os.path.getsize(path))
in_model.save()
# insert soft-linked source models in input table
if 'SOURCE_MODEL_LOGIC_TREE_FILE' in params:
for path in get_source_models(params['SOURCE_MODEL_LOGIC_TREE_FILE']):
in_model = Input(input_set=input_set, path=path,
input_type='source', size=os.path.getsize(path))
in_model.save()
def _insert_input_files(params, job, force_inputs):
"""Create uiapi.input records for all input files
:param dict params: The job config params.
:param job: The :class:`openquake.db.models.OqJob` instance to use
:param bool force_inputs: If `True` the model input files will be parsed
and the resulting content written to the database no matter what.
"""
inputs_seen = set()
def ln_input2job(path, input_type):
"""Link identical or newly created input to the given job."""
digest = _file_digest(path)
linked_inputs = inputs4job(job.id)
if any(li.digest == digest and li.input_type == input_type
for li in linked_inputs):
return
in_model = (_identical_input(input_type, digest, job.owner.id)
if not force_inputs else None)
if in_model is None:
# Save the raw input file contents to the DB:
model_content = ModelContent()
with open(path, 'rb') as fh:
model_content.raw_content = fh.read()
# Try to guess the content type:
model_content.content_type = _get_content_type(path)
model_content.save()
in_model = Input(path=path, input_type=input_type, owner=job.owner,
size=os.path.getsize(path), digest=digest,
model_content=model_content)
in_model.save()
# Make sure we don't link to the same input more than once.
if in_model.id not in inputs_seen:
inputs_seen.add(in_model.id)
i2j = Input2job(input=in_model, oq_job=job)
i2j.save()
return in_model
# insert input files in input table
for param_key, file_type in INPUT_FILE_TYPES.items():
if param_key not in params:
continue
path = os.path.join(params['BASE_PATH'], params[param_key])
input_obj = ln_input2job(path, file_type)
if file_type == "lt_source":
# collect relative paths to source model files
# that are referenced by the logic tree
source_models = logictree.read_logic_trees(
params['BASE_PATH'], params['SOURCE_MODEL_LOGIC_TREE_FILE'],
params['GMPE_LOGIC_TREE_FILE']
)
# insert source model files
for srcrelpath in source_models:
abspath = os.path.join(params['BASE_PATH'], srcrelpath)
hzrd_src_input = ln_input2job(abspath, "source")
Src2ltsrc.objects.get_or_create(
hzrd_src=hzrd_src_input, lt_src=input_obj,
defaults={'filename': srcrelpath}
)
def _insert_input_files(params, job, force_inputs):
"""Create uiapi.input records for all input files
:param dict params: The job config params.
:param job: The :class:`openquake.db.models.OqJob` instance to use
:param bool force_inputs: If `True` the model input files will be parsed
and the resulting content written to the database no matter what.
"""
inputs_seen = []
def ln_input2job(path, input_type):
"""Link identical or newly created input to the given job."""
digest = _file_digest(path)
linked_inputs = inputs4job(job.id)
if any(li.digest == digest and li.input_type == input_type
for li in linked_inputs):
return
in_model = (_identical_input(input_type, digest, job.owner.id)
if not force_inputs else None)
if in_model is None:
# Save the raw input file contents to the DB:
model_content = ModelContent()
with open(path, 'rb') as fh:
model_content.raw_content = fh.read()
# Try to guess the content type:
model_content.content_type = _get_content_type(path)
model_content.save()
in_model = Input(path=path, input_type=input_type, owner=job.owner,
size=os.path.getsize(path), digest=digest,
model_content=model_content)
in_model.save()
# Make sure we don't link to the same input more than once.
if in_model.id not in inputs_seen:
inputs_seen.append(in_model.id)
i2j = Input2job(input=in_model, oq_job=job)
i2j.save()
return in_model
def _insert_referenced_sources(lt_src_input, lt_src_path):
"""
:param lt_src_input: an `:class:openquake.db.models.Input` of type
"lt_source"
:param str lt_src_path: path to the current logic tree source
"""
# insert source models referenced in the logic tree
for path in _get_source_models(lt_src_path):
hzrd_src_input = ln_input2job(path, "source")
one_or_none = Src2ltsrc.objects.filter(hzrd_src=hzrd_src_input,
lt_src=lt_src_input)
if len(one_or_none) == 0:
# No link table entry found, create one.
src_link = Src2ltsrc(
hzrd_src=hzrd_src_input, lt_src=lt_src_input,
filename=os.path.basename(path))
src_link.save()
# insert input files in input table
for param_key, file_type in INPUT_FILE_TYPES.items():
if param_key not in params:
continue
path = params[param_key]
input_obj = ln_input2job(path, file_type)
if file_type == "lt_source":
_insert_referenced_sources(input_obj, path)
def _insert_input_files(params, job, force_inputs):
"""Create uiapi.input records for all input files
:param dict params: The job config params.
:param job: The :class:`openquake.db.models.OqJob` instance to use
:param bool force_inputs: If `True` the model input files will be parsed
and the resulting content written to the database no matter what.
"""
inputs_seen = []
def ln_input2job(path, input_type):
"""Link identical or newly created input to the given job."""
digest = _file_digest(path)
linked_inputs = inputs4job(job.id)
if any(li.digest == digest and li.input_type == input_type
for li in linked_inputs):
return
in_model = (_identical_input(input_type, digest, job.owner.id)
if not force_inputs else None)
if in_model is None:
# Save the raw input file contents to the DB:
model_content = ModelContent()
with open(path, 'rb') as fh:
model_content.raw_content = fh.read()
# Try to guess the content type:
model_content.content_type = _get_content_type(path)
model_content.save()
in_model = Input(path=path,
input_type=input_type,
owner=job.owner,
size=os.path.getsize(path),
digest=digest,
model_content=model_content)
in_model.save()
# Make sure we don't link to the same input more than once.
if in_model.id not in inputs_seen:
inputs_seen.append(in_model.id)
i2j = Input2job(input=in_model, oq_job=job)
i2j.save()
return in_model
def _insert_referenced_sources(lt_src_input, lt_src_path):
"""
:param lt_src_input: an `:class:openquake.db.models.Input` of type
"lt_source"
:param str lt_src_path: path to the current logic tree source
"""
# insert source models referenced in the logic tree
for path in _get_source_models(lt_src_path):
hzrd_src_input = ln_input2job(path, "source")
one_or_none = Src2ltsrc.objects.filter(hzrd_src=hzrd_src_input,
lt_src=lt_src_input)
if len(one_or_none) == 0:
# No link table entry found, create one.
src_link = Src2ltsrc(hzrd_src=hzrd_src_input,
lt_src=lt_src_input,
filename=os.path.basename(path))
src_link.save()
# insert input files in input table
for param_key, file_type in INPUT_FILE_TYPES.items():
if param_key not in params:
continue
path = params[param_key]
input_obj = ln_input2job(path, file_type)
if file_type == "lt_source":
_insert_referenced_sources(input_obj, path)
