def test_model_content_many_files(self): slt_path = os.path.join(self.PARAMS['BASE_PATH'], self.SLT) slt_content = open(slt_path, 'r').read() glt_path = os.path.join(self.PARAMS['BASE_PATH'], self.GLT) glt_content = open(glt_path, 'r').read() engine._insert_input_files(self.PARAMS, self.job, True) [slt] = models.inputs4job(self.job.id, input_type="source_model_logic_tree") [glt] = models.inputs4job(self.job.id, input_type="gsim_logic_tree") self.assertEqual('xml', slt.model_content.content_type) self.assertEqual(slt_content, slt.model_content.raw_content) self.assertEqual('xml', glt.model_content.content_type) self.assertEqual(glt_content, glt.model_content.raw_content)
def test_inputs4job_with_wrong_path(self): # No input is returned. inp = models.Input(owner=self.job.owner, path=self.paths.next(), input_type="exposure", size=self.sizes.next()) inp.save() models.Input2job(oq_job=self.job, input=inp).save() self.assertEqual([], models.inputs4job(self.job.id, path="xyz"))
def ln_input2job(path, input_type): """Link identical or newly created input to the given job.""" digest = _file_digest(path) linked_inputs = inputs4job(job.id) if any(li.digest == digest and li.input_type == input_type for li in linked_inputs): return in_model = (_identical_input(input_type, digest, job.owner.id) if not force_inputs else None) if in_model is None: # Save the raw input file contents to the DB: model_content = ModelContent() with open(path, 'rb') as fh: model_content.raw_content = fh.read() # Try to guess the content type: model_content.content_type = _get_content_type(path) model_content.save() in_model = Input(path=path, input_type=input_type, owner=job.owner, size=os.path.getsize(path), digest=digest, model_content=model_content) in_model.save() # Make sure we don't link to the same input more than once. if in_model.id not in inputs_seen: inputs_seen.add(in_model.id) i2j = Input2job(input=in_model, oq_job=job) i2j.save() return in_model
def test_inputs4job_with_single_input(self): # The single input is returned. inp = models.Input(owner=self.job.owner, path=self.paths.next(), input_type="exposure", size=self.sizes.next()) inp.save() models.Input2job(oq_job=self.job, input=inp).save() self.assertEqual([inp], models.inputs4job(self.job.id))
def test_model_content_unknown_content_type(self): test_file = helpers.touch() params = dict(GMPE_LOGIC_TREE_FILE=test_file, BASE_PATH='/') engine._insert_input_files(params, self.job, True) [glt] = models.inputs4job(self.job.id, input_type="gsim_logic_tree") self.assertEqual('unknown', glt.model_content.content_type)
def test_model_content_single_file(self): # The contents of input files (such as logic trees, exposure models, # etc.) should be saved to the uiapi.model_content table. slt_path = os.path.join(self.PARAMS['BASE_PATH'], self.SLT) expected_content = open(slt_path, 'r').read() engine._insert_input_files(self.PARAMS, self.job, True) [slt] = models.inputs4job(self.job.id, input_type="source_model_logic_tree") self.assertEqual('xml', slt.model_content.content_type) self.assertEqual(expected_content, slt.model_content.raw_content)
def test_inputs4job_with_correct_path(self): # The exposure inputs are returned. inp1 = models.Input(owner=self.job.owner, path=self.paths.next(), input_type="exposure", size=self.sizes.next()) inp1.save() models.Input2job(oq_job=self.job, input=inp1).save() path = self.paths.next() inp2 = models.Input(owner=self.job.owner, path=path, input_type="rupture_model", size=self.sizes.next()) inp2.save() models.Input2job(oq_job=self.job, input=inp2).save() self.assertEqual([inp2], models.inputs4job(self.job.id, path=path))
def test_model_content_detect_content_type(self): # Test detection of the content type (using the file extension). test_file = helpers.touch(suffix=".html") # We use the gmpe logic tree as our test target because there is no # parsing required in the function under test. Thus, we can put # whatever test garbage we want in the file, or just use an empty file # (which is the case here). params = dict(GMPE_LOGIC_TREE_FILE=test_file, BASE_PATH='/') engine._insert_input_files(params, self.job, True) [glt] = models.inputs4job(self.job.id, input_type="gsim_logic_tree") self.assertEqual('html', glt.model_content.content_type)
def test_inputs4job_with_correct_input_type_and_path(self): # The source inputs are returned. inp1 = models.Input(owner=self.job.owner, path=self.paths.next(), input_type="source", size=self.sizes.next()) inp1.save() models.Input2job(oq_job=self.job, input=inp1).save() path = self.paths.next() inp2 = models.Input(owner=self.job.owner, path=path, input_type="source", size=self.sizes.next()) inp2.save() models.Input2job(oq_job=self.job, input=inp2).save() inp3 = models.Input(owner=self.job.owner, path=self.paths.next(), input_type="source", size=self.sizes.next()) inp3.save() models.Input2job(oq_job=self.job, input=inp3).save() self.assertEqual([inp2], models.inputs4job(self.job.id, input_type="source", path=path))
def test_inputs4job_with_correct_input_type(self): # The exposure inputs are returned. inp1 = models.Input(owner=self.job.owner, path=self.paths.next(), input_type="exposure", size=self.sizes.next()) inp1.save() models.Input2job(oq_job=self.job, input=inp1).save() inp2 = models.Input( owner=self.job.owner, path=self.paths.next(), input_type="rupture_model", size=self.sizes.next() ) inp2.save() models.Input2job(oq_job=self.job, input=inp2).save() inp3 = models.Input(owner=self.job.owner, path=self.paths.next(), input_type="exposure", size=self.sizes.next()) inp3.save() models.Input2job(oq_job=self.job, input=inp3).save() actual = sorted(models.inputs4job(self.job.id, input_type="exposure"), key=lambda input: input.id) self.assertEqual([inp1, inp3], actual)
def read_sites_from_exposure(job_ctxt): """Given a :class:`JobContext` object, get all of the sites in the exposure model which are contained by the region of interest (defined in the `JobContext`). It is assumed that exposure model is already loaded into the database. :param job_ctxt: :class:`JobContext` instance. :returns: `list` of :class:`openquake.engine.shapes.Site` objects, with no duplicates """ em_inputs = inputs4job(job_ctxt.job_id, input_type="exposure") exp_points = ExposureData.objects.filter( exposure_model__input__id__in=[em.id for em in em_inputs], site__contained=job_ctxt.oq_job_profile.region).values( 'site').distinct() sites = [shapes.Site(p['site'].x, p['site'].y) for p in exp_points] return sites
def teardown_job(cls, job, filesystem_only=True): """ Tear down the file system (and potentially db) artefacts for the given job. :param job: a :py:class:`db.models.OqJob` instance :param bool filesystem_only: if set the oq_job/oq_param/ input database records will be left intact. This saves time and the test db will be dropped/recreated prior to the next db test suite run anyway. """ DbTestCase.teardown_inputs(models.inputs4job(job.id), filesystem_only=filesystem_only) if filesystem_only: return job.delete() try: oqjp = models.profile4job(job.id) oqjp.delete() except ValueError: # no job profile for this job pass
def test_inputs4job_with_no_input(self): # No inputs exist, an empty list is returned. self.assertEqual([], models.inputs4job(self.job.id))
def _get_inputs(self, job): inputs = [dict(path=i.path, type=i.input_type) for i in models.inputs4job(self.job.id)] return sorted(inputs, key=lambda i: (i["type"], i["path"]))