def __init__(self): self.user = User.objects.create(username='******', password='******', is_superuser=True) self.org = Organization.objects.create(name="Test Organization") self.record_group = RecordGroup.objects.create( organization=self.org, name="Test Record Group") self.job = Job.objects.create(record_group=self.record_group, user=self.user, job_type="HarvestJob", job_details='{"test_key": "test value"}', name="Test Job") self.downstream_job = Job.objects.create( record_group=self.record_group, user=self.user, job_type="TransformJob", job_details='{"test_key": "test value"}', name="Test Transform Job") JobInput.objects.create(job=self.downstream_job, input_job=self.job) # TODO: the test framework should be clearing all the dbs, not just mysql old_records = Record.objects(job_id=self.job.id) for record in old_records: record.delete() self.record = Record.objects.create(job_id=self.job.id, record_id='testrecord', document=TEST_DOCUMENT) self.job.update_record_count()
def job_details(request, org_id, record_group_id, job_id): LOGGER.debug('details for job id: %s', job_id) # get CombineJob cjob = CombineJob.get_combine_job(job_id) # update status cjob.job.update_status() # detailed record count record_count_details = cjob.job.get_detailed_job_record_count() # get job lineage job_lineage = cjob.job.get_lineage() # get dpla_bulk_data_match dpla_bulk_data_matches = cjob.job.get_dpla_bulk_data_matches() # check if limiting to one, pre-existing record get_q = request.GET.get('q', None) # job details and job type specific augment job_detail = cjob.job.job_details_dict # mapped field analysis, generate if not part of job_details if 'mapped_field_analysis' in job_detail.keys(): field_counts = job_detail['mapped_field_analysis'] else: if cjob.job.finished: field_counts = cjob.count_indexed_fields() cjob.job.update_job_details( {'mapped_field_analysis': field_counts}, save=True) else: LOGGER.debug('job not finished, not setting') field_counts = {} # TODO: What is this accomplishing? # OAI Harvest if isinstance(cjob, HarvestOAIJob): pass # Static Harvest elif isinstance(cjob, HarvestStaticXMLJob): pass # Transform elif isinstance(cjob, TransformJob): pass # Merge/Duplicate elif isinstance(cjob, MergeJob): pass # Analysis elif isinstance(cjob, AnalysisJob): pass # get published records, primarily for published sets pub_records = PublishedRecords() oai_sets = Record.objects(job_id=cjob.job.id).item_frequencies(field='oai_set') # get published subsets with PublishedRecords static method published_subsets = PublishedRecords.get_subsets() # loop through subsets and enrich for _ in published_subsets: # add counts counts = mc_handle.combine.misc.find_one( {'_id': 'published_field_counts_%s' % _['name']}) # if counts not yet calculated, do now if counts is None: counts = PublishedRecords( subset=_['name']).count_indexed_fields() _['counts'] = counts # get field mappers field_mappers = FieldMapper.objects.all() # return return render(request, 'core/job_details.html', { 'cjob': cjob, 'record_group': cjob.job.record_group, 'record_count_details': record_count_details, 'field_counts': field_counts, 'field_mappers': field_mappers, 'xml2kvp_handle': xml2kvp.XML2kvp(), 'job_lineage_json': json.dumps(job_lineage), 'dpla_bulk_data_matches': dpla_bulk_data_matches, 'q': get_q, 'job_details': job_detail, 'pr': pub_records, 'published_subsets': published_subsets, 'es_index_str': cjob.esi.es_index_str, 'breadcrumbs': breadcrumb_parser(request), 'oai_sets': dict(oai_sets) })