def create_many_family_mock_event(files, parser=None): # TODO: this will break for matio mock_event = dict() fam_batch = FamilyBatch() family_id = None for file in files: if type(file) is dict: family_id = str(file['family_id']) file = file['filename'] test_fam_1 = Family() group_file_objs = [] base_path = file group_file_objs.append({'path': base_path, 'metadata': dict()}) test_fam_1.download_type = "LOCAL" test_fam_1.add_group(files=group_file_objs, parser=parser) if family_id is not None: test_fam_1.family_id = family_id fam_batch.add_family(test_fam_1) mock_event['family_batch'] = fam_batch return mock_event
def create_mock_event(files, parser=None): mock_event = dict() fam_batch = FamilyBatch() test_fam_1 = Family() group_file_objs = [] for file in files: base_path = file group_file_objs.append({'path': base_path, 'metadata': dict()}) test_fam_1.download_type = "LOCAL" test_fam_1.add_group(files=group_file_objs, parser=parser) fam_batch.add_family(test_fam_1) mock_event['family_batch'] = fam_batch return mock_event
# Execute the extractor on our family_batch. xtra.execute_extractions(family_batch=event['family_batch'], input_type=str) # All metadata are held in XtractAgent's memory. Flush to disk! xtra.flush_metadata_to_files(writer='json') return xtra.get_completion_stats() mock_event = dict() test_fam_1 = Family() test_fam_2 = Family() base_path = "/Users/tylerskluzacek/xtract-sdk/tests/xtract-tabular/tests/test_files" test_fam_1.add_group(files=[{ 'path': os.path.join(base_path, 'comma_delim'), 'metadata': dict() }], parser=None) test_fam_1.download_type = "LOCAL" print(test_fam_1.to_dict()) fam_batch = FamilyBatch() fam_batch.add_family(test_fam_1) mock_event['family_batch'] = fam_batch data = extract_tabular(mock_event) print(data)
family_2 = Family() family_1.add_group(files=[{ 'path': file_id, 'is_gdoc': True, 'mimeType': "text/csv" }], parser='xtract-tabular') family_1.base_url = "" family_2.add_group(files=[{ 'path': file_id2, 'is_gdoc': False }], parser='xtract-tabular') family_2.download_type = "GDRIVE" fam_batch = FamilyBatch() fam_batch.add_family(family_1) fam_batch.add_family(family_2) def test(event): import os return os.environ['container_version'] def main(fxc, ep_id): container_uuid = fxc.register_container('xtract-tabular.img', 'singularity') print("Container UUID: {}".format(container_uuid))