def create_many_family_mock_event(files, parser=None):
# TODO: this will break for matio
mock_event = dict()
fam_batch = FamilyBatch()
family_id = None
for file in files:
if type(file) is dict:
family_id = str(file['family_id'])
file = file['filename']
test_fam_1 = Family()
group_file_objs = []
base_path = file
group_file_objs.append({'path': base_path, 'metadata': dict()})
test_fam_1.download_type = "LOCAL"
test_fam_1.add_group(files=group_file_objs, parser=parser)
if family_id is not None:
test_fam_1.family_id = family_id
fam_batch.add_family(test_fam_1)
mock_event['family_batch'] = fam_batch
return mock_event
def create_mock_event(files, parser=None):
mock_event = dict()
fam_batch = FamilyBatch()
test_fam_1 = Family()
group_file_objs = []
for file in files:
base_path = file
group_file_objs.append({'path': base_path, 'metadata': dict()})
test_fam_1.download_type = "LOCAL"
test_fam_1.add_group(files=group_file_objs, parser=parser)
fam_batch.add_family(test_fam_1)
mock_event['family_batch'] = fam_batch
return mock_event
# Execute the extractor on our family_batch.
xtra.execute_extractions(family_batch=event['family_batch'],
input_type=str)
# All metadata are held in XtractAgent's memory. Flush to disk!
xtra.flush_metadata_to_files(writer='json')
return xtra.get_completion_stats()
mock_event = dict()
test_fam_1 = Family()
test_fam_2 = Family()
base_path = "/Users/tylerskluzacek/xtract-sdk/tests/xtract-tabular/tests/test_files"
test_fam_1.add_group(files=[{
'path': os.path.join(base_path, 'comma_delim'),
'metadata': dict()
}],
parser=None)
test_fam_1.download_type = "LOCAL"
print(test_fam_1.to_dict())
fam_batch = FamilyBatch()
fam_batch.add_family(test_fam_1)
mock_event['family_batch'] = fam_batch
data = extract_tabular(mock_event)
print(data)
family_2 = Family()
family_1.add_group(files=[{
'path': file_id,
'is_gdoc': True,
'mimeType': "text/csv"
}],
parser='xtract-tabular')
family_1.base_url = ""
family_2.add_group(files=[{
'path': file_id2,
'is_gdoc': False
}],
parser='xtract-tabular')
family_2.download_type = "GDRIVE"
fam_batch = FamilyBatch()
fam_batch.add_family(family_1)
fam_batch.add_family(family_2)
def test(event):
import os
return os.environ['container_version']
def main(fxc, ep_id):
container_uuid = fxc.register_container('xtract-tabular.img',
'singularity')
print("Container UUID: {}".format(container_uuid))