def index_file(request):
file_format = json.loads(request.POST.get('fileFormat', '{}'))
_convert_format(file_format["format"], inverse = True)
collection_name = file_format["name"]
indexer = Indexer(request.user, request.fs)
unique_field = indexer.get_uuid_name(file_format)
schema_fields = [{"name": unique_field, "type": "string"}] + \
indexer.get_kept_field_list(file_format['columns'])
morphline = indexer.generate_morphline_config(collection_name, file_format, unique_field)
collection_manager = CollectionManagerController(request.user)
if not collection_manager.collection_exists(collection_name):
collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field)
job_id = indexer.run_morphline(collection_name, morphline, file_format["path"])
return JsonResponse({"jobId": job_id})
def test_end_to_end(self):
fs = cluster.get_hdfs()
collection_name = "test_collection"
indexer = Indexer("test", fs)
input_loc = "/tmp/test.csv"
# upload the test file to hdfs
fs.create(input_loc, data=IndexerTest.simpleCSVString, overwrite=True)
# open a filestream for the file on hdfs
stream = fs.open(input_loc)
# guess the format of the file
file_type_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})
field_types = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": file_type_format})
format_ = field_types.copy()
format_['format'] = file_type_format
# find a field name available to use for the record's uuid
unique_field = indexer.get_uuid_name(format_)
# generate morphline
morphline = indexer.generate_morphline_config(collection_name, format_, unique_field)
schema_fields = [{"name": unique_field, "type": "string"}] + indexer.get_kept_field_list(format_['columns'])
# create the collection from the specified fields
collection_manager = CollectionManagerController("test")
if collection_manager.collection_exists(collection_name):
collection_manager.delete_collection(collection_name, None)
collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field)
# index the file
indexer.run_morphline(collection_name, morphline, input_loc)
