def test_guess_format_invalid_csv_format(self): indexer = Indexer("test", None) stream = StringIO.StringIO(IndexerTest.simpleCSVString) guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}}) guessed_format["fieldSeparator"] = "invalid separator" fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns'] assert_equal(fields, []) stream.seek(0) guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}}) guessed_format["recordSeparator"] = "invalid separator" fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns'] assert_equal(fields, []) stream.seek(0) guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}}) guessed_format["quoteChar"] = "invalid quoteChar" fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns'] assert_equal(fields, [])
def test_guess_format_invalid_csv_format(self): indexer = Indexer("test", None) stream = StringIO.StringIO(IndexerTest.simpleCSVString) guessed_format = indexer.guess_format( {'file': { "stream": stream, "name": "test.csv" }}) guessed_format["fieldSeparator"] = "invalid separator" fields = indexer.guess_field_types({ "file": { "stream": stream, "name": "test.csv" }, "format": guessed_format })['columns'] assert_equal(fields, []) stream.seek(0) guessed_format = indexer.guess_format( {'file': { "stream": stream, "name": "test.csv" }}) guessed_format["recordSeparator"] = "invalid separator" fields = indexer.guess_field_types({ "file": { "stream": stream, "name": "test.csv" }, "format": guessed_format })['columns'] assert_equal(fields, []) stream.seek(0) guessed_format = indexer.guess_format( {'file': { "stream": stream, "name": "test.csv" }}) guessed_format["quoteChar"] = "invalid quoteChar" fields = indexer.guess_field_types({ "file": { "stream": stream, "name": "test.csv" }, "format": guessed_format })['columns'] assert_equal(fields, [])
def guess_field_types(request): file_format = json.loads(request.POST.get('fileFormat', '{}')) if file_format['inputFormat'] == 'file': indexer = Indexer(request.user, request.fs) stream = request.fs.open(file_format["path"]) _convert_format(file_format["format"], inverse=True) format_ = indexer.guess_field_types({ "file": { "stream": stream, "name": file_format['path'] }, "format": file_format['format'] }) elif file_format['inputFormat'] == 'table': sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName']) db = dbms.get(request.user) table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName']) format_ = { "sample": sample['rows'][:4], "columns": [ Field(col.name, HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')).to_dict() for col in table_metadata.cols ] } elif file_format['inputFormat'] == 'query': #TODO get schema from explain query pass return JsonResponse(format_)
def test_guess_csv_format(self): stream = StringIO.StringIO(IndexerTest.simpleCSVString) indexer = Indexer("test", None) guessed_format = indexer.guess_format( {'file': { "stream": stream, "name": "test.csv" }}) fields = indexer.guess_field_types({ "file": { "stream": stream, "name": "test.csv" }, "format": guessed_format })['columns'] # test format expected_format = self.simpleCSVFormat assert_equal(expected_format, guessed_format) # test fields expected_fields = self.simpleCSVFields for expected, actual in zip(expected_fields, fields): for key in ("name", "type"): assert_equal(expected[key], actual[key])
def guess_field_types(request): file_format = json.loads(request.POST.get('fileFormat', '{}')) if file_format['inputFormat'] == 'file': indexer = Indexer(request.user, request.fs) stream = request.fs.open(file_format["path"]) _convert_format(file_format["format"], inverse=True) format_ = indexer.guess_field_types({ "file": { "stream": stream, "name": file_format['path'] }, "format": file_format['format'] }) elif file_format['inputFormat'] == 'table': sample = get_api(request, { 'type': 'hive' }).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName']) db = dbms.get(request.user) table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName']) format_ = { "sample": sample['rows'][:4], "columns": [ Field(col.name, HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')).to_dict() for col in table_metadata.cols ] } elif file_format[ 'inputFormat'] == 'query': # Only support open query history # TODO get schema from explain query, which is not possible notebook = Notebook(document=Document2.objects.get( id=file_format['query'])).get_data() snippet = notebook['snippets'][0] sample = get_api(request, snippet).fetch_result(notebook, snippet, 4, start_over=True) format_ = { "sample": sample['rows'][:4], "columns": [ Field( col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict() for col in sample.meta ] } return JsonResponse(format_)
def guess_field_types(request): file_format = json.loads(request.POST.get('fileFormat', '{}')) indexer = Indexer(request.user, request.fs) stream = request.fs.open(file_format["path"]) _convert_format(file_format["format"], inverse = True) format_ = indexer.guess_field_types({"file":stream, "format":file_format['format']}) return JsonResponse(format_)
def test_end_to_end(self): if not is_live_cluster(): raise SkipTest() cluster = shared_cluster() fs = cluster.fs collection_name = "test_collection" indexer = Indexer("test", fs=fs, jt=cluster.jt) input_loc = "/tmp/test.csv" # upload the test file to hdfs fs.create(input_loc, data=TestIndexer.simpleCSVString, overwrite=True) # open a filestream for the file on hdfs stream = fs.open(input_loc) # guess the format of the file file_type_format = indexer.guess_format( {'file': { "stream": stream, "name": "test.csv" }}) field_types = indexer.guess_field_types({ "file": { "stream": stream, "name": "test.csv" }, "format": file_type_format }) format_ = field_types.copy() format_['format'] = file_type_format # find a field name available to use for the record's uuid unique_field = indexer.get_unique_field(format_) is_unique_generated = indexer.is_unique_generated(format_) # generate morphline morphline = indexer.generate_morphline_config(collection_name, format_, unique_field) schema_fields = indexer.get_kept_field_list(format_['columns']) if is_unique_generated: schema_fields += [{"name": unique_field, "type": "string"}] # create the collection from the specified fields collection_manager = CollectionManagerController("test") if collection_manager.collection_exists(collection_name): collection_manager.delete_collection(collection_name, None) collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field) # index the file indexer.run_morphline(collection_name, morphline, input_loc)
def guess_field_types(request): file_format = json.loads(request.POST.get('fileFormat', '{}')) indexer = Indexer(request.user, request.fs) stream = request.fs.open(file_format["path"]) _convert_format(file_format["format"], inverse=True) format_ = indexer.guess_field_types({ "file": { "stream": stream, "name": file_format['path'] }, "format": file_format['format'] }) return JsonResponse(format_)
def test_end_to_end(self): if not is_live_cluster() or True: # Skipping as requires morplines libs to be setup raise SkipTest() cluster = shared_cluster() fs = cluster.fs make_logged_in_client(username="******", groupname="default", recreate=True, is_superuser=False) user = User.objects.get(username="******") collection_name = "test_collection" indexer = Indexer("test", fs=fs, jt=cluster.jt) input_loc = "/tmp/test.csv" # upload the test file to hdfs fs.create(input_loc, data=TestIndexer.simpleCSVString, overwrite=True) # open a filestream for the file on hdfs stream = fs.open(input_loc) # guess the format of the file file_type_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}}) field_types = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": file_type_format}) format_ = field_types.copy() format_['format'] = file_type_format # find a field name available to use for the record's uuid unique_field = indexer.get_unique_field(format_) is_unique_generated = indexer.is_unique_generated(format_) # generate morphline morphline = indexer.generate_morphline_config(collection_name, format_, unique_field) schema_fields = indexer.get_kept_field_list(format_['columns']) if is_unique_generated: schema_fields += [{"name": unique_field, "type": "string"}] # create the collection from the specified fields collection_manager = CollectionManagerController("test") if collection_manager.collection_exists(collection_name): collection_manager.delete_collection(collection_name, None) collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field) # index the file indexer.run_morphline(MockedRequest(user=user, fs=cluster.fs, jt=cluster.jt), collection_name, morphline, input_loc)
def test_guess_csv_format(self): stream = StringIO.StringIO(TestIndexer.simpleCSVString) indexer = Indexer("test") guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}}) fields = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns'] # test format expected_format = self.simpleCSVFormat assert_equal(expected_format, guessed_format) # test fields expected_fields = self.simpleCSVFields for expected, actual in zip(expected_fields, fields): for key in ("name", "type"): assert_equal(expected[key], actual[key])
def guess_field_types(request): file_format = json.loads(request.POST.get('fileFormat', '{}')) if file_format['inputFormat'] == 'file': indexer = Indexer(request.user, request.fs) stream = request.fs.open(file_format["path"]) _convert_format(file_format["format"], inverse=True) format_ = indexer.guess_field_types({ "file": { "stream": stream, "name": file_format['path'] }, "format": file_format['format'] }) elif file_format['inputFormat'] == 'table': sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName']) db = dbms.get(request.user) table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName']) format_ = { "sample": sample['rows'][:4], "columns": [ Field(col.name, HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')).to_dict() for col in table_metadata.cols ] } elif file_format['inputFormat'] == 'query': # Only support open query history # TODO get schema from explain query, which is not possible notebook = Notebook(document=Document2.objects.get(id=file_format['query'])).get_data() snippet = notebook['snippets'][0] sample = get_api(request, snippet).fetch_result(notebook, snippet, 4, start_over=True) format_ = { "sample": sample['rows'][:4], "sample_cols": sample.meta, "columns": [ Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict() for col in sample.meta ] } return JsonResponse(format_)
def test_guess_format(self): stream = StringIO.StringIO(IndexerTest.simpleCSVString) indexer = Indexer("test", None) guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}}) fields = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns'] # test format assert_equal('csv', guessed_format['type']) assert_equal(',', guessed_format['fieldSeparator']) assert_equal('\n', guessed_format['recordSeparator']) # test fields expected_fields = [ { "name": "id", "type": "long" }, { "name": "Rating", "type": "long" }, { "name": "Location", "type": "string" }, { "name": "Name", "type": "string" }, { "name": "Time", "type": "string" } ] for expected, actual in zip(expected_fields, fields): for key in ("name", "type"): assert_equal(expected[key], actual[key])
def test_end_to_end(self): fs = cluster.get_hdfs() collection_name = "test_collection" indexer = Indexer("test", fs) input_loc = "/tmp/test.csv" # upload the test file to hdfs fs.create(input_loc, data=IndexerTest.simpleCSVString, overwrite=True) # open a filestream for the file on hdfs stream = fs.open(input_loc) # guess the format of the file file_type_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}}) field_types = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": file_type_format}) format_ = field_types.copy() format_['format'] = file_type_format # find a field name available to use for the record's uuid unique_field = indexer.get_uuid_name(format_) # generate morphline morphline = indexer.generate_morphline_config(collection_name, format_, unique_field) schema_fields = [{"name": unique_field, "type": "string"}] + indexer.get_kept_field_list(format_['columns']) # create the collection from the specified fields collection_manager = CollectionManagerController("test") if collection_manager.collection_exists(collection_name): collection_manager.delete_collection(collection_name, None) collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field) # index the file indexer.run_morphline(collection_name, morphline, input_loc)