def test_guess_format_invalid_csv_format(self):
indexer = MorphlineIndexer("test", solr_client=self.solr_client)
stream = StringIO.StringIO(TestIndexer.simpleCSVString)
guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})
guessed_format["fieldSeparator"] = "invalid separator"
fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
assert_equal(fields, [])
stream.seek(0)
guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})
guessed_format["recordSeparator"] = "invalid separator"
fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
assert_equal(fields, [])
stream.seek(0)
guessed_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})
guessed_format["quoteChar"] = "invalid quoteChar"
fields = indexer.guess_field_types({"file": {"stream": stream, "name": "test.csv"}, "format": guessed_format})['columns']
assert_equal(fields, [])
def test_guess_csv_format(self):
stream = StringIO.StringIO(TestIndexer.simpleCSVString)
indexer = MorphlineIndexer("test", solr_client=self.solr_client)
guessed_format = indexer.guess_format(
{'file': {
"stream": stream,
"name": "test.csv"
}})
fields = indexer.guess_field_types({
"file": {
"stream": stream,
"name": "test.csv"
},
"format": guessed_format
})['columns']
# test format
expected_format = self.simpleCSVFormat
assert_equal(expected_format, guessed_format)
# test fields
expected_fields = self.simpleCSVFields
for expected, actual in zip(expected_fields, fields):
for key in ("name", "type"):
assert_equal(expected[key], actual[key])
def guess_field_types(request):
file_format = json.loads(request.POST.get('fileFormat', '{}'))
if file_format['inputFormat'] == 'file':
indexer = MorphlineIndexer(request.user, request.fs)
stream = request.fs.open(file_format["path"])
_convert_format(file_format["format"], inverse=True)
format_ = indexer.guess_field_types({
"file": {
"stream": stream,
"name": file_format['path']
},
"format": file_format['format']
})
elif file_format['inputFormat'] == 'table':
sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName'])
db = dbms.get(request.user)
table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName'])
format_ = {
"sample": sample['rows'][:4],
"columns": [
Field(col.name, HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')).to_dict()
for col in table_metadata.cols
]
}
elif file_format['inputFormat'] == 'query': # Only support open query history
# TODO get schema from explain query, which is not possible
notebook = Notebook(document=Document2.objects.get(id=file_format['query'])).get_data()
snippet = notebook['snippets'][0]
sample = get_api(request, snippet).fetch_result(notebook, snippet, 4, start_over=True)
format_ = {
"sample": sample['rows'][:4],
"sample_cols": sample.meta,
"columns": [
Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
for col in sample.meta
]
}
elif file_format['inputFormat'] == 'rdbms':
query_server = rdbms.get_query_server_config(server=file_format['rdbmsType'])
db = rdbms.get(request.user, query_server=query_server)
sample = RdbmsIndexer(request.user, file_format['rdbmsType']).get_sample_data(mode=file_format['rdbmsMode'], database=file_format['rdbmsDatabaseName'], table=file_format['rdbmsTableName'])
table_metadata = db.get_columns(file_format['rdbmsDatabaseName'], file_format['rdbmsTableName'], names_only=False)
format_ = {
"sample": list(sample['rows'])[:4],
"columns": [
Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
for col in table_metadata
]
}
return JsonResponse(format_)
def guess_field_types(request):
file_format = json.loads(request.POST.get('fileFormat', '{}'))
if file_format['inputFormat'] == 'file':
indexer = MorphlineIndexer(request.user, request.fs)
stream = request.fs.open(file_format["path"])
_convert_format(file_format["format"], inverse=True)
format_ = indexer.guess_field_types({
"file": {
"stream": stream,
"name": file_format['path']
},
"format": file_format['format']
})
elif file_format['inputFormat'] == 'table':
sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName'])
db = dbms.get(request.user)
table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName'])
format_ = {
"sample": sample['rows'][:4],
"columns": [
Field(col.name, HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')).to_dict()
for col in table_metadata.cols
]
}
elif file_format['inputFormat'] == 'query':
query_id = file_format['query']['id'] if file_format['query'].get('id') else file_format['query']
notebook = Notebook(document=Document2.objects.document(user=request.user, doc_id=query_id)).get_data()
snippet = notebook['snippets'][0]
db = get_api(request, snippet)
if file_format.get('sampleCols'):
columns = file_format.get('sampleCols')
sample = file_format.get('sample')
else:
snippet['query'] = snippet['statement']
try:
sample = db.fetch_result(notebook, snippet, 4, start_over=True)['rows'][:4]
except Exception, e:
LOG.warn('Skipping sample data as query handle might be expired: %s' % e)
sample = [[], [], [], [], []]
columns = db.autocomplete(snippet=snippet, database='', table='')
columns = [
Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
for col in columns['extended_columns']
]
format_ = {
"sample": sample,
"columns": columns,
}
def test_end_to_end(self):
if not is_live_cluster(): # Skipping as requires morplines libs to be setup
raise SkipTest()
cluster = shared_cluster()
fs = cluster.fs
make_logged_in_client(username="******", groupname="default", recreate=True, is_superuser=False)
user = User.objects.get(username="******")
collection_name = "test_collection"
indexer = MorphlineIndexer("test", fs=fs, jt=cluster.jt, solr_client=self.solr_client)
input_loc = "/tmp/test.csv"
# upload the test file to hdfs
fs.create(input_loc, data=TestIndexer.simpleCSVString, overwrite=True)
# open a filestream for the file on hdfs
stream = fs.open(input_loc)
# guess the format of the file
file_type_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}})
field_types = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": file_type_format})
format_ = field_types.copy()
format_['format'] = file_type_format
# find a field name available to use for the record's uuid
unique_field = indexer.get_unique_field(format_)
is_unique_generated = indexer.is_unique_generated(format_)
# generate morphline
morphline = indexer.generate_morphline_config(collection_name, format_, unique_field)
schema_fields = indexer.get_kept_field_list(format_['columns'])
if is_unique_generated:
schema_fields += [{"name": unique_field, "type": "string"}]
# create the collection from the specified fields
collection_manager = CollectionManagerController("test")
if collection_manager.collection_exists(collection_name):
collection_manager.delete_collection(collection_name, None)
collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field)
# index the file
indexer.run_morphline(MockedRequest(user=user, fs=cluster.fs, jt=cluster.jt), collection_name, morphline, input_loc)
def guess_field_types(request):
file_format = json.loads(request.POST.get('fileFormat', '{}'))
if file_format['inputFormat'] == 'file':
indexer = MorphlineIndexer(request.user, request.fs)
path = urllib_unquote(file_format["path"])
stream = request.fs.open(path)
encoding = check_encoding(stream.read(10000))
stream.seek(0)
_convert_format(file_format["format"], inverse=True)
format_ = indexer.guess_field_types({
"file": {
"stream": stream,
"name": path
},
"format": file_format['format']
})
# Note: Would also need to set charset to table (only supported in Hive)
if 'sample' in format_ and format_['sample']:
format_['sample'] = escape_rows(format_['sample'],
nulls_only=True,
encoding=encoding)
for col in format_['columns']:
col['name'] = smart_unicode(col['name'],
errors='replace',
encoding=encoding)
elif file_format['inputFormat'] == 'table':
sample = get_api(request, {
'type': 'hive'
}).get_sample_data({'type': 'hive'},
database=file_format['databaseName'],
table=file_format['tableName'])
db = dbms.get(request.user)
table_metadata = db.get_table(database=file_format['databaseName'],
table_name=file_format['tableName'])
format_ = {
"sample":
sample['rows'][:4],
"columns": [
Field(col.name,
HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type,
'string')).to_dict()
for col in table_metadata.cols
]
}
elif file_format['inputFormat'] == 'query':
query_id = file_format['query']['id'] if file_format['query'].get(
'id') else file_format['query']
notebook = Notebook(document=Document2.objects.document(
user=request.user, doc_id=query_id)).get_data()
snippet = notebook['snippets'][0]
db = get_api(request, snippet)
if file_format.get('sampleCols'):
columns = file_format.get('sampleCols')
sample = file_format.get('sample')
else:
snippet['query'] = snippet['statement']
try:
sample = db.fetch_result(notebook, snippet, 4,
start_over=True)['rows'][:4]
except Exception as e:
LOG.warning(
'Skipping sample data as query handle might be expired: %s'
% e)
sample = [[], [], [], [], []]
columns = db.autocomplete(snippet=snippet, database='', table='')
columns = [
Field(
col['name'],
HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'],
'string')).to_dict()
for col in columns['extended_columns']
]
format_ = {
"sample": sample,
"columns": columns,
}
elif file_format['inputFormat'] == 'rdbms':
api = _get_api(request)
sample = api.get_sample_data(None,
database=file_format['rdbmsDatabaseName'],
table=file_format['tableName'])
format_ = {
"sample":
list(sample['rows'])[:4],
"columns": [
Field(col['name'], col['type']).to_dict()
for col in sample['full_headers']
]
}
elif file_format['inputFormat'] == 'stream':
if file_format['streamSelection'] == 'kafka':
data = get_topic_data(request.user,
file_format.get('kafkaSelectedTopics'))
kafkaFieldNames = [col['name'] for col in data['full_headers']]
kafkaFieldTypes = [col['type'] for col in data['full_headers']]
topics_data = data['rows']
format_ = {
"sample":
topics_data,
"columns": [
Field(col, 'string', unique=False).to_dict()
for col in kafkaFieldNames
]
}
# data = """%(kafkaFieldNames)s
# %(data)s""" % {
# 'kafkaFieldNames': ','.join(kafkaFieldNames),
# 'data': '\n'.join([','.join(cols) for cols in topics_data])
# }
# stream = string_io()
# stream.write(data)
# _convert_format(file_format["format"], inverse=True)
# indexer = MorphlineIndexer(request.user, request.fs)
# format_ = indexer.guess_field_types({
# "file": {
# "stream": stream,
# "name": file_format['path']
# },
# "format": file_format['format']
# })
# type_mapping = dict(
# list(
# zip(kafkaFieldNames, kafkaFieldTypes)
# )
# )
# for col in format_['columns']:
# col['keyType'] = type_mapping[col['name']]
# col['type'] = type_mapping[col['name']]
elif file_format['streamSelection'] == 'flume':
if 'hue-httpd/access_log' in file_format['channelSourcePath']:
columns = [{
'name': 'id',
'type': 'string',
'unique': True
}, {
'name': 'client_ip',
'type': 'string'
}, {
'name': 'time',
'type': 'date'
}, {
'name': 'request',
'type': 'string'
}, {
'name': 'code',
'type': 'plong'
}, {
'name': 'bytes',
'type': 'plong'
}, {
'name': 'method',
'type': 'string'
}, {
'name': 'url',
'type': 'string'
}, {
'name': 'protocol',
'type': 'string'
}, {
'name': 'app',
'type': 'string'
}, {
'name': 'subapp',
'type': 'string'
}]
else:
columns = [{'name': 'message', 'type': 'string'}]
format_ = {
"sample": [['...'] * len(columns)] * 4,
"columns": [
Field(col['name'],
HiveFormat.FIELD_TYPE_TRANSLATE.get(
col['type'], 'string'),
unique=col.get('unique')).to_dict()
for col in columns
]
}
elif file_format['inputFormat'] == 'connector':
if file_format['connectorSelection'] == 'sfdc':
sf = Salesforce(username=file_format['streamUsername'],
password=file_format['streamPassword'],
security_token=file_format['streamToken'])
table_metadata = [{
'name': column['name'],
'type': column['type']
} for column in sf.restful('sobjects/%(streamObject)s/describe/' %
file_format)['fields']]
query = 'SELECT %s FROM %s LIMIT 4' % (', '.join(
[col['name']
for col in table_metadata]), file_format['streamObject'])
print(query)
try:
records = sf.query_all(query)
except SalesforceRefusedRequest as e:
raise PopupException(message=str(e))
format_ = {
"sample":
[list(row.values())[1:] for row in records['records']],
"columns": [
Field(
col['name'],
HiveFormat.FIELD_TYPE_TRANSLATE.get(
col['type'], 'string')).to_dict()
for col in table_metadata
]
}
else:
raise PopupException(
_('Connector format not recognized: %(connectorSelection)s') %
file_format)
else:
raise PopupException(
_('Input format not recognized: %(inputFormat)s') % file_format)
return JsonResponse(format_)
def guess_field_types(request):
file_format = json.loads(request.POST.get('fileFormat', '{}'))
if file_format['inputFormat'] == 'file':
indexer = MorphlineIndexer(request.user, request.fs)
path = urllib.unquote(file_format["path"])
stream = request.fs.open(path)
encoding = chardet.detect(stream.read(10000)).get('encoding')
stream.seek(0)
_convert_format(file_format["format"], inverse=True)
format_ = indexer.guess_field_types({
"file": {
"stream": stream,
"name": path
},
"format": file_format['format']
})
# Note: Would also need to set charset to table (only supported in Hive)
if 'sample' in format_:
format_['sample'] = escape_rows(format_['sample'], nulls_only=True, encoding=encoding)
for col in format_['columns']:
col['name'] = smart_unicode(col['name'], errors='replace', encoding=encoding)
elif file_format['inputFormat'] == 'table':
sample = get_api(request, {'type': 'hive'}).get_sample_data({'type': 'hive'}, database=file_format['databaseName'], table=file_format['tableName'])
db = dbms.get(request.user)
table_metadata = db.get_table(database=file_format['databaseName'], table_name=file_format['tableName'])
format_ = {
"sample": sample['rows'][:4],
"columns": [
Field(col.name, HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type, 'string')).to_dict()
for col in table_metadata.cols
]
}
elif file_format['inputFormat'] == 'query':
query_id = file_format['query']['id'] if file_format['query'].get('id') else file_format['query']
notebook = Notebook(document=Document2.objects.document(user=request.user, doc_id=query_id)).get_data()
snippet = notebook['snippets'][0]
db = get_api(request, snippet)
if file_format.get('sampleCols'):
columns = file_format.get('sampleCols')
sample = file_format.get('sample')
else:
snippet['query'] = snippet['statement']
try:
sample = db.fetch_result(notebook, snippet, 4, start_over=True)['rows'][:4]
except Exception, e:
LOG.warn('Skipping sample data as query handle might be expired: %s' % e)
sample = [[], [], [], [], []]
columns = db.autocomplete(snippet=snippet, database='', table='')
columns = [
Field(col['name'], HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'], 'string')).to_dict()
for col in columns['extended_columns']
]
format_ = {
"sample": sample,
"columns": columns,
}
if file_format['streamSelection'] == 'kafka':
data = """%(kafkaFieldNames)s
%(data)s""" % {
'kafkaFieldNames': file_format.get('kafkaFieldNames', ''),
'data': '\n'.join([','.join(['...'] * len(file_format.get('kafkaFieldTypes', '').split(',')))] * 5)
}
stream = StringIO.StringIO()
stream.write(data)
_convert_format(file_format["format"], inverse=True)
indexer = MorphlineIndexer(request.user, request.fs)
format_ = indexer.guess_field_types({
"file": {
"stream": stream,
"name": file_format['path']
},
"format": file_format['format']
})
type_mapping = dict(zip(file_format['kafkaFieldNames'].split(','), file_format['kafkaFieldTypes'].split(',')))
for col in format_['columns']:
col['keyType'] = type_mapping[col['name']]
col['type'] = type_mapping[col['name']]
elif file_format['streamSelection'] == 'sfdc':
sf = Salesforce(
username=file_format['streamUsername'],
password=file_format['streamPassword'],
security_token=file_format['streamToken']
)
table_metadata = [{
def guess_field_types(request):
file_format = json.loads(request.POST.get('fileFormat', '{}'))
if file_format['inputFormat'] == 'file':
indexer = MorphlineIndexer(request.user, request.fs)
path = urllib_unquote(file_format["path"])
stream = request.fs.open(path)
encoding = chardet.detect(stream.read(10000)).get('encoding')
stream.seek(0)
_convert_format(file_format["format"], inverse=True)
format_ = indexer.guess_field_types({
"file": {
"stream": stream,
"name": path
},
"format": file_format['format']
})
# Note: Would also need to set charset to table (only supported in Hive)
if 'sample' in format_ and format_['sample']:
format_['sample'] = escape_rows(format_['sample'],
nulls_only=True,
encoding=encoding)
for col in format_['columns']:
col['name'] = smart_unicode(col['name'],
errors='replace',
encoding=encoding)
elif file_format['inputFormat'] == 'table':
sample = get_api(request, {
'type': 'hive'
}).get_sample_data({'type': 'hive'},
database=file_format['databaseName'],
table=file_format['tableName'])
db = dbms.get(request.user)
table_metadata = db.get_table(database=file_format['databaseName'],
table_name=file_format['tableName'])
format_ = {
"sample":
sample['rows'][:4],
"columns": [
Field(col.name,
HiveFormat.FIELD_TYPE_TRANSLATE.get(col.type,
'string')).to_dict()
for col in table_metadata.cols
]
}
elif file_format['inputFormat'] == 'query':
query_id = file_format['query']['id'] if file_format['query'].get(
'id') else file_format['query']
notebook = Notebook(document=Document2.objects.document(
user=request.user, doc_id=query_id)).get_data()
snippet = notebook['snippets'][0]
db = get_api(request, snippet)
if file_format.get('sampleCols'):
columns = file_format.get('sampleCols')
sample = file_format.get('sample')
else:
snippet['query'] = snippet['statement']
try:
sample = db.fetch_result(notebook, snippet, 4,
start_over=True)['rows'][:4]
except Exception as e:
LOG.warn(
'Skipping sample data as query handle might be expired: %s'
% e)
sample = [[], [], [], [], []]
columns = db.autocomplete(snippet=snippet, database='', table='')
columns = [
Field(
col['name'],
HiveFormat.FIELD_TYPE_TRANSLATE.get(col['type'],
'string')).to_dict()
for col in columns['extended_columns']
]
format_ = {
"sample": sample,
"columns": columns,
}
elif file_format['inputFormat'] == 'rdbms':
api = _get_api(request)
sample = api.get_sample_data(None,
database=file_format['rdbmsDatabaseName'],
table=file_format['tableName'])
format_ = {
"sample":
list(sample['rows'])[:4],
"columns": [
Field(col['name'], col['type']).to_dict()
for col in sample['full_headers']
]
}
elif file_format['inputFormat'] == 'stream':
if file_format['streamSelection'] == 'kafka':
if file_format.get(
'kafkaSelectedTopics') == 'NavigatorAuditEvents':
kafkaFieldNames = [
'id', 'additionalInfo', 'allowed', 'collectionName',
'databaseName', 'db', 'DELEGATION_TOKEN_ID', 'dst',
'entityId', 'family', 'impersonator', 'ip', 'name',
'objectType', 'objType', 'objUsageType', 'operationParams',
'operationText', 'op', 'opText', 'path', 'perms',
'privilege', 'qualifier', 'QUERY_ID', 'resourcePath',
'service', 'SESSION_ID', 'solrVersion', 'src', 'status',
'subOperation', 'tableName', 'table', 'time', 'type',
'url', 'user'
]
kafkaFieldTypes = ['string'] * len(kafkaFieldNames)
kafkaFieldNames.append('timeDate')
kafkaFieldTypes.append('date')
else:
# Note: mocked here, should come from SFDC or Kafka API or sampling job
kafkaFieldNames = file_format.get('kafkaFieldNames',
'').split(',')
kafkaFieldTypes = file_format.get('kafkaFieldTypes',
'').split(',')
data = """%(kafkaFieldNames)s
%(data)s""" % {
'kafkaFieldNames': ','.join(kafkaFieldNames),
'data': '\n'.join(
[','.join(['...'] * len(kafkaFieldTypes))] * 5)
}
stream = string_io()
stream.write(data)
_convert_format(file_format["format"], inverse=True)
indexer = MorphlineIndexer(request.user, request.fs)
format_ = indexer.guess_field_types({
"file": {
"stream": stream,
"name": file_format['path']
},
"format": file_format['format']
})
type_mapping = dict(list(zip(kafkaFieldNames, kafkaFieldTypes)))
for col in format_['columns']:
col['keyType'] = type_mapping[col['name']]
col['type'] = type_mapping[col['name']]
elif file_format['streamSelection'] == 'flume':
if 'hue-httpd/access_log' in file_format['channelSourcePath']:
columns = [{
'name': 'id',
'type': 'string',
'unique': True
}, {
'name': 'client_ip',
'type': 'string'
}, {
'name': 'time',
'type': 'date'
}, {
'name': 'request',
'type': 'string'
}, {
'name': 'code',
'type': 'plong'
}, {
'name': 'bytes',
'type': 'plong'
}, {
'name': 'method',
'type': 'string'
}, {
'name': 'url',
'type': 'string'
}, {
'name': 'protocol',
'type': 'string'
}, {
'name': 'app',
'type': 'string'
}, {
'name': 'subapp',
'type': 'string'
}]
else:
columns = [{'name': 'message', 'type': 'string'}]
format_ = {
"sample": [['...'] * len(columns)] * 4,
"columns": [
Field(col['name'],
HiveFormat.FIELD_TYPE_TRANSLATE.get(
col['type'], 'string'),
unique=col.get('unique')).to_dict()
for col in columns
]
}
elif file_format['inputFormat'] == 'connector':
if file_format['connectorSelection'] == 'sfdc':
sf = Salesforce(username=file_format['streamUsername'],
password=file_format['streamPassword'],
security_token=file_format['streamToken'])
table_metadata = [{
'name': column['name'],
'type': column['type']
} for column in sf.restful('sobjects/%(streamObject)s/describe/' %
file_format)['fields']]
query = 'SELECT %s FROM %s LIMIT 4' % (', '.join(
[col['name']
for col in table_metadata]), file_format['streamObject'])
print(query)
try:
records = sf.query_all(query)
except SalesforceRefusedRequest as e:
raise PopupException(message=str(e))
format_ = {
"sample":
[list(row.values())[1:] for row in records['records']],
"columns": [
Field(
col['name'],
HiveFormat.FIELD_TYPE_TRANSLATE.get(
col['type'], 'string')).to_dict()
for col in table_metadata
]
}
else:
raise PopupException(
_('Connector format not recognized: %(connectorSelection)s') %
file_format)
else:
raise PopupException(
_('Input format not recognized: %(inputFormat)s') % file_format)
return JsonResponse(format_)
