def transform(this, raw_metadata, metadata_output, field_metadata_output):
input_json_file = open(raw_metadata, 'r')
schema_file_writer = FileWriter(metadata_output)
field_file_writer = FileWriter(field_metadata_output)
i = 0
this.sort_id = 0
o_urn = ''
for line in input_json_file:
try:
j = json.loads(line)
except:
this.logger.error(" Invalid JSON:\n%s" % line)
continue
i += 1
o_field_list_ = []
this.sort_id = 0
if not j.has_key('attributes'):
o_properties = {"doc": null}
else:
o_properties = dict(j['attributes'].items())
del j['attributes']
if j.has_key('uri'):
o_urn = j['uri']
o_name = o_urn[o_urn.rfind('/') + 1:]
o_source = 'Hbase'
else:
this.logger.info('*** Warning: "uri" is not found in %s' %
j['name'])
o_urn = ''
o_name = ''
if not j.has_key('fields'):
o_fields = {"doc": None}
else:
o_fields = {}
for f in j['fields']:
o_field_name = f['name']
o_fields[o_field_name] = dict(f)
acp = AvroColumnParser(j, o_urn)
o_field_list_ += acp.get_column_list_result()
dataset_schema_record = DatasetSchemaRecord(
o_name, json.dumps(j, sort_keys=True),
json.dumps(o_properties, sort_keys=True), json.dumps(o_fields),
o_urn, o_source, 'HBase', 'Table', None, None, None)
schema_file_writer.append(dataset_schema_record)
for fields in o_field_list_:
field_record = DatasetFieldRecord(fields)
field_file_writer.append(field_record)
field_file_writer.close()
schema_file_writer.close()
input_json_file.close()
def transform(self, input, hive_metadata, hive_field_metadata):
"""
convert from json to csv
:param input: input json file
:param hive_metadata: output data file for hive table metadata
:param hive_field_metadata: output data file for hive field metadata
:return:
"""
f_json = open(input)
all_data = json.load(f_json)
f_json.close()
schema_file_writer = FileWriter(hive_metadata)
field_file_writer = FileWriter(hive_field_metadata)
lineageInfo = LineageInfo()
# one db info : 'type', 'database', 'tables'
# one table info : required : 'name' , 'type', 'serializationFormat' ,'createTime', 'DB_ID', 'TBL_ID', 'SD_ID'
# optional : 'schemaLiteral', 'schemaUrl', 'fieldDelimiter', 'fieldList'
for one_db_info in all_data:
i = 0
for table in one_db_info['tables']:
i += 1
schema_json = {}
prop_json = {} # set the prop json
for prop_name in TableInfo.optional_prop:
if prop_name in table and table[prop_name] is not None:
prop_json[prop_name] = table[prop_name]
if TableInfo.view_expended_text in prop_json:
text = prop_json[TableInfo.view_expended_text].replace(
'`', '')
array = HiveViewDependency.getViewDependency(text)
l = []
for a in array:
l.append(a)
prop_json['view_depends_on'] = l
# process either schema
flds = {}
field_detail_list = []
if TableInfo.schema_literal in table and table[
TableInfo.schema_literal] is not None:
sort_id = 0
try:
schema_data = json.loads(
table[TableInfo.schema_literal])
except ValueError:
self.logger.error("Schema json error for table : \n" +
str(table))
schema_json = schema_data
# extract fields to field record
urn = "hive:///%s/%s" % (one_db_info['database'],
table['name'])
acp = AvroColumnParser(schema_data, urn=urn)
result = acp.get_column_list_result()
field_detail_list += result
elif TableInfo.field_list in table:
# Convert to avro
uri = "hive:///%s/%s" % (one_db_info['database'],
table['name'])
hcp = HiveColumnParser(table, urn=uri)
schema_json = {
'fields': hcp.column_type_dict['fields'],
'type': 'record',
'name': table['name'],
'uri': uri
}
field_detail_list += hcp.column_type_list
dataset_scehma_record = DatasetSchemaRecord(
table['name'], json.dumps(schema_json),
json.dumps(prop_json), json.dumps(flds),
"hive:///%s/%s" % (one_db_info['database'], table['name']),
'Hive', '', (table[TableInfo.create_time] if table.has_key(
TableInfo.create_time) else None),
(table["lastAlterTime"])
if table.has_key("lastAlterTime") else None)
schema_file_writer.append(dataset_scehma_record)
for fields in field_detail_list:
field_record = DatasetFieldRecord(fields)
field_file_writer.append(field_record)
schema_file_writer.flush()
field_file_writer.flush()
self.logger.info("%20s contains %6d tables" %
(one_db_info['database'], i))
schema_file_writer.close()
field_file_writer.close()
def transform(self, input, hive_metadata, hive_field_metadata):
"""
convert from json to csv
:param input: input json file
:param hive_metadata: output data file for hive table metadata
:param hive_field_metadata: output data file for hive field metadata
:return:
"""
f_json = open(input)
all_data = json.load(f_json)
f_json.close()
schema_file_writer = FileWriter(hive_metadata)
field_file_writer = FileWriter(hive_field_metadata)
lineageInfo = LineageInfo()
# one db info : 'type', 'database', 'tables'
# one table info : required : 'name' , 'type', 'serializationFormat' ,'createTime', 'DB_ID', 'TBL_ID', 'SD_ID'
# optional : 'schemaLiteral', 'schemaUrl', 'fieldDelimiter', 'fieldList'
for one_db_info in all_data:
i = 0
for table in one_db_info['tables']:
i += 1
schema_json = {}
prop_json = {} # set the prop json
for prop_name in TableInfo.optional_prop:
if prop_name in table and table[prop_name] is not None:
prop_json[prop_name] = table[prop_name]
if TableInfo.view_expended_text in prop_json:
text = prop_json[TableInfo.view_expended_text].replace(
'`', '')
array = HiveViewDependency.getViewDependency(text)
l = []
for a in array:
l.append(a)
prop_json['view_depends_on'] = l
# process either schema
flds = {}
field_detail_list = []
if TableInfo.schema_literal in table and table[
TableInfo.schema_literal] is not None:
sort_id = 0
try:
schema_data = json.loads(
table[TableInfo.schema_literal])
except ValueError:
self.logger.error("Schema json error for table : \n" +
str(table))
schema_json = schema_data
# process each field
for field in schema_data['fields']:
field_name = field['name']
type = field['type'] # could be a list
default_value = field[
'default'] if 'default' in field else None
doc = field['doc'] if 'doc' in field else None
attributes_json = json.loads(
field['attributes_json']
) if 'attributes_json' in field else None
pk = delta = is_nullable = is_indexed = is_partitioned = inside_type = format = data_size = None
if attributes_json:
pk = attributes_json[
'pk'] if 'pk' in attributes_json else None
delta = attributes_json[
'delta'] if 'delta' in attributes_json else None
is_nullable = attributes_json[
'nullable'] if 'nullable' in attributes_json else None
inside_type = attributes_json[
'type'] if 'type' in attributes_json else None
format = attributes_json[
'format'] if 'format' in attributes_json else None
flds[field_name] = {'type': type}
# String urn, Integer sortId, Integer parentSortId, String parentPath, String fieldName,
#String dataType, String isNullable, String defaultValue, Integer dataSize, String namespace, String description
sort_id += 1
field_detail_list.append([
"hive:///%s/%s" %
(one_db_info['database'], table['name']),
str(sort_id), '0', None, field_name, '', type,
data_size, None, None, is_nullable, is_indexed,
is_partitioned, default_value, None,
json.dumps(attributes_json)
])
elif TableInfo.field_list in table:
schema_json = {
'type': 'record',
'name': table['name'],
'fields': table[TableInfo.field_list]
} # construct a schema for data came from COLUMN_V2
for field in table[TableInfo.field_list]:
field_name = field['ColumnName']
type = field['TypeName']
# ColumnName, IntegerIndex, TypeName, Comment
flds[field_name] = {'type': type}
pk = delta = is_nullable = is_indexed = is_partitioned = inside_type = format = data_size = default_value = None # TODO ingest
field_detail_list.append([
"hive:///%s/%s" %
(one_db_info['database'], table['name']),
field['IntegerIndex'], '0', None, field_name, '',
field['TypeName'], None, None, None, is_nullable,
is_indexed, is_partitioned, default_value, None,
None
])
dataset_scehma_record = DatasetSchemaRecord(
table['name'], json.dumps(schema_json),
json.dumps(prop_json), json.dumps(flds),
"hive:///%s/%s" % (one_db_info['database'], table['name']),
'Hive', '', (table[TableInfo.create_time] if table.has_key(
TableInfo.create_time) else None),
(table["lastAlterTime"])
if table.has_key("lastAlterTime") else None)
schema_file_writer.append(dataset_scehma_record)
for fields in field_detail_list:
field_record = DatasetFieldRecord(fields)
field_file_writer.append(field_record)
schema_file_writer.flush()
field_file_writer.flush()
self.logger.info("%20s contains %6d tables" %
(one_db_info['database'], i))
schema_file_writer.close()
field_file_writer.close()
def transform(self, input, hive_instance, hive_metadata,
hive_field_metadata):
"""
convert from json to csv
:param input: input json file
:param hive_metadata: output data file for hive table metadata
:param hive_field_metadata: output data file for hive field metadata
:return:
"""
all_data = []
with open(input) as input_file:
for line in input_file:
all_data.append(json.loads(line))
dataset_idx = -1
instance_file_writer = FileWriter(hive_instance)
schema_file_writer = FileWriter(hive_metadata)
field_file_writer = FileWriter(hive_field_metadata)
lineageInfo = LineageInfo()
depends_sql = """
SELECT d.NAME DB_NAME, case when t.TBL_NAME regexp '_[0-9]+_[0-9]+_[0-9]+$'
then concat(substring(t.TBL_NAME, 1, length(t.TBL_NAME) - length(substring_index(t.TBL_NAME, '_', -3)) - 1),'_{version}')
else t.TBL_NAME
end dataset_name,
concat('/', d.NAME, '/', t.TBL_NAME) object_name,
case when (d.NAME like '%\_mp' or d.NAME like '%\_mp\_versioned') and d.NAME not like 'dalitest%' and t.TBL_TYPE = 'VIRTUAL_VIEW'
then 'dalids'
else 'hive'
end object_type,
case when (d.NAME like '%\_mp' or d.NAME like '%\_mp\_versioned') and d.NAME not like 'dalitest%' and t.TBL_TYPE = 'VIRTUAL_VIEW'
then 'View'
else
case when LOCATE('view', LOWER(t.TBL_TYPE)) > 0 then 'View'
when LOCATE('index', LOWER(t.TBL_TYPE)) > 0 then 'Index'
else 'Table'
end
end object_sub_type,
case when (d.NAME like '%\_mp' or d.NAME like '%\_mp\_versioned') and t.TBL_TYPE = 'VIRTUAL_VIEW'
then 'dalids'
else 'hive'
end prefix
FROM TBLS t JOIN DBS d on t.DB_ID = d.DB_ID
WHERE d.NAME = '{db_name}' and t.TBL_NAME = '{table_name}'
"""
# one db info : 'type', 'database', 'tables'
# one table info : required : 'name' , 'type', 'serializationFormat' ,'createTime', 'DB_ID', 'TBL_ID', 'SD_ID'
# optional : 'schemaLiteral', 'schemaUrl', 'fieldDelimiter', 'fieldList'
for one_db_info in all_data:
i = 0
for table in one_db_info['tables']:
i += 1
schema_json = {}
prop_json = {} # set the prop json
for prop_name in TableInfo.optional_prop:
if prop_name in table and table[prop_name] is not None:
prop_json[prop_name] = table[prop_name]
view_expanded_text = ''
if TableInfo.view_expended_text in prop_json:
view_expanded_text = prop_json[
TableInfo.view_expended_text]
text = prop_json[TableInfo.view_expended_text].replace(
'`',
'') # this will be fixed after switching to Hive AST
array = []
try:
array = HiveViewDependency.getViewDependency(text)
except:
self.logger.error(
"HiveViewDependency.getViewDependency(%s) failed!"
% (table['name']))
l = []
for a in array:
l.append(a)
names = str(a).split('.')
if names and len(names) >= 2:
db_name = names[0].lower()
table_name = names[1].lower()
if db_name and table_name:
self.curs.execute(
depends_sql.format(db_name=db_name,
table_name=table_name,
version='{version}'))
rows = self.curs.fetchall()
self.conn_hms.commit()
if rows and len(rows) > 0:
for row_index, row_value in enumerate(
rows):
dependent_record = HiveDependencyInstanceRecord(
one_db_info['type'], table['type'],
"/%s/%s" %
(one_db_info['database'],
table['name']), 'dalids:///' +
one_db_info['database'] + '/' +
table['dataset_name']
if one_db_info['type'].lower()
== 'dalids' else 'hive:///' +
one_db_info['database'] + '/' +
table['dataset_name'],
'depends on', 'Y', row_value[3],
row_value[4], row_value[2],
row_value[5] + ':///' +
row_value[0] + '/' + row_value[1],
'')
self.instance_writer.append(
dependent_record)
prop_json['view_depends_on'] = l
self.instance_writer.flush()
# process either schema
flds = {}
field_detail_list = []
if TableInfo.schema_literal in table and \
table[TableInfo.schema_literal] is not None and \
table[TableInfo.schema_literal].startswith('{'):
sort_id = 0
urn = "hive:///%s/%s" % (one_db_info['database'],
table['dataset_name'])
self.logger.info("Getting schema literal for: %s" % (urn))
try:
schema_data = json.loads(
table[TableInfo.schema_literal])
schema_json = schema_data
acp = AvroColumnParser(schema_data, urn=urn)
result = acp.get_column_list_result()
field_detail_list += result
except ValueError:
self.logger.error(
"Schema Literal JSON error for table: " +
str(table))
elif TableInfo.field_list in table:
# Convert to avro
uri = "hive:///%s/%s" % (one_db_info['database'],
table['dataset_name'])
if one_db_info['type'].lower() == 'dalids':
uri = "dalids:///%s/%s" % (one_db_info['database'],
table['dataset_name'])
else:
uri = "hive:///%s/%s" % (one_db_info['database'],
table['dataset_name'])
self.logger.info("Getting column definition for: %s" %
(uri))
try:
hcp = HiveColumnParser(table, urn=uri)
schema_json = {
'fields': hcp.column_type_dict['fields'],
'type': 'record',
'name': table['name'],
'uri': uri
}
field_detail_list += hcp.column_type_list
except:
self.logger.error("HiveColumnParser(%s) failed!" %
(uri))
schema_json = {
'fields': {},
'type': 'record',
'name': table['name'],
'uri': uri
}
if one_db_info['type'].lower() == 'dalids':
dataset_urn = "dalids:///%s/%s" % (one_db_info['database'],
table['dataset_name'])
else:
dataset_urn = "hive:///%s/%s" % (one_db_info['database'],
table['dataset_name'])
dataset_instance_record = DatasetInstanceRecord(
'dalids:///' + one_db_info['database'] + '/' +
table['name'] if one_db_info['type'].lower() == 'dalids'
else 'hive:///' + one_db_info['database'] + '/' +
table['name'], 'grid', '', '', '*', 0,
table['native_name'], table['logical_name'],
table['version'], table['create_time'],
json.dumps(schema_json), json.dumps(view_expanded_text),
dataset_urn)
instance_file_writer.append(dataset_instance_record)
if dataset_urn not in self.dataset_dict:
dataset_scehma_record = DatasetSchemaRecord(
table['dataset_name'], json.dumps(schema_json),
json.dumps(prop_json), json.dumps(flds), dataset_urn,
'Hive', one_db_info['type'], table['type'], '',
table.get(TableInfo.create_time),
(int(table.get(TableInfo.source_modified_time, "0"))))
schema_file_writer.append(dataset_scehma_record)
dataset_idx += 1
self.dataset_dict[dataset_urn] = dataset_idx
for fields in field_detail_list:
field_record = DatasetFieldRecord(fields)
field_file_writer.append(field_record)
instance_file_writer.flush()
schema_file_writer.flush()
field_file_writer.flush()
self.logger.info("%20s contains %6d tables" %
(one_db_info['database'], i))
instance_file_writer.close()
schema_file_writer.close()
field_file_writer.close()
def transform(self, input, td_metadata, td_field_metadata):
'''
convert from json to csv
:param input: input json file
:param td_metadata: output data file for teradata metadata
:param td_field_metadata: output data file for teradata field metadata
:return:
'''
f_json = open(input)
data = json.load(f_json)
f_json.close()
schema_file_writer = FileWriter(td_metadata)
field_file_writer = FileWriter(td_field_metadata)
for d in data:
i = 0
for k in d.keys():
if k not in ['tables', 'views']:
continue
self.logger.info("%s %4d %s" %
(datetime.datetime.now().strftime("%H:%M:%S"),
len(d[k]), k))
for t in d[k]:
self.logger.info("%4d %s" % (i, t['name']))
if t['name'] == 'HDFStoTD_2464_ERR_1':
continue
i += 1
output = {}
prop_json = {}
output['name'] = t['name']
output['original_name'] = t['original_name']
prop_json["createTime"] = t["createTime"] if t.has_key(
"createTime") else None
prop_json[
"lastAlterTime"] = t["lastAlterTime"] if t.has_key(
"lastAlterTime") else None
prop_json[
"lastAccessTime"] = t["lastAccessTime"] if t.has_key(
"lastAccessTime") else None
prop_json["accessCount"] = t["accessCount"] if t.has_key(
"accessCount") else None
prop_json["sizeInMbytes"] = t["sizeInMbytes"] if t.has_key(
"sizeInMbytes") else None
if "type" in t:
prop_json["storage_type"] = t["type"]
if "partition" in t:
prop_json["partition"] = t["partition"]
if "partitions" in t:
prop_json["partitions"] = t["partitions"]
if "hashKey" in t:
prop_json["hashKey"] = t["hashKey"]
if "indices" in t:
prop_json["indices"] = t["indices"]
if "referenceTables" in t:
prop_json["referenceTables"] = t["referenceTables"]
if "viewSqlText" in t:
prop_json["viewSqlText"] = t["viewSqlText"]
output['fields'] = []
flds = {}
field_detail_list = []
sort_id = 0
for c in t['columns']:
# output['fields'].append(
# { 'name' : t['name'].encode('latin-1'),
# 'type' : None if c['data_type'] is None else c['data_type'].encode('latin-1'),
# 'attributes_json' : c}
# output['fields'][c['name'].encode('latin-1')].append({ "doc" : "", "type" : [None if c['data_type'] is None else c['data_type'].encode('latin-1')]})
sort_id += 1
output['fields'].append({
"name":
c['name'],
"doc":
'',
"type":
c['dataType'] if c['dataType'] else None,
"nullable":
c['nullable'],
"maxByteLength":
c['maxByteLength'],
"format":
c['columnFormat']
if c.has_key('columnFormat') else None,
"accessCount":
c['accessCount']
if c.has_key('accessCount') else None,
"lastAccessTime":
c['lastAccessTime']
if c.has_key("lastAccessTime") else None
})
flds[c['name']] = {
'type': c['dataType'],
"maxByteLength": c['maxByteLength']
}
field_detail_list.append([
"teradata:///%s/%s" %
(d['database'], output['name']),
str(sort_id), '0', '', c['name'], '',
c['dataType'] if 'dataType' in c
and c['dataType'] is not None else '',
str(c['maxByteLength'])
if 'maxByteLength' in c else '0',
str(c['precision']) if 'precision' in c
and c['precision'] is not None else '',
str(c['scale'])
if 'scale' in c and c['scale'] is not None else '',
c['nullable'] if 'nullable' in c
and c['nullable'] is not None else 'Y', '', '', '',
'', '', '', ''
])
dataset_scehma_record = DatasetSchemaRecord(
output['name'], json.dumps(output),
json.dumps(prop_json), json.dumps(flds),
"teradata:///%s/%s" % (d['database'], output['name']),
'Teradata', output['original_name'],
(self.convert_timestamp(t["createTime"])
if t.has_key("createTime") else None),
(self.convert_timestamp(t["lastAlterTime"])
if t.has_key("lastAlterTime") else None))
schema_file_writer.append(dataset_scehma_record)
for fields in field_detail_list:
field_record = DatasetFieldRecord(fields)
field_file_writer.append(field_record)
schema_file_writer.flush()
field_file_writer.flush()
self.logger.info("%20s contains %6d %s" %
(d['database'], i, k))
schema_file_writer.close()
field_file_writer.close()
def transform(self, raw_metadata, metadata_output, field_metadata_output):
# sys.setdefaultencoding("UTF-8")
input_json_file = open(raw_metadata, 'r')
schema_file_writer = FileWriter(metadata_output)
field_file_writer = FileWriter(field_metadata_output)
i = 0
self.sort_id = 0
o_urn = ''
p = ''
def fields_json_to_csv(output_list_, parent_field_path, field_list_):
# string, list, int, optional int
self.sort_id
parent_field_path
parent_id = self.sort_id
for f in field_list_:
self.sort_id += 1
o_field_name = f['name']
o_field_data_type = ''
o_field_data_size = None
o_field_nullable = 'N'
o_field_default = ''
o_field_namespace = ''
o_field_doc = ''
effective_type_index_in_type = -1
if f.has_key('namespace'):
o_field_namespace = f['namespace']
if f.has_key('default') and type(f['default']) != None:
o_field_default = f['default']
if not f.has_key('type'):
o_field_data_type = None
elif type(f['type']) == list:
i = effective_type_index = -1
for data_type in f['type']:
i += 1 # current index
if type(data_type) is None or (data_type == 'null'):
o_field_nullable = 'Y'
elif type(data_type) == dict:
o_field_data_type = data_type['type']
effective_type_index_in_type = i
if data_type.has_key('namespace'):
o_field_namespace = data_type['namespace']
elif data_type.has_key('name'):
o_field_namespace = data_type['name']
if data_type.has_key('size'):
o_field_data_size = data_type['size']
else:
o_field_data_size = None
else:
o_field_data_type = data_type
effective_type_index_in_type = i
elif type(f['type']) == dict:
o_field_data_type = f['type']['type']
else:
o_field_data_type = f['type']
if f.has_key('attributes') and f['attributes'].has_key(
'nullable'):
o_field_nullable = 'Y' if f['attributes'][
'nullable'] else 'N'
if f.has_key('attributes') and f['attributes'].has_key(
'size'):
o_field_data_size = f['attributes']['size']
if f.has_key('doc'):
if len(f['doc']) == 0 and f.has_key('attributes'):
o_field_doc = json.dumps(f['attributes'])
else:
o_field_doc = f['doc']
elif f.has_key('comment'):
o_field_doc = f['comment']
output_list_.append([
o_urn, self.sort_id, parent_id, parent_field_path,
o_field_name, o_field_data_type, o_field_nullable,
o_field_default, o_field_data_size, o_field_namespace,
o_field_doc.replace("\n", ' ')
if o_field_doc is not None else None
])
# check if this field is a nested record
if type(f['type']) == dict and f['type'].has_key('fields'):
current_field_path = o_field_name if parent_field_path == '' else parent_field_path + '.' + o_field_name
fields_json_to_csv(output_list_, current_field_path,
f['type']['fields'])
elif type(f['type']) == dict and f['type'].has_key(
'items') and type(
f['type']['items']
) == dict and f['type']['items'].has_key('fields'):
current_field_path = o_field_name if parent_field_path == '' else parent_field_path + '.' + o_field_name
fields_json_to_csv(output_list_, current_field_path,
f['type']['items']['fields'])
if effective_type_index_in_type >= 0 and type(
f['type'][effective_type_index_in_type]) == dict:
if f['type'][effective_type_index_in_type].has_key(
'items') and type(
f['type'][effective_type_index_in_type]
['items']) == list:
for item in f['type'][effective_type_index_in_type][
'items']:
if type(item) == dict and item.has_key('fields'):
current_field_path = o_field_name if parent_field_path == '' else parent_field_path + '.' + o_field_name
fields_json_to_csv(output_list_,
current_field_path,
item['fields'])
elif f['type'][effective_type_index_in_type].has_key(
'items') and f['type'][
effective_type_index_in_type]['items'].has_key(
'fields'):
# type: [ null, { type: array, items: { name: xxx, type: record, fields: [] } } ]
current_field_path = o_field_name if parent_field_path == '' else parent_field_path + '.' + o_field_name
fields_json_to_csv(
output_list_, current_field_path, f['type']
[effective_type_index_in_type]['items']['fields'])
elif f['type'][effective_type_index_in_type].has_key(
'fields'):
# if f['type'][effective_type_index_in_type].has_key('namespace'):
# o_field_namespace = f['type'][effective_type_index_in_type]['namespace']
current_field_path = o_field_name if parent_field_path == '' else parent_field_path + '.' + o_field_name
fields_json_to_csv(
output_list_, current_field_path,
f['type'][effective_type_index_in_type]['fields'])
# End of function
for line in input_json_file:
try:
j = json.loads(line)
except:
self.logger.error(" Invalid JSON:\n%s" % line)
continue
i += 1
o_field_list_ = []
parent_field_path = ''
self.sort_id = 0
if not (j.has_key('attributes_json') or j.has_key('attributes')):
o_properties = {"doc": null}
else:
o_properties = {}
if j.has_key('attributes_json'):
o_properties = json.loads(j['attributes_json'])
del j['attributes_json']
if j.has_key('attributes'):
o_properties = dict(j['attributes'].items() +
o_properties.items())
del j['attributes']
if j.has_key('uri'):
o_urn = j['uri']
elif o_properties.has_key('uri'):
o_urn = o_properties['uri']
else:
self.logger.info('*** Warning: "uri" is not found in %s' %
j['name'])
o_urn = ''
if o_urn.find('hdfs://') == 0:
o_name = o_urn[o_urn.rfind('/') + 1:]
elif o_properties.has_key('table_name'):
o_name = o_properties['table_name']
elif j.has_key('name') and j['name'][0:5] != 'TUPLE':
o_name = j['name']
else:
o_name = o_urn[o_urn.rfind('/') + 1:]
if j.has_key('id') or not j.has_key('fields'): # esWritable schema
o_fields = {}
for k in j:
if not (k == 'uri' or k == 'attributes' or k == 'doc'):
if type(j[k]) == list:
o_fields[k] = {
"name": k,
"type": 'list',
"doc": str(j[k])
}
elif type(j[k]) == dict:
o_fields[k] = {
"name": k,
"type": 'dict',
"doc": str(j[k])
}
else:
o_fields[k] = {
"name": k,
"type": j[k],
"doc": None
}
self.sort_id += 1
o_field_list_.append([
o_urn, self.sort_id, 0, '', k, o_fields[k]['type'],
'', '', '', o_fields[k]['doc'].replace("\n", ' ')
if o_fields[k]['doc'] is not None else None
])
elif j.has_key('fields'):
o_fields = {}
for f in j['fields']:
o_field_name = f['name']
o_fields[o_field_name] = dict(f) # for schema output
if f.has_key('attributes_json'):
f['attributes'] = json.loads(f['attributes_json'])
del f['attributes_json']
fields_json_to_csv(o_field_list_, '', j['fields'])
else:
o_fields = {"doc": None}
if j.has_key('attributes') and not o_properties.has_key('source'):
o_properties['source'] = j['attributes']['source']
if o_urn.startswith(
'hdfs:///') and self.file_regex_source_map is not None:
o_source = self.get_source(o_urn[7:])
else:
self.logger.warn(
"property : " + Constant.HDFS_FILE_SOURCE_MAP_KEY +
" is None, will use default source for all dataset")
o_source = 'Hdfs'
self.logger.info(
"%4i (%6i): %4i fields, %4i total fields(including nested) found in [%s]@%s with source %s"
% (i, len(j), len(o_fields), len(o_field_list_), o_name, o_urn,
o_source))
dataset_schema_record = DatasetSchemaRecord(
o_name, json.dumps(j, sort_keys=True),
json.dumps(o_properties, sort_keys=True), json.dumps(o_fields),
o_urn, o_source, None, None, None)
schema_file_writer.append(dataset_schema_record)
for fields in o_field_list_:
field_record = DatasetFieldRecord(fields)
field_file_writer.append(field_record)
schema_file_writer.close()
field_file_writer.close()
input_json_file.close()
def transform(self, raw_metadata, metadata_output, field_metadata_output):
# sys.setdefaultencoding("UTF-8")
input_json_file = open(raw_metadata, 'r')
schema_file_writer = FileWriter(metadata_output)
field_file_writer = FileWriter(field_metadata_output)
i = 0
self.sort_id = 0
o_urn = ''
p = ''
for line in input_json_file:
try:
j = json.loads(line)
except:
self.logger.error(" Invalid JSON:\n%s" % line)
continue
i += 1
o_field_list_ = []
parent_field_path = ''
self.sort_id = 0
if not (j.has_key('attributes_json') or j.has_key('attributes')):
o_properties = {"doc": null}
else:
o_properties = {}
if j.has_key('attributes_json'):
o_properties = json.loads(j['attributes_json'])
del j['attributes_json']
if j.has_key('attributes'):
o_properties = dict(j['attributes'].items() +
o_properties.items())
del j['attributes']
if j.has_key('uri'):
o_urn = j['uri']
elif o_properties.has_key('uri'):
o_urn = o_properties['uri']
else:
self.logger.info('*** Warning: "uri" is not found in %s' %
j['name'])
o_urn = ''
if o_urn.find('hdfs://') == 0:
o_name = o_urn[o_urn.rfind('/') + 1:]
elif o_properties.has_key('table_name'):
o_name = o_properties['table_name']
elif j.has_key('name') and j['name'][0:5] != 'TUPLE':
o_name = j['name']
else:
o_name = o_urn[o_urn.rfind('/') + 1:]
if j.has_key('id') or not j.has_key('fields'): # esWritable schema
o_fields = {}
for k in j:
if not (k == 'uri' or k == 'attributes' or k == 'doc'):
if type(j[k]) == list:
o_fields[k] = {
"name": k,
"type": 'list',
"doc": str(j[k])
}
elif type(j[k]) == dict:
o_fields[k] = {
"name": k,
"type": 'dict',
"doc": str(j[k])
}
else:
o_fields[k] = {
"name": k,
"type": j[k],
"doc": None
}
self.sort_id += 1
o_field_list_.append([
o_urn, self.sort_id, 0, '', k, o_fields[k]['type'],
'', '', '', o_fields[k]['doc'].replace("\n", ' ')
if o_fields[k]['doc'] is not None else None
])
elif j.has_key('fields'):
o_fields = {}
for f in j['fields']:
o_field_name = f['name']
o_fields[o_field_name] = dict(f) # for schema output
if f.has_key('attributes_json'):
f['attributes'] = json.loads(f['attributes_json'])
del f['attributes_json']
acp = AvroColumnParser(j, o_urn)
o_field_list_ += acp.get_column_list_result()
else:
o_fields = {"doc": None}
if j.has_key('attributes') and not o_properties.has_key('source'):
o_properties['source'] = j['attributes']['source']
if o_urn.startswith(
'hdfs:///') and self.file_regex_source_map is not None:
o_source = self.get_source(o_urn[7:])
else:
self.logger.warn(
"property : " + Constant.HDFS_FILE_SOURCE_MAP_KEY +
" is None, will use default source for all dataset")
o_source = 'Hdfs'
self.logger.info(
"%4i (%6i): %4i fields, %4i total fields(including nested) found in [%s]@%s with source %s"
% (i, len(j), len(o_fields), len(o_field_list_), o_name, o_urn,
o_source))
dataset_schema_record = DatasetSchemaRecord(
o_name, json.dumps(j, sort_keys=True),
json.dumps(o_properties, sort_keys=True), json.dumps(o_fields),
o_urn, o_source, None, None, None)
schema_file_writer.append(dataset_schema_record)
for fields in o_field_list_:
field_record = DatasetFieldRecord(fields)
field_file_writer.append(field_record)
schema_file_writer.close()
field_file_writer.close()
input_json_file.close()