def table_from_csv(table_name=None, column_names=None):
standardized_column_names = []
for column_name in column_names:
if column_name.lower() not in ['', 'date']:
standardized_column_names.append(
clean_data.standardize_column_name(column_name, table_name))
if len(standardized_column_names) < 1:
return
try:
# probably a bad way to format, but it's a known data source
pg_cursor.execute(""" DROP TABLE IF EXISTS {table_name};
CREATE TABLE {table_name} (
"DATE" date not null,
{data_columns}
);
""".format(table_name=table_name,
data_columns=reduce(
lambda column_list, column: column_list + column, [
'{} float,\n'.format(column_name)
for column_name in standardized_column_names
])[0:-2]))
except Exception as e:
print(table_name + ' Error: ' + str(e))
def table_from_csv(table_name=None, column_names=None):
standardized_column_names = []
for column_name in column_names:
if column_name.lower() not in ['', 'date']:
standardized_column_names.append(clean_data.standardize_column_name(column_name, table_name))
if len(standardized_column_names) < 1:
return
try:
# probably a bad way to format, but it's a known data source
pg_cursor.execute(
""" DROP TABLE IF EXISTS {table_name};
CREATE TABLE {table_name} (
"DATE" date not null,
{data_columns}
);
""".format(
table_name=table_name,
data_columns=reduce(
lambda column_list, column: column_list + column,
[
'{} float,\n'.format(column_name)
for column_name in standardized_column_names
]
)[0:-2]
)
)
except Exception as e:
print(table_name + ' Error: ' + str(e))
def populate_row(table_name=None, row=None):
metric_values = []
metric_date = None
for key, value in row.items():
cleaned_key = clean_data.standardize_column_name(key, table_name)
cleaned_value = clean_data.standardize_value(value)
if str(cleaned_key) == '"DATE"':
metric_date = cleaned_value
elif cleaned_key is not None and cleaned_value is not None and cleaned_value != "NULL":
metric_values.append({
'release_date': cleaned_key,
'value': cleaned_value
})
metric_values = sorted(metric_values,
key=lambda release: release['release_date'])
final_value = None
value_finalization_index = None
for index, value_dict in enumerate(reversed(metric_values)):
release_value = value_dict['value']
if final_value is None:
final_value = release_value
if release_value != final_value:
break
value_finalization_index = index
if value_finalization_index is not None:
metric_values = metric_values[:-(value_finalization_index)]
if len(metric_values) > 0:
try:
# probably a bad way to format, but it's a known data source
pg_cursor.execute(""" INSERT INTO {table_name}
(metric_date, metric_values, initial_release_date, final_release_date)
VALUES
({metric_date}, {metric_values}, {initial_release_date}, {final_release_date});
""".format(table_name=table_name,
metric_date=metric_date,
metric_values=Json(metric_values),
initial_release_date="'" +
metric_values[0]['release_date'][1:] + "'::DATE",
final_release_date="'" +
metric_values[-1]['release_date'][1:] + "'::DATE"))
except Exception as e:
print(table_name + ' Error: ' + str(e))
def populate_row(table_name=None, row=None):
cleaned_row = OrderedDict()
for key, value in row.items():
cleaned_row[clean_data.standardize_column_name(
key, table_name)] = clean_data.standardize_value(value)
if cleaned_row.get(None):
cleaned_row.pop(None)
if len(cleaned_row) > 0:
try:
# probably a bad way to format, but it's a known data source
pg_cursor.execute(""" INSERT INTO {table_name} ({columns})
VALUES ({vals});
""".format(table_name=table_name,
columns=clean_data.string_list(cleaned_row.keys()),
vals=clean_data.string_list(cleaned_row.values())))
except Exception as e:
print(table_name + ' Error: ' + str(e))
def populate_row(table_name=None, row=None):
cleaned_row = OrderedDict()
for key, value in row.items():
cleaned_row[clean_data.standardize_column_name(key, table_name)] = clean_data.standardize_value(value)
if cleaned_row.get(None):
cleaned_row.pop(None)
if len(cleaned_row) > 0:
try:
# probably a bad way to format, but it's a known data source
pg_cursor.execute(
""" INSERT INTO {table_name} ({columns})
VALUES ({vals});
""".format(
table_name=table_name,
columns=clean_data.string_list(cleaned_row.keys()),
vals=clean_data.string_list(cleaned_row.values())
)
)
except Exception as e:
print(table_name + ' Error: ' + str(e))
