def write_csv(inid, df, ftype='data', site_dir=''):
"""
For a given ID and data set, write out as csv
Args:
inid: str. The indicator identifier
df: DataFrame. The pandas data frame of the data
ftype: Sets directory path
site_dir: str. The site directory to build to.
Returns:
bool: Status
"""
status = True
# If the csv dir isn't there, make it
csv_dir = output_path(ftype=ftype, format='csv', site_dir=site_dir)
if not os.path.exists(csv_dir):
os.makedirs(csv_dir, exist_ok=True)
# The path within the csv dir
out_path = output_path(inid, ftype=ftype, format='csv', site_dir=site_dir)
try:
df.to_csv(out_path, index=False)
except Exception as e:
print(inid, e)
return False
return status
def test_out_path():
"""Check that output_path is as expected"""
out_path = output_path(inid="1-2-1",
ftype='data',
format='json',
site_dir='_site')
assert out_path == os.path.join('_site', 'data', '1-2-1.json')
def compare_reload_data(inid, src_dir, site_dir):
"""Load the original csv and compare to reloading the JSON you wrote out
which = 'edges' or 'data'
"""
csv_path = input_path(inid, ftype='data', src_dir=src_dir)
jsn_path = output_path(inid,
ftype='comb',
format='json',
site_dir=site_dir)
jsn = json.load(open(jsn_path))
df_csv = pd.read_csv(csv_path, encoding='utf-8')
df_jsn = pd.DataFrame(jsn['data']).replace({None: np.nan})
# Account for empty data
if df_jsn.shape[0] == df_csv.shape[0] == 0:
return True
df_jsn = df_jsn[df_csv.columns.values]
status = isclose_df(df_csv, df_jsn)
if not status:
print("reload error in " + inid)
return status
def test_built_json(test_site_dir):
meta9 = json.load(
open(
output_path('9-3-1',
ftype='meta',
format='json',
site_dir=test_site_dir)))
assert meta9['indicator'] == '9.3.1'
def test_reporting_json(test_site_dir):
stats_reporting = json.load(
open(
output_path('reporting',
ftype='stats',
format='json',
site_dir=test_site_dir)))
assert stats_reporting['goals'][7]['totals']['total'] == 2
def test_built_translations(test_site_dir):
translations = json.load(
open(
output_path('translations',
ftype='translations',
format='json',
site_dir=test_site_dir)))
assert translations['de']['global_goals']['1-short'] == 'Keine Armut'
def test_built_schema(test_site_dir):
schema_output = json.load(
open(
output_path('schema',
ftype='meta',
format='json',
site_dir=test_site_dir)))
reporting_status_field = next(
(item for item in schema_output if item['name'] == 'reporting_status'),
None)
assert reporting_status_field
assert reporting_status_field['field']['label'] == 'Reporting status'
def write_json(inid, obj, ftype='data', gz=False, site_dir=''):
"""Write out the supplied object as a single json file. This can
either be as records (orient='records') or as columns (orient='list').
Args:
inid -- str: The indicator id, e.g. '1-1-1'
obj -- dict or list: A json ready dict/list
ftype -- str: Output type. Used to find the path
gz -- bool: if True then compress the output with gzip
Return:
status. bool.
"""
try:
out_json = pd.io.json.dumps(obj)
out_json = out_json.replace("\\/", "/") # why does it double escape?
json_dir = output_path(ftype=ftype, format='json', site_dir=site_dir)
if not os.path.exists(json_dir):
os.makedirs(json_dir, exist_ok=True)
json_path = output_path(inid,
ftype=ftype,
format='json',
site_dir=site_dir)
# Write out
if gz:
json_bytes = out_json.encode('utf-8')
with gzip.open(json_path + '.gz', 'w') as outfile:
outfile.write(json_bytes)
else:
with open(json_path, 'w', encoding='utf-8') as outfile:
outfile.write(out_json)
except Exception as e:
print(inid, e)
return False
return True