def build_data(src_dir='', site_dir='_site', git=True):
"""Read each input file and edge file and write out json.
Args:
src_dir: str. Directory root for the project where data and meta data
folders are
site_dir: str. Directory to build the site to
git: bool. Do you want to check git for last updated dates?"""
status = True
ids = sdg.path.get_ids(src_dir=src_dir)
if len(ids) < 1:
raise IOError('No ids found in src_dir: ' + src_dir)
print("Processing data for " + str(len(ids)) + " indicators...")
all_meta = dict()
all_headline = dict()
# Schema
schema = sdg.schema.get_schema(prose_file='_prose.yml', src_dir=src_dir)
status = status & write_json('schema', schema, ftype='meta', gz=False, site_dir=site_dir)
for inid in ids:
# Load the raw
data = sdg.data.get_inid_data(inid, src_dir=src_dir)
# Compute derived datasets
edges = sdg.edges.edge_detection(inid, data)
headline = sdg.data.filter_headline(data)
# Output all the csvs
status = status & write_csv(inid, data, ftype='data', site_dir=site_dir)
status = status & write_csv(inid, edges, ftype='edges', site_dir=site_dir)
status = status & write_csv(inid, headline, ftype='headline', site_dir=site_dir)
# And JSON
data_dict = df_to_list_dict(data, orient='list')
edges_dict = df_to_list_dict(edges, orient='list')
headline_dict = df_to_list_dict(headline, orient='records')
status = status & write_json(inid, data_dict, ftype='data', gz=False, site_dir=site_dir)
status = status & write_json(inid, edges_dict, ftype='edges', gz=False, site_dir=site_dir)
status = status & write_json(inid, headline_dict, ftype='edges', gz=False, site_dir=site_dir)
# combined
comb = {'data': data_dict, 'edges': edges_dict}
status = status & write_json(inid, comb, ftype='comb', gz=False, site_dir=site_dir)
# Metadata
meta = sdg.meta.read_meta(inid, git=git, src_dir=src_dir)
status = status & sdg.json.write_json(inid, meta, ftype='meta', site_dir=site_dir)
# Append to the build-time "all" output
all_meta[inid] = meta
all_headline[inid] = headline_dict
status = status & sdg.json.write_json('all', all_meta, ftype='meta', site_dir=site_dir)
status = status & sdg.json.write_json('all', all_headline, ftype='headline', site_dir=site_dir)
return(status)
def build(self, language=None):
"""Write the JSON output expected by Open SDG. Overrides parent."""
status = True
all_meta = dict()
all_headline = dict()
site_dir = self.output_folder
# Write the schema.
schema_output = sdg.schemas.SchemaOutputOpenSdg(schema=self.schema)
schema_output_folder = os.path.join(site_dir, 'meta')
schema_output.write_schema(output_folder=schema_output_folder,
filename='schema.json')
# Write the translations.
translation_output = sdg.translations.TranslationOutputJson(
self.translations)
translation_folder = os.path.join(site_dir, 'translations')
translation_output.write_translations(language=language,
output_folder=translation_folder,
filename='translations.json')
for indicator_id in self.get_indicator_ids():
indicator = self.get_indicator_by_id(indicator_id).language(
language)
# Output all the csvs
status = status & write_csv(
indicator_id, indicator.data, ftype='data', site_dir=site_dir)
status = status & write_csv(indicator_id,
indicator.edges,
ftype='edges',
site_dir=site_dir)
status = status & write_csv(indicator_id,
indicator.headline,
ftype='headline',
site_dir=site_dir)
# And JSON
data_dict = df_to_list_dict(indicator.data, orient='list')
edges_dict = df_to_list_dict(indicator.edges, orient='list')
headline_dict = df_to_list_dict(indicator.headline,
orient='records')
status = status & write_json(indicator_id,
data_dict,
ftype='data',
gz=False,
site_dir=site_dir)
status = status & write_json(indicator_id,
edges_dict,
ftype='edges',
gz=False,
site_dir=site_dir)
status = status & write_json(indicator_id,
headline_dict,
ftype='headline',
gz=False,
site_dir=site_dir)
# combined
comb = {'data': data_dict, 'edges': edges_dict}
status = status & write_json(
indicator_id, comb, ftype='comb', gz=False, site_dir=site_dir)
# Metadata
status = status & sdg.json.write_json(
indicator_id, indicator.meta, ftype='meta', site_dir=site_dir)
# Append to the build-time "all" output
all_meta[indicator_id] = indicator.meta
all_headline[indicator_id] = headline_dict
status = status & sdg.json.write_json(
'all', all_meta, ftype='meta', site_dir=site_dir)
status = status & sdg.json.write_json(
'all', all_headline, ftype='headline', site_dir=site_dir)
stats_reporting = sdg.stats.reporting_status(
self.schema, all_meta, self.reporting_status_grouping_fields)
status = status & sdg.json.write_json(
'reporting', stats_reporting, ftype='stats', site_dir=site_dir)
indicator_export_service = OVRIndicatorExportService(
site_dir, self.indicators)
indicator_export_service.export_all_indicator_data_as_zip_archive()
return (status)
def build(self, language=None):
"""Write the JSON output expected by Open SDG. Overrides parent."""
status = True
all_meta = dict()
all_headline = dict()
site_dir = self.output_folder
# Write the schema.
schema_output = sdg.schemas.SchemaOutputOpenSdg(schema=self.schema)
schema_output_folder = os.path.join(site_dir, 'meta')
schema_output.write_schema(output_folder=schema_output_folder,
filename='schema.json')
# Write the translations.
translation_output = sdg.translations.TranslationOutputJson(
self.translations)
translation_folder = os.path.join(site_dir, 'translations')
translation_output.write_translations(language=language,
output_folder=translation_folder,
filename='translations.json')
for indicator_id in self.get_indicator_ids():
indicator = self.get_indicator_by_id(indicator_id).language(
language)
# Output all the csvs
status = status & write_csv(
indicator_id, indicator.data, ftype='data', site_dir=site_dir)
status = status & write_csv(indicator_id,
indicator.edges,
ftype='edges',
site_dir=site_dir)
status = status & write_csv(indicator_id,
indicator.headline,
ftype='headline',
site_dir=site_dir)
# And JSON
data_dict = df_to_list_dict(indicator.data, orient='list')
edges_dict = df_to_list_dict(indicator.edges, orient='list')
headline_dict = df_to_list_dict(indicator.headline,
orient='records')
status = status & write_json(indicator_id,
data_dict,
ftype='data',
gz=False,
site_dir=site_dir)
status = status & write_json(indicator_id,
edges_dict,
ftype='edges',
gz=False,
site_dir=site_dir)
status = status & write_json(indicator_id,
headline_dict,
ftype='headline',
gz=False,
site_dir=site_dir)
# combined
comb = {'data': data_dict, 'edges': edges_dict}
status = status & write_json(
indicator_id, comb, ftype='comb', gz=False, site_dir=site_dir)
# Metadata
status = status & sdg.json.write_json(
indicator_id, indicator.meta, ftype='meta', site_dir=site_dir)
# Append to the build-time "all" output
all_meta[indicator_id] = indicator.meta
all_headline[indicator_id] = headline_dict
status = status & sdg.json.write_json(
'all', all_meta, ftype='meta', site_dir=site_dir)
status = status & sdg.json.write_json(
'all', all_headline, ftype='headline', site_dir=site_dir)
# Reporting status.
reporting_status_types = []
if self.schema.get('reporting_status') is not None:
status_values = self.schema.get_values('reporting_status')
value_translation = self.schema.get_value_translation(
'reporting_status')
reporting_status_types = [{
'value': value,
'label': value_translation[value]
} for value in status_values]
stats_reporting = sdg.stats.reporting_status(
reporting_status_types, all_meta,
self.reporting_status_grouping_fields)
status = status & sdg.json.write_json(
'reporting', stats_reporting, ftype='stats', site_dir=site_dir)
disaggregation_status_service = sdg.DisaggregationStatusService(
site_dir, self.indicators, self.reporting_status_grouping_fields)
disaggregation_status_service.write_json()
indicator_export_service = sdg.IndicatorExportService(
site_dir, self.indicators, filename=self.indicator_export_filename)
indicator_export_service.export_all_indicator_data_as_zip_archive()
# Write the indicator downloads.
if self.indicator_downloads is not None:
download_service = sdg.IndicatorDownloadService(self.output_folder)
for download in self.indicator_downloads:
download_service.write_downloads(
download['button_label'], download['source_pattern'],
download['indicator_id_pattern'] if 'indicator_id_pattern'
in download else None, download['output_folder'])
download_service.write_index()
return (status)