def currency_pipeline_environmental_dict(env_dict: EnvironmentDict,
sj_config: Dict) -> EnvironmentDict:
currency_config = sj_config['currency']
# environment dictionary
env_dict.add_solid_input('read_currency_codes', 'cur_config', currency_config) \
.add_solid_input('read_currency_codes', 'xml_path', currency_config['currency_codes']['cc_xml_path']) \
.add_solid_input('generate_currency_table_fields_str',
'table_data_columns', currency_config['currency_data_columns']) \
.add_composite_solid_input('read_sdr_per_currency', 'read_sdr_currencies', 'cur_config', currency_config) \
.add_composite_solid_input('read_sdr_per_currency', 'read_sdr_currencies',
'tsv_path', currency_config['tsv_path']) \
.add_composite_solid_input('transform_sdr_per_currency', 'transform_sdr_currencies', 'cur_config',
currency_config) \
.add_composite_solid_input('read_sdr_per_usd', 'read_usd_sdr', 'cur_config', currency_config) \
.add_composite_solid_input('read_sdr_per_usd', 'read_usd_sdr',
'tsv_path', currency_config['sdr_per_usd_path']) \
.add_composite_solid_input('transform_usd_sdr', 'transform_usd_sdr_currency', 'cur_config',
currency_config) \
.add_solid_input('read_sdr_valuation', 'cur_config', currency_config) \
.add_solid_input('read_sdr_valuation', 'sdrv_path', currency_config['sdrv_path']) \
.add_solid_input('read_ex_rates_per_usd', 'cur_config', currency_config) \
.add_solid_input('read_ex_rates_per_usd', 'ex_per_usd_path', currency_config['ex_per_usd_path']) \
.add_solid_input('transform_rates', 'cur_config', currency_config) \
.add_solid_input('transform_ex_rates_per_usd', 'cur_config', currency_config) \
return env_dict
def execute_create_sales_data_postgres_pipeline(sj_config: dict, postgres_warehouse: dict):
"""
Execute the pipeline to create the sales journal tables in Postgres
:param sj_config: app configuration
:param postgres_warehouse: postgres server resource
"""
# environment dictionary
env_dict = EnvironmentDict() \
.add_solid_input('load_csv', 'csv_path', sj_config['sales_data_desc']) \
.add_solid_input('load_csv', 'kwargs', {}, is_kwargs=True) \
.add_solid('generate_table_fields_str') \
.add_solid_input('generate_tracking_table_fields_str',
'tracking_data_columns', sj_config['tracking_data_columns']) \
.add_composite_solid_input('create_tables', 'create_data_table', 'table_name',
sj_config['sales_data_table']) \
.add_composite_solid_input('create_tables', 'create_tracking_table', 'table_name',
sj_config['tracking_data_table']) \
.add_resource('postgres_warehouse', postgres_warehouse) \
.build()
pp = pprint.PrettyPrinter(indent=2)
pp.pprint(env_dict)
result = execute_pipeline(create_sales_data_postgres_pipeline, environment_dict=env_dict)
assert result.success
def execute_file_ip_sql_to_plot_pipeline(sj_config: Dict, plotly_cfg: String,
postgres_warehouse: Dict,
plot_cfg_path: String,
plot_name: String):
"""
Execute the pipeline to create a plot
:param sj_config: app configuration
:param plotly_cfg: plotly configuration
:param postgres_warehouse: postgres server resource
:param plot_cfg_path: path to plot configuration file
:param plot_name: name of plot to produce
"""
if not isinstance(plotly_cfg, str):
plotly_cfg = ''
# environment dictionary
env_dict = EnvironmentDict() \
.add_solid_input('initialise_plot', 'yaml_path', plot_cfg_path) \
.add_solid_input('initialise_plot', 'plot_name', plot_name) \
.add_solid_input('process_sql_plot', 'plotly_cfg', plotly_cfg) \
.add_resource('postgres_warehouse', postgres_warehouse) \
.build()
pp = pprint.PrettyPrinter(indent=2)
pp.pprint(env_dict)
result = execute_pipeline(sql_to_plot_pipeline, environment_dict=env_dict)
assert result.success
def execute_mongo_to_postgres_pipeline():
"""
Execute the pipeline to retrieve the data from mongoDB, process and
save the result to Postgres
"""
# environment dictionary
env_dict = EnvironmentDict() \
.add_solid_input('download_from_mongo', 'sel_filter', {}) \
.add_solid_input('download_from_mongo', 'projection',
exclude_fields) \
.add_resource('postgres_warehouse', postgres_warehouse) \
.add_resource('mongo_warehouse', mongo_warehouse) \
.build()
result = execute_pipeline(mongo_to_postgres_pipeline,
run_config=env_dict)
assert result.success
def execute_csv_to_mongo_pipeline():
"""
Execute the pipeline to upload the UN/LOCODE data from the zipped csv
files to mongoDB
"""
# environment dictionary
env_dict = EnvironmentDict() \
.add_solid_input('load_csv_from_zip', 'zip_path',
app_cfg['airport_codes']['unlocode_zip']) \
.add_solid_input('load_csv_from_zip', 'pattern',
r'.*UNLOCODE CodeListPart\d*\.csv') \
.add_solid_input('load_csv_from_zip', 'encoding', 'latin_1') \
.add_solid_input('load_csv_from_zip', 'header', unlocode_header) \
.add_resource('mongo_warehouse', mongo_warehouse) \
.build()
result = execute_pipeline(csv_to_mongo_pipeline, run_config=env_dict)
assert result.success
def execute_create_currency_data_postgres_pipeline(cur_config: dict, postgres_warehouse: dict):
"""
Execute the pipeline to create the currency tables in Postgres
:param cur_config: app configuration
:param postgres_warehouse: postgres server resource
"""
# environment dictionary
env_dict = EnvironmentDict() \
.add_solid_input('generate_currency_table_fields_str',
'table_data_columns', cur_config['currency_data_columns']) \
.add_composite_solid_input('create_currency_tables', 'create_currency_table', 'table_name',
cur_config['currency_data_table']) \
.add_resource('postgres_warehouse', postgres_warehouse) \
.build()
pp = pprint.PrettyPrinter(indent=2)
pp.pprint(env_dict)
result = execute_pipeline(create_currency_data_postgres_pipeline, environment_dict=env_dict)
assert result.success
def execute_currency_to_postgres_pipeline(sj_config: Dict,
postgres_warehouse: Dict):
"""
Execute the pipeline to create a plot
:param sj_config:
:param sj_config: app configuration
:param postgres_warehouse: postgres server resource
:param plot_cfg_path: path to plot configuration file
:param plot_name: name of plot to produce
"""
currency_config = sj_config['currency']
# environment dictionary
env_dict = currency_pipeline_environmental_dict(EnvironmentDict(), sj_config)\
.add_solid_input('upload_currency_table', 'table_name', currency_config['currency_data_table']) \
.add_resource('postgres_warehouse', postgres_warehouse) \
.build()
pp = pprint.PrettyPrinter(indent=2)
pp.pprint(env_dict)
result = execute_pipeline(currency_to_postgres_pipeline,
environment_dict=env_dict)
assert result.success