def get_bu_graph(**options):
graph = bonobo.Graph()
graph.add_chain(
get_business_units,
join_cost_centers,
centerstone_BU_SupOrg_Merge_remap,
centerstone_BussUnit_remap,
)
graph.add_chain(
#bonobo.Limit(3),
#bonobo.PrettyPrinter(),
productLineLevel1_remap,
unique_product_line,
bonobo.UnpackItems(0),
bonobo.PrettyPrinter(),
bonobo.CsvWriter('/etl/centerstone/downloads/ProductLineLevel1.txt' +
options['suffix'],
lineterminator="\n",
delimiter="\t",
fs="brickftp"),
_input=centerstone_BussUnit_remap)
graph.add_chain(
teamLevel3_remap,
bonobo.UnpackItems(0),
bonobo.CsvWriter('/etl/centerstone/downloads/TeamLevel3.txt' +
options['suffix'],
lineterminator="\n",
delimiter="\t",
fs="brickftp"),
_input=centerstone_BussUnit_remap)
return graph
def get_graph(**options):
graph = bonobo.Graph()
graph.add_chain(
bonobo_sqlalchemy.Select('SELECT * FROM example', limit=100),
bonobo.PrettyPrinter(),
)
return graph
def get_graph(**options):
graph = bonobo.Graph()
graph.add_chain(
extract_fablabs,
bonobo.Limit(10),
bonobo.PrettyPrinter(),
)
return graph
def get_graph_options(options):
_limit = options.pop('limit', None)
_print = options.pop('print', False)
return {
'_limit': (bonobo.Limit(_limit), ) if _limit else (),
'_print': (bonobo.PrettyPrinter(), ) if _print else (),
}
def get_graph_options(options):
_limit = options.pop("limit", None)
_print = options.pop("print", False)
return {
"_limit": (bonobo.Limit(_limit), ) if _limit else (),
"_print": (bonobo.PrettyPrinter(), ) if _print else ()
}
def get_graph(**options):
graph = bonobo.Graph()
graph.add_chain(
getotherdata,
bonobo.Limit(10),
bonobo.PrettyPrinter(),
)
return graph
def get_graph():
graph = bonobo.Graph()
graph.add_chain(
bonobo_sqlalchemy.Select(
'SELECT * FROM dbo.Location where PublicRelease=1',
engine='nm_aquifer'),
bonobo.PrettyPrinter(),
)
return graph
def get_graph_options(options):
logger.debug("Unpacking command line options %s.", options)
_limit = options.pop("limit", None)
_print = options.pop("print", False)
graph_options = {
"_limit": (bonobo.Limit(_limit), ) if _limit else (),
"_print": (bonobo.PrettyPrinter(), ) if _print else ()
}
logger.debug("Created graph options %s.", graph_options)
return graph_options
def get_graph(**options):
"""
This function builds the graph that needs to be executed.
:return: bonobo.Graph
"""
graph = bonobo.Graph()
graph.add_chain(extract, bonobo.PrettyPrinter(), write_repr_to_file)
return graph
def get_graph(**options):
"""
This function builds the graph that needs to be executed.
:return: bonobo.Graph
"""
graph = bonobo.Graph()
graph.add_chain(extract_fablabs,bonobo.Limit(10),bonobo.PrettyPrinter())
return graph
def get_cs_graph(**options):
"""
This function builds the graph that needs to be executed.
:return: bonobo.Graph
"""
graph = bonobo.Graph()
split_employees = bonobo.noop
graph.add_chain(bonobo.FileReader(path='HrExport.txt',
fs='centerstone',
encoding='latin-1',
eol="\r\n"),
split_tabs,
prefix_desk_ids,
join_wpr,
mismatch,
split_employees,
_name="main")
# Process regular employees
if options['dry_run']:
update = ()
else:
update = (update_employee_record, )
graph.add_chain(bonobo.PrettyPrinter(),
*update,
bonobo.PrettyPrinter(),
_input=split_employees)
# Dump out outlier employees
#graph.add_chain(
# bonobo.Filter(filter=odd_employee),
# bonobo.UnpackItems(0),
# bonobo.PrettyPrinter(),
# _input=split_employees)
#
return graph
def get_graph(*, _limit=None, _print=False):
graph = bonobo.Graph()
trunk = graph.add_chain(bonobo.JsonReader('datasets/theaters.json'), *((bonobo.Limit(_limit),) if _limit else ()))
if _print:
graph.add_chain(bonobo.PrettyPrinter(), _input=trunk.output)
graph.add_chain(bonobo.JsonWriter('theaters.json', fs='fs.output'), _input=trunk.output)
graph.add_chain(bonobo.LdjsonWriter('theaters.ldjson', fs='fs.output'), _input=trunk.output)
return graph
def get_graph(*, _limit=None, _print=False):
graph = bonobo.Graph()
trunk = graph.add_chain(bonobo.JsonReader("theaters.json", fs="fs.static"),
*((bonobo.Limit(_limit), ) if _limit else ()))
if _print:
graph.add_chain(bonobo.PrettyPrinter(), _input=trunk.output)
graph.add_chain(bonobo.JsonWriter("theaters.output.json", fs="fs.output"),
_input=trunk.output)
graph.add_chain(bonobo.LdjsonWriter("theaters.output.ldjson",
fs="fs.output"),
_input=trunk.output)
return graph
def get_sched_graph(**options):
"""
This function builds the graph that needs to be executed.
:return: bonobo.Graph
"""
graph = bonobo.Graph(
get_sched,
bonobo.UnpackItems(0),
modified_events,
sync_event,
bonobo.UnpackItems(0),
bonobo.PrettyPrinter(),
bonobo.count,
)
return graph
def get_interesting_fields(record):
result = {
"line_id": record["fields"]["mnemoligne"],
"stop_id": record["fields"]["idarret"],
"line_name": record["fields"]["nomligne"],
"stop_name": record["fields"]["nomarret"]
}
yield result
# Fonction utilisant un générateur qui se connecte à la base et tente d'enregistrer les données dans la table des lignes
def insert_in_relationnal(record):
with psycopg2.connect(**relationnal_connection_data) as connection:
with connection.cursor() as cursor:
try:
cursor.execute(
"""
INSERT INTO line (id, name) VALUES (%s, %s);
""", (record["line_id"], record["line_name"]))
except psycopg2.IntegrityError as e:
print("line already exists", record["line_id"])
yield record
# Génération du Graph bonobo
graph = bonobo.Graph()
graph.add_chain(fetch_desserte_data, get_interesting_fields,
insert_in_relationnal, bonobo.PrettyPrinter())
graph.add_chain(graph, _input=fetch_desserte_data)
r = bonobo.run(graph)
import bonobo
from bonobo.commands.run import get_default_services
from bonobo.nodes.io.json import JsonDictItemsReader
from bonobo_factories import Factory
normalize = Factory()
normalize[0].str().title()
normalize.move(0, 'title')
normalize.move(0, 'address')
graph = bonobo.Graph(
JsonDictItemsReader('datasets/coffeeshops.json'),
normalize,
bonobo.PrettyPrinter(),
)
if __name__ == '__main__':
bonobo.run(graph, services=get_default_services(__file__))
def get_graph(*, _limit=None, _print=False):
return bonobo.Graph(bonobo.CsvReader("coffeeshops.csv"),
*((bonobo.Limit(_limit), ) if _limit else ()),
*((bonobo.PrettyPrinter(), ) if _print else ()),
bonobo.CsvWriter("coffeeshops.csv", fs="fs.output"))
def get_graph():
graph = bonobo.Graph()
graph.add_chain(extract, transform, load_csv, load_postgres,
bonobo.Limit(1000), bonobo.PrettyPrinter())
return graph
def get_graph(*, _limit=None, _print=False):
return bonobo.Graph(bonobo.CsvReader('datasets/coffeeshops.txt'),
*((bonobo.Limit(_limit), ) if _limit else ()),
*((bonobo.PrettyPrinter(), ) if _print else ()),
bonobo.CsvWriter('coffeeshops.csv', fs='fs.output'))
