def get_graph(graph=None, *, _limit=(), _print=()):
"""
Extracts a list of cafes with on euro in Paris, renames the name, address and zipcode fields,
reorders the fields and formats to json and csv files.
"""
graph = graph or bonobo.Graph()
producer = (
graph.get_cursor() >> ODSReader(dataset="liste-des-cafes-a-un-euro",
netloc="opendata.paris.fr") >>
PartialGraph(*_limit) >> bonobo.UnpackItems(0) >> bonobo.Rename(
name="nom_du_cafe", address="adresse", zipcode="arrondissement") >>
bonobo.Format(city="Paris", country="France") >> bonobo.OrderFields([
"name", "address", "zipcode", "city", "country", "geometry",
"geoloc"
]) >> PartialGraph(*_print))
# Comma separated values.
graph.get_cursor(producer.output) >> bonobo.CsvWriter(
"coffeeshops.csv",
fields=["name", "address", "zipcode", "city"],
delimiter=",")
# Standard JSON
graph.get_cursor(
producer.output) >> bonobo.JsonWriter(path="coffeeshops.json")
# Line-delimited JSON
graph.get_cursor(
producer.output) >> bonobo.LdjsonWriter(path="coffeeshops.ldjson")
return graph
def get_inventory_graph(**options):
"""
This function builds the graph that needs to be executed.
:return: bonobo.Graph
"""
graph = bonobo.Graph()
graph.add_chain(
bonobo.CsvReader('Deckbox-inventory.csv'),
bonobo.Filter(lambda *args: args[-1] != 'English'),
inventory,
bonobo.Rename(Card_Number='Card Number',
Tradelist_Count='Tradelist Count'),
# bonobo_sqlalchemy.InsertOrUpdate(
# 'cards',
# discriminant=(
# 'Name',
# 'Edition',
# 'Card_Number',
# 'Foil',
# ),
# engine='cards'),
_name='main',
)
return graph
def get_graph(**options):
"""
This function builds the graph that needs to be executed.
:return: bonobo.Graph
"""
graph = bonobo.Graph()
split_dbs = bonobo.noop
graph.add_chain(
GetOrderXML(prefix="/etl/ivm",
glob=[
'Mozilla_Corporation{timestamp:%Y_%m_%d}*.xml'.format(
timestamp=options['now'])
]),
ParseDates(['Transactionlog_Tranenddatetime']),
truncate_description,
bonobo.UnpackItems(0),
bonobo.Rename(transaction_date='Transactionlog_Tranenddatetime',
item_number='Transactionlog_Itemnumber',
transaction_id='Transactionlog_Tlid',
item_description='Transactionlog_Itemdesc'),
bonobo.Rename(
user_id='Transactionlog_User',
quantity='Transactionlog_Qty',
transaction_code='Transactionlog_Transcode',
description='Vendingmachines_Descr',
),
split_dbs,
_name="main")
#insert into ivm (description, transaction_id, item_number, item_description, user_id, quantity, transaction_date, transaction_code) values
for engine in list(set(options['engine'])):
graph.add_chain(bonobo_sqlalchemy.InsertOrUpdate(
table_name=options['table_name'] + options['table_suffix'],
discriminant=('transaction_id', ),
engine=engine),
_input=split_dbs)
return graph
def get_graph(graph=None, *, _limit=(), _print=()):
"""
Extracts a list of cafes with on euro in Paris, renames the name, address and zipcode fields,
reorders the fields and formats to json and csv files.
"""
graph = graph or bonobo.Graph()
producer = graph.add_chain(
ODSReader(dataset='liste-des-cafes-a-un-euro',
netloc='opendata.paris.fr'),
*_limit,
bonobo.UnpackItems(0),
bonobo.Rename(name='nom_du_cafe',
address='adresse',
zipcode='arrondissement'),
bonobo.Format(city='Paris', country='France'),
bonobo.OrderFields([
'name', 'address', 'zipcode', 'city', 'country', 'geometry',
'geoloc'
]),
*_print,
)
# Comma separated values.
graph.add_chain(
bonobo.CsvWriter('coffeeshops.csv',
fields=['name', 'address', 'zipcode', 'city'],
delimiter=','),
_input=producer.output,
)
# Standard JSON
graph.add_chain(
bonobo.JsonWriter(path='coffeeshops.json'),
_input=producer.output,
)
# Line-delimited JSON
graph.add_chain(
bonobo.LdjsonWriter(path='coffeeshops.ldjson'),
_input=producer.output,
)
return graph
def get_graph(**options):
"""
This function builds the graph that needs to be executed.
:return: bonobo.Graph
"""
graph = bonobo.Graph()
split = bonobo.noop
graph.add_chain(
bonobo.CsvWriter('DeckedBuilder.csv'),
# bonobo.Limit(10),
metadata,
# bonobo.UnpackItems(0),
split,
_input=None,
_name='main',
)
graph.add_chain(
bonobo.CsvReader('main-en.csv'),
bonobo.Format(Language='English'),
_output='main',
)
graph.add_chain(
bonobo.CsvReader('main-de.csv'),
bonobo.Format(Language='German'),
_output='main',
)
graph.add_chain(
bonobo.CsvReader('main-ru.csv'),
bonobo.Format(Language='Russian'),
_output='main',
)
graph.add_chain(
bonobo.CsvReader('main-it.csv'),
bonobo.Format(Language='Italian'),
_output='main',
)
graph.add_chain(
bonobo.CsvReader('main-jp.csv'),
bonobo.Format(Language='Japanese'),
_output='main',
)
graph.add_chain(
bonobo.CsvReader('main-fr.csv'),
bonobo.Format(Language='French'),
_output='main',
)
graph.add_chain(
bonobo.CsvReader('main-kr.csv'),
bonobo.Format(Language='Korean'),
_output='main',
)
graph.add_chain(
bonobo.CsvReader('main-cs.csv'),
bonobo.Format(Language='Chinese'),
_output='main',
)
graph.add_chain(
bonobo.CsvReader('Deckbox-extras.csv'),
bonobo.Format(Language='English'),
_output='main',
)
if ECHO_MTG:
# Reg Qty,Foil Qty,Name,Set,Acquired,Language
echomtg = {'Acquired For': '0.004', 'Language': 'en'}
graph.add_chain(
# echomtg specific fiddling
remove_metadata,
bonobo.UnpackItems(0),
# bonobo.PrettyPrinter(),
bonobo.Rename(Name='Card'),
bonobo.Format(**echomtg),
bonobo.CsvWriter('EchoMTG.csv'),
_input=split,
)
# MTG Studio
if MTG_STUDIO:
graph.add_chain(
mtg_studio,
remove_metadata,
bonobo.UnpackItems(0),
# bonobo.Format(Edition='{Set}'),
bonobo.Rename(Edition='Set'),
# bonobo.Rename(Name='Card'),
# bonobo.Rename(Qty='Reg Qty'),
# bonobo.Rename(Foil='Foil Qty'),
# bonobo.PrettyPrinter(),
bonobo.CsvWriter('MTG-Studio.csv'),
_input=split,
)
# graph.add_chain(
# tradeable,
# bonobo.UnpackItems(0),
# #bonobo.PrettyPrinter(),
# #bonobo.Limit(3000),
# bonobo.CsvWriter("DeckedBuilder-tradelist.csv"),
# bonobo.OrderFields([
# 'Card',
# 'Set',
# 'Foil',
# 'Quantity',
# ]),
# bonobo.CsvWriter("CardKingdom-buylist.csv"),
# bonobo.OrderFields([
# 'Quantity',
# 'Card',
# 'Set',
# ]),
# bonobo.CsvWriter(
# "mtgprice-buylist.csv",
# delimiter="\t",
# ),
# _input=split,
# )
#
if DECKBOX:
csv_out = bonobo.CsvWriter('Deckbox-inventory.csv')
graph.add_chain(
# # metadata,
# #bonobo.UnpackItems(0),
deckbox,
bonobo.UnpackItems(0),
csv_out,
_input=split,
)
graph.add_chain(bonobo.CsvReader('Deckbox-specials.csv'),
_output=csv_out)
return graph