def add_default_services(services, options):
services['mysql'] = create_engine('mysql+mysqldb://localhost/aws',
echo=False)
services['redshift'] = create_engine(options['redshift'].format(
host=options['redshift_host'],
port=options['redshift_port'],
name=options['redshift_name'],
username=options['redshift_username'],
password=options['redshift_password']),
echo=False)
services['vertica'] = create_engine(options['vertica'].format(
host=options['vertica_host'],
port=options['vertica_port'],
name=options['vertica_name'],
username=options['vertica_username'],
password=options['vertica_password']),
echo=False)
if options['local']:
services['brickftp'] = fs.open_fs("file:///tmp/etl")
services['centerstone'] = fs.open_fs("file:///tmp/etl")
else:
services['brickftp'] = fs.open_fs(
"ssh://%s@%s" %
(options['brickftp_username'], options['brickftp_host']))
# Bug workaround to brickftp's sftp-only server
services['brickftp']._platform = "Linux"
services['centerstone'] = fs.open_fs(
"ssh://[email protected]:/Out/"),
if options['use_cache']:
from requests_cache import CachedSession
services['servicenow'] = CachedSession('http.cache')
services['workday'] = CachedSession('http.cache')
else:
services['servicenow'] = requests.Session()
services['workday'] = requests.Session()
services['servicenow'].headers = {'User-Agent': 'Mozilla/ETL/v1'}
services['servicenow'].auth = HTTPBasicAuth(options['sn_username'],
options['sn_password'])
services['servicenow'].headers.update({'Accept-encoding': 'text/json'})
services['workday'].headers = {'User-Agent': 'Mozilla/ETL/v1'}
services['workday'].auth = HTTPBasicAuth(options['wd_username'],
options['wd_password'])
services['workday'].headers.update({'Accept-encoding': 'text/json'})
# Set a file suffix for non-prod jobs
if options['environment'] == "prod":
options['suffix'] = ""
options['table_suffix'] = ""
else:
options['suffix'] = '.' + options['environment']
options['table_suffix'] = '_' + options['environment']
return
def get_services(**options):
"""
This function builds the services dictionary, which is a simple dict of names-to-implementation used by bonobo
for runtime injection.
It will be used on top of the defaults provided by bonobo (fs, http, ...). You can override those defaults, or just
let the framework define them. You can also define your own services and naming is up to you.
:return: dict
"""
if options['use_cache']:
from requests_cache import CachedSession
servicenow = CachedSession('http.cache')
else:
servicenow = requests.Session()
servicenow.headers = {'User-Agent': 'Mozilla/ETL/v1'}
servicenow.auth = HTTPBasicAuth(options['sn_username'],
options['sn_password'])
servicenow.headers.update({'Accept-encoding': 'text/json'})
return {
'servicenow':
servicenow,
'db':
create_engine('sqlite:///test.sqlite', echo=False),
'vertica':
create_engine(options['vertica'].format(
username=options['vertica_username'],
password=options['vertica_password']),
echo=False)
}
