def add_default_services(services, options): services['mysql'] = create_engine('mysql+mysqldb://localhost/aws', echo=False) services['redshift'] = create_engine(options['redshift'].format( host=options['redshift_host'], port=options['redshift_port'], name=options['redshift_name'], username=options['redshift_username'], password=options['redshift_password']), echo=False) services['vertica'] = create_engine(options['vertica'].format( host=options['vertica_host'], port=options['vertica_port'], name=options['vertica_name'], username=options['vertica_username'], password=options['vertica_password']), echo=False) if options['local']: services['brickftp'] = fs.open_fs("file:///tmp/etl") services['centerstone'] = fs.open_fs("file:///tmp/etl") else: services['brickftp'] = fs.open_fs( "ssh://%s@%s" % (options['brickftp_username'], options['brickftp_host'])) # Bug workaround to brickftp's sftp-only server services['brickftp']._platform = "Linux" services['centerstone'] = fs.open_fs( "ssh://[email protected]:/Out/"), if options['use_cache']: from requests_cache import CachedSession services['servicenow'] = CachedSession('http.cache') services['workday'] = CachedSession('http.cache') else: services['servicenow'] = requests.Session() services['workday'] = requests.Session() services['servicenow'].headers = {'User-Agent': 'Mozilla/ETL/v1'} services['servicenow'].auth = HTTPBasicAuth(options['sn_username'], options['sn_password']) services['servicenow'].headers.update({'Accept-encoding': 'text/json'}) services['workday'].headers = {'User-Agent': 'Mozilla/ETL/v1'} services['workday'].auth = HTTPBasicAuth(options['wd_username'], options['wd_password']) services['workday'].headers.update({'Accept-encoding': 'text/json'}) # Set a file suffix for non-prod jobs if options['environment'] == "prod": options['suffix'] = "" options['table_suffix'] = "" else: options['suffix'] = '.' + options['environment'] options['table_suffix'] = '_' + options['environment'] return
def get_services(**options): """ This function builds the services dictionary, which is a simple dict of names-to-implementation used by bonobo for runtime injection. It will be used on top of the defaults provided by bonobo (fs, http, ...). You can override those defaults, or just let the framework define them. You can also define your own services and naming is up to you. :return: dict """ if options['use_cache']: from requests_cache import CachedSession servicenow = CachedSession('http.cache') else: servicenow = requests.Session() servicenow.headers = {'User-Agent': 'Mozilla/ETL/v1'} servicenow.auth = HTTPBasicAuth(options['sn_username'], options['sn_password']) servicenow.headers.update({'Accept-encoding': 'text/json'}) return { 'servicenow': servicenow, 'db': create_engine('sqlite:///test.sqlite', echo=False), 'vertica': create_engine(options['vertica'].format( username=options['vertica_username'], password=options['vertica_password']), echo=False) }