def _build_resource_metadata(rootdir, pmd={}, rmd={}, user_md=dict()): d = merge(pmd, rmd) d['provider_path'] = pmd.get('path', '') d['resource_path'] = rmd.get('path', '') d['provider_alias'] = pmd.get('alias') d['resource_alias'] = rmd.get('alias') d.pop('alias', None) d.pop('path', None) d['rootdir'] = rootdir if not d.get('service'): parts = d['provider_path'].split('://') if len(parts) > 1: d['service'] = parts[0] d['provider_path'] = parts[1] if not d.get('service'): parts = d['resource_path'].split('://') if len(parts) > 1: d['service'] = parts[0] d['resource_path'] = parts[1] if not d.get('service'): d['service'] = 'file' # if service is local or sqlite, # relative path is allowed, and prefixed with rootpath if d['service'] in ['file', 'sqlite'] and \ not os.path.isabs(d['provider_path']) and \ not os.path.isabs(d['resource_path']): d['provider_path'] = os.path.realpath( os.path.join(d['rootdir'], d['provider_path'])) d['format'] = _format(d) d['driver'] = _driver(d) #default hostname is localhost d['host'] = d.get('hostname', d.get('host', '127.0.0.1')) # provider path can be use as database name if not d.get('database') and d.get('format') in ['jdbc', 'nosql']: d['database'] = d.get('path') d['port'] = d.get('port', _port(d['service'])) d['url'] = _url(d) d['options'] = d['options'] if d.get('options') else {} d['mapping'] = d['mapping'] if d.get('mapping') else {} # override with function provided metadata d = merge(d, user_md) d['hash'] = hash(d['url']) ^ hash(d['format']) ^ hash(d['resource_path']) d['hash'] = hex(ctypes.c_size_t(d['hash']).value) return d
def read(file_paths=None): """ Return all profiles, stored in a nested dictionary profiles are merged over the list provided profiles. list order determines override each profile name :param file_paths: list of yaml files :return: dict of profiles """ profiles = {} if not file_paths: file_paths = [] for filename in file_paths: if os.path.isfile(filename): with open(filename, 'r') as f: try: docs = list(yaml.load_all(f)) except yaml.YAMLError as e: if hasattr(e, 'problem_mark'): mark = e.problem_mark logging.error( "Error loading yml file {} at position: (%s:%s): skipping file" .format(filename, mark.line + 1, mark.column + 1)) docs = [] finally: for doc in docs: doc['profile'] = doc.get('profile', 'default') profiles[doc['profile']] = merge( profiles.get(doc['profile'], {}), doc) return profiles
def read(file_paths=None): """ Return all profiles, stored in a nested dictionary Profiles are merged over the list provided of provided metadata files to read. The order in the list of metadata files determines how profile properties are override :param file_paths: list of yaml files paths :return: dict of profiles """ global loaded_md_files, profiles # empty profiles, before start reading profiles = {} if not file_paths: file_paths = [] loaded_md_files = [] for filename in file_paths: if os.path.isfile(filename): with open(filename, 'r') as f: try: docs = list(yaml.load_all(f)) loaded_md_files.append(filename) except yaml.YAMLError as e: if hasattr(e, 'problem_mark'): mark = e.problem_mark logging.error("Error loading yml file {} at position: (%s:%s): skipping file".format(filename, mark.line+1, mark.column+1)) docs = [] finally: for doc in docs: doc['profile'] = doc.get('profile', 'default') profiles[doc['profile']] = merge(profiles.get(doc['profile'],{}), doc) return profiles
def inherit(profiles): """ Modify profiles to inherit from default profile :param profiles: input dict of profiles :return: profile """ # inherit from default for all other profiles for k in profiles.get('default', {}).keys(): for p in profiles.keys() - 'default': profiles[p][k] = merge(profiles['default'][k], profiles[p].get(k)) return profiles
def inherit(profiles): """ Profiles inherit from a default profile. Inherit merges each profile with the configuration of the default profile. :param profiles: dict of profiles :return: dict of profiles """ # inherit from default for all other profiles for k in profiles.get('default', {}).keys(): for p in set(profiles.keys()) - {'default'}: profiles[p][k] = merge(profiles['default'][k], profiles[p].get(k)) return profiles
def test_minimal(self, tempdir): pmd = {'service': 'local', 'format': 'csv', 'path': tempdir.path} # noinspection PyProtectedMember d = resource._build_resource_metadata(tempdir.path, pmd=pmd) m = self.empty(tempdir).copy() u = { 'provider_path': tempdir.path, 'service': 'local', 'format': 'csv', 'url': f'{tempdir.path}' } m = merge(m, u) assert (d == m)
def to_resource(url_alias=None, *args, **kwargs): md = None # if a dict, create from dictionary if isinstance(url_alias, dict): md = resource_from_dict(url_alias) # if a string, and a metadata profile is loaded, check for aliases if metadata.profile(): if not md and url_alias in metadata.profile().get('resources', {}).keys(): md = metadata.profile()['resources'][url_alias] if not md and url_alias in metadata.profile().get('providers', {}).keys(): md = metadata.profile()['providers'][url_alias] # if nothing found yet, interpret as a urn/path if not md and url_alias: md = resource_from_urn(urnparse(url_alias)) # empty default if not md: md = get_default_md() # sanitize path if it's a url or a query if md['path']: url_md = resource_from_urn(urnparse(md['path'])) md = merge(url_md, md) md['path'] = url_md['path'] # override using kwargs md = metadata_overrides(md, **kwargs) if 'hostname' in md: del md['hostname'] if 'username' in md: del md['username'] return md
def process(self, msg, kwargs): """ Process the logging message and keyword arguments passed in to a logging call to insert contextual information. You can either manipulate the message itself, the keyword args or both. Return the message and kwargs modified (or not) to suit your needs. Normally, you'll only need to override this one method in a LoggerAdapter subclass for your specific needs. """ d = self.extra d.update({'dlf_funcname': func_name(5)}) if isinstance(msg, dict): d.update({'dlf_data': merge(msg, kwargs.get('extra', {}))}) msg = 'data' elif isinstance(msg, str): d.update({'dlf_data': kwargs.get('extra', {})}) else: raise ValueError('log message must be a str or a dict') kwargs["extra"] = d return msg, kwargs
def metadata_overrides(md, host=None, service=None, port=None, user=None, password=None, driver=None, database=None, schema=None, table=None, format=None, hostname=None, username=None, **options): d = {} d['path'] = md.get('url') or md.get('path') d['provider'] = md.get('provider') d['host'] = host or hostname or md['host'] or md.get('hostname') d['port'] = port or md['port'] d['service'] = service or md['service'] d['format'] = format or md['format'] d['user'] = user or username or md['user'] or md.get('username') d['password'] = password or md['password'] d['database'] = database or md['database'] d['schema'] = schema or md['schema'] d['table'] = table or md['table'] d['driver'] = driver or md['driver'] d['options'] = merge(md['options'], options) if database or table: d['path'] = None return d
def test_resource_provider_2path_absolute(self, tempdir): pmd = { 'alias': 'p', 'service': 'local', 'format': 'csv', 'path': '/absolute/path' } rmd = {'alias': 'r', 'path': 'abc/def'} # noinspection PyProtectedMember d = resource._build_resource_metadata(tempdir.path, pmd=pmd, rmd=rmd) m = self.empty(tempdir).copy() u = { 'provider_path': '/absolute/path', 'provider_alias': 'p', 'resource_alias': 'r', 'resource_path': 'abc/def', 'service': 'local', 'format': 'csv', 'url': f'/absolute/path/abc/def' } m = merge(m, u) assert (d == m)
def test_merge(): a = {'a': 1, 'b': 4, 'c': {'merge1': 2}} b = {'d': 'add', 'b': 'override', 'c': {'merge2': 4}} r1 = merge(a, b) r2 = {'a': 1, 'd': 'add', 'b': 'override', 'c': {'merge2': 4, 'merge1': 2}} assert (r1 == r2)
def _build_resource_metadata(rootdir, pmd={}, rmd={}, user_md=dict()): d = merge(pmd, rmd) d['provider_alias'] = pmd.get('alias', pmd.get('path', '')) d['resource_alias'] = rmd.get('alias', rmd.get('path', '')) d['provider_path'] = pmd.get('path', pmd.get('alias', '')) d['resource_path'] = rmd.get('path', rmd.get('alias', '')) d.pop('alias', None) d.pop('path', None) d['rootdir'] = rootdir if not d.get('service'): parts = d['provider_path'].split('://') if len(parts) > 1: d['service'] = parts[0] d['provider_path'] = parts[1] if not d.get('service'): parts = d['resource_path'].split('://') if len(parts) > 1: d['service'] = parts[0] d['resource_path'] = parts[1] if not d.get('service'): d['service'] = 'file' # if service is local or sqlite, # relative path is allowed, and prefixed with rootpath if d['service'] in ['file', 'sqlite'] and \ not os.path.isabs(d['provider_path']) and \ not os.path.isabs(d['resource_path']): d['provider_path'] = os.path.realpath( os.path.join(d['rootdir'], d['provider_path'])) d['format'] = _format(d) d['driver'] = _driver(d) #default hostname is localhost d['host'] = d.get('hostname', d.get('host', '127.0.0.1')) # provider path can be use as database name, if database is undefined # for some special rdbms, database and path are both required # if both database and path are provided, path is interpreted as a database schema # if only path or database is provided, assume that the schema is 'public' if d['format'] == 'jdbc': d['table'] = d['resource_path'] d['table'] = d['table'] if d[ 'table'] else 'SELECT 0 as result where 1 = 0' if d.get('database'): d['database'] = d.get('database') d['schema'] = d['provider_path'] if d['provider_path'] else '' else: d['database'] = d['provider_path'] d['schema'] = '' #if schema is not yet defined, take the default for each service if not d['schema']: if d.get('service') == 'mysql': d['schema'] = d['database'] elif d.get('service') == 'mssql': d['schema'] = 'dbo' elif d.get('service') == 'postgres': d['schema'] = 'public' elif d.get('service') == 'oracle': d['schema'] = d.get('username', '') else: #use postgres default if service unkown d['schema'] = 'public' # if format is jdbc and an SQL query is detected, # wrap the resource path as a temp table sql_query = d['table'] sql_query = sql_query.replace('\n', ' ') sql_query = sql_query.replace('\t', ' ') sql_query = sql_query.replace('\r', ' ') sql_query = ' '.join(sql_query.split()) sql_query = sql_query.rstrip(' ') sql_query = sql_query.rstrip(';') if ' from ' in sql_query.lower(): d['table'] = '( {} ) as _query'.format(sql_query) d['port'] = d.get('port', _port(d['service'])) d['url'] = _url(d) d['options'] = d['options'] if d.get('options') else {} d['mapping'] = d['mapping'] if d.get('mapping') else {} # override with function provided metadata d = merge(d, user_md) d['hash'] = hash(d['url']) ^ hash(d['format']) ^ hash(d['resource_path']) d['hash'] = hex(ctypes.c_size_t(d['hash']).value) return d
def _override_metadata(access, param, pmd=dict(), rmd=dict()): d = merge( pmd.get(access, {}).get(param, {}), rmd.get(access, {}).get(param, {})) return d
def test_resource_provider_2path_absolute(self, tempdir): pmd = { 'alias': 'p', 'service': 'local', 'format': 'csv', 'path': '/absolute/path' } rmd = {'alias': 'r', 'path': 'abc/def'} # noinspection PyProtectedMember d = resource._build_resource_metadata(tempdir.path, pmd=pmd, rmd=rmd) m = self.empty(tempdir).copy() u = { 'provider_path': '/absolute/path', 'provider_alias': 'p', 'resource_alias': 'r', 'resource_path': 'abc/def', 'service': 'local', 'format': 'csv', 'url': f'/absolute/path/abc/def' } m = merge(m, u) assert (d == m) # resource('SELECT 0 as result where 1 = 0', 'pagila') # resource('foo.csv', '/bar') # resource('foo.csv', 'bar') # resource('foo.csv', 'hdfs') # resource('/foo.abc', 'hdfs') # resource('/foo.abc', 'test') # resource('hello/foo.abc', 'test') # resource('foo.abc', 'hdfs://*****:*****@1.2.3.4:3306/sakila?useSSL=false&serverTimezone=UTC&zeroDateTimeBehavior=CONVERT_TO_NULL') # resource('staff', 'jdbc:mysql://1.2.3.4/sakila', useSSL='false', serverTimezone='UTC', zeroDateTimeBehavior='CONVERT_TO_NULL') # resource('staff', service='mysql', database='sakila', serverTimezone='UTC') # resource('sakila/staff', service='mysql', serverTimezone='UTC', user='******', password='******') # resource('foo/bar.tsv', service='s3a') # resource('/foo/bar.tsv', service='s3a') # resource('/apples/orange', service='minio') # resource('SELECT count(*) as cnt from employees;', 'jdbc:mysql://1.2.3.4:3306/sakila?useSSL=false&serverTimezone=UTC&zeroDateTimeBehavior=CONVERT_TO_NULL', user='******', password='******') # resource('ascombe') # resource('ascombe', 'saywhat') # resource('ascombe', 'hdfs://*****:*****@//123.123.123:345/schema/database' # parsed = Urn(scheme=['jdbc', 'oracle', 'thin', 'name/pass@'], user='******', password='******', host='123.123.123', port='345', path='/schema/database', params=[], query='', fragment='') # assert(parsed == urnparse(urn)) # urn = 'jdbc:oracle:thin:name@//123.123.123:345/schema/database' # parsed = Urn(scheme=['jdbc', 'oracle', 'thin', 'name@'], user='******', password='', host='123.123.123', port='345', path='/schema/database', params=[], query='', fragment='') # assert(parsed == urnparse(urn)) # urn = 'jdbc:oracle:thin:@//123.123.123/schema/database' # parsed = Urn(scheme=['jdbc', 'oracle', 'thin', '@'], user='', password='', host='123.123.123', port='', path='/schema/database', params=[], query='', fragment='') # assert(parsed == urnparse(urn)) # urn = 'hdfs://123.123.123/schema/database' # parsed = Urn(scheme=['hdfs'], user='', password='', host='123.123.123', port='', path='/schema/database', params=[], query='', fragment='') # assert(parsed == urnparse(urn)) # urn = '/schema/database' # parsed = Urn(scheme=[], user='', password='', host='', port='', path='/schema/database', params=[], query='', fragment='') # assert(parsed == urnparse(urn)) # urn = 's3a://schema/database' # parsed = Urn(scheme=['s3a'], user='', password='', host='', port='', path='schema/database', params=[], query='', fragment='') # assert(parsed == urnparse(urn)) # urn = '1.2.34/schema/database' # parsed = Urn(scheme=[], user='', password='', host='', port='', path='1.2.34/schema/database', params=[], query='', fragment='') # assert(parsed == urnparse(urn)) # urn = 'file://1.2.34/schema/database' # parsed = Urn(scheme=['file'], user='', password='', host='1.2.34', port='', path='/schema/database', params=[], query='', fragment='') # assert(parsed == urnparse(urn)) # urn = 'jdbc:sqlite://localdir/a/b/c' # parsed = Urn(scheme=['jdbc', 'sqlite'], user='', password='', host='', port='', path='localdir/a/b/c', params=[], query='', fragment='') # assert(parsed == urnparse(urn)) # urn = 'jdbc:oracle:thin:@ldap://xyz.acme.com:7777/sales,cn=salesdept,cn=OracleContext,dc=com' # parsed = Urn(scheme=['jdbc', 'oracle', 'thin', '@ldap'], user='', password='', host='xyz.acme.com', port='7777', path='/sales', params=[('cn', 'salesdept'), ('cn', 'OracleContext'), ('dc', 'com')], query='cn=salesdept&cn=OracleContext&dc=com', fragment='') # assert(parsed == urnparse(urn)) # urn = 'http://xyz.acme.com:7777/foo/bar?a=1&edf=abc#anchor1' # parsed = Urn(scheme=['http'], user='', password='', host='xyz.acme.com', port='7777', path='/foo/bar', params=[('a', '1'), ('edf', 'abc')], query='a=1&edf=abc', fragment='anchor1') # assert(parsed == urnparse(urn)) # urn = 'jdbc:sqlserver://localhost:1433;databaseName=AdventureWorks;integratedSecurity=true;' # parsed = Urn(scheme=['jdbc', 'sqlserver'], user='', password='', host='localhost', port='1433', path='', params=[('databaseName', 'AdventureWorks'), ('integratedSecurity', 'true')], query='databaseName=AdventureWorks&integratedSecurity=true', fragment='') # assert(parsed == urnparse(urn)) # urn = 'jdbc:sqlserver://localhost;databaseName=AdventureWorks;integratedSecurity=true;' # parsed = Urn(scheme=['jdbc', 'sqlserver'], user='', password='', host='localhost', port='', path='', params=[('databaseName', 'AdventureWorks'), ('integratedSecurity', 'true')], query='databaseName=AdventureWorks&integratedSecurity=true', fragment='') # assert(parsed == urnparse(urn)) # urn = 'jdbc:postgresql://localhost/test?user=fred&password=secret&ssl=false' # parsed = Urn(scheme=['jdbc', 'postgresql'], user='', password='', host='localhost', port='', path='/test', params=[('user', 'fred'), ('password', 'secret'), ('ssl', 'false')], query='user=fred&password=secret&ssl=false', fragment='') # assert(parsed == urnparse(urn)) # urn = 'jdbc:mysql://localhost:3306/youdatabase?useSSL=false&serverTimezone=UTC&zeroDateTimeBehavior=CONVERT_TO_NULL' # parsed = Urn(scheme=['jdbc', 'mysql'], user='', password='', host='localhost', port='3306', path='/youdatabase', params=[('useSSL', 'false'), ('serverTimezone', 'UTC'), ('zeroDateTimeBehavior', 'CONVERT_TO_NULL')], query='useSSL=false&serverTimezone=UTC&zeroDateTimeBehavior=CONVERT_TO_NULL', fragment='') # assert(parsed == urnparse(urn))
def Resource(path_or_alias_or_url=None, provider_path_or_alias_or_url=None, host=None, service=None, port=None, user=None, password=None, driver=None, database=None, schema=None, table=None, format=None, hostname=None, username=None, **options): prov = provider_path_or_alias_or_url path = path_or_alias_or_url # get the resource, by alias metadata or by url rmd = to_resource(path, host=host, service=service, port=port, user=user, password=password, driver=driver, database=database, schema=schema, table=table, format=format, hostname=hostname, username=username, **options) # get the provider by reference from the resource, if available prov = prov or rmd.get('provider') # get the provider, by alias metadata or by url pmd = to_resource(prov) # check if the provider is a jdbc connection, if so set it pmd['database'], pmd['table'], pmd['path'] = path_to_jdbc(pmd, True) # merge provider and resource metadata md = merge(pmd, rmd) # concatenate paths, if no table is defined if md['table']: md['path'] = None else: md['path'] = os.path.join(pmd['path'] or '', rmd['path'] or '') #process metadata md = process_metadata(md) #todo: verify resource # check format and other minimum requirements are met # assemble output md = assemble_metadata(md) return md