def _dict_formatting(d):
keys = (
'hash',
'url',
'service',
'format',
'host',
'port',
'driver',
'database',
'schema',
'table',
'username',
'password',
'resource_path',
'provider_path',
'provider_alias',
'resource_alias',
'cache',
'date_column',
'date_start',
'date_end',
'date_window',
'date_partition',
'update_column',
'hash_column',
'state_column',
'options',
'mapping',
)
return to_ordered_dict(d, keys)
def formatted(md):
keys = (
'profile',
'variables',
('engine',(
'type',
'master',
'jobname',
'timezone',
('submit',(
'detect',
'jars',
'packages',
'py-files',
)),
'config',
)
),
'providers',
'resources',
('loggers',(
('root',('severity',)),
('datalabframework',(
'name',
('stream',(
'severity',
'enable',
)),
('stdout',(
'severity',
'enable',
)),
('file',(
'severity',
'enable',
'path',
)),
('kafka',(
'severity',
'enable',
'hosts',
'topic',
))
))
)),
)
d = to_ordered_dict(md, keys)
if d['variables']:
d['variables'] = dict(sorted(d['variables'].items()))
return d
def assemble_metadata(md):
keys = ['hash', 'url', 'service', 'format', 'host']
if md['service'] != 'file':
keys.append('port')
if md['service'] == 's3a' or md['format'] == 'jdbc':
keys.extend(['user', 'password'])
if md['format'] == 'jdbc':
keys.extend(['driver', 'database', 'schema', 'table'])
keys.append('options')
return YamlDict(to_ordered_dict(md, keys))
def config(self):
keys = [
'type', 'name', 'version', 'config', 'env', 'rootdir', 'timezone'
]
d = {
'type': self._type,
'name': self._name,
'version': self._version,
'config': self._config,
'env': self._env,
'rootdir': self._rootdir,
'timezone': self._timezone,
'timestamps': self._timestamps
}
return YamlDict(to_ordered_dict(d, keys))
def metadata(self):
keys = (
'profile',
'variables',
('engine',(
'type',
'master'
)
),
'providers',
'resources',
('loggers',(
'root',
('datalabframework',(
'name',
'stream',
'kafka'
)
)
)
)
)
return YamlDict(to_ordered_dict(self._metadata, keys))