def test_update_with_append_to(self):
# Include d in both to have a collision
d = slovar(self.sample_d).subset(['a', 'b', 'd', 'g', 'h'])
e = slovar(self.sample_d).subset(['c', 'd', 'e', 'f'])
a = []
assert d.update_with(e, append_to=a) == self.sample_d
d1 = slovar(
a = [],
aa = [1],
aaa = [{'b':1, 'c':2}]
)
d2 = slovar(
a = 1,
aa = 2,
aaa= [{'b':3, 'c': 3}, {'b':2, 'c': 33}]
)
d3 = d1.update_with(d2, append_to='aaa')
assert d3.aaa == [{'b':1, 'c':2}, {'b':3, 'c': 3}, {'b':2, 'c': 33}]
d3 = d1.update_with(d2, append_to='aaa:b')
assert d3.aaa == [{'b':1, 'c':2}, {'b':2, 'c': 33}, {'b':3, 'c': 3}]
d3 = d1.update_with(d2, append_to='aaa:-b')
assert d3.aaa == [{'b':3, 'c': 3}, {'b':2, 'c': 33}, {'b':1, 'c':2}]
def test_update_with_append_to_set(self):
d1 = slovar(
a = [],
aa = [1],
aaa = [{'b':1, 'c':2}]
)
d2 = slovar(
a = 1,
aa = 2,
aaa= [{'b':1, 'c': 3}, {'b':2, 'c': 33}]
)
for ix in range(2):
d3 = d1.update_with(d2, append_to_set=[
'a',
'aa',
'aaa:b'
])
assert len(d3.a) == 1
assert len(d3.aa) == 2
assert len(d3.aaa) == 2
d3 = d1.update_with({'x':1}, append_to='x')
assert 'x' in d3
def test_update_with_flat(self):
d1 = slovar(a={'b': {'c': 1}, 'bb': 2}, )
d2 = slovar(a={'b': {'d': 2}})
d3 = d1.update_with(d2, flatten=True)
assert d3.a.b.d == 2
assert d3.a.b.c == 1
def __init__(self, settings):
self.settings = slovar(settings)
self.url = self.settings.OPEN_WEATHER_API_URL
self.default_params = slovar({
'units':
self.settings.TEMP_UNITS,
'appid':
self.settings.OPEN_WEATHER_API_KEY,
})
def setup_ds(self, ds_name, field_name, required=False, **overwrites):
params = slovar({field_name:self.parse_ds(ds_name, **overwrites)})
if not params[field_name]:
if required:
self.error('%s is required' % field_name)
else:
return slovar()
self._datasets[field_name] = params.extract('%s.*' % field_name)
return params
def test_remove(self):
dset = slovar(a=1, b=2, c=3)
assert dset.remove([]) == dset
assert dset.remove(['NOTTHERE']) == dset
assert dset.remove(['b', 'c']) == dict(a=1)
dd = slovar(a=dict(b=1, d=2), c=1)
assert dd.remove('a') == dict(c=1)
assert dd.remove(['a']) == dict(c=1)
assert dd.remove('a.b') == dd
assert dd.remove('a.b', flat=True) == dict(a=dict(d=2), c=1)
def test_merge(self):
d1 = {}
merge(d1, {})
assert d1 == {}
merge(d1, dict(a=1))
assert d1 == dict(a=1)
# XXX This doesn't raise anymore. It should.
d1 = dict(a={})
# with pytest.raises(ValueError):
# merge(d1, dict(a=1))
merge(d1, dict(a={}))
assert d1 == dict(a={})
merge(d1, dict(a=dict(b=1)))
assert d1 == dict(a=dict(b=1))
d1 = dict(a=dict(c=1))
merge(d1, dict(a=dict(b=1)))
assert d1 == {'a': {'c': 1, 'b': 1}}
d1 = slovar(a={})
d1.merge({})
def add_mongo_indices(self, args, name, indices):
if args.skip_index:
return
if isinstance(args[name], list):
ds_name = args[name][0]
else:
ds_name = args[name]
ds = self.parse_ds(ds_name)
if ds.backend != 'mongo':
log.warning('`%s` is not a mongo dataset' % ds)
return
_indices = slovar()
for ind in indices:
op,_,margs = ind.partition('=')
op = op or 'index'
_indices.add_to_list(op, margs)
uri = ''
for kk, vv in _indices.items():
uri += '%s=%s' % (kk,','.join(vv))
args['mongo_index'].append('%s/%s?%s' % (ds.ns, ds.name, uri))
return args
def test_update_with_flat(self):
d1 = slovar(
a = {'b':
{'c': 1},
'bb': 2},
)
d2 = slovar(
a = {'b':
{'d': 2}}
)
d3 = d1.flat().update_with(d2.flat()).unflat()
assert d3.a.b.d == 2
assert d3.a.b.c == 1
def test_extract_exclude(self):
d = slovar(self.sample_d)
args = ['-a', '-b', '-c', '-e', '-g', '-h']
assert d.extract(args) == {
'd': self.sample_d['d'],
'f': self.sample_d['f'],
}
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
with Configurator(settings=settings) as config:
config = Configurator(settings=settings)
config.include('prf') # pyramid way of adding external packages.
config.add_tween('prf.tweens.GET_tunneling')
config.add_error_view(KeyError, error='Missing param: %s', error_attr='args')
config.add_error_view(ValueError, error='Missing values: %s', error_attr='args')
root = config.get_root_resource() # acquire root resource.
root.add_singular('init', view='sixds.views.InitView')
root.add('movie', 'movies', view='sixds.views.MoviesView')
root.add_singular('distance', view='sixds.views.DistanceView')
root.add_singular('random', view='sixds.views.RandomView')
config.scan()
print('Frying the bacons...')
settings = slovar(settings)
settings.has('movie_db_file', err='`movie_db_file` is missing in configuration')
api.init(settings['movie_db_file'], settings.get('db_size'))
print('Your bacon is served !')
return config.make_wsgi_app()
def run(self, params, path='', info=''):
f_params = slovar(params).flat()
self.show_info(params, path=path, info=info)
#if not target, read from source and print
if not f_params.get('target.name'):
query = params.extract('source.query.*', {'_limit':1})
pp(self.query_source(f_params, **query))
return
def ask4workers():
def _auto_workers():
if not self.source_total:
return
if params.get('workers'):
return params.workers
ret = math.ceil(self.source_total/1000)
if ret > 100:
return 100
return ret
if params.args.dev:
print('Running in DEV mode')
return
elif not params.args.dry:
aworkers = _auto_workers()
if not params.args.silent:
workers = input('Enter number of workers (%s):' % aworkers)
if workers == '':
workers = aworkers
else:
workers = aworkers
workers = int(workers)
if workers:
params['workers'] = workers
params['async'] = 1
print('Running with %s workers' % params['workers'])
else:
params['async'] = 0
print('Running in DEV mode')
if self.ask_input('RUN COMMAND ', ignore_enter=True):
if not self.source_total:
print('Empty Source')
return
self.run_mongo_indices(params.args)
ask4workers()
return self._run(path, params)
else:
print('Skipped\n')
def test_copy(self):
dset = slovar(a=1, b=2, c=3)
dset_copy = dset.copy()
dset_alias = dset
assert dset == dset_copy
assert id(dset) == id(dset_alias)
assert id(dset) != id(dset_copy)
def test_subset(self):
d = slovar(self.sample_d)
args = ['a', 'b', 'd']
assert d.subset(args) == {
'a': self.sample_d['a'],
'b': self.sample_d['b'],
'd': self.sample_d['d'],
}
def test_extract(self):
d = slovar(self.sample_d)
args = ['a', 'b', 'c']
assert d.extract(args) == {
'a': self.sample_d['a'],
'b': self.sample_d['b'],
'c': self.sample_d['c'],
}
def test_pop_by_values(self):
dset = slovar(a=1, b=2, c=2)
dset_copy = dset.copy()
dset.pop_by_values(666)
assert dset == dset_copy
dset.pop_by_values(2)
assert list(dset.keys()) == ['a']
assert dset != dset_copy
def test_has(self):
d1 = slovar(a=1)
with pytest.raises(ValueError):
d1.has('a', check_type=str)
assert d1.has('a', check_type=int) == True
with pytest.raises(ValueError):
d1.has('b', check_type=int)
def test_update_with_append_to_set2(self):
d1 = slovar(
a = [],
aa = [1],
aaa = [{'b':1, 'c':2}]
)
d3 = d1.update_with({'aaa':{'b': 1, 'c':11, 'd':33}}, append_to_set=['aaa:b'])
assert d3.aaa[0]['c'] == 11
assert d3.aaa[0]['d'] == 33
def test_extract_nested(self):
d = slovar(self.sample_d)
args = ['a', 'b', 'c', 'd.ii.*']
assert d.extract(['a', 'b', 'c', 'd.ii.*']) == {
'a': self.sample_d['a'],
'b': self.sample_d['b'],
'c': self.sample_d['c'],
'aa': self.sample_d['d']['ii']['aa'],
'ab': self.sample_d['d']['ii']['ab'],
}
def create(self):
if api.INITIALIZED and not self._params.asbool('reload',
default=False):
raise ValueError(
'Already initialized. Use `reload=1` to re-initialize')
settings = slovar(self.resource.config.get_settings())
settings.has('movie_db_file',
err='`movie_db_file` is missing in configuration')
api.init(settings['movie_db_file'])
def processor(msg):
_d = slovar(msg).data.extract(
[
's__as__Symbol',
'p__as__Price:float',
]
)
if _d.Symbol in PRICE_ALERT_MAP and _d.Price > PRICE_ALERT_MAP[_d.Symbol]:
print(
f'!!! Price alert !!! Ticker: {_d.Symbol}: Price:{_d.Price}, Alert Price: {PRICE_ALERT_MAP[_d.Symbol]}')
TICKER_FILES[_d.Symbol].write(json.dumps(msg)+',\n')
def qlsit2query(self, qs_list, prefix=''):
_pp = slovar()
for qs in qs_list:
for kk,vv in qs2dict(qs).items():
fld_name = '%s%s'%(prefix, kk)
if kk == '_fields':
_pp.add_to_list(fld_name, vv)
else:
_pp[fld_name] = vv
if '%s_fields' % prefix in _pp:
fname = '%s_fields' % prefix
_pp[fname] = ','.join(_pp[fname])
return _pp
def test_update_with_overwrite(self):
d1 = slovar(a={'b': 1}, c=2, d=[2])
d2 = d1.update_with({'c': 1, 'd': 3})
assert d2.c == 1
assert d2.d == 3
d2 = d1.update_with({'c': 1, 'd': 3}, overwrite=False)
assert d2.c == 2
assert d2.d == [2]
d2 = d1.update_with({'d': 1}, append_to='d')
assert d2.d == [2, 1]
d2 = d1.update_with({'d': 4}, append_to='d', overwrite=False)
assert d2.d == [2, 4]
def query_source(self, params, **query):
query = slovar(query).flat()
ds = params.unflat().get('source')
if not ds:
return
if ds.backend == 'http':
return
url = ('{backend}/{ns}/{name}').format(**ds)
url = '%s/%s' % (self.args.etl_api, url)
api = PRFRequest(url, auth=params.get('auth'))
print('Querying %s' % api.base_url)
pp(query)
resp = api.get(params=query)
return api.get_data(resp)
def test_subset(self):
dset = slovar(a=1, b=2, c=3)
assert set(dset.subset(['a', 'c']).keys()) == set(['a', 'c'])
assert set(dset.subset(['-a']).keys()) == set(['b', 'c'])
# can not have both negative and positive.
with pytest.raises(Exception):
dset.subset(['-a', 'b'])
assert dset.subset(['NOTTHERE']) == {}
assert dset.subset(['-NOTTHERE']) == dset
assert dset.subset([]) == {}
assert set(dset.subset(['a', 'NOTTHERE']).keys()) == set(['a'])
assert set(dset.subset(['-a', '-NOTTHERE']).keys()) == set(['b', 'c'])
def __init__(self):
self.add_argument('--etl_api', default='localhost:6544/api',
help='base url of etl service')
self.add_argument('--drop', help='drop target first', action='store_true')
self.add_argument('--drop-ns', help='drop target namespace', action='store_true')
self.add_argument('--dry', help='print out params', action='store_true')
self.add_argument('--workers', help='nb of workers', type=int)
self.add_argument('--batch', help='batch_size')
self.add_argument('--dev', action='store_true', help='dev mode will run request in async mode on server')
self.add_argument('--silent', action='store_true')
self.add_argument('--log', help='turn on logs', default='-logs,-set__logs')
self.add_argument('--pretty', action='store_true', help='pretty logs')
self.add_argument('-a', '--arg', action='append', default=[],
help= 'extra args can be specified multiple times')
self.add_argument('--no-count', action='store_true', help='skip doing count')
self.add_argument('--pk')
self.add_argument('--info', help='just show info and quit')
self.add_argument('--msg', help='job comment')
self.add_argument('--poll-interval', type=float, default=10, help='sleep seconds between repeats')
self.add_argument('--show-args', help='non-blocking', action='store_true')
self.add_argument('--cron', help='schedule cron')
self.add_argument('--fail-on-error', help='fail on error', default=True)
self.add_argument('--paginate', help='fast pagination')
self.add_argument('--mongo-index', help='create mongo index', action='append', default=[])
self.add_argument('--skip-index', help='skip mongo indexing', action='store_true')
self.add_argument('--verbose', help='verbose logging on server', action='store_true')
self.add_argument('--profile', help='run cProfiler', action='store_true')
self.add_argument('--pylog2es', help='enable python logging to elastic', action='store_true')
self.add_argument('--log_ds', help='elastic ns to save target logs', default='logs')
self.add_argument('--log-level', help='set log level for server')
self.add_argument('--ns', help='source ns')
_parsed_args = self.parser.parse_args()
self.args = Args(vars(_parsed_args), mutables=self.MUTABLE_OPS)
self.api = self.get_api(_raise=True)
self._datasets = slovar()
self.source_total = None
self.job_uid = None
def process_response(self, resp):
if not resp.ok:
if resp.status_code == 401:
raise prf_exc.HTTPForbidden('Bad API key')
raise ValueError(resp.json())
else:
data = slovar(resp.json()).extract([
'main.*',
'dt__as__date:ts2dt',
'coord.*',
'unit:=%s' % self.settings.TEMP_UNITS,
'weather.0.description__as__',
'weather.0.icon__as__',
'sys.country__as__',
'name',
])
return data
def parse_ds(name, **overwrites):
SEP = '/'
if not name or isinstance(name, dict):
return name
if '%TODAY%' in name:
name = name.replace('%TODAY%',
datetime.utcnow().strftime('%Y-%m-%d')
)
params = slovar()
name_parts = name.split(SEP)
params.backend = name_parts[0]
params.ns = SEP.join(name_parts[1:-1])
params.name = name_parts[-1]
params.update(overwrites)
return params
def test(self):
dset = slovar(a=1)
assert isinstance(dset, dict) is True
assert dset.a == dset['a']
dset.a = 10
assert dset['a'] == 10
dset.b = 2
assert dset['b'] == dset.b
dset.d = dict(a=1)
assert dset.d.a == dset['d']['a']
del dset.b
assert 'b' not in dset
def get_source_total(self, params):
qlimit = params.source.flat().asint('query._limit', allow_missing=True)
res = self.query_source(params, _count=1, **params.unflat('source')\
.extract('source.query.*'))
if isinstance(res, int):
if qlimit:
return min(qlimit, res)
else:
return res
elif qlimit:
return qlimit
elif res == slovar():
log.error('source is depleted')
return 0
else:
self.error('Could not determine total in source')
return None
