def session_client(func, *args, **kwargs):
clientname = context.get('clientname')
contributor_type = context.get('contributor_type')
try:
client = Client.select().where(Client.name == clientname).get()
except Client.DoesNotExist:
raise Exception('Client not found {}'.format(clientname or ''))
session = Session.create(client=client, contributor_type=contributor_type)
context.set('session', session)
return func(*args, **kwargs)
def collect_report(func, chunk):
# This is a process reporter instance.
reporter = context.get('reporter')
results = func(*chunk)
reports = reporter._reports.copy()
reporter.clear()
return results, reports
def test_authorized_responses_contain_sessions_data(get):
municipality = MunicipalityFactory(name="Cabour")
resp = get('/municipality/{}'.format(municipality.id))
assert resp.status_code == 200
session = context.get('session')
assert resp.headers['Session-Client'] == session.client.id
assert resp.headers['Session-User'] == session.user.id
def batch(func, iterable, chunksize=1000, total=None, progress=True):
# This is the main reporter instance.
reporter = context.get('reporter')
pool = (ProcessPoolExecutor if config.get('BATCH_EXECUTOR') == 'process'
else ThreadPoolExecutor)
bar = Bar(total=total, throttle=timedelta(seconds=1))
workers = int(config.get('WORKERS', os.cpu_count()))
chunk = []
count = 0
def loop():
for reports in executor.map(collect_report, repeat(func), chunk):
reporter.merge(reports)
if progress:
bar()
with pool(max_workers=workers) as executor:
for item in iterable:
if not item:
continue
chunk.append(item)
count += 1
if count % 10000 == 0:
loop()
chunk = []
if chunk:
loop()
def batch(func, iterable, chunksize=1000, total=None, progress=True):
# This is the main reporter instance.
reporter = context.get('reporter')
pool = (ProcessPoolExecutor if config.get('BATCH_EXECUTOR') == 'process'
else ThreadPoolExecutor)
bar = Bar(total=total, throttle=timedelta(seconds=1))
workers = int(config.get('WORKERS', os.cpu_count()))
chunk = []
count = 0
def loop():
for reports in executor.map(collect_report, repeat(func), chunk):
reporter.merge(reports)
if progress:
bar()
with pool(max_workers=workers) as executor:
for item in iterable:
if not item:
continue
chunk.append(item)
count += 1
if count % 10000 == 0:
loop()
chunk = []
if chunk:
loop()
def dummytoken(token, **kwargs):
"""Create a dummy token for dev."""
session = context.get('session')
Token.delete().where(Token.access_token == token).execute()
Token.create(session=session.pk, access_token=token, expires_in=3600*24,
token_type='Bearer', scope='*')
reporter.notice('Created token', token)
def test_authorized_responses_contain_sessions_data(get):
municipality = MunicipalityFactory(name="Cabour")
resp = get('/municipality/{}'.format(municipality.id))
assert resp.status_code == 200
session = context.get('session')
assert resp.headers['Session-Client'] == session.client.id
assert resp.headers['Session-User'] == session.user.id
def log_headers(resp):
session = context.get('session')
if session:
if session.client:
resp.headers.add('Session-Client', session.client.id)
if session.user:
resp.headers.add('Session-User', session.user.id)
return resp
def dummytoken(**kwargs):
"""Create a dummy token for dev."""
session = context.get('session')
Token.delete().where(Token.access_token == 'token').execute()
Token.create(session=session.id,
access_token="token",
expires_in=3600 * 24)
report('Created token', 'token', report.NOTICE)
def log_headers(resp):
session = context.get('session')
if session:
if session.client:
resp.headers.add('Session-Client', session.client.id)
if session.user:
resp.headers.add('Session-User', session.user.id)
return resp
def dummytoken(token, **kwargs):
"""Create a dummy token for dev."""
session = context.get('session')
Token.delete().where(Token.access_token == token).execute()
Token.create(session=session.pk,
access_token=token,
expires_in=3600 * 24,
token_type='Bearer',
scope='*')
reporter.notice('Created token', token)
def collect_report(func, *args, **kwargs):
# This is a process reporter instance.
reporter = context.get('reporter')
if not reporter:
# In thread mode, reporter is not shared with subthreads.
reporter = Reporter(config.get('VERBOSE'))
context.set('reporter', reporter)
func(*args, **kwargs)
reports = reporter._reports.copy()
reporter.clear()
return reports
def collect_report(func, *args, **kwargs):
# This is a process reporter instance.
reporter = context.get('reporter')
if not reporter:
# In thread mode, reporter is not shared with subthreads.
reporter = Reporter(config.get('VERBOSE'))
context.set('reporter', reporter)
func(*args, **kwargs)
reports = reporter._reports.copy()
reporter.clear()
return reports
def dummytoken(token, **kwargs):
"""Create a dummy token for dev."""
session = context.get('session')
Token.delete().where(Token.access_token == token).execute()
Token.create(session=session.pk,
access_token=token,
expires_in=3600 * 24,
token_type='Bearer',
scopes="municipality_write postcode_write \
group_write housenumber_write position_write bal".split(),
status='dev')
reporter.notice('Created token', token)
def session(func, *args, **kwargs):
session = context.get('session')
if not session:
qs = User.select().where(User.is_staff == True)
username = config.get('SESSION_USER')
if username:
qs = qs.where(User.username == username)
try:
user = qs.get()
except User.DoesNotExist:
abort('Admin user not found {}'.format(username or ''))
session = Session.create(user=user)
context.set('session', session)
return func(*args, **kwargs)
def session(func, *args, **kwargs):
session = context.get('session')
if not session:
qs = User.select().where(User.is_staff == True)
username = config.get('SESSION_USER')
if username:
qs = qs.where(User.username == username)
try:
user = qs.get()
except User.DoesNotExist:
abort('Admin user not found {}'.format(username or ''))
session = Session.create(user=user)
context.set('session', session)
return func(*args, **kwargs)
def batch(func, iterable, chunksize=1000, total=None, progress=True):
# This is the main reporter instance.
reporter = context.get('reporter')
bar = Bar(total=total, throttle=timedelta(seconds=1))
workers = int(config.get('WORKERS', os.cpu_count()))
with ChunkedPool(processes=workers) as pool:
try:
for results, reports in pool.imap_unordered(
func, iterable, chunksize):
reporter.merge(reports)
bar(step=len(results))
yield from results
bar.finish()
except Exception as e:
print("\n" + e.args[0])
pool.terminate()
def on_post_bal(self, req, resp, *args, **kwargs):
"""Import file at BAL format."""
data = req.get_param('data', required=True)
bal(StringIO(data.value.decode('utf-8-sig')))
reporter = context.get('reporter')
resp.json(report=reporter)
def report(name, item, level=1):
reporter = context.get('reporter')
if not reporter:
print("Reporter not set!")
return
reporter(name, item, level=level)
def bal_post():
"""Import file at BAL format."""
data = request.files['data']
bal(StringIO(data.read().decode('utf-8-sig')))
reporter = context.get('reporter')
return dumps({'report': reporter})
def dummytoken(**kwargs):
"""Create a dummy token for dev."""
session = context.get('session')
Token.delete().where(Token.access_token == 'token').execute()
Token.create(session=session.id, access_token="token", expires_in=3600*24)
report('Created token', 'token', report.NOTICE)
def report(name, item, level=1):
reporter = context.get('reporter')
if not reporter:
print("Reporter not set!")
return
reporter(name, item, level=level)
