def test_exact_sequence():
schema = Schema(ExactSequence([int, int]))
try:
schema([1, 2, 3])
except Invalid:
assert True
else:
assert False, "Did not raise Invalid"
assert_equal(schema([1, 2]), [1, 2])
def rangeable(func):
"""Decorator for enabling filtering by a range of values.
Range filtering is expected to be passed in the `_range` header as a list
of triplets with the following values separated by commmas:
- Field: The name of the field to filter by
- From: The minimum value to include in the results
- To: The maxium value to include in the results
The range filters are mapped to a dictionary where the keys are the names
of the fields to filter and the values are dictionaries that have
`from`/`to` as fields with their values.
:param func:
The function to be wrapped. It is assumed that the function will be
implementing an endpoint.
:type func: callable
:returns:
Decorated function in which the `range_filters` parameter is injected
with the values from the `_range` headers mapped to a dictionary as
explained above.
:rtype: callable
"""
def valid_datetime(datetime):
"""Make sure that datetime is parseable.
:param datetime: Datetime value to parse
:type datetime: str
:return: The datetime value after parsing
:rtype: :class:`datetime.datetime`
"""
try:
parsed_datetime = parse_datetime(datetime)
except Exception:
raise Invalid('Datetime parsing error')
# Make sure timestamp is in UTC, but doesn't have any timezone info.
# Passing timezone aware timestamp to PosgreSQL through SQLAlchemy
# doesn't seem to work well in manual tests
if parsed_datetime.tzinfo:
parsed_datetime = (parsed_datetime.astimezone(
pytz.timezone('UTC')).replace(tzinfo=None))
return parsed_datetime
def from_or_to_present(range_param):
"""Make sure that at least one of from or to are present.
:param range_param: Range parameter splitted at the commas
:type range_param: tuple(str, str, str)
:return: The same value that was passed
:rtype: tuple(str, str, str)
"""
field, from_, to = range_param
if not (from_ or to):
raise Invalid('At least one of from/to must be passed')
return range_param
schema = Schema(
All(ExactSequence([
basestring,
Any(valid_datetime, ''),
Any(valid_datetime, ''),
]),
Length(min=3, max=3),
from_or_to_present,
msg=('Range parameter should be formatted as follows: '
'<field:str>,[<from:datetime>],[<to:datetime>]\n'
'Where from/to are optional, '
'but at least one of them must be passed')))
@wraps(func)
def create_range_params(*args, **kw):
range_args = request.args.getlist('_range')
range_params = [
schema(range_arg.split(',')) for range_arg in range_args
]
range_filters = {
range_param[0]: dicttoolz.valfilter(functoolz.identity, {
'from': range_param[1],
'to': range_param[2],
})
for range_param in range_params
}
return func(range_filters=range_filters, *args, **kw)
return create_range_params
"""
# You can directly use or subclass aiida.orm.data.Data
# or any other data type listed under 'verdi data'
import os
from collections import OrderedDict
from aiida.orm import Dict
from voluptuous import Schema, Optional, Any, ExactSequence
# key : [ accepted values, label ]
cmdline_options = OrderedDict([
(Optional('build_grid', default=False), bool),
# true/false input_grid_file dh1sz dh2sz dh3sz vdw_factor_e vdw_factor_f use_vdw_factor offset
# e.g. build_grid_from_scratch 1 none 0.25 0.25 0.25 1.0 2.0 0 3.0
(Optional('build_grid_from_scratch',
default=[True, 'none']), ExactSequence([bool, str])),
(Optional('grid_spacing',
default=[0.25, 0.25, 0.25]), ExactSequence([float, float,
float])),
# where to print the potential (e.g. between 1x vdw radius and 2x vdw radius)
(Optional('vdw_factors',
default=[False, 1.0, 2.0]), ExactSequence([bool, float, float])),
(Optional('offset', default=3.0), float),
(Optional('save_grid', default=[False,
'grid.cube']), ExactSequence([bool, str])),
(Optional('calculate_pot_diff', default=False), bool),
(Optional('calculate_pot',
default=[0, 'repeat.cube']), ExactSequence([int, str])),
(Optional('skip_everything', default=False), bool),
(Optional('point_charges_present', default=False), bool),
(Optional('include_pceq', default=False), bool),
send = {}
else:
send = {
msg: NoData if prototypes is NoData else Schema(prototypes)
for msg, prototypes in send.items()
}
return super().__new__(cls, recv, send)
def __reversed__(self):
return Protocol(self.send, self.recv)
_master_hello = ExactSequence([
dt.datetime, # start timestamp
str, # label (default: hostname)
str, # os name
str, # os version
str, # board revision
str, # board serial
])
_master_stats = ExactSequence([
dt.datetime, # timestamp
int, # packages built
{str: int}, # builds last hour (map of abi: count)
dt.timedelta, # builds time
int, # builds size
{str: int}, # builds pending (map of abi: count)
int, # new last hour
int, # files count
int, # downloads last hour
class MockTask(Thread):
"""
Helper class for testing tasks which interface with REQ/REP sockets. This
spawns a thread which can be tasked with expecting certain inputs and to
respond with certain outputs. Typically used to emulate DbClient and
FsClient to downstream tasks.
"""
ident = 0
protocol = protocols.Protocol(recv={
'QUIT': NoData,
'SEND': ExactSequence([str, Extra]),
'RECV': ExactSequence([str, Extra]),
'TEST': Any(int, float),
'RESET': NoData,
},
send={
'OK': NoData,
'ERROR': str,
})
def __init__(self, ctx, sock_type, sock_addr, sock_protocol):
address = 'inproc://mock-%d' % MockTask.ident
super().__init__(target=self.loop, args=(ctx, address))
MockTask.ident += 1
self.sock_type = sock_type
self.sock_addr = sock_addr
self.control = ctx.socket(transport.REQ,
protocol=reversed(self.protocol))
self.control.hwm = 1
self.control.bind(address)
self.sock = ctx.socket(sock_type, protocol=sock_protocol)
self.sock.hwm = 1
self.sock.bind(sock_addr)
self.daemon = True
self.start()
def __repr__(self):
return '<MockTask sock_addr="%s">' % self.sock_addr
def close(self):
self.control.send_msg('QUIT')
msg, data = self.control.recv_msg()
if msg == 'ERROR':
raise RuntimeError(data)
self.join(10)
self.control.close()
self.control = None
if self.is_alive():
raise RuntimeError('failed to terminate mock task %r' % self)
self.sock.close()
self.sock = None
def expect(self, message, data=NoData):
self.control.send_msg('RECV', (message, data))
assert self.control.recv_msg()[0] == 'OK'
def send(self, message, data=NoData):
self.control.send_msg('SEND', (message, data))
assert self.control.recv_msg()[0] == 'OK'
def check(self, timeout=1):
self.control.send_msg('TEST', timeout)
msg, data = self.control.recv_msg()
if msg == 'ERROR':
assert False, data
def reset(self):
self.control.send_msg('RESET')
assert self.control.recv_msg()[0] == 'OK'
def loop(self, ctx, address):
pending = None
tested = False
queue = []
done = []
socks = {}
def handle_queue():
if self.sock in socks and queue[0].action == 'recv':
queue[0].actual = self.sock.recv_msg()
done.append(queue.pop(0))
elif queue[0].action == 'send':
self.sock.send_msg(*queue[0].expect)
queue[0].actual = queue[0].expect
done.append(queue.pop(0))
control = ctx.socket(transport.REP, protocol=self.protocol)
control.hwm = 1
control.connect(address)
try:
poller = transport.Poller()
poller.register(control, transport.POLLIN)
poller.register(self.sock, transport.POLLIN)
while True:
socks = poller.poll(0.1)
if control in socks:
msg, data = control.recv_msg()
if msg == 'QUIT':
if tested or not queue:
control.send_msg('OK')
else:
control.send_msg('ERROR', 'forgot to call check()')
break
elif msg == 'SEND':
queue.append(MockMessage('send', *data))
control.send_msg('OK')
elif msg == 'RECV':
queue.append(MockMessage('recv', *data))
control.send_msg('OK')
elif msg == 'TEST':
tested = True
if pending is not None:
control.send_msg('ERROR', str(pending))
else:
try:
timeout = timedelta(seconds=data)
start = datetime.now(tz=UTC)
while queue and datetime.now(
tz=UTC) - start < timeout:
socks = dict(poller.poll(10))
handle_queue()
if queue:
assert False, 'Still waiting for %r' % queue[
0]
for item in done:
assert item.expect == item.actual
except Exception as exc:
control.send_msg('ERROR', str(exc))
else:
control.send_msg('OK')
elif msg == 'RESET':
queue = []
done = []
control.send_msg('OK')
if queue:
try:
handle_queue()
except Exception as exc:
pending = exc
finally:
control.close()
from voluptuous import Schema, ExactSequence, Any
from aiida.orm import Dict
import six
from six.moves import map
# These options allow specifying the name of the output file
# key : [ accepted values, label ]
output_options = {
'cssr': (bool, 'structure_cssr'),
'v1': (bool, 'structure_v1'),
'xyz': (bool, 'structure_xyz'),
'nt2': (bool, 'network_nt2'),
'res': (bool, 'free_sphere_res'),
'zvis': (bool, 'network_zvis'),
'axs': (float, 'nodes_axs'),
'sa': (ExactSequence([float, float, int]), 'surface_area_sa'),
'vsa': (ExactSequence([float, float, int]), 'surface_sample_vsa'),
'vol': (ExactSequence([float, float, int]), 'volume_vol'),
'volpo': (ExactSequence([float, float, int]), 'pore_volume_volpo'),
'ray_atom': (ExactSequence([float, float, int]), 'ray_atom'),
'block': (ExactSequence([float, int]), 'block'),
'psd': (ExactSequence([float, float, int]), 'psd'),
'chan': (float, 'channels_chan'),
'gridG': (bool, 'grid_gaussian'),
'gridGBohr': (bool, 'grid_gaussian_bohr'),
'strinfo': (bool, 'str_info'),
'oms': (bool, 'open_metal_sites'),
}
# Currently NOT implemented
# These options produce an output file with a hardcoded name
output = {
'IsCloudTrailOwnerAccount': account_id == event_account_id(world),
'IsOrganizationTrail': trail.get('IsOrganizationTrail'),
'CloudTrailSNSTopicArn': trail_topic,
'CloudTrailTrailArn': trail.get('TrailARN'),
'VisibleCloudTrailArns': visible_trails,
}
return update_in(world, ['output'], lambda x: merge(x or {}, output))
MINIMUM_BILLING_REPORT = Schema(
{
'TimeUnit': 'HOURLY',
'Format': 'textORcsv',
'Compression': 'GZIP',
'AdditionalSchemaElements': ExactSequence(['RESOURCES']),
'S3Bucket': str,
'S3Prefix': str,
'S3Region': str,
'ReportVersioning': 'CREATE_NEW_REPORT',
'RefreshClosedReports': True,
},
extra=ALLOW_EXTRA,
required=True)
def get_first_valid_report_definition(valid_report_definitions, default=None):
return valid_report_definitions[0] if any(
valid_report_definitions) else default
Optional('exit'): All(Any(int, bool), Coerce(int))
})
subgroup = Schema({
Optional('name', default=''): str,
Optional('executable', default=None): Any(str, None),
Required('tests'): [test]
})
group = Schema({
Optional('name', default=''): str,
Optional('executable', default=None): Any(str, None),
Required('tests'): [Any(subgroup, test)]
})
filters = Schema({
Optional('uppercase'): bool,
Optional('lowercase'): bool,
Optional('trim'): bool,
Optional('ignorespaces'): bool,
Optional('regex'): ExactSequence([str, str])
})
schema = Schema({
Optional('version', default=1): 1,
Optional('name', default=''): str,
Optional('executable', default=None): Any(str, None),
Optional('filters', default={}): filters,
Required('tests'): [Any(test, group)]
})
'builds_pending': int,
'files_count': int,
'disk_free': int,
'disk_size': int,
'downloads_last_month': int,
'downloads_all': int,
}
_file_state = ExactSequence([
str, # filename
int, # filesize
str, # filehash
str, # package_tag
str, # package_version_tag
str, # py_version_tag
str, # abi_tag
str, # platform_tag
{
str: [str]
}, # dependencies
# NOTE: the optional transferred field is never included. It is effectively
# internal to whatever is tracking the file state
])
_build_state = ExactSequence([
int, # slave id
str, # package
str, # version
str, # abi_tag
bool, # status
dt.timedelta, # duration