def findKeys(ws, rows):
func_list = OrderedSet()
for i in range(2, rows):
if (ws.cell(i, 0)).value:
func_list.add((ws.cell(i, 0)).value)
else:
i += 1
return func_list
def get_collection_intersection(*args):
ret = []
for ii,arg in enumerate(args):
if ii == 0:
ret = OrderedSet(tf.get_collection(arg))
else:
ret = ret.intersection(OrderedSet(tf.get_collection(arg)))
return list(ret)
class Poller(object):
def __init__(self):
rospy.init_node('poller_node')
self.rate = rospy.Rate(3) # 3hz
self.extractBasestationFromParams()
self.createCommunicators()
self.request_list = OrderedSet([])
def createCommunicators(self):
self.client = Client(10019)
self.measurements_publisher = rospy.Publisher('measurements', MeasurementList, queue_size=10)
self.request_subscriber = rospy.Subscriber("measurements_request", String, self.pushbackRequest)
def extractBasestationFromParams(self):
stations = rospy.get_param("/poller_node/basestations")
self.storeBasestation(stations)
def storeBasestation(self, stations):
self.basestations = []
for station in stations:
self.basestations.append(Basestation(station[0], float(station[1]), float(station[2])))
def pushbackRequest(self, msg):
self.request_list.add(msg.data)
def measurementsLoop(self):
while not rospy.is_shutdown():
while not self.request_list.isEmpty():
station_address = self.request_list.pop()
self.serveRequest(station_address)
self.rate.sleep()
def pollStation(self, station_address):
return self.client.pollBasestation(station_address)
def serveRequest(self, station_address):
try:
data = self.pollStation(station_address)
if containsMeasurements(data):
self.publishMeasuements(extractJson(data), station_address)
except socket.error:
pass
def publishMeasuements(self, measurs, station):
msg = MeasurementList()
for el in measurs:
msg.data.append(self.generateMeasurement(el))
msg.basestation = station
msg.header.stamp = rospy.Time.now()
self.measurements_publisher.publish(msg)
def generateMeasurement(self, element):
tmp = Measurement()
tmp.tag = element['id_tag'].encode('utf-8')
tmp.measurement = int(element['rssid'])
return tmp
def iter_variations(self):
needed_ids = OrderedSet()
for fixture in self._needed_fixtures:
needed_ids.update(self._store.get_all_needed_fixture_ids(fixture))
parametrizations = [self._store.get_fixture_by_id(param_id) for param_id in needed_ids]
if not needed_ids:
yield Variation(self._store, {}, self._name_bindings.copy())
return
for value_indices in itertools.product(*(xrange(len(p.values)) for p in parametrizations)):
yield self._build_variation(parametrizations, value_indices)
class RoundRobin(SchedulingPolicy):
def __init__(self):
self.__coroutines = OrderedSet()
@property
def busy(self):
return bool(self.__coroutines)
def add(self, coroutine):
self.__coroutines.add(coroutine)
def remove(self, coroutine):
self.__coroutines.remove(coroutine)
def freeze(self, coroutine):
self.__coroutines.remove(coroutine)
def unfreeze(self, coroutine):
self.__coroutines.add(coroutine)
def round(self):
for coro in list(self.__coroutines):
assert coro is not None
yield coro
def dump(self):
def dump(c, idt = 0):
print('{}{}{}'.format(' ' * idt, c, ' (frozen)' if c.frozen else ''))
for child in self.__hierarchy.get(c, []):
dump(child, idt + 1)
for root in self.__hierarchy.get(None, []):
dump(root)
class Fixture(FixtureBase):
def __init__(self, store, fixture_func):
super(Fixture, self).__init__()
self.fixture_func = fixture_func
self.info = self.fixture_func.__slash_fixture__
self.scope = self.info.scope
self.namespace = Namespace(store, store.get_current_namespace())
def __repr__(self):
return '<Function Fixture around {0}>'.format(self.fixture_func)
def get_value(self, kwargs, active_fixture):
if self.info.needs_this:
assert 'this' not in kwargs
kwargs['this'] = active_fixture
return self.fixture_func(**kwargs)
def _resolve(self, store):
assert self.fixture_kwargs is None
assert self.parametrization_ids is None
self.parametrization_ids = OrderedSet()
kwargs = OrderedDict()
parametrized = set()
for name, param in iter_parametrization_fixtures(self.fixture_func):
store.register_fixture_id(param)
parametrized.add(name)
self.parametrization_ids.add(param.info.id)
for param_name, arg in self.info.required_args.items():
if param_name in parametrized:
continue
try:
needed_fixture = self.namespace.get_fixture_by_name(get_real_fixture_name_from_argument(arg))
if needed_fixture.scope < self.scope:
raise InvalidFixtureScope('Fixture {0} is dependent on {1}, which has a smaller scope ({2} > {3})'.format(
self.info.name, param_name, self.scope, needed_fixture.scope))
if needed_fixture is self:
raise CyclicFixtureDependency('Cyclic fixture dependency detected in {0}: {1} depends on itself'.format(
self.info.func.__code__.co_filename,
self.info.name))
kwargs[param_name] = needed_fixture.info.id
except LookupError:
raise UnknownFixtures(param_name)
return kwargs
def post(self, request):
if request.data.get('flag') == 'tag':
tag_list = request.data.getlist('tag_list[]')
rules = cache.get("recommend_tag_list")
recommend_list = OrderedSet()
for rule, confidence in rules:
tags, results = rule
if tags and tag_list:
if set(tags) == set(tag_list):
for result in results:
recommend_list.add(result)
else:
return Response({'recommend_tag_list': list(recommend_list)})
return Response({'recommend_tag_list': list(recommend_list[:5])})
def _check_controls(self):
#controlled_region = set.union(*[c.region for c in self.controls])
controlled_region = OrderedSet.union(*[c.region for c in self.controls])
if len(controlled_region) != Plate.SIZE:
assert len(controlled_region) < Plate.SIZE
raise ValueError('layout contains uncontrolled wells')
del controlled_region
def _resolve(self, store):
assert self.fixture_kwargs is None
assert self.parametrization_ids is None
self.parametrization_ids = OrderedSet()
kwargs = OrderedDict()
parametrized = set()
for name, param in iter_parametrization_fixtures(self.fixture_func):
store.register_fixture_id(param)
parametrized.add(name)
self.parametrization_ids.add(param.info.id)
for param_name, arg in self.info.required_args.items():
if param_name in parametrized:
continue
try:
needed_fixture = self.namespace.get_fixture_by_name(get_real_fixture_name_from_argument(arg))
if needed_fixture.scope < self.scope:
raise InvalidFixtureScope('Fixture {0} is dependent on {1}, which has a smaller scope ({2} > {3})'.format(
self.info.name, param_name, self.scope, needed_fixture.scope))
if needed_fixture is self:
raise CyclicFixtureDependency('Cyclic fixture dependency detected in {0}: {1} depends on itself'.format(
self.info.func.__code__.co_filename,
self.info.name))
kwargs[param_name] = needed_fixture.info.id
except LookupError:
raise UnknownFixtures(param_name)
return kwargs
def __Dedupe(items, key=None):
seen = OrderedSet()
num_seen = list()
gn_item = (item for item in items)
while True:
try:
item = gn_item.next()
except Exception as e:
yield (None, num_seen)
break
else:
val = item if key is None else key(item)
if val not in seen:
yield (item, None)
seen.add(val)
num_seen.append(1)
else:
num_seen[seen.index(val)] += 1
def prepare_annotations(self):
# compile annotations data
fields = OrderedSet([
'start_time',
'commit_time',
'commit',
'commit_message',
'working_directory',
'command',
'elapsed_time'])
annotations_list = list()
for annotation in self.repo.get_annotations():
fields.update(annotation)
annotations_list.append(annotation)
annotations_list.sort(key = lambda x: x['start_time'], reverse=True)
return fields, annotations_list
def __init__(self, name):
self.name = name
self.node_set = OrderedSet()
self.edge_set = OrderedSet()
self.h_matrix = None
self.s_matrix = None
self.g_matrix = None
self.pi_vector = None
def _compute_all_needed_parametrization_ids(self, fixtureobj):
stack = [(fixtureobj.info.id, [fixtureobj.info.id], set([fixtureobj.info.id]))]
returned = OrderedSet()
while stack:
fixture_id, path, visited = stack.pop()
if fixture_id in self._all_needed_parametrization_ids_by_fixture_id:
returned.update(self._all_needed_parametrization_ids_by_fixture_id[fixture_id])
continue
fixture = self._fixtures_by_id[fixture_id]
if fixture.parametrization_ids:
assert isinstance(fixture.parametrization_ids, OrderedSet)
returned.update(fixture.parametrization_ids)
if fixture.fixture_kwargs:
for needed_id in itervalues(fixture.fixture_kwargs):
if needed_id in visited:
self._raise_cyclic_dependency_error(fixtureobj, path, needed_id)
stack.append((needed_id, path + [needed_id], visited | set([needed_id])))
return returned
def __init__(self, id_, master_graph):
self.id_ = id_
self.master_graph = master_graph
self.edges = OrderedSet()
self._state = None
# determine ownership
self.determined = False
self.graph = None
def filter_by_children(self, *args):
obj_attrs = []
data = args[0]
for d in data:
names = OrderedSet()
attrs = {}
try:
for dc in d.find_all('div'):
name = dc.find('span')
value = dc.find('div')
if value and name is not None:
if name.text not in names:
names.add(name.text)
attrs[name.text] = value.text
obj_attrs.append(attrs)
except AttributeError:
self.view.display_item('Error filtering data from children.....')
web_objs = self.sanitise_attributes(obj_attrs)
return web_objs
class DepthFirst(SchedulingPolicy):
def __init__(self):
self.__coroutines = OrderedSet()
self.__hierarchy = {}
@property
def busy(self):
return bool(self.__coroutines)
def add(self, coroutine):
parent = coroutine.parent
self.__coroutines.add(coroutine)
self.__hierarchy.setdefault(parent, OrderedSet()).add(coroutine)
def remove(self, coroutine):
self.__coroutines.remove(coroutine)
children = self.__hierarchy.pop(coroutine, None)
if children is not None:
assert len(children) == 0
self.__hierarchy.get(coroutine.parent).remove(coroutine)
def freeze(self, coroutine):
self.__coroutines.remove(coroutine)
def unfreeze(self, coroutine):
self.__coroutines.add(coroutine)
def round(self):
c = self.__round(self.__hierarchy.get(None, ()))
assert c is not None
return (c,)
def __round(self, coroutines):
for coroutine in coroutines:
assert coroutines is not None
active = coroutine in self.__coroutines
if active and coroutine.exception:
return coroutine
sub = self.__round(self.__hierarchy.get(coroutine, ()))
if sub is not None:
return sub
if active:
return coroutine
def dump(self):
def dump(c, idt = 0):
print('{}{}{}'.format(' ' * idt, c, ' (frozen)' if c.frozen else ''))
for child in self.__hierarchy.get(c, []):
dump(child, idt + 1)
for root in self.__hierarchy.get(None, []):
dump(root)
def __init__(self, name = None):
"""Initialize the molecule
Args:
name (str): name of the molecule
"""
if name != None:
self.name = name
else:
# TODO Fix the naming resolution
self.name = "Untitled"
self._atoms = OrderedSet()
def __get_centroid(self, cluster=None):
centroid = ''
# centroid for a particular cluster
if cluster:
for log_id in cluster:
if centroid == '':
centroid = self.preprocessed_logs[log_id]
else:
centroid = ' '.join([centroid, self.preprocessed_logs[log_id]])
# centroid for the whole logs
else:
for log_id in self.preprocessed_logs:
if centroid == '':
centroid = self.preprocessed_logs[log_id]
else:
centroid = ' '.join([centroid, self.preprocessed_logs[log_id]])
centroid = OrderedSet(centroid.split())
centroid = ' '.join(list(centroid))
return centroid
class UNIQUE_ARRAY(_ObjectHook):
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
def __init__(self, name, flag, is_required):
self._name = name
self._flag = flag
self._values = OrderedSet()
self._is_required = is_required
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
def add_value(self, value):
self._values.add(value)
#- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
def close(self, name, flag):
if (self._is_required and
not self._values):
raise Pattern.UnfinishedPattern(
Pattern.UNIQUE_ARRAY, self._flag,
Pattern.EOL() if name is NotImplemented else flag) from None
return self._values
class Molecule(object):
"""An abstract molecule object.
"""
__slots__ = ['name', '_atoms']
def __init__(self, name = None):
"""Initialize the molecule
Args:
name (str): name of the molecule
"""
if name != None:
self.name = name
else:
# TODO Fix the naming resolution
self.name = "Untitled"
self._atoms = OrderedSet()
def addAtom(self, atom):
"""Add and atom
Args:
atom (atom): the atom to add into the molecule
"""
self._atoms.add(atom)
def removeAtom(self, atom):
"""Remove Atom
Args:
atom (atom): the atom to remove from the molecule
"""
self._atoms.remove(atom)
def getAtoms(self):
"""Return an orderedset of atoms
"""
return self._atoms
def __repr__(self):
return self.name
def __init__(self, name):
"""Initialize the MoleculeType container
Args:
name (str): the name of the moleculetype to add
"""
self.name = name
self.moleculeSet = OrderedSet()
self.bondForceSet = HashMap()
self.pairForceSet = HashMap()
self.angleForceSet = HashMap()
self.dihedralForceSet = HashMap()
self.torsiontorsionForceSet = HashMap()
self.constraints = HashMap()
self.exclusions = HashMap()
self.settles = None
self.nrexcl = None
def merge_ordered_set(diff, base, mine, theirs):
mine_merged = mine - diff.deleted
theirs_merged = theirs - diff.deleted
mine_added = mine_merged - theirs_merged
merged = OrderedSet()
theirs_merged_iter = iter(theirs_merged)
mine_merged_iter = iter(mine_merged)
for theirs_el in theirs_merged_iter:
for mine_el in mine_merged_iter:
if mine_el in mine_added:
merged.add(mine_el)
else:
break
merged.add(theirs_el)
for mine_el in mine_merged_iter:
merged.add(mine_el)
return merged
def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False):
torrent_data = upload_form.torrent_file.parsed_data
# The torrent has been validated and is safe to access with ['foo'] etc - all relevant
# keys and values have been checked for (see UploadForm in forms.py for details)
info_dict = torrent_data.torrent_dict['info']
changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict)
# Use uploader-given name or grab it from the torrent
display_name = upload_form.display_name.data.strip(
) or info_dict['name'].decode('utf8').strip()
information = (upload_form.information.data or '').strip()
description = (upload_form.description.data or '').strip()
torrent_filesize = info_dict.get('length') or sum(
f['length'] for f in info_dict.get('files'))
# In case no encoding, assume UTF-8.
torrent_encoding = torrent_data.torrent_dict.get('encoding',
b'utf-8').decode('utf-8')
torrent = models.Torrent(info_hash=torrent_data.info_hash,
display_name=display_name,
torrent_name=torrent_data.filename,
information=information,
description=description,
encoding=torrent_encoding,
filesize=torrent_filesize,
user=uploading_user,
uploader_ip=ip_address(
flask.request.remote_addr).packed)
# Store bencoded info_dict
torrent.info = models.TorrentInfo(
info_dict=torrent_data.bencoded_info_dict)
torrent.stats = models.Statistic()
torrent.has_torrent = True
# Fields with default value will be None before first commit, so set .flags
torrent.flags = 0
torrent.anonymous = upload_form.is_anonymous.data if uploading_user else True
torrent.hidden = upload_form.is_hidden.data
torrent.remake = upload_form.is_remake.data
torrent.complete = upload_form.is_complete.data
# Copy trusted status from user if possible
can_mark_trusted = uploading_user and uploading_user.is_trusted
# To do, automatically mark trusted if user is trusted unless user specifies otherwise
torrent.trusted = upload_form.is_trusted.data if can_mark_trusted else False
# Set category ids
torrent.main_category_id, torrent.sub_category_id = \
upload_form.category.parsed_data.get_category_ids()
# To simplify parsing the filelist, turn single-file torrent into a list
torrent_filelist = info_dict.get('files')
used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding
parsed_file_tree = dict()
if not torrent_filelist:
# If single-file, the root will be the file-tree (no directory)
file_tree_root = parsed_file_tree
torrent_filelist = [{
'length': torrent_filesize,
'path': [info_dict['name']]
}]
else:
# If multi-file, use the directory name as root for files
file_tree_root = parsed_file_tree.setdefault(
info_dict['name'].decode(used_path_encoding), {})
# Parse file dicts into a tree
for file_dict in torrent_filelist:
# Decode path parts from utf8-bytes
path_parts = [
path_part.decode(used_path_encoding)
for path_part in file_dict['path']
]
filename = path_parts.pop()
current_directory = file_tree_root
for directory in path_parts:
current_directory = current_directory.setdefault(directory, {})
# Don't add empty filenames (BitComet directory)
if filename:
current_directory[filename] = file_dict['length']
parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)
json_bytes = json.dumps(parsed_file_tree,
separators=(',', ':')).encode('utf8')
torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)
db.session.add(torrent)
db.session.flush()
# Store the users trackers
trackers = OrderedSet()
announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii')
if announce:
trackers.add(announce)
# List of lists with single item
announce_list = torrent_data.torrent_dict.get('announce-list', [])
for announce in announce_list:
trackers.add(announce[0].decode('ascii'))
# Remove our trackers, maybe? TODO ?
# Search for/Add trackers in DB
db_trackers = OrderedSet()
for announce in trackers:
tracker = models.Trackers.by_uri(announce)
# Insert new tracker if not found
if not tracker:
tracker = models.Trackers(uri=announce)
db.session.add(tracker)
db_trackers.add(tracker)
db.session.flush()
# Store tracker refs in DB
for order, tracker in enumerate(db_trackers):
torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
tracker_id=tracker.id,
order=order)
db.session.add(torrent_tracker)
db.session.commit()
# Store the actual torrent file as well
torrent_file = upload_form.torrent_file.data
if app.config.get('BACKUP_TORRENT_FOLDER'):
torrent_file.seek(0, 0)
torrent_dir = app.config['BACKUP_TORRENT_FOLDER']
if not os.path.exists(torrent_dir):
os.makedirs(torrent_dir)
torrent_path = os.path.join(
torrent_dir,
'{}.{}'.format(torrent.id, secure_filename(torrent_file.filename)))
torrent_file.save(torrent_path)
torrent_file.close()
return torrent
def __init__(self):
self.__coroutines = OrderedSet()
self.__hierarchy = {}
class MoleculeType(object):
"""An abstract container for molecules of one type
"""
def __init__(self, name):
"""Initialize the MoleculeType container
Args:
name (str): the name of the moleculetype to add
"""
self.name = name
self.moleculeSet = OrderedSet()
self.bondForceSet = HashMap()
self.pairForceSet = HashMap()
self.angleForceSet = HashMap()
self.dihedralForceSet = HashMap()
self.torsiontorsionForceSet = HashMap()
self.constraints = HashMap()
self.exclusions = HashMap()
self.settles = None
self.nrexcl = None
def add_molecule(self, molecule):
"""Add a molecule into the moleculetype container
Args:
molecule (Molecule): the molecule to append
"""
self.moleculeSet.add(molecule)
def remove_molecule(self, molecule):
"""Remove a molecule from the system.
Args:
molecule (Molecule): remove a molecule from the moleculeType
"""
self.moleculeSet.remove(molecule)
def getMolecule(self, molecule):
"""Get a molecule from the system
Args:
molecule (Molecule): retrieve an equivalent molecule from the moleculetype
"""
return get_equivalent(self.moleculeSet, molecule, False)
def addForce(self, force):
"""Add a forces to the moleculeType
Args:
forces (AbstractForce): Add a forces or contraint to the moleculeType
"""
self.forceSet.add(force)
def removeForce(self, force):
"""Remove a forces from the moleculeType
Args:
forces (AbstractForce): Remove a forces from the moleculeType
"""
self.forceSet.remove(force)
def getForce(self, force):
"""Get a forces from the moleculeType
Args:
forces (AbstractForce): Retrieve a forces from the moleculeType
"""
return get_equivalent(self.forceSet, force, False)
def setNrexcl(self, nrexcl):
"""Set the nrexcl
Args:
nrexcl (int): the value for nrexcl
"""
self.nrexcl = nrexcl
def getNrexcl(self):
"""Gets the nrexcl
"""
return self.nrexcl
def update_remote_groups(self, dry_run=True, threshold=None, remove=True):
'''Update remote configuration with the local one.'''
# Copy those so that we can modify them even with dry-run
local = OrderedSet(self.local)
remote = OrderedSet(self.remote)
validate_groups(local)
def parse_groups(groups, remote):
if remote:
self._process_remote_groups(groups)
groups = {
group.name: group
for group in groups if group.name != 'default'
}
keys = OrderedSet(groups.keys())
return groups, keys
lgroups, lkeys = parse_groups(local, False)
rgroups, rkeys = parse_groups(remote, True)
changes = 0
unchanged = 0
groups_added = OrderedSet()
groups_updated = OrderedSet()
groups_removed = OrderedSet()
rules_added = OrderedSet()
rules_removed = OrderedSet()
# Added groups
for group in (lgroups[name] for name in lkeys - rkeys):
grp = Group(group.name, group.description)
groups_added.add(grp)
rgroups[group.name] = grp
rkeys.add(grp.name)
changes += 1
# Changed groups
for rgroup, lgroup in ((rgroups[name], lgroups[name])
for name in rkeys & lkeys):
if rgroup not in groups_added:
unchanged += 1
if rgroup.description != lgroup.description:
# XXX: https://review.openstack.org/596609
# groups_updated.add((rgroup, lgroup))
pass
# FIXME: when comparing using OrderedSet, added rules part contains
# all elements rather than different ones.
lrules, rrules = set(lgroup.rules), set(rgroup.rules)
if rrules != lrules:
# Added rules
for rule in lrules - rrules:
rules_added.add((rgroup.name, rule))
changes += 1
# Removed rules
for rule in rrules - lrules:
if remove:
rules_removed.add((rgroup.name, rule))
changes += 1
else:
unchanged += 1
unchanged += len(rrules & lrules)
# Removed groups
for group in (rgroups[name] for name in rkeys - lkeys):
if remove:
if group._project is None:
continue
groups_removed.add(group)
changes += len(group.rules) + 1
else:
unchanged += len(group.rules) + 1
if changes == 0 and not groups_updated:
return
# Report result
logger.info(f'{changes:d} changes to be made:')
for group in groups_added:
logger.info(f' - Create group {group.name!r}')
for rgroup, lgroup in groups_updated:
logger.info(f' - Update description for {rgroup.name!r}:'
f' {rgroup.description!r} → {lgroup.description!r}')
for group_name, rule in rules_added:
logger.info(f' - Create {rule!r} in group {group_name!r}')
for group_name, rule in rules_removed:
logger.info(f' - Remove {rule!r} from group {group_name!r}')
for group in groups_removed:
logger.info(
f' - Remove group {group.name!r} with {len(group.rules)} rules'
)
if threshold is not None:
changes_percentage = changes / (unchanged + changes) * 100
if changes_percentage > threshold:
raise ThresholdException(
f'Amount of changes is {changes_percentage:f}%'
f' which is more than allowed ({threshold:f}%)')
if dry_run:
return
# We've modified 'remote', so copy it again
remote = OrderedSet(self.remote)
rgroups, rkeys = parse_groups(remote, True)
# Added groups
for group in groups_added:
ginfo = self.connection.create_security_group(
name=group.name, description=group.description)
remote.add(Group.from_remote(**ginfo))
rgroups, rkeys = parse_groups(remote, True)
# Updated groups
for rgroup, lgroup in groups_updated:
self.connection.update_security_group(
name_or_id=rgroup._id, description=lgroup.description)
# Updating group should not change its ID
rgroup.description = lgroup.description
# Added rules
for group_name, rule in rules_added:
rgroup = rgroups[group_name]
cidr = str(rule.cidr) if rule.cidr is not None else None
group_id = rgroups[
rule.group]._id if rule.group is not None else None
rinfo = self.connection.create_security_group_rule(
secgroup_name_or_id=rgroup._id,
port_range_min=rule.port_min,
port_range_max=rule.port_max,
protocol=rule.protocol.value,
remote_ip_prefix=cidr,
remote_group_id=group_id,
direction=rule.direction.value,
ethertype=rule.ethertype.value)
rgroup.rules.add(Rule.from_remote(**rinfo))
if remove:
# Removed rules
for group_name, rule in rules_removed:
rgroup = rgroups[group_name]
self.connection.delete_security_group_rule(rule_id=rule._id)
rgroup.rules.remove(rule)
# Removed groups
for group in groups_removed:
self.connection.delete_security_group(name_or_id=group._id)
remote.remove(group)
self.remote = remote
def __init__(self): self.__coroutines = OrderedSet()
class Graph:
def __init__(self, name):
self.name = name
self.node_set = OrderedSet()
self.edge_set = OrderedSet()
self.h_matrix = None
self.s_matrix = None
self.g_matrix = None
self.pi_vector = None
def create_graph_from_file(self, file):
for line in file.read().splitlines():
if "NodeName" in line:
node_name = line.split(" = ")[1]
node = Node(node_name)
self.node_set.add(node)
if "EdgeName" in line:
nodes = line.split(" = ")[1].split("->")
starting_node_name = nodes[0]
target_node_name= nodes[1]
starting_node = self.find_node(starting_node_name)
target_node = self.find_node(target_node_name)
if starting_node is None or target_node is None:
raise RuntimeError
starting_node.add_edge(target_node)
def find_node(self, name):
"""
Since sets do not support getting an item out, loop over and compare each node
for equality in order to find the desired node
:param name: the name of the node to search for
:return: the node with `name` if it is present in the graphs node_set
"""
target_node = Node(name)
if target_node in self.node_set:
for node in self.node_set:
if node == target_node:
return node
return None
def create_h_matrix(self, df=None, store=True):
"""
Adjacency matrix
:return:
"""
# set up default matrix (n x n with all 0 values)
if not df:
df = self.h_matrix
df = self.dataframe_nxn()
for node in self.node_set:
adjacency_map = node.compute_adjacency()
for adjacent_node, probability in adjacency_map.items():
df.set_value(node, adjacent_node, probability)
if store:
self.h_matrix = df
return df
def create_s_matrix(self, df=None, store=True):
"""
Transition Stochastic Matrix
:return:
"""
if not df:
df = self.create_h_matrix(store=False)
def correct_dangling(row):
if row.sum() == 0:
return [1/len(row) for node in self.node_set]
else:
return row
df = df.apply(correct_dangling, axis=1, reduce=False)
if store:
self.s_matrix = df
return df
def create_g_matrix(self, df=None, store=False):
"""
Google Matrix
:return:
"""
scaling_param = .9
if not df:
df = self.create_s_matrix(store=False)
df = np.dot(.9, df)
df += np.dot((1-scaling_param), 1/len(self.node_set))
return df
def compute_page_rank(self, df=None):
if not df:
df = self.create_g_matrix()
vector = [1/len(self.node_set) for x in self.node_set]
for node in self.node_set:
vector = np.dot(vector, df)
vector = vector.round(3)
page_map = {}
for i, node in enumerate(self.node_set):
page_map[node] = vector[i]
return page_map
def describe_graph(self):
print("Details for graph: %s" % self.name)
print("++++ Nodes in the graph")
for node in self.node_set:
print("NodeName = %s" % node.name)
print("++++ Edges in the Graph")
for node in self.node_set:
node.describe()
print("\n\nAdjency Matrix:\n")
print(self.create_h_matrix().round(3))
print("\n\nTransition Stochastic Matrix:\n")
print(self.create_s_matrix().round(3))
print("\n\nGoogle Matrix:\n")
print(self.create_g_matrix().round(3))
print("\n\nRankings:\n")
data = self.compute_page_rank()
sorted_nodes = sorted(data, key=data.get, reverse=True)
for i, node in enumerate(sorted_nodes):
print("Rank #%s: %s - %s" % (i+1, node, data[node]))
def dataframe_nxn(self):
structure = OrderedDict()
blank_data = [0.0 for node in self.node_set]
for node in self.node_set:
structure[node] = pd.Series(blank_data, index=self.node_set)
return pd.DataFrame(structure)
def comm(a, b):
sa, sb = map(OrderedSet, (a, b))
return (sa.difference(sb), sb.difference(sa),
OrderedSet.intersection(sa, sb))
class FileMeta(ValidatorObject):
mandatory = OrderedSet()
optional = OrderedSet()
order = OrderedSet()
enforce_order = False
mappings = {}
from rdfframes.query_builder.querymodel import QueryModel
from orderedset import OrderedSet
if __name__ == '__main__':
####### complete query
subquery = QueryModel()
subquery.add_triple("tweet", "sioc:has_creater", "tweep")
#subquery.add_variable("tweeter")
subquery.add_group_columns(OrderedSet(["tweep"]))
subquery.add_aggregate_pair("tweet", "COUNT", "tweet_count", "distinct")
subquery.add_having_condition("tweet_count", "< 300")
subquery.add_having_condition("tweet_count", "> 250")
subquery.add_select_column("tweep")
twitterquery = QueryModel()
prefixes = {
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"sioc": "http://rdfs.org/sioc/ns#",
"sioct": "http://rdfs.org/sioc/types#",
"to": "http://twitter.com/ontology/",
"dcterms": "http://purl.org/dc/terms/",
"xsd": "http://www.example.org/",
"foaf": "http://xmlns.com/foaf/0.1/",
}
twitterquery.add_prefixes(prefixes)
twitterquery.add_graphs(["http://twitter.com/"])
twitterquery.add_variable("tweep")
twitterquery.add_subquery(subquery)
twitterquery.add_triple("tweet", "sioc:has_creater", "tweep")
twitterquery.add_triple("tweet", " sioc:content", "text")
def add_ops(self, roots, after_exop=None):
"""
Add exops needed to compute ops in roots.
Args:
roots: A collection of ops whose values are needed.
after_exop: Where in the list to add the ops. Defaults to the end.
"""
if after_exop is None:
after_exop = self.prev_exop
# Get computation graph ops that are already inserted.
available = OrderedSet()
counts = dict()
parents = defaultdict(OrderedSet)
ready = OrderedSet()
# Setting the environmental variable below to 0 can be used to disable toposort
# with priorities and switch to naive algo in case something went wrong unexpectedly
algo_num = int(os.getenv('NGRAPH_TOPOSORT_ALGO', 1))
pqueue = PriorityQueue()
op_counter = 0
wait_order = 100000
std_order = 2
start_order = 1
# Some ops in roots may have been replaced by other ops; if so, they
# are in the graph already, although maybe not in this block. Get the
# op from the exop so we have the current version.
for op in roots:
exop = self.computation_decl.get_exop(op, None)
if exop is not None:
op = exop.op
available.add(op)
while available:
op = available.pop()
if algo_num > 0:
if 'priority' in op.metadata:
if op.metadata['priority'] == 'high':
op.metadata['order'] = start_order
else:
op.metadata['order'] = wait_order
elif 'order' not in op.metadata:
op.metadata['order'] = std_order
if op in counts or op in self.all_ops:
continue
nchildren = 0
op_deps = op.all_deps
if (isinstance(op, CPUMlslGatherRecvOp) or isinstance(
op, CPUMlslScatterRecvOp)) and op.send_node() in available:
op_deps.add(op.send_node())
for child in op_deps:
exop = self.computation_decl.get_exop(child, None)
if exop is not None:
child = exop.op
if child not in self.all_ops:
parents[child].add(op)
available.add(child)
if algo_num > 0:
ch_order = child.metadata[
'order'] if 'order' in child.metadata else -1
new_order = op.metadata['order'] + 1
if 'priority' not in child.metadata and \
('order' not in child.metadata or new_order < ch_order):
child.metadata['order'] = new_order
nchildren += 1
if nchildren > 0:
counts[op] = nchildren
else:
if op not in ready:
ready.add(op)
if algo_num > 0:
op_counter = op_counter - 1
pqueue.put((op.metadata['order'], op_counter, op))
if algo_num == 0:
while ready:
op = ready.pop()
after_exop = self.add_op(op, after_exop=after_exop)
for p in parents.get(op, []):
count = counts[p] - 1
if count == 0:
ready.add(p)
del counts[p]
else:
counts[p] = count
else:
while len(pqueue.queue) > 0:
_, _, op = pqueue.get()
after_exop = self.add_op(op, after_exop=after_exop)
for p in parents.get(op, []):
count = counts[p] - 1
if count == 0:
op_counter = op_counter - 1
# Shouldn't happen, but we have a way to get back to naive scheduling
assert 'order' in p.metadata, \
"Something went wrong with the scheduling. \
Please try NGRAPH_TOPOSORT_ALGO=0"
if p.metadata['order'] == wait_order:
pqueue.put(
(p.metadata['order'], int(-op_counter), p))
else:
pqueue.put((p.metadata['order'], op_counter, p))
del counts[p]
else:
counts[p] = count
if len(counts) > 0:
raise ValueError("Graph not a DAG")
def validate(self):
"""
Validate the appinfo and put the results into ``errors``, ``warnings``
and ``info`` in ``self``.
"""
self.load(False)
if self.ini is None:
self.errors.append(self.ini_fail)
return # Don't need "section missing" errors
if not isfile(self.path_abs()):
# If appinfo.ini doesn't exist, we've created an INIConfig which
# will be empty, for the fix routine. Then as we don't want to spew
# a whole heap of errors given that an error about appinfo.ini
# being missing has already been added to the list, we'll give up.
return
ini = self.ini
package = self.package
meta = self.Meta(self, ini, package) # () so properties work
setini = OrderedSet(ini)
for missing in meta.mandatory - setini:
self.errors.append(LANG.INIVALIDATOR.SECTION_MISSING %
dict(filename=self.path(), section=missing))
for extra in setini - meta.order:
self.errors.append(LANG.INIVALIDATOR.SECTION_EXTRA %
dict(filename=self.path(), section=extra))
# Check that they're in the same order
if meta.enforce_order and setini & meta.order != meta.order & setini:
# A warning? We like fancy INI files.
self.warnings.append(LANG.INIVALIDATOR.SECTIONS_OUT_OF_ORDER %
dict(filename=self.path()))
if len(ini) == 0:
self.warnings.append(LANG.INIVALIDATOR.FILE_EMPTY %
dict(filename=self.path()))
for section in setini & meta.order:
if hasattr(self.module, section):
secval_cls = getattr(self.module, section)
else:
for mapping in meta.mappings:
if mapping.match(section):
secval_cls = getattr(self.module, mapping.target)
# TODO: could be nice to know what section is in this case...
break
secval = secval_cls(self, ini, package)
smeta = secval.Meta(self, ini, package, secval)
inisection = ini[section]
inisectionset = OrderedSet(inisection)
if len(inisection) == 0:
self.warnings.append(
LANG.INIVALIDATOR.SECTION_EMPTY %
dict(filename=self.path(), section=section))
for key in smeta.mandatory:
if key not in inisection:
self.errors.append(
LANG.INIVALIDATOR.VALUE_MISSING %
dict(filename=self.path(), section=section, key=key))
elif inisection[key] == '':
self.errors.append(
LANG.INIVALIDATOR.VALUE_EMPTY %
dict(filename=self.path(), section=section, key=key))
for key in inisectionset - smeta.mandatory:
if key not in smeta.optional:
self.errors.append(
LANG.INIVALIDATOR.VALUE_EXTRA %
dict(filename=self.path(), section=section, key=key))
elif inisection[key] == '':
self.errors.append(
LANG.INIVALIDATOR.OMIT_EMPTY %
dict(filename=self.path(), section=section, key=key))
# Check that they're in the same order
enforce_order = meta.enforce_order if smeta.enforce_order is Ellipsis else smeta.enforce_order
if enforce_order and inisectionset & smeta.order != smeta.order & inisectionset:
# A warning? We like fancy INI files.
self.warnings.append(
LANG.INIVALIDATOR.KEYS_OUT_OF_ORDER %
dict(filename=self.path(), section=section))
# Oh yeah, we may as well validate the value. Could be handy.
for key in smeta.order:
if key in inisection:
if hasattr(secval, key):
self._add_item(getattr(secval, key)(inisection[key]))
else:
for mapping in smeta.mappings:
if mapping.match(key):
self._add_item(
getattr(secval,
mapping.target)(inisection[key]))
# TODO: could be nice to know what key is in this case...
break
class ExOpBlock(ExecutionGraphElt):
"""
A list of exops to be executed sequentially.
Attributes:
computation_decl: The associated computation graph.
prev_exop: The latst exop.
next_exop: The first exop.
root_set: Set of exops whose values are needed.
"""
def __init__(self, computation_decl=None, **kwargs):
if computation_decl is None:
raise ValueError("computation_decl must be specified.")
super(ExOpBlock,
self).__init__(execution_graph=computation_decl.execution_graph,
**kwargs)
self.computation_decl = computation_decl
# Doubly linked loop, with self as termination
self.prev_exop = self
self.next_exop = self
# All ops handled by the block.
self.all_ops = set()
self.root_set = OrderedSet()
@property
def is_exop_end_of_list(self):
"""
Returns:
True if this represents the guard past the exop list. See ExecuteOp.
"""
return True
class ExOpForwardIterator(object):
def __init__(self, exop_term):
self.exop_term = exop_term
self.exop = self.exop_term.next_exop
def next(self):
if self.exop.is_exop_end_of_list:
raise StopIteration
result = self.exop
self.exop = result.next_exop
return result
__next__ = next # Python 3.X compatibility
class ExOpReversedIterator(object):
def __init__(self, exop_term):
self.exop_term = exop_term
self.exop = self.exop_term.prev_exop
def __iter__(self):
return self
def next(self):
if self.exop.is_exop_end_of_list:
raise StopIteration
result = self.exop
self.exop = result.prev_exop
return result
__next__ = next # Python 3.X compatibility
def __iter__(self):
return ExOpBlock.ExOpForwardIterator(self)
def __reversed__(self):
return ExOpBlock.ExOpReversedIterator(self)
def add_ops(self, roots, after_exop=None):
"""
Add exops needed to compute ops in roots.
Args:
roots: A collection of ops whose values are needed.
after_exop: Where in the list to add the ops. Defaults to the end.
"""
if after_exop is None:
after_exop = self.prev_exop
# Get computation graph ops that are already inserted.
available = OrderedSet()
counts = dict()
parents = defaultdict(OrderedSet)
ready = OrderedSet()
# Setting the environmental variable below to 0 can be used to disable toposort
# with priorities and switch to naive algo in case something went wrong unexpectedly
algo_num = int(os.getenv('NGRAPH_TOPOSORT_ALGO', 1))
pqueue = PriorityQueue()
op_counter = 0
wait_order = 100000
std_order = 2
start_order = 1
# Some ops in roots may have been replaced by other ops; if so, they
# are in the graph already, although maybe not in this block. Get the
# op from the exop so we have the current version.
for op in roots:
exop = self.computation_decl.get_exop(op, None)
if exop is not None:
op = exop.op
available.add(op)
while available:
op = available.pop()
if algo_num > 0:
if 'priority' in op.metadata:
if op.metadata['priority'] == 'high':
op.metadata['order'] = start_order
else:
op.metadata['order'] = wait_order
elif 'order' not in op.metadata:
op.metadata['order'] = std_order
if op in counts or op in self.all_ops:
continue
nchildren = 0
op_deps = op.all_deps
if (isinstance(op, CPUMlslGatherRecvOp) or isinstance(
op, CPUMlslScatterRecvOp)) and op.send_node() in available:
op_deps.add(op.send_node())
for child in op_deps:
exop = self.computation_decl.get_exop(child, None)
if exop is not None:
child = exop.op
if child not in self.all_ops:
parents[child].add(op)
available.add(child)
if algo_num > 0:
ch_order = child.metadata[
'order'] if 'order' in child.metadata else -1
new_order = op.metadata['order'] + 1
if 'priority' not in child.metadata and \
('order' not in child.metadata or new_order < ch_order):
child.metadata['order'] = new_order
nchildren += 1
if nchildren > 0:
counts[op] = nchildren
else:
if op not in ready:
ready.add(op)
if algo_num > 0:
op_counter = op_counter - 1
pqueue.put((op.metadata['order'], op_counter, op))
if algo_num == 0:
while ready:
op = ready.pop()
after_exop = self.add_op(op, after_exop=after_exop)
for p in parents.get(op, []):
count = counts[p] - 1
if count == 0:
ready.add(p)
del counts[p]
else:
counts[p] = count
else:
while len(pqueue.queue) > 0:
_, _, op = pqueue.get()
after_exop = self.add_op(op, after_exop=after_exop)
for p in parents.get(op, []):
count = counts[p] - 1
if count == 0:
op_counter = op_counter - 1
# Shouldn't happen, but we have a way to get back to naive scheduling
assert 'order' in p.metadata, \
"Something went wrong with the scheduling. \
Please try NGRAPH_TOPOSORT_ALGO=0"
if p.metadata['order'] == wait_order:
pqueue.put(
(p.metadata['order'], int(-op_counter), p))
else:
pqueue.put((p.metadata['order'], op_counter, p))
del counts[p]
else:
counts[p] = count
if len(counts) > 0:
raise ValueError("Graph not a DAG")
def add_op(self, op, after_exop):
"""
Add an exop for op to be executed after after_exop.
Args:
op: The op.
after_exop: The exop to precede op.
Returns:
The new last op. If the op is executable, it will be the added exop,
othwerwise the previous after_exop.
"""
if after_exop is None:
after_exop = self
if op.is_sequencing_op:
return after_exop
exec_op = ExOp(computation_decl=self.computation_decl, op=op)
return self.add_exop(exec_op, after_exop)
def add_exop(self, exop, after_exop=None):
"""
Add exop to the list of exops, after after_exop.
Args:
exop:
The exop to add.
after_exop:
If specified, the exop that should be added after after_exop. Defaults to the
last exop added.
Returns:
The exop.
"""
if after_exop is None:
after_exop = self.prev_exop
# Insert between after_exop and the op after after_exop
before_exop = after_exop.next_exop
# Add after after_exop
after_exop.next_exop = exop
exop.prev_exop = after_exop
# Add before before_exop
before_exop.prev_exop = exop
exop.next_exop = before_exop
self.all_ops.add(exop.op)
return exop
def move_exop_to_after_exop(self, exop, after_exop):
exop.prev_exop.next_exop = exop.next_exop
exop.next_exop.prev_exop = exop.prev_exop
exop.prev_exop = after_exop
exop.next_exop = after_exop.next_exop
after_exop.next_exop = exop
exop.next_exop.prev_exop = exop
def remove_exop(self, exop):
exop.prev_exop.next_exop = exop.next_exop
exop.next_exop.prev_exop = exop.prev_exop
for input_decl in exop.input_decls:
input_decl.source_output_decl.user_input_decls.remove(input_decl)
self.all_ops.remove(exop.op)
def replace_op(self, old_op, new_op):
# TODO Replacing an op can remove ops. For example, (x + 2) * 1 -> x + 2
# replaces the * with +, so * and 1 drop out
# 1 dropping out means one less constant tensor, if it's not used
# anywhere else
# * dropping out means a change to sequencing.
new_op = as_op(new_op)
old_exop = self.computation_decl.get_exop(old_op)
after_exop = old_exop.prev_exop
self.remove_exop(old_exop)
# FIXME: find better way to update dependencies
next_op = old_exop.next_exop.op
if old_op in next_op.control_deps:
next_op.remove_control_dep(old_op)
next_op.add_control_dep(new_op)
# FIXME: find better way to preserve metadata
if hasattr(old_op, 'metadata') and hasattr(new_op, 'metadata') and \
len(old_op.metadata) > len(new_op.metadata):
new_op.metadata = old_op.metadata
if old_op is new_op:
# Hetr bashes some ops. See MutateInsteadOfCopyWithNewArgsMixin, issue #1410
self.add_ops([new_op], after_exop=after_exop)
return
new_exop = self.computation_decl.get_exop(new_op, None)
if new_exop is None:
self.add_ops([new_op], after_exop=after_exop)
new_exop = self.computation_decl.get_exop(new_op, None)
self.replace_users(old_exop, new_exop)
if old_exop in self.root_set:
self.root_set.remove(old_exop)
self.root_set.add(new_exop)
def replace_users(self, old_exop, new_exop):
"""
Replace all users of old_exop with new_exop.
Args:
old_exop: The original exop.
new_exop: The replaceent exop.
"""
for old_output_decl, new_output_decl in zip(old_exop.output_decls,
new_exop.output_decls):
self.replace_output_decl(old_output_decl, new_output_decl)
for op in old_exop.ref_ops:
new_exop.add_ref_op(op)
self.computation_decl.ops[old_exop.op] = new_exop
def replace_output_decl(self, old_output_decl, new_output_decl):
for input_decl in set(old_output_decl.user_input_decls):
input_decl.source_output_decl = new_output_decl
new_output_decl.tensor_decl.merge_flags(old_output_decl.tensor_decl)
old_output_decl.exop.output_decls[
old_output_decl.pos] = new_output_decl
def replace_exop(self, old_exop, new_exop):
prev_exop = old_exop.prev_exop
self.remove_exop(old_exop)
self.add_exop(new_exop, prev_exop)
self.replace_users(old_exop, new_exop)
def merge_exop(self, old_exop, new_exop):
"""
new_exop, which should already exist, takes over for old_exop.
Args:
old_exop:
new_exop:
"""
self.replace_users(old_exop, new_exop)
self.remove_exop(old_exop)
def memory_footprint(self):
max_mem = 0
for node in self:
max_mem = max([node.memory_footprint(), max_mem])
return max_mem
def worst_case_footprint(self):
mem = 0
for var in self.get_temp_vars():
mem += var.tensor_view_decl.tensor_decl.size
return mem
def memory_efficiency(self):
footprint = self.memory_footprint()
usage = 0
for node in self.ops:
usage = max(usage, node.memory_usage())
result = 100
if footprint > 0:
result = int(round((float(usage) / float(footprint)) * 100))
return result
def persistent_size(self):
mem = 0
for var in self.get_persistent_vars():
mem += var.tensor_view_decl.tensor_decl.size
return mem
def get_vars(self):
vars = set()
for exop in self:
vars |= set(input_decl.source_output_decl
for input_decl in exop.input_decls)
vars |= set(exop.output_decls)
return vars
def get_temp_vars(self):
result = list()
for var in self.get_vars():
if not var.tensor_view_decl.tensor_decl.is_persistent:
result.append(var)
return result
def get_persistent_vars(self):
result = list()
for var in self.get_vars():
if var.tensor_view_decl.tensor_decl.is_persistent:
result.append(var)
return result
def send_nodes(self):
"""
:return: iterable of send nodes
"""
from ngraph.transformers.hetr.hetr_utils import get_iterable
return OrderedSet(i for i in get_iterable(self._send_node))
print(morpheme_list)
print(row["MORPHS"])
print(morph_list)
i = i + 1
if len(morpheme_list) != len(morph_list):
print("** line", i, ":", row["MORPHEMES"],
"is incompatible with", row["MORPHS"])
continue
if not morpheme_list:
continue
stem_name_set.add(morpheme_list[0])
name_morph_pair_lst = list(zip(morpheme_list, morph_list))
seg_example_list.append(name_morph_pair_lst)
for morpheme, morph in name_morph_pair_lst:
if morpheme not in morphs_of_morpheme:
morphs_of_morpheme[morpheme] = OrderedSet()
morphs_of_morpheme[morpheme].add(morph.strip())
csvfile.close()
print("-- STEP 1 COMPLETED (seg_example_list, stem_name_set, morphs_of_morpheme done)--")
# STEP 2:
# align the allomorphs of each morpheme
from multialign import aligner
alignments = {}
"""All aligned morphs. index: morpheme name, value: sequence of aligned symbols.
Each aligned symbol has as many characters as there are items in the sequence.
"""
def _check_messages(service_id,
template_id,
upload_id,
preview_row,
letters_as_pdf=False):
try:
# The happy path is that the job doesn’t already exist, so the
# API will return a 404 and the client will raise HTTPError.
job_api_client.get_job(service_id, upload_id)
# the job exists already - so go back to the templates page
# If we just return a `redirect` (302) object here, we'll get
# errors when we try and unpack in the check_messages route.
# Rasing a werkzeug.routing redirect means that doesn't happen.
raise PermanentRedirect(
url_for(".send_messages",
service_id=service_id,
template_id=template_id))
except HTTPError as e:
if e.status_code != 404:
raise
statistics = service_api_client.get_service_statistics(service_id,
today_only=True)
remaining_messages = current_service.message_limit - sum(
stat["requested"] for stat in statistics.values())
contents = s3download(service_id, upload_id)
db_template = current_service.get_template_with_user_permission_or_403(
template_id, current_user)
email_reply_to = None
sms_sender = None
if db_template["template_type"] == "email":
email_reply_to = get_email_reply_to_address_from_session()
elif db_template["template_type"] == "sms":
sms_sender = get_sms_sender_from_session()
template = get_template(
db_template,
current_service,
show_recipient=True,
letter_preview_url=url_for(
".check_messages_preview",
service_id=service_id,
template_id=template_id,
upload_id=upload_id,
filetype="png",
row_index=preview_row,
) if not letters_as_pdf else None,
email_reply_to=email_reply_to,
sms_sender=sms_sender,
page_count=get_page_count_for_letter(db_template),
)
recipients = RecipientCSV(
contents,
template_type=template.template_type,
placeholders=template.placeholders,
max_initial_rows_shown=50,
max_errors_shown=50,
safelist=itertools.chain.from_iterable(
[user.name, user.mobile_number, user.email_address]
for user in Users(service_id))
if current_service.trial_mode else None,
remaining_messages=remaining_messages,
international_sms=current_service.has_permission("international_sms"),
max_rows=get_csv_max_rows(service_id),
)
if request.args.get("from_test"):
# only happens if generating a letter preview test
back_link = url_for(".send_test",
service_id=service_id,
template_id=template.id)
choose_time_form = None
else:
back_link = url_for(".send_messages",
service_id=service_id,
template_id=template.id)
choose_time_form = ChooseTimeForm()
if preview_row < 2:
abort(404)
if preview_row < len(recipients) + 2:
template.values = recipients[preview_row -
2].recipient_and_personalisation
elif preview_row > 2:
abort(404)
return dict(
recipients=recipients,
template=template,
errors=recipients.has_errors,
row_errors=get_errors_for_csv(recipients, template.template_type),
count_of_recipients=len(recipients),
count_of_displayed_recipients=len(list(recipients.displayed_rows)),
original_file_name=request.args.get("original_file_name", ""),
upload_id=upload_id,
form=CsvUploadForm(),
remaining_messages=remaining_messages,
choose_time_form=choose_time_form,
back_link=back_link,
help=get_help_argument(),
trying_to_send_letters_in_trial_mode=all((
current_service.trial_mode,
template.template_type == "letter",
)),
required_recipient_columns=OrderedSet(
recipients.recipient_column_headers) - optional_address_columns,
preview_row=preview_row,
sent_previously=job_api_client.has_sent_previously(
service_id,
template.id,
db_template["version"],
request.args.get("original_file_name", ""),
),
)
def add_areas(self, *new_areas):
areas = list(OrderedSet(
self._dict['areas'] + list(new_areas)
))
simple_polygons = self.get_simple_polygons(areas=self.get_areas(areas=areas))
self._update(areas=areas, simple_polygons=simple_polygons.as_coordinate_pairs_lat_long)
def show_reflections(
reflections,
show_intensities=False,
show_profile_fit=False,
show_centroids=False,
show_all_reflection_data=False,
show_flags=False,
max_reflections=None,
show_identifiers=False,
):
text = []
from orderedset import OrderedSet
formats = {
"miller_index": "%i, %i, %i",
"d": "%.2f",
"qe": "%.3f",
"dqe": "%.3f",
"id": "%i",
"imageset_id": "%i",
"panel": "%i",
"flags": "%i",
"background.mean": "%.1f",
"background.dispersion": "%.1f",
"background.mse": "%.1f",
"background.sum.value": "%.1f",
"background.sum.variance": "%.1f",
"intensity.prf.value": "%.1f",
"intensity.prf.variance": "%.1f",
"intensity.sum.value": "%.1f",
"intensity.sum.variance": "%.1f",
"intensity.cor.value": "%.1f",
"intensity.cor.variance": "%.1f",
"intensity.scale.value": "%.1f",
"intensity.scale.variance": "%.1f",
"Ih_values": "%.1f",
"lp": "%.3f",
"num_pixels.background": "%i",
"num_pixels.background_used": "%i",
"num_pixels.foreground": "%i",
"num_pixels.valid": "%i",
"partial_id": "%i",
"partiality": "%.4f",
"profile.correlation": "%.3f",
"profile.rmsd": "%.3f",
"xyzcal.mm": "%.2f, %.2f, %.2f",
"xyzcal.px": "%.2f, %.2f, %.2f",
"delpsical.rad": "%.3f",
"delpsical2": "%.3f",
"delpsical.weights": "%.3f",
"xyzobs.mm.value": "%.2f, %.2f, %.2f",
"xyzobs.mm.variance": "%.4e, %.4e, %.4e",
"xyzobs.px.value": "%.2f, %.2f, %.2f",
"xyzobs.px.variance": "%.4f, %.4f, %.4f",
"s1": "%.4f, %.4f, %.4f",
"s2": "%.4f, %.4f, %.4f",
"shoebox": "%.1f",
"rlp": "%.4f, %.4f, %.4f",
"zeta": "%.3f",
"x_resid": "%.3f",
"x_resid2": "%.3f",
"y_resid": "%.3f",
"y_resid2": "%.3f",
"kapton_absorption_correction": "%.3f",
"kapton_absorption_correction_sigmas": "%.3f",
"inverse_scale_factor": "%.3f",
"inverse_scale_factor_variance": "%.3f",
}
for rlist in reflections:
from dials.algorithms.shoebox import MaskCode
foreground_valid = MaskCode.Valid | MaskCode.Foreground
text.append("")
text.append(f"Reflection list contains {len(rlist)} reflections")
if len(rlist) == 0:
continue
rows = [["Column", "min", "max", "mean"]]
for k, col in rlist.cols():
if k in formats and "%" not in formats.get(k, "%s"):
# Allow blanking out of entries that wouldn't make sense
rows.append([
k,
formats.get(k, "%s"),
formats.get(k, "%s"),
formats.get(k, "%s"),
])
elif type(col) in (flex.double, flex.int, flex.size_t):
if type(col) in (flex.int, flex.size_t):
col = col.as_double()
rows.append([
k,
formats.get(k, "%s") % flex.min(col),
formats.get(k, "%s") % flex.max(col),
formats.get(k, "%s") % flex.mean(col),
])
elif type(col) in (flex.vec3_double, flex.miller_index):
if isinstance(col, flex.miller_index):
col = col.as_vec3_double()
rows.append([
k,
formats.get(k, "%s") % col.min(),
formats.get(k, "%s") % col.max(),
formats.get(k, "%s") % col.mean(),
])
elif isinstance(col, flex.shoebox):
rows.append([k, "", "", ""])
si = col.summed_intensity().observed_value()
rows.append([
" summed I",
formats.get(k, "%s") % flex.min(si),
formats.get(k, "%s") % flex.max(si),
formats.get(k, "%s") % flex.mean(si),
])
x1, x2, y1, y2, z1, z2 = col.bounding_boxes().parts()
bbox_sizes = ((z2 - z1) * (y2 - y1) * (x2 - x1)).as_double()
rows.append([
" N pix",
formats.get(k, "%s") % flex.min(bbox_sizes),
formats.get(k, "%s") % flex.max(bbox_sizes),
formats.get(k, "%s") % flex.mean(bbox_sizes),
])
fore_valid = col.count_mask_values(
foreground_valid).as_double()
rows.append([
" N valid foreground pix",
formats.get(k, "%s") % flex.min(fore_valid),
formats.get(k, "%s") % flex.max(fore_valid),
formats.get(k, "%s") % flex.mean(fore_valid),
])
text.append(tabulate(rows, headers="firstrow"))
if show_flags:
text.append(_create_flag_count_table(rlist))
if show_identifiers:
if rlist.experiment_identifiers():
text.append(
"""Experiment identifiers id-map values:\n%s""" %
("\n".join(
"id:" + str(k) + " -> experiment identifier:" +
str(rlist.experiment_identifiers()[k])
for k in rlist.experiment_identifiers().keys())))
intensity_keys = (
"miller_index",
"d",
"intensity.prf.value",
"intensity.prf.variance",
"intensity.sum.value",
"intensity.sum.variance",
"background.mean",
"profile.correlation",
"profile.rmsd",
)
profile_fit_keys = ("miller_index", "d")
centroid_keys = (
"miller_index",
"d",
"xyzcal.mm",
"xyzcal.px",
"xyzobs.mm.value",
"xyzobs.mm.variance",
"xyzobs.px.value",
"xyzobs.px.variance",
)
keys_to_print = OrderedSet()
if show_intensities:
for k in intensity_keys:
keys_to_print.add(k)
if show_profile_fit:
for k in profile_fit_keys:
keys_to_print.add(k)
if show_centroids:
for k in centroid_keys:
keys_to_print.add(k)
if show_all_reflection_data:
for k in formats:
keys_to_print.add(k)
def format_column(key, data, format_strings=None):
if isinstance(data, flex.vec3_double):
c_strings = [
c.as_string(format_strings[i].strip())
for i, c in enumerate(data.parts())
]
elif isinstance(data, flex.miller_index):
c_strings = [
c.as_string(format_strings[i].strip())
for i, c in enumerate(data.as_vec3_double().parts())
]
elif isinstance(data, flex.size_t):
c_strings = [data.as_int().as_string(format_strings[0].strip())]
elif isinstance(data, flex.shoebox):
x1, x2, y1, y2, z1, z2 = data.bounding_boxes().parts()
bbox_sizes = ((z2 - z1) * (y2 - y1) * (x2 - x1)).as_double()
c_strings = [bbox_sizes.as_string(format_strings[0].strip())]
key += " (N pix)"
else:
c_strings = [data.as_string(format_strings[0].strip())]
column = flex.std_string()
max_element_lengths = [c.max_element_length() for c in c_strings]
for i in range(len(c_strings[0])):
column.append(f"%{len(key)}s" % ", ".join(
("%%%is" % max_element_lengths[j]) % c_strings[j][i]
for j in range(len(c_strings))))
return column
if keys_to_print:
keys = [k for k in keys_to_print if k in rlist]
if max_reflections is not None:
max_reflections = min(len(rlist), max_reflections)
else:
max_reflections = len(rlist)
columns = []
for k in keys:
columns.append(
format_column(k,
rlist[k],
format_strings=formats[k].split(",")))
text.append("")
text.append("Printing %i of %i reflections:" %
(max_reflections, len(rlist)))
line = []
for j in range(len(columns)):
key = keys[j]
if key == "shoebox":
key += " (N pix)"
width = max(len(key), columns[j].max_element_length())
line.append("%%%is" % width % key)
text.append(" ".join(line))
for i in range(max_reflections):
line = (c[i] for c in columns)
text.append(" ".join(line))
return "\n".join(text)
class SimpleBreadthFirstLearner(TemplateLearner):
def __init__(self, solver_instance: Prolog, max_body_literals=4):
super().__init__(solver_instance)
self._max_body_literals = max_body_literals
def initialise_pool(self):
self._candidate_pool = OrderedSet()
def put_into_pool(
self, candidates: typing.Union[Clause, Procedure,
typing.Sequence]) -> None:
if isinstance(candidates, Clause):
self._candidate_pool.add(candidates)
else:
self._candidate_pool |= candidates
def get_from_pool(self) -> Clause:
return self._candidate_pool.pop(0)
def evaluate(self, examples: Task,
clause: Clause) -> typing.Union[int, float]:
covered = self._execute_program(examples, clause)
pos, neg = examples.get_examples()
covered_pos = pos.intersection(covered)
covered_neg = neg.intersection(covered)
if len(covered_neg) > 0:
return 0
else:
return len(covered_pos)
def stop_inner_search(self, eval: typing.Union[int, float], examples: Task,
clause: Clause) -> bool:
if eval > 0:
return True
else:
return False
def process_expansions(
self, examples: Task, exps: typing.Sequence[Clause],
hypothesis_space: TopDownHypothesisSpace
) -> typing.Sequence[Clause]:
# eliminate every clause with more body literals than allowed
exps = [cl for cl in exps if len(cl) <= self._max_body_literals]
# check if every clause has solutions
exps = [(cl, self._solver.has_solution(*cl.get_body().get_literals()))
for cl in exps]
new_exps = []
for ind in range(len(exps)):
if exps[ind][1]:
# keep it if it has solutions
new_exps.append(exps[ind][0])
else:
# remove from hypothesis space if it does not
hypothesis_space.remove(exps[ind][0])
return new_exps
def keys(self):
return OrderedSet(super().keys())
def __init__(self, filename):
super().__init__()
triggers = OrderedDict()
queries = OrderedDict()
actions = OrderedDict()
functions = dict(trigger=triggers, query=queries, action=actions)
self.functions = functions
self.entities = OrderedSet()
devices = []
trigger_or_query_params = set()
enum_types = OrderedDict()
# Token order:
# first the padding, go and end of sentence
# then the begin tokens
# then triggers - queries - actions
# in this order
# then parameters names
# then operators
# then values
# then entity tokens
#
# This order is important as it affects the 3-part aligner
# algorithm
tokens = ['<<PAD>>', '<<EOS>>', '<<GO>>']
self.num_control_tokens = 3
tokens += BEGIN_TOKENS
self.num_begin_tokens = len(BEGIN_TOKENS)
# add the special functions
functions['trigger']['tt:$builtin.now'] = []
functions['query']['tt:$builtin.noop'] = []
functions['action']['tt:$builtin.notify'] = []
param_tokens = OrderedSet()
param_tokens.add('tt-param:$event')
trigger_or_query_params.add('tt-param:$event')
with open(filename, 'r') as fp:
for line in fp.readlines():
line = line.strip().split()
function_type = line[0]
function = line[1]
if function_type == 'device':
devices.append(function)
continue
if function_type == 'entity':
self.entities.add(function)
continue
parameters = line[2:]
paramlist = []
functions[function_type][function] = paramlist
for i in range(len(parameters) // 2):
param = parameters[2 * i]
type = parameters[2 * i + 1]
paramlist.append((param, type))
param_tokens.add('tt-param:' + param)
if function_type != 'action':
trigger_or_query_params.add('tt-param:' + param)
if type.startswith('Array('):
elementtype = type[len('Array('):-1]
else:
elementtype = type
if elementtype.startswith('Enum('):
enums = elementtype[len('Enum('):-1].split(',')
if not elementtype in enum_types:
enum_types[elementtype] = enums
for function_type in ('trigger', 'query', 'action'):
for function in functions[function_type]:
tokens.append(function)
self.num_functions = len(
tokens) - self.num_control_tokens - self.num_begin_tokens
tokens += param_tokens
self.num_params = len(param_tokens)
tokens += OPERATORS
tokens += VALUES
tokens += SPECIAL_TOKENS
tokens += BOOKKEEPING_TOKENS
tokens += devices
enumtokenset = set()
for enum_type in enum_types.values():
for enum in enum_type:
if enum in enumtokenset:
continue
enumtokenset.add(enum)
tokens.append(enum)
for unitlist in UNITS.values():
tokens += unitlist
for i in range(MAX_ARG_VALUES):
for entity in ENTITIES:
tokens.append(entity + "_" + str(i))
for generic_entity in self.entities:
for i in range(MAX_ARG_VALUES):
tokens.append('GENERIC_ENTITY_' + generic_entity + "_" +
str(i))
print('num functions', self.num_functions)
print('num triggers', len(self.functions['trigger']))
print('num queries', len(self.functions['query']))
print('num actions', len(self.functions['action']))
print('num params', self.num_params)
first_value_token = self.num_functions + self.num_begin_tokens + self.num_control_tokens
print('num value tokens', len(tokens) - first_value_token)
self.tokens = tokens
self.dictionary = dict()
for i, token in enumerate(self.tokens):
self.dictionary[token] = i
# build a DFA that will parse the thingtalk-ish code
states = []
transitions = []
state_names = []
def to_ids(tokens, words):
return list([words[x] for x in tokens])
def add_allowed_tokens(state, tokens):
state[to_ids(tokens, self.dictionary)] = 1
def new_state(name):
state = np.zeros((self.output_size, ))
states.append(state)
state_names.append(name)
return len(states) - 1
# start with one of the begin tokens
self.start_state = new_state('start')
# in the before end state we just wait for EOS
self.before_end_state = new_state('before_end')
# in the end state we are done
self.end_state = new_state('end')
transitions.append((self.before_end_state, self.end_state, '<<EOS>>'))
transitions.append((self.end_state, self.end_state, '<<PAD>>'))
# bookkeeping
bookkeeping_id = new_state('bookkeeping')
transitions.append((self.start_state, bookkeeping_id, 'bookkeeping'))
transitions.append((bookkeeping_id, self.end_state, '<<EOS>>'))
self.bookeeping_state_id = bookkeeping_id
# special
special_id = new_state('special')
transitions.append((bookkeeping_id, special_id, 'special'))
for t in SPECIAL_TOKENS:
transitions.append((special_id, self.before_end_state, t))
# command
command_id = new_state('command')
transitions.append((bookkeeping_id, command_id, 'command'))
# help/configure/discover command
help_id = new_state('device_or_generic')
transitions.append((command_id, help_id, 'help'))
transitions.append((help_id, self.before_end_state, 'generic'))
for d in devices:
transitions.append((help_id, self.before_end_state, d))
# answers
answer_id = new_state('answer')
transitions.append((bookkeeping_id, answer_id, 'answer'))
for v in VALUES:
if v != '0' and v != '1':
transitions.append((answer_id, self.before_end_state, v))
for v in ENTITIES:
if v != 'NUMBER':
for i in range(MAX_ARG_VALUES):
transitions.append(
(answer_id, self.before_end_state, v + '_' + str(i)))
before_unit = new_state('answer_before_unit')
for i in range(MAX_ARG_VALUES):
transitions.append((answer_id, before_unit, 'NUMBER_' + str(i)))
transitions.append((answer_id, before_unit, '0'))
transitions.append((answer_id, before_unit, '1'))
transitions.append((before_unit, self.end_state, '<<EOS>>'))
for base_unit in UNITS:
for unit in UNITS[base_unit]:
transitions.append((before_unit, self.before_end_state, unit))
def do_invocation(invocation_name, params, for_action=False):
state_id = new_state(invocation_name)
# allow one USERNAME_ parameter to follow the invocation immediately
for i in range(MAX_ARG_VALUES):
transitions.append((state_id, state_id, 'USERNAME_' + str(i)))
# go to each parameter
for param_name, param_type in params:
if param_type in ('Any'):
continue
elementtype = param_type
is_array = False
is_measure = False
if param_type.startswith('Array('):
is_array = True
elementtype = param_type[len('Array('):-1]
if elementtype in TYPE_RENAMES:
elementtype = TYPE_RENAMES[elementtype]
if elementtype.startswith('Measure('):
is_measure = True
operators = ['is', '<', '>']
base_unit = elementtype[len('Measure('):-1]
values = UNITS[base_unit]
elif elementtype.startswith('Enum('):
operators = ['is']
values = enum_types[elementtype]
elif elementtype == 'Entity(tt:device)':
operators = ['is']
values = devices
elif elementtype in TYPES:
operators, values = TYPES[elementtype]
elif elementtype.startswith('Entity('):
operators = ['is']
values = [
'GENERIC_ENTITY_' + elementtype[len('Entity('):-1],
'QUOTED_STRING'
]
else:
operators, values = TYPES[elementtype]
if is_array:
if for_action:
continue
else:
operators = ['has']
elif for_action:
operators = ['is']
before_op = new_state(invocation_name + '_tt-param:.' +
param_name)
transitions.append(
(state_id, before_op, 'tt-param:' + param_name))
before_value = new_state(invocation_name + '_tt-param:' +
param_name + ':value')
for op in operators:
transitions.append((before_op, before_value, op))
if is_measure:
before_unit = new_state(invocation_name + '_tt-param:' +
param_name + ':unit')
for i in range(MAX_ARG_VALUES):
transitions.append((before_value, before_unit, '0'))
transitions.append((before_value, before_unit, '1'))
transitions.append(
(before_value, before_unit, 'NUMBER_' + str(i)))
for unit in values:
transitions.append((before_unit, state_id, unit))
else:
for v in values:
if v[0].isupper():
for i in range(MAX_ARG_VALUES):
transitions.append(
(before_value, state_id, v + '_' + str(i)))
else:
transitions.append((before_value, state_id, v))
if is_measure and base_unit == 'ms':
for i in range(MAX_ARG_VALUES):
transitions.append(
(before_value, state_id, 'DURATION_' + str(i)))
for v in trigger_or_query_params:
transitions.append((before_value, state_id, v))
return state_id
# rules
rule_id = new_state('rule')
transitions.append((self.start_state, rule_id, 'rule'))
trigger_ids = []
query_ids = []
for trigger_name, params in triggers.items():
state_id = do_invocation(trigger_name, params, for_action=False)
transitions.append((rule_id, state_id, trigger_name))
trigger_ids.append(state_id)
for query_name, params in queries.items():
state_id = do_invocation(query_name, params, for_action=False)
for trigger_id in trigger_ids:
transitions.append((trigger_id, state_id, query_name))
query_ids.append(state_id)
for action_name, params in actions.items():
state_id = do_invocation(action_name, params, for_action=True)
for query_id in query_ids:
transitions.append((query_id, state_id, action_name))
transitions.append((state_id, self.end_state, '<<EOS>>'))
# do a second copy of the transition matrix for split sequences
self.function_states = np.zeros((self.num_functions, ), dtype=np.int32)
self.function_states.fill(-1)
for part in ('trigger', 'query', 'action'):
for function_name, params in self.functions[part].items():
token = self.dictionary[
function_name] - self.num_control_tokens - self.num_begin_tokens
state_id = do_invocation(function_name,
params,
for_action=(part == 'action'))
transitions.append((state_id, self.end_state, '<<EOS>>'))
self.function_states[token] = state_id
# now build the actual DFA
num_states = len(states)
self.num_states = num_states
print("num states", num_states)
print("num tokens", self.output_size)
self.transition_matrix = np.zeros((num_states, self.output_size),
dtype=np.int32)
self.transition_matrix.fill(-1)
self.allowed_token_matrix = np.zeros((num_states, self.output_size),
dtype=np.bool8)
for from_state, to_state, token in transitions:
token_id = self.dictionary[token]
if self.transition_matrix[from_state, token_id] != -1 and \
self.transition_matrix[from_state, token_id] != to_state:
raise ValueError("Ambiguous transition around token " + token +
" in state " + state_names[from_state])
self.transition_matrix[from_state, token_id] = to_state
self.allowed_token_matrix[from_state, token_id] = True
if True:
visited = set()
def dfs(state):
visited.add(state)
any_out = False
for next_state in self.transition_matrix[state]:
if next_state == -1:
continue
any_out = True
if next_state in visited:
continue
dfs(next_state)
if not any_out:
raise ValueError(
'Reachable state %d (%s) has no outgoing states' %
(state, state_names[state]))
dfs(self.start_state)
self.state_names = state_names
def local(self, groups):
self._local = OrderedSet(groups)
class ThingtalkGrammar(AbstractGrammar):
def __init__(self, filename):
super().__init__()
triggers = OrderedDict()
queries = OrderedDict()
actions = OrderedDict()
functions = dict(trigger=triggers, query=queries, action=actions)
self.functions = functions
self.entities = OrderedSet()
devices = []
trigger_or_query_params = set()
enum_types = OrderedDict()
# Token order:
# first the padding, go and end of sentence
# then the begin tokens
# then triggers - queries - actions
# in this order
# then parameters names
# then operators
# then values
# then entity tokens
#
# This order is important as it affects the 3-part aligner
# algorithm
tokens = ['<<PAD>>', '<<EOS>>', '<<GO>>']
self.num_control_tokens = 3
tokens += BEGIN_TOKENS
self.num_begin_tokens = len(BEGIN_TOKENS)
# add the special functions
functions['trigger']['tt:$builtin.now'] = []
functions['query']['tt:$builtin.noop'] = []
functions['action']['tt:$builtin.notify'] = []
param_tokens = OrderedSet()
param_tokens.add('tt-param:$event')
trigger_or_query_params.add('tt-param:$event')
with open(filename, 'r') as fp:
for line in fp.readlines():
line = line.strip().split()
function_type = line[0]
function = line[1]
if function_type == 'device':
devices.append(function)
continue
if function_type == 'entity':
self.entities.add(function)
continue
parameters = line[2:]
paramlist = []
functions[function_type][function] = paramlist
for i in range(len(parameters) // 2):
param = parameters[2 * i]
type = parameters[2 * i + 1]
paramlist.append((param, type))
param_tokens.add('tt-param:' + param)
if function_type != 'action':
trigger_or_query_params.add('tt-param:' + param)
if type.startswith('Array('):
elementtype = type[len('Array('):-1]
else:
elementtype = type
if elementtype.startswith('Enum('):
enums = elementtype[len('Enum('):-1].split(',')
if not elementtype in enum_types:
enum_types[elementtype] = enums
for function_type in ('trigger', 'query', 'action'):
for function in functions[function_type]:
tokens.append(function)
self.num_functions = len(
tokens) - self.num_control_tokens - self.num_begin_tokens
tokens += param_tokens
self.num_params = len(param_tokens)
tokens += OPERATORS
tokens += VALUES
tokens += SPECIAL_TOKENS
tokens += BOOKKEEPING_TOKENS
tokens += devices
enumtokenset = set()
for enum_type in enum_types.values():
for enum in enum_type:
if enum in enumtokenset:
continue
enumtokenset.add(enum)
tokens.append(enum)
for unitlist in UNITS.values():
tokens += unitlist
for i in range(MAX_ARG_VALUES):
for entity in ENTITIES:
tokens.append(entity + "_" + str(i))
for generic_entity in self.entities:
for i in range(MAX_ARG_VALUES):
tokens.append('GENERIC_ENTITY_' + generic_entity + "_" +
str(i))
print('num functions', self.num_functions)
print('num triggers', len(self.functions['trigger']))
print('num queries', len(self.functions['query']))
print('num actions', len(self.functions['action']))
print('num params', self.num_params)
first_value_token = self.num_functions + self.num_begin_tokens + self.num_control_tokens
print('num value tokens', len(tokens) - first_value_token)
self.tokens = tokens
self.dictionary = dict()
for i, token in enumerate(self.tokens):
self.dictionary[token] = i
# build a DFA that will parse the thingtalk-ish code
states = []
transitions = []
state_names = []
def to_ids(tokens, words):
return list([words[x] for x in tokens])
def add_allowed_tokens(state, tokens):
state[to_ids(tokens, self.dictionary)] = 1
def new_state(name):
state = np.zeros((self.output_size, ))
states.append(state)
state_names.append(name)
return len(states) - 1
# start with one of the begin tokens
self.start_state = new_state('start')
# in the before end state we just wait for EOS
self.before_end_state = new_state('before_end')
# in the end state we are done
self.end_state = new_state('end')
transitions.append((self.before_end_state, self.end_state, '<<EOS>>'))
transitions.append((self.end_state, self.end_state, '<<PAD>>'))
# bookkeeping
bookkeeping_id = new_state('bookkeeping')
transitions.append((self.start_state, bookkeeping_id, 'bookkeeping'))
transitions.append((bookkeeping_id, self.end_state, '<<EOS>>'))
self.bookeeping_state_id = bookkeeping_id
# special
special_id = new_state('special')
transitions.append((bookkeeping_id, special_id, 'special'))
for t in SPECIAL_TOKENS:
transitions.append((special_id, self.before_end_state, t))
# command
command_id = new_state('command')
transitions.append((bookkeeping_id, command_id, 'command'))
# help/configure/discover command
help_id = new_state('device_or_generic')
transitions.append((command_id, help_id, 'help'))
transitions.append((help_id, self.before_end_state, 'generic'))
for d in devices:
transitions.append((help_id, self.before_end_state, d))
# answers
answer_id = new_state('answer')
transitions.append((bookkeeping_id, answer_id, 'answer'))
for v in VALUES:
if v != '0' and v != '1':
transitions.append((answer_id, self.before_end_state, v))
for v in ENTITIES:
if v != 'NUMBER':
for i in range(MAX_ARG_VALUES):
transitions.append(
(answer_id, self.before_end_state, v + '_' + str(i)))
before_unit = new_state('answer_before_unit')
for i in range(MAX_ARG_VALUES):
transitions.append((answer_id, before_unit, 'NUMBER_' + str(i)))
transitions.append((answer_id, before_unit, '0'))
transitions.append((answer_id, before_unit, '1'))
transitions.append((before_unit, self.end_state, '<<EOS>>'))
for base_unit in UNITS:
for unit in UNITS[base_unit]:
transitions.append((before_unit, self.before_end_state, unit))
def do_invocation(invocation_name, params, for_action=False):
state_id = new_state(invocation_name)
# allow one USERNAME_ parameter to follow the invocation immediately
for i in range(MAX_ARG_VALUES):
transitions.append((state_id, state_id, 'USERNAME_' + str(i)))
# go to each parameter
for param_name, param_type in params:
if param_type in ('Any'):
continue
elementtype = param_type
is_array = False
is_measure = False
if param_type.startswith('Array('):
is_array = True
elementtype = param_type[len('Array('):-1]
if elementtype in TYPE_RENAMES:
elementtype = TYPE_RENAMES[elementtype]
if elementtype.startswith('Measure('):
is_measure = True
operators = ['is', '<', '>']
base_unit = elementtype[len('Measure('):-1]
values = UNITS[base_unit]
elif elementtype.startswith('Enum('):
operators = ['is']
values = enum_types[elementtype]
elif elementtype == 'Entity(tt:device)':
operators = ['is']
values = devices
elif elementtype in TYPES:
operators, values = TYPES[elementtype]
elif elementtype.startswith('Entity('):
operators = ['is']
values = [
'GENERIC_ENTITY_' + elementtype[len('Entity('):-1],
'QUOTED_STRING'
]
else:
operators, values = TYPES[elementtype]
if is_array:
if for_action:
continue
else:
operators = ['has']
elif for_action:
operators = ['is']
before_op = new_state(invocation_name + '_tt-param:.' +
param_name)
transitions.append(
(state_id, before_op, 'tt-param:' + param_name))
before_value = new_state(invocation_name + '_tt-param:' +
param_name + ':value')
for op in operators:
transitions.append((before_op, before_value, op))
if is_measure:
before_unit = new_state(invocation_name + '_tt-param:' +
param_name + ':unit')
for i in range(MAX_ARG_VALUES):
transitions.append((before_value, before_unit, '0'))
transitions.append((before_value, before_unit, '1'))
transitions.append(
(before_value, before_unit, 'NUMBER_' + str(i)))
for unit in values:
transitions.append((before_unit, state_id, unit))
else:
for v in values:
if v[0].isupper():
for i in range(MAX_ARG_VALUES):
transitions.append(
(before_value, state_id, v + '_' + str(i)))
else:
transitions.append((before_value, state_id, v))
if is_measure and base_unit == 'ms':
for i in range(MAX_ARG_VALUES):
transitions.append(
(before_value, state_id, 'DURATION_' + str(i)))
for v in trigger_or_query_params:
transitions.append((before_value, state_id, v))
return state_id
# rules
rule_id = new_state('rule')
transitions.append((self.start_state, rule_id, 'rule'))
trigger_ids = []
query_ids = []
for trigger_name, params in triggers.items():
state_id = do_invocation(trigger_name, params, for_action=False)
transitions.append((rule_id, state_id, trigger_name))
trigger_ids.append(state_id)
for query_name, params in queries.items():
state_id = do_invocation(query_name, params, for_action=False)
for trigger_id in trigger_ids:
transitions.append((trigger_id, state_id, query_name))
query_ids.append(state_id)
for action_name, params in actions.items():
state_id = do_invocation(action_name, params, for_action=True)
for query_id in query_ids:
transitions.append((query_id, state_id, action_name))
transitions.append((state_id, self.end_state, '<<EOS>>'))
# do a second copy of the transition matrix for split sequences
self.function_states = np.zeros((self.num_functions, ), dtype=np.int32)
self.function_states.fill(-1)
for part in ('trigger', 'query', 'action'):
for function_name, params in self.functions[part].items():
token = self.dictionary[
function_name] - self.num_control_tokens - self.num_begin_tokens
state_id = do_invocation(function_name,
params,
for_action=(part == 'action'))
transitions.append((state_id, self.end_state, '<<EOS>>'))
self.function_states[token] = state_id
# now build the actual DFA
num_states = len(states)
self.num_states = num_states
print("num states", num_states)
print("num tokens", self.output_size)
self.transition_matrix = np.zeros((num_states, self.output_size),
dtype=np.int32)
self.transition_matrix.fill(-1)
self.allowed_token_matrix = np.zeros((num_states, self.output_size),
dtype=np.bool8)
for from_state, to_state, token in transitions:
token_id = self.dictionary[token]
if self.transition_matrix[from_state, token_id] != -1 and \
self.transition_matrix[from_state, token_id] != to_state:
raise ValueError("Ambiguous transition around token " + token +
" in state " + state_names[from_state])
self.transition_matrix[from_state, token_id] = to_state
self.allowed_token_matrix[from_state, token_id] = True
if True:
visited = set()
def dfs(state):
visited.add(state)
any_out = False
for next_state in self.transition_matrix[state]:
if next_state == -1:
continue
any_out = True
if next_state in visited:
continue
dfs(next_state)
if not any_out:
raise ValueError(
'Reachable state %d (%s) has no outgoing states' %
(state, state_names[state]))
dfs(self.start_state)
self.state_names = state_names
def get_embeddings(self, use_types=False):
if not use_types:
return np.identity(self.output_size, np.float32)
num_entity_tokens = (len(ENTITIES) +
len(self.entities)) * MAX_ARG_VALUES
num_other_tokens = len(self.tokens) - num_entity_tokens
num_entities = len(ENTITIES) + len(self.entities)
embed_size = num_other_tokens + num_entities + MAX_ARG_VALUES
embedding = np.zeros((len(self.tokens), embed_size), dtype=np.float32)
for token_id, token in enumerate(self.tokens):
if '_' in token and token[0].isupper():
continue
embedding[token_id, token_id] = 1
for i, entity in enumerate(ENTITIES):
assert not np.any(embedding[:, num_other_tokens + i] > 0)
for j in range(MAX_ARG_VALUES):
token_id = self.dictionary[entity + '_' + str(j)]
embedding[token_id, num_other_tokens + i] = 1
embedding[token_id, num_other_tokens + num_entities + j] = 1
for i, entity in enumerate(self.entities):
assert not np.any(
embedding[:, num_other_tokens + len(ENTITIES) + i] > 0)
for j in range(MAX_ARG_VALUES):
token_id = self.dictionary['GENERIC_ENTITY_' + entity + '_' +
str(j)]
embedding[token_id, num_other_tokens + len(ENTITIES) + i] = 1
embedding[token_id, num_other_tokens + num_entities + j] = 1
for i in range(len(embedding)):
assert np.any(embedding[i] > 0)
return embedding
def dump_tokens(self):
for token in self.tokens:
print(token)
def split_batch_in_parts(self, labels_batch):
batch_size = len(labels_batch)
top_batch = np.empty((batch_size, ), dtype=np.int32)
special_label_batch = np.zeros((batch_size, MAX_SPECIAL_LENGTH),
dtype=np.int32)
part_function_batches = dict()
part_sequence_batches = dict()
part_sequence_length_batches = dict()
for part in ('trigger', 'query', 'action'):
part_function_batches[part] = np.zeros((batch_size, ),
dtype=np.int32)
part_sequence_batches[part] = np.zeros(
(batch_size, MAX_PRIMITIVE_LENGTH), dtype=np.int32)
part_sequence_length_batches[part] = np.zeros((batch_size, ),
dtype=np.int32)
rule_token = self.dictionary['rule']
first_value_token = self.num_functions + self.num_begin_tokens + self.num_control_tokens
for i, label in enumerate(labels_batch):
top_batch[i] = label[0]
if top_batch[i] != rule_token:
special_label_batch[i] = label[1:1 + MAX_SPECIAL_LENGTH]
for part in ('trigger', 'query', 'action'):
if part == 'trigger':
function_offset = self.num_begin_tokens + self.num_control_tokens
elif part == 'query':
function_offset = self.num_begin_tokens + self.num_control_tokens + len(
self.functions['trigger'])
else:
function_offset = self.num_begin_tokens + self.num_control_tokens + len(
self.functions['trigger']) + len(
self.functions['query'])
# add dummy values to the sequences to preserve the ability to compute
# losses and grammar constraints
part_function_batches[part][i] = function_offset
part_sequence_batches[part][i, 0] = self.end
else:
special_label_batch[i, 0] = self.end
j = 1
for part in ('trigger', 'query', 'action'):
if part == 'trigger':
function_offset = self.num_begin_tokens + self.num_control_tokens
elif part == 'query':
function_offset = self.num_begin_tokens + self.num_control_tokens + len(
self.functions['trigger'])
else:
function_offset = self.num_begin_tokens + self.num_control_tokens + len(
self.functions['trigger']) + len(
self.functions['query'])
function_max = len(self.functions[part])
assert function_offset <= label[
j] < function_offset + function_max, (
function_offset, function_max, label[j],
self.tokens[label[j]])
part_function_batches[part][i] = label[j]
j += 1
start = j
while label[j] >= first_value_token:
j += 1
end = j
assert end - start + 1 < MAX_PRIMITIVE_LENGTH
part_sequence_batches[part][i, 0:end -
start] = label[start:end]
part_sequence_batches[part][i, end - start] = self.end
part_sequence_length_batches[part][i] = end - start + 1
return top_batch, special_label_batch, part_function_batches, part_sequence_batches, part_sequence_length_batches
def vectorize_program(self, program, max_length=60):
vector, length = super().vectorize_program(program, max_length)
self.normalize_sequence(vector)
return vector, length
def parse(self, program):
curr_state = self.start_state
for token_id in program:
next = self.transition_matrix[curr_state, token_id]
if next == -1:
raise ValueError("Unexpected token " + self.tokens[token_id] +
" in " + (' '.join(self.tokens[x]
for x in program)) +
" (in state " + self.state_names[curr_state] +
")")
#print("transition", self.state_names[curr_state], "->", self.state_names[next])
curr_state = next
if curr_state != self.end_state:
raise ValueError("Premature end of program in " +
(' '.join(self.tokens[x]
for x in program)) + " (in state " +
self.state_names[curr_state] + ")")
#print(*(self.tokens[x] for x in program))
def parse_all(self, fp):
vectors = []
for line in fp.readlines():
try:
program = line.strip().split()
vector = self.vectorize_program(program)[0]
self.parse(vector)
vectors.append(vector)
except ValueError as e:
print(e)
return np.array(vectors, dtype=np.int32)
def get_function_init_state(self, function_tokens):
next_state = tf.gather(
self.function_states, function_tokens -
(self.num_begin_tokens + self.num_control_tokens))
assert2 = tf.Assert(tf.reduce_all(next_state >= 0), [function_tokens])
with tf.control_dependencies([assert2]):
return tf.identity(next_state)
def get_init_state(self, batch_size):
return tf.ones((batch_size, ), dtype=tf.int32) * self.start_state
def constrain_value_logits(self, logits, curr_state):
first_value_token = self.num_functions + self.num_begin_tokens + self.num_control_tokens
num_value_tokens = self.output_size - first_value_token
value_allowed_token_matrix = np.concatenate(
(self.allowed_token_matrix[:, :self.num_control_tokens],
self.allowed_token_matrix[:, first_value_token:]),
axis=1)
with tf.name_scope('constrain_logits'):
allowed_tokens = tf.gather(tf.constant(value_allowed_token_matrix),
curr_state)
assert allowed_tokens.get_shape()[1:] == (self.num_control_tokens +
num_value_tokens, )
constrained_logits = logits - tf.to_float(
tf.logical_not(allowed_tokens)) * 1e+10
return constrained_logits
def constrain_logits(self, logits, curr_state):
with tf.name_scope('constrain_logits'):
allowed_tokens = tf.gather(tf.constant(self.allowed_token_matrix),
curr_state)
assert allowed_tokens.get_shape()[1:] == (self.output_size, )
constrained_logits = logits - tf.to_float(
tf.logical_not(allowed_tokens)) * 1e+10
return constrained_logits
def value_transition(self, curr_state, next_symbols, batch_size):
first_value_token = self.num_functions + self.num_begin_tokens + self.num_control_tokens
num_value_tokens = self.output_size - first_value_token
with tf.name_scope('grammar_transition'):
adjusted_next_symbols = tf.where(
next_symbols >= self.num_control_tokens,
next_symbols + (first_value_token - self.num_control_tokens),
next_symbols)
assert1 = tf.Assert(
tf.reduce_all(
tf.logical_and(next_symbols < num_value_tokens,
next_symbols >= 0)),
[curr_state, next_symbols])
with tf.control_dependencies([assert1]):
transitions = tf.gather(tf.constant(self.transition_matrix),
curr_state)
assert transitions.get_shape()[1:] == (self.output_size, )
indices = tf.stack(
(tf.range(0, batch_size), adjusted_next_symbols), axis=1)
next_state = tf.gather_nd(transitions, indices)
assert2 = tf.Assert(
tf.reduce_all(next_state >= 0),
[curr_state, adjusted_next_symbols, next_state])
with tf.control_dependencies([assert2]):
return tf.identity(next_state)
def transition(self, curr_state, next_symbols, batch_size):
with tf.name_scope('grammar_transition'):
transitions = tf.gather(tf.constant(self.transition_matrix),
curr_state)
assert transitions.get_shape()[1:] == (self.output_size, )
indices = tf.stack((tf.range(0, batch_size), next_symbols), axis=1)
next_state = tf.gather_nd(transitions, indices)
return next_state
def _normalize_invocation(self, seq, start):
assert self.tokens[seq[start]].startswith('tt:')
if self.tokens[seq[start]].startswith('USERNAME_'):
start += 1
end = start
params = []
while end < len(seq) and seq[end] != self.end and self.tokens[
seq[end]].startswith('tt-param:'):
param_id = seq[end]
end += 1
if end >= len(seq) or seq[end] == self.end:
# truncated output
return end
operator = seq[end]
end += 1
if end >= len(seq) or seq[end] == self.end:
# this can occur at training time, if the output is truncated
#raise AssertionError("missing value for " + self.tokens[param_id])
params.append((param_id, operator, []))
continue
param_value = [seq[end]]
end += 1
while end < len(seq) and seq[end] != self.end and not self.tokens[
seq[end]].startswith('tt:'):
param_value.append(seq[end])
end += 1
params.append((param_id, operator, param_value))
params.sort(key=lambda x: x[0])
assert end <= len(seq)
i = start
for param_id, operator, param_value in params:
seq[i] = param_id
seq[i + 1] = operator
seq[i + 2:i + 2 + len(param_value)] = param_value
i += 2 + len(param_value)
assert i <= end
return end
def normalize_sequence(self, seq):
i = 0
if seq[0] == self.dictionary['rule']:
i += 1
i = self._normalize_invocation(seq, i)
if i < len(seq) and seq[i] != self.end:
i = self._normalize_invocation(seq, i)
if i < len(seq) and seq[i] != self.end:
i = self._normalize_invocation(seq, i)
def compare(self, gold, decoded):
decoded = list(decoded)
#self._normalize_sequence(decoded)
return gold == decoded
def __init__(self, advance_func=None):
self._player_queue = OrderedSet()
self.advance_func = advance_func or self._default_advance_turn
def placeholders(self):
return OrderedSet(
Placeholder(body).name
for body in re.findall(self.placeholder_pattern, self.content))
def _deserialize_graph(graph_pb):
"""
Will deserialize a graph and return the list of all ops in that graph. Does not bother
filtering down to only the original set of ops the user passed in for serialization
(if that's what the user desired upon serializing with the serialization
only_return_handle_ops parameter).
"""
# For safety we clear this registry
GLOBAL_AXIS_REGISTRY.clear()
ops = list(map(protobuf_to_op, graph_pb.ops))
uuid_lookup = {op.uuid.bytes: op for op in ops}
for edge in graph_pb.edges:
head_op = uuid_lookup[edge.from_uuid.uuid]
tail_op = uuid_lookup[edge.to_uuid.uuid]
if edge.edge_type == ops_pb.Edge.DATA: # args
tail_op._args = tail_op._args + (head_op, )
elif edge.edge_type == ops_pb.Edge.CONTROL: # control_deps
head_op._control_deps.add(tail_op)
elif edge.edge_type == ops_pb.Edge.CONTAINER:
if '_ngraph_forward' in edge.attrs: # forward
head_op._forward = tail_op
else: # ops
if not hasattr(head_op, '_ops'):
head_op._ops = []
head_op._ops.append(tail_op)
elif edge.edge_type == ops_pb.Edge.OTHER:
if '_ngraph_attribute' in edge.attrs:
setattr(head_op,
edge.attrs['_ngraph_attribute'].scalar.string_val,
tail_op)
elif '_ngraph_list_attribute' in edge.attrs:
key = edge.attrs['_ngraph_list_attribute'].scalar.string_val
# import pdb; pdb.set_trace()
if hasattr(head_op, key):
getattr(head_op, key).add(tail_op)
else:
setattr(head_op, key, OrderedSet([tail_op]))
else:
raise ValueError("Edge not mapped to op: {}".format(edge))
# This must come after tensor has been set which occurs after edges
# op.dtype
for py_op, pb_op in zip(ops, graph_pb.ops):
py_op.dtype = pb_to_dtype(pb_op.dtype)
# Done with this and don't want it to bleed to subsequent serializations
GLOBAL_AXIS_REGISTRY.clear()
# Assemble list of nodes to return depending on if the user wanted to
# return all ops or only those that were originally serialized (and
# implicitly those upstream)
final_ops = []
for op in ops:
if hasattr(op, '_ngraph_ser_handle'):
del op._ngraph_ser_handle
final_ops.append(op)
if len(final_ops) > 0:
return final_ops
else:
return ops
class HetrComputation(Computation):
"""
Lightweight wrapper class for handling runtime execution of child computations for Hetr
"""
def __init__(self, hetr, computation_op):
self.child_computations = dict()
self.transformer = hetr
self.send_nodes = hetr.send_nodes
self.computation_op = computation_op
# self.returns could be replaced by comp_op.returns if it were expressed as a set
self.returns = OrderedSet()
if isinstance(computation_op.returns, collections.Container):
self.returns.update(list(computation_op.returns))
elif isinstance(computation_op.returns, Op):
self.returns.update(list([computation_op.returns]))
# if one of the requested results is marked as distributed across devices,
# wrap it in a ResultOp to facilitate DistributedPass inserting a gather operation
new_returns = OrderedSet()
for op in self.returns:
if 'device_id' in op.metadata and \
isinstance(op.metadata['device_id'], (list, tuple)):
op.metadata['is_split_op'] = True
new_result = ResultOp(device_id=0, args=tuple([op]))
op.metadata['hetr_replaced_by'] = new_result
new_result.metadata['replaces_op'] = op
new_returns.add(new_result)
else:
new_returns.add(op)
# Do Hetr passes
pass_ops = new_returns | OrderedSet(self.computation_op.parameters)
for graph_pass in self.transformer.graph_passes:
pass_ops = pass_ops | OrderedSet(hetr.send_nodes)
graph_pass.do_pass(ops=pass_ops)
# hack around new TensorValueOp that wraps AssignableTensorOp
# autogenerated by creating a ComputationOp:
for p in self.computation_op.parameters:
if isinstance(p, TensorValueOp):
p.metadata.update(p.states_read[0].metadata)
# simplify by already having asynctrans made by passes
for t_name, trans in iteritems(self.transformer.child_transformers):
my_params = [
(g_pos, p)
for g_pos, p in enumerate(self.computation_op.parameters)
if p.metadata['transformer'] == t_name
]
my_ops = [
op for op in self.send_nodes | new_returns
if op.metadata['transformer'] == t_name
]
transform_ops = [
op.args[0] if isinstance(op, ResultOp) else op for op in my_ops
]
comp = trans.computation(transform_ops,
tuple([p for pos, p in my_params]))
comp.param_idx = [g_pos for g_pos, p in my_params]
# when there is a ResultOp, hack around it
comp.returns = dict()
for i, op in enumerate(my_ops):
if op in self.returns and 'hetr_replaced_by' not in op.metadata:
comp.returns[op] = i
elif 'replaces_op' in op.metadata and op.metadata[
'replaces_op'] in self.returns:
comp.returns[op.metadata['replaces_op']] = i
self.child_computations[t_name] = comp
def __call__(self, *args, **kwargs):
"""
Executes child computations in parallel.
:arg args: list of values to the placeholders specified in __init__ *args
:return: tuple of return values, one per return specified in __init__ returns list.
"""
args = self.unpack_args_or_feed_dict(args, kwargs)
for child in itervalues(self.child_computations):
child.feed_input([args[i] for i in child.param_idx])
return_vals = dict()
for child in itervalues(self.child_computations):
return_vals.update(child.get_results())
if isinstance(self.computation_op.returns, Op):
return return_vals[self.computation_op.returns]
elif isinstance(self.computation_op.returns, collections.Set):
return return_vals
elif isinstance(self.computation_op.returns, collections.Sequence):
return tuple(return_vals[op] for op in self.computation_op.returns)
else:
return None
def __init__(self, hetr, computation_op):
self.child_computations = dict()
self.transformer = hetr
self.send_nodes = hetr.send_nodes
self.computation_op = computation_op
# self.returns could be replaced by comp_op.returns if it were expressed as a set
self.returns = OrderedSet()
if isinstance(computation_op.returns, collections.Container):
self.returns.update(list(computation_op.returns))
elif isinstance(computation_op.returns, Op):
self.returns.update(list([computation_op.returns]))
# if one of the requested results is marked as distributed across devices,
# wrap it in a ResultOp to facilitate DistributedPass inserting a gather operation
new_returns = OrderedSet()
for op in self.returns:
if 'device_id' in op.metadata and \
isinstance(op.metadata['device_id'], (list, tuple)):
op.metadata['is_split_op'] = True
new_result = ResultOp(device_id=0, args=tuple([op]))
op.metadata['hetr_replaced_by'] = new_result
new_result.metadata['replaces_op'] = op
new_returns.add(new_result)
else:
new_returns.add(op)
# Do Hetr passes
pass_ops = new_returns | OrderedSet(self.computation_op.parameters)
for graph_pass in self.transformer.graph_passes:
pass_ops = pass_ops | OrderedSet(hetr.send_nodes)
graph_pass.do_pass(ops=pass_ops)
# hack around new TensorValueOp that wraps AssignableTensorOp
# autogenerated by creating a ComputationOp:
for p in self.computation_op.parameters:
if isinstance(p, TensorValueOp):
p.metadata.update(p.states_read[0].metadata)
# simplify by already having asynctrans made by passes
for t_name, trans in iteritems(self.transformer.child_transformers):
my_params = [
(g_pos, p)
for g_pos, p in enumerate(self.computation_op.parameters)
if p.metadata['transformer'] == t_name
]
my_ops = [
op for op in self.send_nodes | new_returns
if op.metadata['transformer'] == t_name
]
transform_ops = [
op.args[0] if isinstance(op, ResultOp) else op for op in my_ops
]
comp = trans.computation(transform_ops,
tuple([p for pos, p in my_params]))
comp.param_idx = [g_pos for g_pos, p in my_params]
# when there is a ResultOp, hack around it
comp.returns = dict()
for i, op in enumerate(my_ops):
if op in self.returns and 'hetr_replaced_by' not in op.metadata:
comp.returns[op] = i
elif 'replaces_op' in op.metadata and op.metadata[
'replaces_op'] in self.returns:
comp.returns[op.metadata['replaces_op']] = i
self.child_computations[t_name] = comp
def calc_build(self,_seen=None):
"""decides if it needs to be built by recursively asking it's prerequisites
the same question.
_seen is an internal variable (a set) for optimising the search. I'll be relying
on the set being a mutable container in order to not have to pass it explicitly
back up the call stack."""
#There is an oportunity to optimise calculations to occur only once for rules that are called multiple
#times by using a shared (global) buildseq + _already_seen set, or by passing those structures into
#the calc_build method call.
#i.e. if (self in buildseq) or (self in _already_seen): return buildseq
#Or we can memoize this method
#updated_only should be calculated during build calculation time (rather than build time) for consistancy.
self.updated_only #force evaluation of lazy property
buildseq = OrderedSet()
_seen = set() if not _seen else _seen
_seen.add(self) # this will also solve any circular dependency issues!
for req in self.order_only:
if not os.path.exists(req):
reqrule = Rule.get(req,None) #super(ExplicitRule,self).get(req,None)
if reqrule:
if reqrule not in _seen:
buildseq.update(reqrule.calc_build())
else:
warnings.warn('rule for %r has already been processed' %req,stacklevel=2)
else:
warnings.warn('%r has an order_only prerequisite with no rule' %self,stacklevel=2)
for req in self.reqs:
reqrule = Rule.get(req,None) #super(ExplicitRule,self).get(req,None)
if reqrule:
if reqrule not in _seen:
buildseq.update(reqrule.calc_build())
else:
warnings.warn('rule for %r has already been processed' %req,stacklevel=2)
else: #perform checks
try:
self.get_mtime(req) #get_mtime is cached to reduce number of file accesses
except OSError as e:
raise AssertionError("No rule or file found for %r for targets: %r" %(req,self.targets))
if len(buildseq)==0:
if self.PHONY or any([not os.path.exists(target) for target in self.targets]):
buildseq.add(self)
else:
oldest_target = self._oldest_target
#Since none of the prerequisites have rules that need to update, we can assume
#that all prerequisites should be real files (phony rules always update which
#should skip this section of code). Hence non-existing files imply an malformed build
#file.
for req in self.reqs:
try:
req_mtime = self.get_mtime(req)
if req_mtime > oldest_target:
buildseq.add(self)
break
except OSError as e:
raise AssertionError("A non file prerequisite was found (%r) for targets %r in wrong code path" %(req,self.targets))
else:
buildseq.add(self)
return buildseq
class ComputationGraphTransformer(Transformer):
def __init__(self, **kwargs):
super(ComputationGraphTransformer, self).__init__(**kwargs)
self.computations = OrderedSet()
self.device_buffers = None
self.op_tensors = None
self.op_tensor_views = None
self.initialize_allocations()
self.finalized = False
self.allocated = False
self.initialized = False
def run_registered_graph_passes(self, ops, **kwargs):
for graph_pass in self.graph_passes:
graph_pass.wrapped_do_pass(ops=ops, **kwargs)
return ops
def initialize_allocations(self):
"""
Inititializes allocation caches.
"""
self.op_tensors = dict()
self.op_tensor_views = dict()
self.device_buffers = OrderedSet()
def get_op_tensor(self, op):
"""
Returns the tensor allocated for this op.
Args:
op: A computation graph op.
Returns:
A device tensor (DeviceBuffer).
"""
return self.op_tensors[
op.forwarded.tensor.forwarded.tensor_description().base]
def has_op_tensor(self, op):
"""
Returns true if the op has a device tensor.
Args:
op: A computation graph op.
Returns:
True if the op has a device tensor.
"""
return op.forwarded.tensor.forwarded.tensor_description(
).base in self.op_tensors
def get_op_tensor_view(self, op):
"""
Returns the tensor view for this op.
Args:
op: A computation graph op.
Returns:
A device tensor view.
"""
return self.op_tensor_views[
op.forwarded.tensor.forwarded.tensor_description()]
def get_tensor_view_value(self, op, host_tensor=None):
"""
Returns the contents of the tensor view for op.
Args:
op: The computation graph op.
host_tensor: Optional tensor to copy value into.
Returns:
A NumPy tensor with the elements associated with op.
"""
return self.get_op_tensor_view(op).get(host_tensor)
def get_tensor_description_tensor(self, tensor_description):
"""
Returns a tensor for a tensor description.
Deprecated. Use op version.
Args:
tensor_description:
Returns:
A device tensor (DeviceBuffer).
"""
return self.op_tensors[tensor_description.base]
def has_tensor_description_tensor(self, tensor_description):
"""
Tests if a tensor description has a device tensor.
Deprecated, only used by flex.
Args:
tensor_description:
Returns:
True if the tensor description has a device tensor.
"""
return tensor_description.base in self.op_tensors
def get_tensor_description_tensor_view(self, tensor_description):
"""
Returns a tensor for a tensor description.
Deprecated, only used by flex.
Args:
tensor_description:
Returns:
A device tensor view.
"""
return self.op_tensor_views[tensor_description]
def host_to_device(self, computation, parameters, args):
"""
Copy args to parameters in computation.
Args:
computation: The computation.
parameters: Parameters of the computation.
args: Values for the parameters.
"""
for param, arg in zip(parameters, args):
self.get_op_tensor_view(param)[...] = arg
def device_to_host(self, computation, op, tensor=None):
"""
Copy a computation result from the device back to the host.
Args:
computation: The computation.
op: The op associated with the value.
tensor: Optional tensor for returned value.
Returns:
The value of op.
"""
if self.has_op_tensor(op):
return self.get_op_tensor_view(op).get(tensor)
@property
def use_exop(self):
"""
Returns: True if this transformer uses the execution graph.
"""
return False
@abc.abstractmethod
def start_transform_allocate(self):
"""
Called just before allocation code is transformed.
"""
@abc.abstractmethod
def finish_transform_allocate(self):
"""
Called after last allocation is transformed.
"""
@abc.abstractmethod
def transform_ordered_ops(self, computation, ordered_ops, name):
"""
Generate code to compute ordered_ops.
Arguments:
computation: The computation being compiled.
ordered_ops: Ops to compute
name: The name of the computation.
Returns: Handle for generated code
"""
@abc.abstractmethod
def finish_transform(self):
"""
Finish generating the model.
"""
@abc.abstractmethod
def allocate_storage(self):
"""
Allocate storage on the device.
"""
@abc.abstractmethod
def device_buffer_storage(self, bytes, dtype, name):
"""
Make a DeviceBuffer.
Arguments:
bytes: Size of buffer.
dtype: dtype of buffer.
name: Name of the storage variable
returns: A DeviceBuffer.
"""
def allocate(self):
"""
Allocate storage and then initializes constants.
Will finalize if not already done.
"""
if self.allocated:
return
if not self.finalized:
logging.info("Finalizing transformer.")
self._transform_computations()
self.allocate_storage()
init_states = OrderedSet()
for op in OrderedSet(self.ops):
op = op.forwarded
states = op.states_read | op.states_written
for state in states:
state = state.forwarded
if state.initial_value is not None:
init_states.add(state)
for state in init_states:
tensor_description = state.tensor.tensor_description()
self.get_tensor_description_tensor_view(tensor_description)[
...] = state.initial_value
self.allocated = True
def add_computation(self, computation):
"""
Adds a computation to the transformer.
Arguments:
computation: A computation Op.
Returns:
Callable.
"""
if self.finalized:
raise ValueError(
'Cannot create computations from a finalized transformer')
computation = super(ComputationGraphTransformer,
self).add_computation(computation)
self.computations.add(computation)
return computation
def initialize(self):
"""
Initialize storage. Will allocate if not already performed.
"""
if self.initialized:
return
self.allocate()
# Need to set initialized before we are done because the init computation will
# try to initialize.
self.initialized = True
def _transform_computations(self):
"""
Transform computation graphs to a form that can be run.
"""
# Run passes on the computation graphs
all_results = []
for comp in self.computations:
all_results.append(comp.computation_op)
all_ops = self.run_registered_graph_passes(ops=all_results)
# Collect up all ops from the graph and obtain the init graph
all_ops = OrderedSet(Op.ordered_ops(all_ops))
def ensure_tensor(op):
op = op.forwarded
tensor_description = op.tensor_description()
base = tensor_description.base
tensor = self.op_tensors.get(base, None)
if tensor is None:
tensor = self.device_buffer_storage(base.tensor_size,
base.dtype, base.name)
self.op_tensors[base] = tensor
self.device_buffers.add(tensor)
tensor_view = tensor.device_tensor(tensor_description)
self.op_tensor_views[tensor_description] = tensor_view
self.ops = Op.ordered_ops(all_ops)
for op in self.ops:
if op.is_tensor_op:
ensure_tensor(op)
self.start_transform_allocate()
for device_buffer in self.device_buffers:
device_buffer.transform_allocate()
self.finish_transform_allocate()
# Compile the computations now that we know their storage
for comp in self.computations:
comp.computation_name = \
self.transform_ordered_ops(comp,
Op.ordered_ops([comp.computation_op]),
name=comp.name)
self.finish_transform()
self.finalized = True
from construct import *
except ImportError as e:
from modules.easydependencies import handle_importerror
handle_importerror(e)
from modules.easydependencies import setup_third_party
setup_third_party()
from modules.handyfunctions import get_frames_folder, os_adapt, get_modules_path, to_command, InfiniteTimer
import sdl2.ext
from sdl2 import *
from sdl2.ext import colorpalettes
from orderedset import OrderedSet
from construct import *
kb_state = SDL_GetKeyboardState(None)
pressed_key = OrderedSet()
class DynamicBG:
"""Make a dynamic background that changes colors"""
def __init__(self,
factory,
renderer,
ext_window,
on_top_sprite=None,
i=188,
fps=60):
self._factory = factory
self._renderer = renderer
self._on_top_sprite = on_top_sprite
self._switch_on = True
class Node(object):
UNKNOWN_STATE = "_UNKNOWN_"
def __init__(self, id_, master_graph):
self.id_ = id_
self.master_graph = master_graph
self.edges = OrderedSet()
self._state = None
# determine ownership
self.determined = False
self.graph = None
def __repr__(self):
if self.determined:
if self.graph:
if self.state is not self.UNKNOWN_STATE:
return "<Node#%s: %s %r>" \
% (self.id_, self.graph.name, self.state)
else:
return "<Node#%s: %s>" % (self.id_, self.graph.name)
else:
return "<!Node#%s: DETERMINE NONE!>" % self.id_
elif self.determined is False:
if self.graph is self.master_graph:
return "<!Node#%s: GUESS MASTER!>" % self.id_
elif self.graph:
if self.state is not self.UNKNOWN_STATE:
return "<Node#%s: ?%s %r>" \
% (self.id_, self.graph.name, self.state)
else:
return "<Node#%s: ?%s>" % (self.id_, self.graph.name)
else:
return "<!Node#%s: NEW!>" % self.id_
@property
def state(self):
if self._state is None:
return self.UNKNOWN_STATE
else:
return self._state
@state.setter
def state(self, val):
self._state = val
@property
def correct(self):
# Allowed combinations:
# 1. determined = True, graph is master
# 2. determined = True, graph is not master
# 3. determined = False, graph is not master
if self.determined and self.graph is not None:
return True
if self.determined is False and isinstance(self.graph, LeafGraph):
return True
print("%s is not correct!" % self)
return False
@property
def guessed(self):
return self.determined is False and self.graph is not None
@property
def determined_master(self):
return self.determined and self.graph is self.master_graph
@property
def is_new(self):
return self.determined is False and self.graph is None
def add_edge(self, edge):
# NOTE: Order matters
self.edges.add(edge)
def guess_graph(self, graph):
assert graph is not self.master_graph
if self.determined:
pass
elif self.graph is None:
self.graph = graph
elif self.graph is graph:
pass
else:
self.determine_graph(self.master_graph)
def determine_graph(self, graph):
if not self.determined:
if self.graph is graph:
graph.determine_node(self)
else:
self.master_graph.determine_node(self)
elif self.determined_master is False and self.graph is not graph:
raise RuntimeError("Node#%s is determined %s, but assigned %s"
% (self, self.graph.name, graph.name))
def decide_edge(self, log):
for edge in self.edges:
if edge.accept(log):
return edge
return None
def accept_edge(self, edge):
return edge in self.edges
def __init__(self):
self.state_initialization_ops = OrderedSet()
def get_players(self):
return OrderedSet(self._player_queue)
def remote(self, groups):
self._remote = OrderedSet(self._process_remote_groups(groups))
# encoding: cinje
: from orderedset import OrderedSet
: from marrow.util.url import QueryString
: def render_video_block block
: """Render a YouTube video embed."""
: classes = OrderedSet(block.properties.get('cls', '').split() + ['block', 'video'])
: if 'width' in block.properties
: classes.add('col-md-' + str(block.properties.width))
: end
<div&{id=block.properties.get('id', block.id), class_=classes}>
<iframe width="560" height="315" src="https://www.youtube.com/embed/${block.video}" frameborder="0" allowfullscreen></iframe>
</div>
def __init__(self, src_dataset_name, grouping_cols, new_dataset_name):
super(GroupByOperator, self).__init__(src_dataset_name)
self.grouping_cols = OrderedSet(grouping_cols)
self.new_dataset_name = new_dataset_name
self.grouped_dataset = None
self._id = self.create_id()
class City(simpy.Environment):
def __init__(self, env, n_people, rng, x_range, y_range, start_time,
init_percent_sick, Human):
self.env = env
self.rng = rng
self.x_range = x_range
self.y_range = y_range
self.total_area = (x_range[1] - x_range[0]) * (y_range[1] - y_range[0])
self.n_people = n_people
self.start_time = start_time
self.init_percent_sick = init_percent_sick
self.last_date_to_check_tests = self.env.timestamp.date()
self.test_count_today = defaultdict(int)
self.test_type_preference = list(
zip(*sorted(TEST_TYPES.items(),
key=lambda x: x[1]['preference'])))[0]
print("Initializing locations ...")
self.initialize_locations()
self.humans = []
self.households = OrderedSet()
print("Initializing humans ...")
self.initialize_humans(Human)
self.log_static_info()
print("Computing their preferences")
self._compute_preferences()
self.tracker = Tracker(env, self)
# self.tracker.track_initialized_covid_params(self.humans)
self.intervention = None
def create_location(self, specs, type, name, area=None):
_cls = Location
if type in ['household', 'senior_residency']:
_cls = Household
if type == 'hospital':
_cls = Hospital
return _cls(
env=self.env,
rng=self.rng,
name=f"{type}:{name}",
location_type=type,
lat=self.rng.randint(*self.x_range),
lon=self.rng.randint(*self.y_range),
area=area,
social_contact_factor=specs['social_contact_factor'],
capacity=None if not specs['rnd_capacity'] else self.rng.randint(
*specs['rnd_capacity']),
surface_prob=specs['surface_prob'])
@property
def tests_available(self):
if self.last_date_to_check_tests != self.env.timestamp.date():
self.last_date_to_check_tests = self.env.timestamp.date()
for k in self.test_count_today.keys():
self.test_count_today[k] = 0
return any(self.test_count_today[test_type] < TEST_TYPES[test_type]
['capacity'] for test_type in self.test_type_preference)
def get_available_test(self):
for test_type in self.test_type_preference:
if self.test_count_today[test_type] < TEST_TYPES[test_type][
'capacity']:
self.test_count_today[test_type] += 1
return test_type
def initialize_locations(self):
for location, specs in LOCATION_DISTRIBUTION.items():
if location in ['household']:
continue
n = math.ceil(self.n_people / specs["n"])
area = _get_random_area(n, specs['area'] * self.total_area,
self.rng)
locs = [
self.create_location(specs, location, i, area[i])
for i in range(n)
]
setattr(self, f"{location}s", locs)
def initialize_humans(self, Human):
# allocate humans to houses such that (unsolved)
# 1. average number of residents in a house is (approx.) 2.6
# 2. not all residents are below 15 years of age
# 3. age occupancy distribution follows HUMAN_DSITRIBUTION.residence_preference.house_size
# current implementation is an approximate heuristic
# make humans
count_humans = 0
house_allocations = {2: [], 3: [], 4: [], 5: []}
n_houses = 0
for age_bin, specs in HUMAN_DISTRIBUTION.items():
n = math.ceil(specs['p'] * self.n_people)
ages = self.rng.randint(*age_bin, size=n)
senior_residency_preference = specs['residence_preference'][
'senior_residency']
professions = ['healthcare', 'school', 'others', 'retired']
p = [specs['profession_profile'][x] for x in professions]
profession = self.rng.choice(professions, p=p, size=n)
for i in range(n):
count_humans += 1
age = ages[i]
# residence
res = None
if self.rng.random() < senior_residency_preference:
res = self.rng.choice(self.senior_residencys)
# workplace
if profession[i] == "healthcare":
workplace = self.rng.choice(self.hospitals +
self.senior_residencys)
elif profession[i] == 'school':
workplace = self.rng.choice(self.schools)
elif profession[i] == 'others':
type_of_workplace = self.rng.choice(
[0, 1, 2], p=OTHERS_WORKPLACE_CHOICE, size=1).item()
type_of_workplace = [
self.workplaces, self.stores, self.miscs
][type_of_workplace]
workplace = self.rng.choice(type_of_workplace)
else:
workplace = res
self.humans.append(
Human(
env=self.env,
city=self,
rng=self.rng,
name=count_humans,
age=age,
household=res,
workplace=workplace,
profession=profession[i],
rho=RHO,
gamma=GAMMA,
infection_timestamp=self.start_time if
self.rng.random() < self.init_percent_sick else None))
# assign houses
# stores tuples - (location, current number of residents, maximum number of residents allowed)
remaining_houses = []
for human in self.humans:
if human.household is not None:
continue
if len(remaining_houses) == 0:
cap = self.rng.choice(range(1, 6),
p=HOUSE_SIZE_PREFERENCE,
size=1)
x = self.create_location(LOCATION_DISTRIBUTION['household'],
'household', len(self.households))
remaining_houses.append((x, cap))
# get_best_match
res = None
for c, (house, n_vacancy) in enumerate(remaining_houses):
new_avg_age = (human.age + sum(x.age
for x in house.residents)) / (
len(house.residents) + 1)
if new_avg_age > MIN_AVG_HOUSE_AGE:
res = house
n_vacancy -= 1
if n_vacancy == 0:
remaining_houses = remaining_houses[:
c] + remaining_houses[
c + 1:]
break
if res is None:
for i, (l, u) in enumerate(HUMAN_DISTRIBUTION.keys()):
if l <= human.age < u:
bin = (l, u)
break
house_size_preference = HUMAN_DISTRIBUTION[(
l, u)]['residence_preference']['house_size']
cap = self.rng.choice(range(1, 6),
p=house_size_preference,
size=1)
res = self.create_location(LOCATION_DISTRIBUTION['household'],
'household', len(self.households))
if cap - 1 > 0:
remaining_houses.append((res, cap - 1))
# FIXME: there is some circular reference here
res.residents.append(human)
human.assign_household(res)
self.households.add(res)
# assign area to house
area = _get_random_area(
len(self.households),
LOCATION_DISTRIBUTION['household']['area'] * self.total_area,
self.rng)
for i, house in enumerate(self.households):
house.area = area[i]
# this allows for easy O(1) access of humans for message passing
self.hd = {human.name: human for human in self.humans}
def log_static_info(self):
for h in self.humans:
Event.log_static_info(self, h, self.env.timestamp)
@property
def events(self):
return list(itertools.chain(*[h.events for h in self.humans]))
def events_slice(self, begin, end):
return list(
itertools.chain(*[h.events_slice(begin, end)
for h in self.humans]))
def pull_events_slice(self, end):
return list(
itertools.chain(*[h.pull_events_slice(end) for h in self.humans]))
def _compute_preferences(self):
""" compute preferred distribution of each human for park, stores, etc."""
for h in self.humans:
h.stores_preferences = [
(compute_distance(h.household, s) + 1e-1)**-1
for s in self.stores
]
h.parks_preferences = [
(compute_distance(h.household, s) + 1e-1)**-1
for s in self.parks
]
def run(self, duration, outfile, start_time, all_possible_symptoms, port,
n_jobs):
self.current_day = 0
print(f"INTERVENTION_DAY: {INTERVENTION_DAY}")
while True:
#
if INTERVENTION_DAY >= 0 and self.current_day == INTERVENTION_DAY:
self.intervention = get_intervention(INTERVENTION)
_ = [h.notify(self.intervention) for h in self.humans]
print(self.intervention)
# update risk every day
if isinstance(self.intervention, Tracing):
self.intervention.update_human_risks(
city=self,
symptoms=all_possible_symptoms,
port=port,
n_jobs=n_jobs,
data_path=outfile)
#
# if (COLLECT_TRAINING_DATA or GET_RISK_PREDICTOR_METRICS) and (self.current_day == 0 and INTERVENTION_DAY < 0):
# _ = [h.notify(collect_training_data=True) for h in self.humans]
# print("naive risk calculation without changing behavior... Humans notified!")
self.tracker.increment_day()
self.current_day += 1
# Let the day pass
yield self.env.timeout(duration / TICK_MINUTE)
def initialise_pool(self):
self._candidate_pool = OrderedSet()