Exemple #1
0
def findKeys(ws, rows):
    func_list = OrderedSet()
    for i in range(2, rows):
        if (ws.cell(i, 0)).value:
            func_list.add((ws.cell(i, 0)).value)
        else:
            i += 1
    return func_list
Exemple #2
0
def get_collection_intersection(*args):
    ret = []
    for ii,arg in enumerate(args):
        if ii == 0:
            ret = OrderedSet(tf.get_collection(arg))
        else:
            ret = ret.intersection(OrderedSet(tf.get_collection(arg)))
    return list(ret)
class Poller(object):
    def __init__(self):
        rospy.init_node('poller_node')
        self.rate = rospy.Rate(3)   # 3hz
        self.extractBasestationFromParams()
        self.createCommunicators()
        self.request_list = OrderedSet([])

    def createCommunicators(self):
        self.client = Client(10019)
        self.measurements_publisher = rospy.Publisher('measurements', MeasurementList, queue_size=10)
        self.request_subscriber = rospy.Subscriber("measurements_request", String, self.pushbackRequest)

    def extractBasestationFromParams(self):
        stations = rospy.get_param("/poller_node/basestations")
        self.storeBasestation(stations)

    def storeBasestation(self, stations):
        self.basestations = []
        for station in stations:
            self.basestations.append(Basestation(station[0], float(station[1]), float(station[2])))

    def pushbackRequest(self, msg):
        self.request_list.add(msg.data)

    def measurementsLoop(self):
        while not rospy.is_shutdown():
            while not self.request_list.isEmpty():
                station_address = self.request_list.pop()
                self.serveRequest(station_address)
            self.rate.sleep()

    def pollStation(self, station_address):
        return self.client.pollBasestation(station_address)

    def serveRequest(self, station_address):
        try:
            data = self.pollStation(station_address)
            if containsMeasurements(data):
                self.publishMeasuements(extractJson(data), station_address)
        except socket.error:
            pass

    def publishMeasuements(self, measurs, station):
        msg = MeasurementList()
        for el in measurs:
            msg.data.append(self.generateMeasurement(el))
        msg.basestation = station
        msg.header.stamp = rospy.Time.now()
        self.measurements_publisher.publish(msg)

    def generateMeasurement(self, element):
        tmp = Measurement()
        tmp.tag = element['id_tag'].encode('utf-8')
        tmp.measurement = int(element['rssid'])
        return tmp
Exemple #4
0
    def iter_variations(self):
        needed_ids = OrderedSet()

        for fixture in self._needed_fixtures:
            needed_ids.update(self._store.get_all_needed_fixture_ids(fixture))

        parametrizations = [self._store.get_fixture_by_id(param_id) for param_id in needed_ids]
        if not needed_ids:
            yield Variation(self._store, {}, self._name_bindings.copy())
            return
        for value_indices in itertools.product(*(xrange(len(p.values)) for p in parametrizations)):
            yield self._build_variation(parametrizations, value_indices)
Exemple #5
0
class RoundRobin(SchedulingPolicy):

  def __init__(self):
    self.__coroutines = OrderedSet()

  @property
  def busy(self):
    return bool(self.__coroutines)

  def add(self, coroutine):
    self.__coroutines.add(coroutine)

  def remove(self, coroutine):
    self.__coroutines.remove(coroutine)

  def freeze(self, coroutine):
    self.__coroutines.remove(coroutine)

  def unfreeze(self, coroutine):
    self.__coroutines.add(coroutine)

  def round(self):
    for coro in list(self.__coroutines):
      assert coro is not None
      yield coro

  def dump(self):
    def dump(c, idt = 0):
      print('{}{}{}'.format('  ' * idt, c, ' (frozen)' if c.frozen else ''))
      for child in self.__hierarchy.get(c, []):
        dump(child, idt + 1)
    for root in self.__hierarchy.get(None, []):
      dump(root)
Exemple #6
0
class Fixture(FixtureBase):

    def __init__(self, store, fixture_func):
        super(Fixture, self).__init__()
        self.fixture_func = fixture_func
        self.info = self.fixture_func.__slash_fixture__
        self.scope = self.info.scope
        self.namespace = Namespace(store, store.get_current_namespace())

    def __repr__(self):
        return '<Function Fixture around {0}>'.format(self.fixture_func)

    def get_value(self, kwargs, active_fixture):
        if self.info.needs_this:
            assert 'this' not in kwargs
            kwargs['this'] = active_fixture
        return self.fixture_func(**kwargs)

    def _resolve(self, store):
        assert self.fixture_kwargs is None

        assert self.parametrization_ids is None
        self.parametrization_ids = OrderedSet()

        kwargs = OrderedDict()
        parametrized = set()

        for name, param in iter_parametrization_fixtures(self.fixture_func):
            store.register_fixture_id(param)
            parametrized.add(name)
            self.parametrization_ids.add(param.info.id)

        for param_name, arg in self.info.required_args.items():
            if param_name in parametrized:
                continue
            try:
                needed_fixture = self.namespace.get_fixture_by_name(get_real_fixture_name_from_argument(arg))

                if needed_fixture.scope < self.scope:
                    raise InvalidFixtureScope('Fixture {0} is dependent on {1}, which has a smaller scope ({2} > {3})'.format(
                        self.info.name, param_name, self.scope, needed_fixture.scope))

                if needed_fixture is self:
                    raise CyclicFixtureDependency('Cyclic fixture dependency detected in {0}: {1} depends on itself'.format(
                        self.info.func.__code__.co_filename,
                        self.info.name))
                kwargs[param_name] = needed_fixture.info.id
            except LookupError:
                raise UnknownFixtures(param_name)
        return kwargs
    def post(self, request):
        if request.data.get('flag') == 'tag':
            tag_list = request.data.getlist('tag_list[]')
            rules = cache.get("recommend_tag_list")

            recommend_list = OrderedSet()
            for rule, confidence in rules:
                tags, results = rule
                if tags and tag_list:
                    if set(tags) == set(tag_list):
                        for result in results:
                            recommend_list.add(result)
                else:
                    return Response({'recommend_tag_list': list(recommend_list)})

            return Response({'recommend_tag_list': list(recommend_list[:5])})
 def _check_controls(self):
     #controlled_region = set.union(*[c.region for c in self.controls])
     controlled_region = OrderedSet.union(*[c.region for c in self.controls])
     if len(controlled_region) != Plate.SIZE:
         assert len(controlled_region) < Plate.SIZE
         raise ValueError('layout contains uncontrolled wells')
     del controlled_region
Exemple #9
0
    def _resolve(self, store):
        assert self.fixture_kwargs is None

        assert self.parametrization_ids is None
        self.parametrization_ids = OrderedSet()

        kwargs = OrderedDict()
        parametrized = set()

        for name, param in iter_parametrization_fixtures(self.fixture_func):
            store.register_fixture_id(param)
            parametrized.add(name)
            self.parametrization_ids.add(param.info.id)

        for param_name, arg in self.info.required_args.items():
            if param_name in parametrized:
                continue
            try:
                needed_fixture = self.namespace.get_fixture_by_name(get_real_fixture_name_from_argument(arg))

                if needed_fixture.scope < self.scope:
                    raise InvalidFixtureScope('Fixture {0} is dependent on {1}, which has a smaller scope ({2} > {3})'.format(
                        self.info.name, param_name, self.scope, needed_fixture.scope))

                if needed_fixture is self:
                    raise CyclicFixtureDependency('Cyclic fixture dependency detected in {0}: {1} depends on itself'.format(
                        self.info.func.__code__.co_filename,
                        self.info.name))
                kwargs[param_name] = needed_fixture.info.id
            except LookupError:
                raise UnknownFixtures(param_name)
        return kwargs
Exemple #10
0
def __Dedupe(items, key=None):
    seen = OrderedSet()
    num_seen = list()
    gn_item = (item for item in items)
    while True:
        try:
            item = gn_item.next()
        except Exception as e:
            yield (None, num_seen)
            break
        else:
            val = item if key is None else key(item)
            if val not in seen:
                yield (item, None)
                seen.add(val)
                num_seen.append(1)
            else:
                num_seen[seen.index(val)] += 1
Exemple #11
0
    def prepare_annotations(self):
        # compile annotations data
        fields = OrderedSet([
            'start_time',
            'commit_time',
            'commit',
            'commit_message',
            'working_directory',
            'command',
            'elapsed_time'])

        annotations_list = list()
        for annotation in self.repo.get_annotations():
            fields.update(annotation)
            annotations_list.append(annotation)

        annotations_list.sort(key = lambda x: x['start_time'], reverse=True)
        return fields, annotations_list
Exemple #12
0
    def __init__(self, name):
        self.name = name
        self.node_set = OrderedSet()
        self.edge_set = OrderedSet()

        self.h_matrix = None
        self.s_matrix = None
        self.g_matrix = None
        self.pi_vector = None
Exemple #13
0
 def _compute_all_needed_parametrization_ids(self, fixtureobj):
     stack = [(fixtureobj.info.id, [fixtureobj.info.id], set([fixtureobj.info.id]))]
     returned = OrderedSet()
     while stack:
         fixture_id, path, visited = stack.pop()
         if fixture_id in self._all_needed_parametrization_ids_by_fixture_id:
             returned.update(self._all_needed_parametrization_ids_by_fixture_id[fixture_id])
             continue
         fixture = self._fixtures_by_id[fixture_id]
         if fixture.parametrization_ids:
             assert isinstance(fixture.parametrization_ids, OrderedSet)
             returned.update(fixture.parametrization_ids)
         if fixture.fixture_kwargs:
             for needed_id in itervalues(fixture.fixture_kwargs):
                 if needed_id in visited:
                     self._raise_cyclic_dependency_error(fixtureobj, path, needed_id)
                 stack.append((needed_id, path + [needed_id], visited | set([needed_id])))
     return returned
    def __init__(self, id_, master_graph):
        self.id_ = id_
        self.master_graph = master_graph
        self.edges = OrderedSet()
        self._state = None

        # determine ownership
        self.determined = False
        self.graph = None
Exemple #15
0
 def filter_by_children(self, *args):
     obj_attrs = []
     data = args[0]
     for d in data:
         names = OrderedSet()
         attrs = {}
         try:
             for dc in d.find_all('div'):
                 name = dc.find('span')
                 value = dc.find('div')
                 if value and name is not None:
                     if name.text not in names:
                         names.add(name.text)
                         attrs[name.text] = value.text
             obj_attrs.append(attrs)
         except AttributeError:
             self.view.display_item('Error filtering data from children.....')
     web_objs = self.sanitise_attributes(obj_attrs)
     return web_objs
Exemple #16
0
class DepthFirst(SchedulingPolicy):

  def __init__(self):
    self.__coroutines = OrderedSet()
    self.__hierarchy = {}

  @property
  def busy(self):
    return bool(self.__coroutines)

  def add(self, coroutine):
    parent = coroutine.parent
    self.__coroutines.add(coroutine)
    self.__hierarchy.setdefault(parent, OrderedSet()).add(coroutine)

  def remove(self, coroutine):
    self.__coroutines.remove(coroutine)
    children = self.__hierarchy.pop(coroutine, None)
    if children is not None:
      assert len(children) == 0
    self.__hierarchy.get(coroutine.parent).remove(coroutine)

  def freeze(self, coroutine):
    self.__coroutines.remove(coroutine)

  def unfreeze(self, coroutine):
    self.__coroutines.add(coroutine)

  def round(self):
    c = self.__round(self.__hierarchy.get(None, ()))
    assert c is not None
    return (c,)

  def __round(self, coroutines):
    for coroutine in coroutines:
      assert coroutines is not None
      active = coroutine in self.__coroutines
      if active and coroutine.exception:
        return coroutine
      sub = self.__round(self.__hierarchy.get(coroutine, ()))
      if sub is not None:
        return sub
      if active:
        return coroutine

  def dump(self):
    def dump(c, idt = 0):
      print('{}{}{}'.format('  ' * idt, c, ' (frozen)' if c.frozen else ''))
      for child in self.__hierarchy.get(c, []):
        dump(child, idt + 1)
    for root in self.__hierarchy.get(None, []):
      dump(root)
Exemple #17
0
    def __init__(self, name = None):
        """Initialize the molecule

        Args:
            name (str): name of the molecule
        """
        if name != None:
            self.name = name
        else:
            # TODO Fix the naming resolution
            self.name = "Untitled"
        self._atoms = OrderedSet()
    def __get_centroid(self, cluster=None):
        centroid = ''

        # centroid for a particular cluster
        if cluster:
            for log_id in cluster:
                if centroid == '':
                    centroid = self.preprocessed_logs[log_id]
                else:
                    centroid = ' '.join([centroid, self.preprocessed_logs[log_id]])
        # centroid for the whole logs
        else:
            for log_id in self.preprocessed_logs:
                if centroid == '':
                    centroid = self.preprocessed_logs[log_id]
                else:
                    centroid = ' '.join([centroid, self.preprocessed_logs[log_id]])

        centroid = OrderedSet(centroid.split())
        centroid = ' '.join(list(centroid))
        return centroid
Exemple #19
0
    class UNIQUE_ARRAY(_ObjectHook):

        #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
        def __init__(self, name, flag, is_required):
            self._name        = name
            self._flag        = flag
            self._values      = OrderedSet()
            self._is_required = is_required

        #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
        def add_value(self, value):
            self._values.add(value)

        #- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
        def close(self, name, flag):
            if (self._is_required and
                not self._values):
                    raise Pattern.UnfinishedPattern(
                        Pattern.UNIQUE_ARRAY, self._flag,
                        Pattern.EOL() if name is NotImplemented else flag) from None
            return self._values
Exemple #20
0
class Molecule(object):
    """An abstract molecule object.
    """
    __slots__ = ['name', '_atoms']
    def __init__(self, name = None):
        """Initialize the molecule

        Args:
            name (str): name of the molecule
        """
        if name != None:
            self.name = name
        else:
            # TODO Fix the naming resolution
            self.name = "Untitled"
        self._atoms = OrderedSet()

    def addAtom(self, atom):
        """Add and atom

        Args:
            atom (atom): the atom to add into the molecule
        """
        self._atoms.add(atom)

    def removeAtom(self, atom):
        """Remove Atom

        Args:
            atom (atom): the atom to remove from the molecule
        """
        self._atoms.remove(atom)

    def getAtoms(self):
        """Return an orderedset of atoms
        """
        return self._atoms

    def __repr__(self):
        return self.name
Exemple #21
0
    def __init__(self, name):
        """Initialize the MoleculeType container

        Args:
            name (str): the name of the moleculetype to add
        """
        self.name = name
        self.moleculeSet = OrderedSet()
        self.bondForceSet = HashMap()
        self.pairForceSet = HashMap()
        self.angleForceSet = HashMap()
        self.dihedralForceSet = HashMap()
        self.torsiontorsionForceSet = HashMap()
        self.constraints = HashMap()
        self.exclusions = HashMap()
        self.settles = None
        self.nrexcl = None
def merge_ordered_set(diff, base, mine, theirs):
    mine_merged = mine - diff.deleted
    theirs_merged = theirs - diff.deleted
    mine_added = mine_merged - theirs_merged
    merged = OrderedSet()

    theirs_merged_iter = iter(theirs_merged)
    mine_merged_iter = iter(mine_merged)

    for theirs_el in theirs_merged_iter:
        for mine_el in mine_merged_iter:
            if mine_el in mine_added:
                merged.add(mine_el)
            else:
                break
        merged.add(theirs_el)

    for mine_el in mine_merged_iter:
        merged.add(mine_el)

    return merged
Exemple #23
0
def handle_torrent_upload(upload_form, uploading_user=None, fromAPI=False):
    torrent_data = upload_form.torrent_file.parsed_data

    # The torrent has been  validated and is safe to access with ['foo'] etc - all relevant
    # keys and values have been checked for (see UploadForm in forms.py for details)
    info_dict = torrent_data.torrent_dict['info']

    changed_to_utf8 = _replace_utf8_values(torrent_data.torrent_dict)

    # Use uploader-given name or grab it from the torrent
    display_name = upload_form.display_name.data.strip(
    ) or info_dict['name'].decode('utf8').strip()
    information = (upload_form.information.data or '').strip()
    description = (upload_form.description.data or '').strip()

    torrent_filesize = info_dict.get('length') or sum(
        f['length'] for f in info_dict.get('files'))

    # In case no encoding, assume UTF-8.
    torrent_encoding = torrent_data.torrent_dict.get('encoding',
                                                     b'utf-8').decode('utf-8')

    torrent = models.Torrent(info_hash=torrent_data.info_hash,
                             display_name=display_name,
                             torrent_name=torrent_data.filename,
                             information=information,
                             description=description,
                             encoding=torrent_encoding,
                             filesize=torrent_filesize,
                             user=uploading_user,
                             uploader_ip=ip_address(
                                 flask.request.remote_addr).packed)

    # Store bencoded info_dict
    torrent.info = models.TorrentInfo(
        info_dict=torrent_data.bencoded_info_dict)
    torrent.stats = models.Statistic()
    torrent.has_torrent = True

    # Fields with default value will be None before first commit, so set .flags
    torrent.flags = 0

    torrent.anonymous = upload_form.is_anonymous.data if uploading_user else True
    torrent.hidden = upload_form.is_hidden.data
    torrent.remake = upload_form.is_remake.data
    torrent.complete = upload_form.is_complete.data
    # Copy trusted status from user if possible
    can_mark_trusted = uploading_user and uploading_user.is_trusted
    # To do, automatically mark trusted if user is trusted unless user specifies otherwise
    torrent.trusted = upload_form.is_trusted.data if can_mark_trusted else False

    # Set category ids
    torrent.main_category_id, torrent.sub_category_id = \
        upload_form.category.parsed_data.get_category_ids()

    # To simplify parsing the filelist, turn single-file torrent into a list
    torrent_filelist = info_dict.get('files')

    used_path_encoding = changed_to_utf8 and 'utf-8' or torrent_encoding

    parsed_file_tree = dict()
    if not torrent_filelist:
        # If single-file, the root will be the file-tree (no directory)
        file_tree_root = parsed_file_tree
        torrent_filelist = [{
            'length': torrent_filesize,
            'path': [info_dict['name']]
        }]
    else:
        # If multi-file, use the directory name as root for files
        file_tree_root = parsed_file_tree.setdefault(
            info_dict['name'].decode(used_path_encoding), {})

    # Parse file dicts into a tree
    for file_dict in torrent_filelist:
        # Decode path parts from utf8-bytes
        path_parts = [
            path_part.decode(used_path_encoding)
            for path_part in file_dict['path']
        ]

        filename = path_parts.pop()
        current_directory = file_tree_root

        for directory in path_parts:
            current_directory = current_directory.setdefault(directory, {})

        # Don't add empty filenames (BitComet directory)
        if filename:
            current_directory[filename] = file_dict['length']

    parsed_file_tree = utils.sorted_pathdict(parsed_file_tree)

    json_bytes = json.dumps(parsed_file_tree,
                            separators=(',', ':')).encode('utf8')
    torrent.filelist = models.TorrentFilelist(filelist_blob=json_bytes)

    db.session.add(torrent)
    db.session.flush()

    # Store the users trackers
    trackers = OrderedSet()
    announce = torrent_data.torrent_dict.get('announce', b'').decode('ascii')
    if announce:
        trackers.add(announce)

    # List of lists with single item
    announce_list = torrent_data.torrent_dict.get('announce-list', [])
    for announce in announce_list:
        trackers.add(announce[0].decode('ascii'))

    # Remove our trackers, maybe? TODO ?

    # Search for/Add trackers in DB
    db_trackers = OrderedSet()
    for announce in trackers:
        tracker = models.Trackers.by_uri(announce)

        # Insert new tracker if not found
        if not tracker:
            tracker = models.Trackers(uri=announce)
            db.session.add(tracker)

        db_trackers.add(tracker)

    db.session.flush()

    # Store tracker refs in DB
    for order, tracker in enumerate(db_trackers):
        torrent_tracker = models.TorrentTrackers(torrent_id=torrent.id,
                                                 tracker_id=tracker.id,
                                                 order=order)
        db.session.add(torrent_tracker)

    db.session.commit()

    # Store the actual torrent file as well
    torrent_file = upload_form.torrent_file.data
    if app.config.get('BACKUP_TORRENT_FOLDER'):
        torrent_file.seek(0, 0)

        torrent_dir = app.config['BACKUP_TORRENT_FOLDER']
        if not os.path.exists(torrent_dir):
            os.makedirs(torrent_dir)

        torrent_path = os.path.join(
            torrent_dir,
            '{}.{}'.format(torrent.id, secure_filename(torrent_file.filename)))
        torrent_file.save(torrent_path)
    torrent_file.close()

    return torrent
Exemple #24
0
 def __init__(self):
   self.__coroutines = OrderedSet()
   self.__hierarchy = {}
Exemple #25
0
class MoleculeType(object):
    """An abstract container for molecules of one type
    """
    def __init__(self, name):
        """Initialize the MoleculeType container

        Args:
            name (str): the name of the moleculetype to add
        """
        self.name = name
        self.moleculeSet = OrderedSet()
        self.bondForceSet = HashMap()
        self.pairForceSet = HashMap()
        self.angleForceSet = HashMap()
        self.dihedralForceSet = HashMap()
        self.torsiontorsionForceSet = HashMap()
        self.constraints = HashMap()
        self.exclusions = HashMap()
        self.settles = None
        self.nrexcl = None

    def add_molecule(self, molecule):
        """Add a molecule into the moleculetype container

        Args:
            molecule (Molecule): the molecule to append
        """
        self.moleculeSet.add(molecule)

    def remove_molecule(self, molecule):
        """Remove a molecule from the system.

        Args:
            molecule (Molecule): remove a molecule from the moleculeType
        """
        self.moleculeSet.remove(molecule)

    def getMolecule(self, molecule):
        """Get a molecule from the system

        Args:
            molecule (Molecule): retrieve an equivalent molecule from the moleculetype
        """
        return get_equivalent(self.moleculeSet, molecule, False)

    def addForce(self, force):
        """Add a forces to the moleculeType

        Args:
            forces (AbstractForce): Add a forces or contraint to the moleculeType
        """
        self.forceSet.add(force)

    def removeForce(self, force):
        """Remove a forces from the moleculeType

        Args:
            forces (AbstractForce): Remove a forces from the moleculeType
        """
        self.forceSet.remove(force)

    def getForce(self, force):
        """Get a forces from the moleculeType

        Args:
            forces (AbstractForce): Retrieve a forces from the moleculeType
        """
        return get_equivalent(self.forceSet, force, False)

    def setNrexcl(self, nrexcl):
        """Set the nrexcl

        Args:
            nrexcl (int): the value for nrexcl
        """
        self.nrexcl = nrexcl

    def getNrexcl(self):
        """Gets the nrexcl
        """
        return self.nrexcl
Exemple #26
0
    def update_remote_groups(self, dry_run=True, threshold=None, remove=True):
        '''Update remote configuration with the local one.'''
        # Copy those so that we can modify them even with dry-run
        local = OrderedSet(self.local)
        remote = OrderedSet(self.remote)

        validate_groups(local)

        def parse_groups(groups, remote):
            if remote:
                self._process_remote_groups(groups)
            groups = {
                group.name: group
                for group in groups if group.name != 'default'
            }
            keys = OrderedSet(groups.keys())
            return groups, keys

        lgroups, lkeys = parse_groups(local, False)
        rgroups, rkeys = parse_groups(remote, True)

        changes = 0
        unchanged = 0
        groups_added = OrderedSet()
        groups_updated = OrderedSet()
        groups_removed = OrderedSet()
        rules_added = OrderedSet()
        rules_removed = OrderedSet()

        # Added groups
        for group in (lgroups[name] for name in lkeys - rkeys):
            grp = Group(group.name, group.description)
            groups_added.add(grp)
            rgroups[group.name] = grp
            rkeys.add(grp.name)
            changes += 1

        # Changed groups
        for rgroup, lgroup in ((rgroups[name], lgroups[name])
                               for name in rkeys & lkeys):
            if rgroup not in groups_added:
                unchanged += 1

            if rgroup.description != lgroup.description:
                # XXX: https://review.openstack.org/596609
                # groups_updated.add((rgroup, lgroup))
                pass

            # FIXME: when comparing using OrderedSet, added rules part contains
            #        all elements rather than different ones.
            lrules, rrules = set(lgroup.rules), set(rgroup.rules)

            if rrules != lrules:
                # Added rules
                for rule in lrules - rrules:
                    rules_added.add((rgroup.name, rule))
                    changes += 1

                # Removed rules
                for rule in rrules - lrules:
                    if remove:
                        rules_removed.add((rgroup.name, rule))
                        changes += 1
                    else:
                        unchanged += 1
            unchanged += len(rrules & lrules)

        # Removed groups
        for group in (rgroups[name] for name in rkeys - lkeys):
            if remove:
                if group._project is None:
                    continue
                groups_removed.add(group)
                changes += len(group.rules) + 1
            else:
                unchanged += len(group.rules) + 1

        if changes == 0 and not groups_updated:
            return

        # Report result
        logger.info(f'{changes:d} changes to be made:')
        for group in groups_added:
            logger.info(f'  - Create group {group.name!r}')
        for rgroup, lgroup in groups_updated:
            logger.info(f'  - Update description for {rgroup.name!r}:'
                        f' {rgroup.description!r} → {lgroup.description!r}')
        for group_name, rule in rules_added:
            logger.info(f'  - Create {rule!r} in group {group_name!r}')
        for group_name, rule in rules_removed:
            logger.info(f'  - Remove {rule!r} from group {group_name!r}')
        for group in groups_removed:
            logger.info(
                f'  - Remove group {group.name!r} with {len(group.rules)} rules'
            )

        if threshold is not None:
            changes_percentage = changes / (unchanged + changes) * 100
            if changes_percentage > threshold:
                raise ThresholdException(
                    f'Amount of changes is {changes_percentage:f}%'
                    f' which is more than allowed ({threshold:f}%)')

        if dry_run:
            return

        # We've modified 'remote', so copy it again
        remote = OrderedSet(self.remote)
        rgroups, rkeys = parse_groups(remote, True)

        # Added groups
        for group in groups_added:
            ginfo = self.connection.create_security_group(
                name=group.name, description=group.description)
            remote.add(Group.from_remote(**ginfo))
        rgroups, rkeys = parse_groups(remote, True)

        # Updated groups
        for rgroup, lgroup in groups_updated:
            self.connection.update_security_group(
                name_or_id=rgroup._id, description=lgroup.description)
            # Updating group should not change its ID
            rgroup.description = lgroup.description

        # Added rules
        for group_name, rule in rules_added:
            rgroup = rgroups[group_name]
            cidr = str(rule.cidr) if rule.cidr is not None else None
            group_id = rgroups[
                rule.group]._id if rule.group is not None else None
            rinfo = self.connection.create_security_group_rule(
                secgroup_name_or_id=rgroup._id,
                port_range_min=rule.port_min,
                port_range_max=rule.port_max,
                protocol=rule.protocol.value,
                remote_ip_prefix=cidr,
                remote_group_id=group_id,
                direction=rule.direction.value,
                ethertype=rule.ethertype.value)
            rgroup.rules.add(Rule.from_remote(**rinfo))

        if remove:
            # Removed rules
            for group_name, rule in rules_removed:
                rgroup = rgroups[group_name]
                self.connection.delete_security_group_rule(rule_id=rule._id)
                rgroup.rules.remove(rule)

            # Removed groups
            for group in groups_removed:
                self.connection.delete_security_group(name_or_id=group._id)
                remote.remove(group)

        self.remote = remote
Exemple #27
0
 def __init__(self):
   self.__coroutines = OrderedSet()
Exemple #28
0
class Graph:

    def __init__(self, name):
        self.name = name
        self.node_set = OrderedSet()
        self.edge_set = OrderedSet()

        self.h_matrix = None
        self.s_matrix = None
        self.g_matrix = None
        self.pi_vector = None

    def create_graph_from_file(self, file):
        for line in file.read().splitlines():

            if "NodeName" in line:
                node_name = line.split(" = ")[1]
                node = Node(node_name)

                self.node_set.add(node)

            if "EdgeName" in line:
                nodes = line.split(" = ")[1].split("->")
                starting_node_name = nodes[0]
                target_node_name= nodes[1]

                starting_node = self.find_node(starting_node_name)
                target_node = self.find_node(target_node_name)

                if starting_node is None or target_node is None:
                    raise RuntimeError

                starting_node.add_edge(target_node)

    def find_node(self, name):
        """
        Since sets do not support getting an item out, loop over and compare each node
        for equality in order to find the desired node

        :param name: the name of the node to search for
        :return: the node with `name` if it is present in the graphs node_set
        """
        target_node = Node(name)

        if target_node in self.node_set:
            for node in self.node_set:
                if node == target_node:
                    return node

        return None

    def create_h_matrix(self, df=None, store=True):
        """
        Adjacency matrix
        :return:
        """
        # set up default matrix (n x n with all 0 values)
        if not df:
            df = self.h_matrix

        df = self.dataframe_nxn()

        for node in self.node_set:
            adjacency_map = node.compute_adjacency()

            for adjacent_node, probability in adjacency_map.items():
                df.set_value(node, adjacent_node, probability)

        if store:
            self.h_matrix = df

        return df


    def create_s_matrix(self, df=None, store=True):
        """
        Transition Stochastic Matrix
        :return:
        """
        if not df:
            df = self.create_h_matrix(store=False)

        def correct_dangling(row):
            if row.sum() == 0:
                return [1/len(row) for node in self.node_set]

            else:
                return row

        df = df.apply(correct_dangling, axis=1, reduce=False)


        if store:
            self.s_matrix = df

        return df


    def create_g_matrix(self, df=None, store=False):
        """
        Google Matrix
        :return:
        """
        scaling_param = .9
        if not df:
            df = self.create_s_matrix(store=False)

        df = np.dot(.9, df)
        df += np.dot((1-scaling_param), 1/len(self.node_set))

        return df

    def compute_page_rank(self, df=None):
        if not df:
            df = self.create_g_matrix()

        vector = [1/len(self.node_set) for x in self.node_set]

        for node in self.node_set:
            vector = np.dot(vector, df)

        vector = vector.round(3)
        page_map = {}
        for i, node in enumerate(self.node_set):
            page_map[node] = vector[i]

        return page_map


    def describe_graph(self):
        print("Details for graph: %s" % self.name)
        print("++++ Nodes in the graph")
        for node in self.node_set:
            print("NodeName = %s" % node.name)

        print("++++ Edges in the Graph")
        for node in self.node_set:
            node.describe()

        print("\n\nAdjency Matrix:\n")
        print(self.create_h_matrix().round(3))

        print("\n\nTransition Stochastic Matrix:\n")
        print(self.create_s_matrix().round(3))

        print("\n\nGoogle Matrix:\n")
        print(self.create_g_matrix().round(3))

        print("\n\nRankings:\n")

        data = self.compute_page_rank()

        sorted_nodes = sorted(data, key=data.get, reverse=True)
        for i, node in enumerate(sorted_nodes):
            print("Rank #%s: %s - %s" % (i+1, node, data[node]))


    def dataframe_nxn(self):
        structure = OrderedDict()
        blank_data = [0.0 for node in self.node_set]

        for node in self.node_set:
            structure[node] = pd.Series(blank_data, index=self.node_set)

        return pd.DataFrame(structure)
Exemple #29
0
def comm(a, b):
    sa, sb = map(OrderedSet, (a, b))
    return (sa.difference(sb), sb.difference(sa),
            OrderedSet.intersection(sa, sb))
Exemple #30
0
class FileMeta(ValidatorObject):
    mandatory = OrderedSet()
    optional = OrderedSet()
    order = OrderedSet()
    enforce_order = False
    mappings = {}
Exemple #31
0
from rdfframes.query_builder.querymodel import QueryModel
from orderedset import OrderedSet

if __name__ == '__main__':

    ####### complete query
    subquery = QueryModel()
    subquery.add_triple("tweet", "sioc:has_creater", "tweep")
    #subquery.add_variable("tweeter")
    subquery.add_group_columns(OrderedSet(["tweep"]))
    subquery.add_aggregate_pair("tweet", "COUNT", "tweet_count", "distinct")
    subquery.add_having_condition("tweet_count", "< 300")
    subquery.add_having_condition("tweet_count", "> 250")
    subquery.add_select_column("tweep")

    twitterquery = QueryModel()
    prefixes = {
        "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
        "sioc": "http://rdfs.org/sioc/ns#",
        "sioct": "http://rdfs.org/sioc/types#",
        "to": "http://twitter.com/ontology/",
        "dcterms": "http://purl.org/dc/terms/",
        "xsd": "http://www.example.org/",
        "foaf": "http://xmlns.com/foaf/0.1/",
    }
    twitterquery.add_prefixes(prefixes)
    twitterquery.add_graphs(["http://twitter.com/"])
    twitterquery.add_variable("tweep")
    twitterquery.add_subquery(subquery)
    twitterquery.add_triple("tweet", "sioc:has_creater", "tweep")
    twitterquery.add_triple("tweet", " sioc:content", "text")
Exemple #32
0
    def add_ops(self, roots, after_exop=None):
        """
        Add exops needed to compute ops in roots.

        Args:
            roots: A collection of ops whose values are needed.
            after_exop: Where in the list to add the ops. Defaults to the end.

        """
        if after_exop is None:
            after_exop = self.prev_exop

        # Get computation graph ops that are already inserted.
        available = OrderedSet()
        counts = dict()
        parents = defaultdict(OrderedSet)
        ready = OrderedSet()

        # Setting the environmental variable below to 0 can be used to disable toposort
        # with priorities and switch to naive algo in case something went wrong unexpectedly
        algo_num = int(os.getenv('NGRAPH_TOPOSORT_ALGO', 1))
        pqueue = PriorityQueue()
        op_counter = 0
        wait_order = 100000
        std_order = 2
        start_order = 1

        # Some ops in roots may have been replaced by other ops; if so, they
        # are in the graph already, although maybe not in this block. Get the
        # op from the exop so we have the current version.
        for op in roots:
            exop = self.computation_decl.get_exop(op, None)
            if exop is not None:
                op = exop.op
            available.add(op)

        while available:
            op = available.pop()
            if algo_num > 0:
                if 'priority' in op.metadata:
                    if op.metadata['priority'] == 'high':
                        op.metadata['order'] = start_order
                    else:
                        op.metadata['order'] = wait_order
                elif 'order' not in op.metadata:
                    op.metadata['order'] = std_order
            if op in counts or op in self.all_ops:
                continue

            nchildren = 0

            op_deps = op.all_deps
            if (isinstance(op, CPUMlslGatherRecvOp) or isinstance(
                    op, CPUMlslScatterRecvOp)) and op.send_node() in available:
                op_deps.add(op.send_node())
            for child in op_deps:
                exop = self.computation_decl.get_exop(child, None)
                if exop is not None:
                    child = exop.op
                if child not in self.all_ops:
                    parents[child].add(op)
                    available.add(child)
                    if algo_num > 0:
                        ch_order = child.metadata[
                            'order'] if 'order' in child.metadata else -1
                        new_order = op.metadata['order'] + 1
                        if 'priority' not in child.metadata and \
                                ('order' not in child.metadata or new_order < ch_order):
                            child.metadata['order'] = new_order
                    nchildren += 1
            if nchildren > 0:
                counts[op] = nchildren
            else:
                if op not in ready:
                    ready.add(op)
                    if algo_num > 0:
                        op_counter = op_counter - 1
                        pqueue.put((op.metadata['order'], op_counter, op))

        if algo_num == 0:
            while ready:
                op = ready.pop()
                after_exop = self.add_op(op, after_exop=after_exop)
                for p in parents.get(op, []):
                    count = counts[p] - 1
                    if count == 0:
                        ready.add(p)
                        del counts[p]
                    else:
                        counts[p] = count

        else:
            while len(pqueue.queue) > 0:
                _, _, op = pqueue.get()
                after_exop = self.add_op(op, after_exop=after_exop)
                for p in parents.get(op, []):
                    count = counts[p] - 1
                    if count == 0:
                        op_counter = op_counter - 1
                        # Shouldn't happen, but we have a way to get back to naive scheduling
                        assert 'order' in p.metadata, \
                            "Something went wrong with the scheduling. \
                                 Please try NGRAPH_TOPOSORT_ALGO=0"

                        if p.metadata['order'] == wait_order:
                            pqueue.put(
                                (p.metadata['order'], int(-op_counter), p))
                        else:
                            pqueue.put((p.metadata['order'], op_counter, p))
                        del counts[p]
                    else:
                        counts[p] = count
        if len(counts) > 0:
            raise ValueError("Graph not a DAG")
Exemple #33
0
    def validate(self):
        """
        Validate the appinfo and put the results into ``errors``, ``warnings``
        and ``info`` in ``self``.
        """
        self.load(False)
        if self.ini is None:
            self.errors.append(self.ini_fail)
            return  # Don't need "section missing" errors

        if not isfile(self.path_abs()):
            # If appinfo.ini doesn't exist, we've created an INIConfig which
            # will be empty, for the fix routine. Then as we don't want to spew
            # a whole heap of errors given that an error about appinfo.ini
            # being missing has already been added to the list, we'll give up.
            return

        ini = self.ini
        package = self.package
        meta = self.Meta(self, ini, package)  # () so properties work
        setini = OrderedSet(ini)

        for missing in meta.mandatory - setini:
            self.errors.append(LANG.INIVALIDATOR.SECTION_MISSING %
                               dict(filename=self.path(), section=missing))

        for extra in setini - meta.order:
            self.errors.append(LANG.INIVALIDATOR.SECTION_EXTRA %
                               dict(filename=self.path(), section=extra))

        # Check that they're in the same order
        if meta.enforce_order and setini & meta.order != meta.order & setini:
            # A warning? We like fancy INI files.
            self.warnings.append(LANG.INIVALIDATOR.SECTIONS_OUT_OF_ORDER %
                                 dict(filename=self.path()))

        if len(ini) == 0:
            self.warnings.append(LANG.INIVALIDATOR.FILE_EMPTY %
                                 dict(filename=self.path()))

        for section in setini & meta.order:
            if hasattr(self.module, section):
                secval_cls = getattr(self.module, section)
            else:
                for mapping in meta.mappings:
                    if mapping.match(section):
                        secval_cls = getattr(self.module, mapping.target)
                        # TODO: could be nice to know what section is in this case...
                        break
            secval = secval_cls(self, ini, package)
            smeta = secval.Meta(self, ini, package, secval)
            inisection = ini[section]
            inisectionset = OrderedSet(inisection)

            if len(inisection) == 0:
                self.warnings.append(
                    LANG.INIVALIDATOR.SECTION_EMPTY %
                    dict(filename=self.path(), section=section))

            for key in smeta.mandatory:
                if key not in inisection:
                    self.errors.append(
                        LANG.INIVALIDATOR.VALUE_MISSING %
                        dict(filename=self.path(), section=section, key=key))
                elif inisection[key] == '':
                    self.errors.append(
                        LANG.INIVALIDATOR.VALUE_EMPTY %
                        dict(filename=self.path(), section=section, key=key))

            for key in inisectionset - smeta.mandatory:
                if key not in smeta.optional:
                    self.errors.append(
                        LANG.INIVALIDATOR.VALUE_EXTRA %
                        dict(filename=self.path(), section=section, key=key))
                elif inisection[key] == '':
                    self.errors.append(
                        LANG.INIVALIDATOR.OMIT_EMPTY %
                        dict(filename=self.path(), section=section, key=key))

            # Check that they're in the same order
            enforce_order = meta.enforce_order if smeta.enforce_order is Ellipsis else smeta.enforce_order
            if enforce_order and inisectionset & smeta.order != smeta.order & inisectionset:
                # A warning? We like fancy INI files.
                self.warnings.append(
                    LANG.INIVALIDATOR.KEYS_OUT_OF_ORDER %
                    dict(filename=self.path(), section=section))

            # Oh yeah, we may as well validate the value. Could be handy.
            for key in smeta.order:
                if key in inisection:
                    if hasattr(secval, key):
                        self._add_item(getattr(secval, key)(inisection[key]))
                    else:
                        for mapping in smeta.mappings:
                            if mapping.match(key):
                                self._add_item(
                                    getattr(secval,
                                            mapping.target)(inisection[key]))
                                # TODO: could be nice to know what key is in this case...
                                break
Exemple #34
0
class ExOpBlock(ExecutionGraphElt):
    """
    A list of exops to be executed sequentially.

    Attributes:
        computation_decl: The associated computation graph.
        prev_exop: The latst exop.
        next_exop: The first exop.
        root_set: Set of exops whose values are needed.

    """
    def __init__(self, computation_decl=None, **kwargs):
        if computation_decl is None:
            raise ValueError("computation_decl must be specified.")
        super(ExOpBlock,
              self).__init__(execution_graph=computation_decl.execution_graph,
                             **kwargs)
        self.computation_decl = computation_decl
        # Doubly linked loop, with self as termination
        self.prev_exop = self
        self.next_exop = self
        # All ops handled by the block.
        self.all_ops = set()

        self.root_set = OrderedSet()

    @property
    def is_exop_end_of_list(self):
        """

        Returns:
            True if this represents the guard past the exop list. See ExecuteOp.

        """
        return True

    class ExOpForwardIterator(object):
        def __init__(self, exop_term):
            self.exop_term = exop_term
            self.exop = self.exop_term.next_exop

        def next(self):
            if self.exop.is_exop_end_of_list:
                raise StopIteration
            result = self.exop
            self.exop = result.next_exop
            return result

        __next__ = next  # Python 3.X compatibility

    class ExOpReversedIterator(object):
        def __init__(self, exop_term):
            self.exop_term = exop_term
            self.exop = self.exop_term.prev_exop

        def __iter__(self):
            return self

        def next(self):
            if self.exop.is_exop_end_of_list:
                raise StopIteration
            result = self.exop
            self.exop = result.prev_exop
            return result

        __next__ = next  # Python 3.X compatibility

    def __iter__(self):
        return ExOpBlock.ExOpForwardIterator(self)

    def __reversed__(self):
        return ExOpBlock.ExOpReversedIterator(self)

    def add_ops(self, roots, after_exop=None):
        """
        Add exops needed to compute ops in roots.

        Args:
            roots: A collection of ops whose values are needed.
            after_exop: Where in the list to add the ops. Defaults to the end.

        """
        if after_exop is None:
            after_exop = self.prev_exop

        # Get computation graph ops that are already inserted.
        available = OrderedSet()
        counts = dict()
        parents = defaultdict(OrderedSet)
        ready = OrderedSet()

        # Setting the environmental variable below to 0 can be used to disable toposort
        # with priorities and switch to naive algo in case something went wrong unexpectedly
        algo_num = int(os.getenv('NGRAPH_TOPOSORT_ALGO', 1))
        pqueue = PriorityQueue()
        op_counter = 0
        wait_order = 100000
        std_order = 2
        start_order = 1

        # Some ops in roots may have been replaced by other ops; if so, they
        # are in the graph already, although maybe not in this block. Get the
        # op from the exop so we have the current version.
        for op in roots:
            exop = self.computation_decl.get_exop(op, None)
            if exop is not None:
                op = exop.op
            available.add(op)

        while available:
            op = available.pop()
            if algo_num > 0:
                if 'priority' in op.metadata:
                    if op.metadata['priority'] == 'high':
                        op.metadata['order'] = start_order
                    else:
                        op.metadata['order'] = wait_order
                elif 'order' not in op.metadata:
                    op.metadata['order'] = std_order
            if op in counts or op in self.all_ops:
                continue

            nchildren = 0

            op_deps = op.all_deps
            if (isinstance(op, CPUMlslGatherRecvOp) or isinstance(
                    op, CPUMlslScatterRecvOp)) and op.send_node() in available:
                op_deps.add(op.send_node())
            for child in op_deps:
                exop = self.computation_decl.get_exop(child, None)
                if exop is not None:
                    child = exop.op
                if child not in self.all_ops:
                    parents[child].add(op)
                    available.add(child)
                    if algo_num > 0:
                        ch_order = child.metadata[
                            'order'] if 'order' in child.metadata else -1
                        new_order = op.metadata['order'] + 1
                        if 'priority' not in child.metadata and \
                                ('order' not in child.metadata or new_order < ch_order):
                            child.metadata['order'] = new_order
                    nchildren += 1
            if nchildren > 0:
                counts[op] = nchildren
            else:
                if op not in ready:
                    ready.add(op)
                    if algo_num > 0:
                        op_counter = op_counter - 1
                        pqueue.put((op.metadata['order'], op_counter, op))

        if algo_num == 0:
            while ready:
                op = ready.pop()
                after_exop = self.add_op(op, after_exop=after_exop)
                for p in parents.get(op, []):
                    count = counts[p] - 1
                    if count == 0:
                        ready.add(p)
                        del counts[p]
                    else:
                        counts[p] = count

        else:
            while len(pqueue.queue) > 0:
                _, _, op = pqueue.get()
                after_exop = self.add_op(op, after_exop=after_exop)
                for p in parents.get(op, []):
                    count = counts[p] - 1
                    if count == 0:
                        op_counter = op_counter - 1
                        # Shouldn't happen, but we have a way to get back to naive scheduling
                        assert 'order' in p.metadata, \
                            "Something went wrong with the scheduling. \
                                 Please try NGRAPH_TOPOSORT_ALGO=0"

                        if p.metadata['order'] == wait_order:
                            pqueue.put(
                                (p.metadata['order'], int(-op_counter), p))
                        else:
                            pqueue.put((p.metadata['order'], op_counter, p))
                        del counts[p]
                    else:
                        counts[p] = count
        if len(counts) > 0:
            raise ValueError("Graph not a DAG")

    def add_op(self, op, after_exop):
        """
        Add an exop for op to be executed after after_exop.

        Args:
            op: The op.
            after_exop: The exop to precede op.

        Returns:
            The new last op. If the op is executable, it will be the added exop,
            othwerwise the previous after_exop.

        """
        if after_exop is None:
            after_exop = self
        if op.is_sequencing_op:
            return after_exop

        exec_op = ExOp(computation_decl=self.computation_decl, op=op)
        return self.add_exop(exec_op, after_exop)

    def add_exop(self, exop, after_exop=None):
        """
        Add exop to the list of exops, after after_exop.

        Args:
            exop:
                The exop to add.

            after_exop:
                If specified, the exop that should be added after after_exop. Defaults to the
                last exop added.

        Returns:
            The exop.

        """
        if after_exop is None:
            after_exop = self.prev_exop

        # Insert between after_exop and the op after after_exop
        before_exop = after_exop.next_exop

        # Add after after_exop
        after_exop.next_exop = exop
        exop.prev_exop = after_exop

        # Add before before_exop
        before_exop.prev_exop = exop
        exop.next_exop = before_exop

        self.all_ops.add(exop.op)

        return exop

    def move_exop_to_after_exop(self, exop, after_exop):
        exop.prev_exop.next_exop = exop.next_exop
        exop.next_exop.prev_exop = exop.prev_exop
        exop.prev_exop = after_exop
        exop.next_exop = after_exop.next_exop
        after_exop.next_exop = exop
        exop.next_exop.prev_exop = exop

    def remove_exop(self, exop):
        exop.prev_exop.next_exop = exop.next_exop
        exop.next_exop.prev_exop = exop.prev_exop
        for input_decl in exop.input_decls:
            input_decl.source_output_decl.user_input_decls.remove(input_decl)
        self.all_ops.remove(exop.op)

    def replace_op(self, old_op, new_op):
        # TODO Replacing an op can remove ops. For example, (x + 2) * 1 -> x + 2
        # replaces the * with +, so * and 1 drop out
        # 1 dropping out means one less constant tensor, if it's not used
        # anywhere else
        # * dropping out means a change to sequencing.
        new_op = as_op(new_op)
        old_exop = self.computation_decl.get_exop(old_op)
        after_exop = old_exop.prev_exop
        self.remove_exop(old_exop)

        # FIXME: find better way to update dependencies
        next_op = old_exop.next_exop.op
        if old_op in next_op.control_deps:
            next_op.remove_control_dep(old_op)
            next_op.add_control_dep(new_op)

        # FIXME: find better way to preserve metadata
        if hasattr(old_op, 'metadata') and hasattr(new_op, 'metadata') and \
           len(old_op.metadata) > len(new_op.metadata):
            new_op.metadata = old_op.metadata

        if old_op is new_op:
            # Hetr bashes some ops. See MutateInsteadOfCopyWithNewArgsMixin, issue #1410
            self.add_ops([new_op], after_exop=after_exop)
            return
        new_exop = self.computation_decl.get_exop(new_op, None)
        if new_exop is None:
            self.add_ops([new_op], after_exop=after_exop)
            new_exop = self.computation_decl.get_exop(new_op, None)
        self.replace_users(old_exop, new_exop)
        if old_exop in self.root_set:
            self.root_set.remove(old_exop)
            self.root_set.add(new_exop)

    def replace_users(self, old_exop, new_exop):
        """
        Replace all users of old_exop with new_exop.

        Args:
            old_exop: The original exop.
            new_exop: The replaceent exop.

        """
        for old_output_decl, new_output_decl in zip(old_exop.output_decls,
                                                    new_exop.output_decls):
            self.replace_output_decl(old_output_decl, new_output_decl)
        for op in old_exop.ref_ops:
            new_exop.add_ref_op(op)
        self.computation_decl.ops[old_exop.op] = new_exop

    def replace_output_decl(self, old_output_decl, new_output_decl):
        for input_decl in set(old_output_decl.user_input_decls):
            input_decl.source_output_decl = new_output_decl
        new_output_decl.tensor_decl.merge_flags(old_output_decl.tensor_decl)
        old_output_decl.exop.output_decls[
            old_output_decl.pos] = new_output_decl

    def replace_exop(self, old_exop, new_exop):
        prev_exop = old_exop.prev_exop
        self.remove_exop(old_exop)
        self.add_exop(new_exop, prev_exop)
        self.replace_users(old_exop, new_exop)

    def merge_exop(self, old_exop, new_exop):
        """
        new_exop, which should already exist, takes over for old_exop.

        Args:
            old_exop:
            new_exop:

        """
        self.replace_users(old_exop, new_exop)
        self.remove_exop(old_exop)

    def memory_footprint(self):
        max_mem = 0
        for node in self:
            max_mem = max([node.memory_footprint(), max_mem])
        return max_mem

    def worst_case_footprint(self):
        mem = 0
        for var in self.get_temp_vars():
            mem += var.tensor_view_decl.tensor_decl.size
        return mem

    def memory_efficiency(self):
        footprint = self.memory_footprint()
        usage = 0
        for node in self.ops:
            usage = max(usage, node.memory_usage())
        result = 100
        if footprint > 0:
            result = int(round((float(usage) / float(footprint)) * 100))
        return result

    def persistent_size(self):
        mem = 0
        for var in self.get_persistent_vars():
            mem += var.tensor_view_decl.tensor_decl.size
        return mem

    def get_vars(self):
        vars = set()
        for exop in self:
            vars |= set(input_decl.source_output_decl
                        for input_decl in exop.input_decls)
            vars |= set(exop.output_decls)
        return vars

    def get_temp_vars(self):
        result = list()
        for var in self.get_vars():
            if not var.tensor_view_decl.tensor_decl.is_persistent:
                result.append(var)
        return result

    def get_persistent_vars(self):
        result = list()
        for var in self.get_vars():
            if var.tensor_view_decl.tensor_decl.is_persistent:
                result.append(var)
        return result
Exemple #35
0
 def send_nodes(self):
     """
     :return: iterable of send nodes
     """
     from ngraph.transformers.hetr.hetr_utils import get_iterable
     return OrderedSet(i for i in get_iterable(self._send_node))
Exemple #36
0
        print(morpheme_list)
        print(row["MORPHS"])
        print(morph_list)
    i = i + 1
    if len(morpheme_list) != len(morph_list):
        print("** line", i, ":", row["MORPHEMES"],
                "is incompatible with", row["MORPHS"])
        continue
    if not morpheme_list:
        continue
    stem_name_set.add(morpheme_list[0])
    name_morph_pair_lst = list(zip(morpheme_list, morph_list))
    seg_example_list.append(name_morph_pair_lst)
    for morpheme, morph in name_morph_pair_lst:
        if morpheme not in morphs_of_morpheme:
            morphs_of_morpheme[morpheme] = OrderedSet()
        morphs_of_morpheme[morpheme].add(morph.strip())
csvfile.close()

print("-- STEP 1 COMPLETED (seg_example_list, stem_name_set, morphs_of_morpheme done)--")

# STEP 2:
# align the allomorphs of each morpheme

from multialign import aligner

alignments = {}
"""All aligned morphs. index: morpheme name, value: sequence of aligned symbols.
Each aligned symbol has as many characters as there are  items in the sequence. 
"""
Exemple #37
0
def _check_messages(service_id,
                    template_id,
                    upload_id,
                    preview_row,
                    letters_as_pdf=False):

    try:
        # The happy path is that the job doesn’t already exist, so the
        # API will return a 404 and the client will raise HTTPError.
        job_api_client.get_job(service_id, upload_id)

        # the job exists already - so go back to the templates page
        # If we just return a `redirect` (302) object here, we'll get
        # errors when we try and unpack in the check_messages route.
        # Rasing a werkzeug.routing redirect means that doesn't happen.
        raise PermanentRedirect(
            url_for(".send_messages",
                    service_id=service_id,
                    template_id=template_id))
    except HTTPError as e:
        if e.status_code != 404:
            raise

    statistics = service_api_client.get_service_statistics(service_id,
                                                           today_only=True)
    remaining_messages = current_service.message_limit - sum(
        stat["requested"] for stat in statistics.values())

    contents = s3download(service_id, upload_id)

    db_template = current_service.get_template_with_user_permission_or_403(
        template_id, current_user)

    email_reply_to = None
    sms_sender = None
    if db_template["template_type"] == "email":
        email_reply_to = get_email_reply_to_address_from_session()
    elif db_template["template_type"] == "sms":
        sms_sender = get_sms_sender_from_session()
    template = get_template(
        db_template,
        current_service,
        show_recipient=True,
        letter_preview_url=url_for(
            ".check_messages_preview",
            service_id=service_id,
            template_id=template_id,
            upload_id=upload_id,
            filetype="png",
            row_index=preview_row,
        ) if not letters_as_pdf else None,
        email_reply_to=email_reply_to,
        sms_sender=sms_sender,
        page_count=get_page_count_for_letter(db_template),
    )
    recipients = RecipientCSV(
        contents,
        template_type=template.template_type,
        placeholders=template.placeholders,
        max_initial_rows_shown=50,
        max_errors_shown=50,
        safelist=itertools.chain.from_iterable(
            [user.name, user.mobile_number, user.email_address]
            for user in Users(service_id))
        if current_service.trial_mode else None,
        remaining_messages=remaining_messages,
        international_sms=current_service.has_permission("international_sms"),
        max_rows=get_csv_max_rows(service_id),
    )

    if request.args.get("from_test"):
        # only happens if generating a letter preview test
        back_link = url_for(".send_test",
                            service_id=service_id,
                            template_id=template.id)
        choose_time_form = None
    else:
        back_link = url_for(".send_messages",
                            service_id=service_id,
                            template_id=template.id)
        choose_time_form = ChooseTimeForm()

    if preview_row < 2:
        abort(404)

    if preview_row < len(recipients) + 2:
        template.values = recipients[preview_row -
                                     2].recipient_and_personalisation
    elif preview_row > 2:
        abort(404)

    return dict(
        recipients=recipients,
        template=template,
        errors=recipients.has_errors,
        row_errors=get_errors_for_csv(recipients, template.template_type),
        count_of_recipients=len(recipients),
        count_of_displayed_recipients=len(list(recipients.displayed_rows)),
        original_file_name=request.args.get("original_file_name", ""),
        upload_id=upload_id,
        form=CsvUploadForm(),
        remaining_messages=remaining_messages,
        choose_time_form=choose_time_form,
        back_link=back_link,
        help=get_help_argument(),
        trying_to_send_letters_in_trial_mode=all((
            current_service.trial_mode,
            template.template_type == "letter",
        )),
        required_recipient_columns=OrderedSet(
            recipients.recipient_column_headers) - optional_address_columns,
        preview_row=preview_row,
        sent_previously=job_api_client.has_sent_previously(
            service_id,
            template.id,
            db_template["version"],
            request.args.get("original_file_name", ""),
        ),
    )
Exemple #38
0
 def add_areas(self, *new_areas):
     areas = list(OrderedSet(
         self._dict['areas'] + list(new_areas)
     ))
     simple_polygons = self.get_simple_polygons(areas=self.get_areas(areas=areas))
     self._update(areas=areas, simple_polygons=simple_polygons.as_coordinate_pairs_lat_long)
Exemple #39
0
def show_reflections(
    reflections,
    show_intensities=False,
    show_profile_fit=False,
    show_centroids=False,
    show_all_reflection_data=False,
    show_flags=False,
    max_reflections=None,
    show_identifiers=False,
):

    text = []

    from orderedset import OrderedSet

    formats = {
        "miller_index": "%i, %i, %i",
        "d": "%.2f",
        "qe": "%.3f",
        "dqe": "%.3f",
        "id": "%i",
        "imageset_id": "%i",
        "panel": "%i",
        "flags": "%i",
        "background.mean": "%.1f",
        "background.dispersion": "%.1f",
        "background.mse": "%.1f",
        "background.sum.value": "%.1f",
        "background.sum.variance": "%.1f",
        "intensity.prf.value": "%.1f",
        "intensity.prf.variance": "%.1f",
        "intensity.sum.value": "%.1f",
        "intensity.sum.variance": "%.1f",
        "intensity.cor.value": "%.1f",
        "intensity.cor.variance": "%.1f",
        "intensity.scale.value": "%.1f",
        "intensity.scale.variance": "%.1f",
        "Ih_values": "%.1f",
        "lp": "%.3f",
        "num_pixels.background": "%i",
        "num_pixels.background_used": "%i",
        "num_pixels.foreground": "%i",
        "num_pixels.valid": "%i",
        "partial_id": "%i",
        "partiality": "%.4f",
        "profile.correlation": "%.3f",
        "profile.rmsd": "%.3f",
        "xyzcal.mm": "%.2f, %.2f, %.2f",
        "xyzcal.px": "%.2f, %.2f, %.2f",
        "delpsical.rad": "%.3f",
        "delpsical2": "%.3f",
        "delpsical.weights": "%.3f",
        "xyzobs.mm.value": "%.2f, %.2f, %.2f",
        "xyzobs.mm.variance": "%.4e, %.4e, %.4e",
        "xyzobs.px.value": "%.2f, %.2f, %.2f",
        "xyzobs.px.variance": "%.4f, %.4f, %.4f",
        "s1": "%.4f, %.4f, %.4f",
        "s2": "%.4f, %.4f, %.4f",
        "shoebox": "%.1f",
        "rlp": "%.4f, %.4f, %.4f",
        "zeta": "%.3f",
        "x_resid": "%.3f",
        "x_resid2": "%.3f",
        "y_resid": "%.3f",
        "y_resid2": "%.3f",
        "kapton_absorption_correction": "%.3f",
        "kapton_absorption_correction_sigmas": "%.3f",
        "inverse_scale_factor": "%.3f",
        "inverse_scale_factor_variance": "%.3f",
    }

    for rlist in reflections:
        from dials.algorithms.shoebox import MaskCode

        foreground_valid = MaskCode.Valid | MaskCode.Foreground
        text.append("")
        text.append(f"Reflection list contains {len(rlist)} reflections")

        if len(rlist) == 0:
            continue

        rows = [["Column", "min", "max", "mean"]]
        for k, col in rlist.cols():
            if k in formats and "%" not in formats.get(k, "%s"):
                # Allow blanking out of entries that wouldn't make sense
                rows.append([
                    k,
                    formats.get(k, "%s"),
                    formats.get(k, "%s"),
                    formats.get(k, "%s"),
                ])
            elif type(col) in (flex.double, flex.int, flex.size_t):
                if type(col) in (flex.int, flex.size_t):
                    col = col.as_double()
                rows.append([
                    k,
                    formats.get(k, "%s") % flex.min(col),
                    formats.get(k, "%s") % flex.max(col),
                    formats.get(k, "%s") % flex.mean(col),
                ])
            elif type(col) in (flex.vec3_double, flex.miller_index):
                if isinstance(col, flex.miller_index):
                    col = col.as_vec3_double()
                rows.append([
                    k,
                    formats.get(k, "%s") % col.min(),
                    formats.get(k, "%s") % col.max(),
                    formats.get(k, "%s") % col.mean(),
                ])
            elif isinstance(col, flex.shoebox):
                rows.append([k, "", "", ""])
                si = col.summed_intensity().observed_value()
                rows.append([
                    "  summed I",
                    formats.get(k, "%s") % flex.min(si),
                    formats.get(k, "%s") % flex.max(si),
                    formats.get(k, "%s") % flex.mean(si),
                ])
                x1, x2, y1, y2, z1, z2 = col.bounding_boxes().parts()
                bbox_sizes = ((z2 - z1) * (y2 - y1) * (x2 - x1)).as_double()
                rows.append([
                    "  N pix",
                    formats.get(k, "%s") % flex.min(bbox_sizes),
                    formats.get(k, "%s") % flex.max(bbox_sizes),
                    formats.get(k, "%s") % flex.mean(bbox_sizes),
                ])
                fore_valid = col.count_mask_values(
                    foreground_valid).as_double()
                rows.append([
                    "  N valid foreground pix",
                    formats.get(k, "%s") % flex.min(fore_valid),
                    formats.get(k, "%s") % flex.max(fore_valid),
                    formats.get(k, "%s") % flex.mean(fore_valid),
                ])

        text.append(tabulate(rows, headers="firstrow"))

        if show_flags:
            text.append(_create_flag_count_table(rlist))

        if show_identifiers:
            if rlist.experiment_identifiers():
                text.append(
                    """Experiment identifiers id-map values:\n%s""" %
                    ("\n".join(
                        "id:" + str(k) + " -> experiment identifier:" +
                        str(rlist.experiment_identifiers()[k])
                        for k in rlist.experiment_identifiers().keys())))

    intensity_keys = (
        "miller_index",
        "d",
        "intensity.prf.value",
        "intensity.prf.variance",
        "intensity.sum.value",
        "intensity.sum.variance",
        "background.mean",
        "profile.correlation",
        "profile.rmsd",
    )

    profile_fit_keys = ("miller_index", "d")

    centroid_keys = (
        "miller_index",
        "d",
        "xyzcal.mm",
        "xyzcal.px",
        "xyzobs.mm.value",
        "xyzobs.mm.variance",
        "xyzobs.px.value",
        "xyzobs.px.variance",
    )

    keys_to_print = OrderedSet()

    if show_intensities:
        for k in intensity_keys:
            keys_to_print.add(k)
    if show_profile_fit:
        for k in profile_fit_keys:
            keys_to_print.add(k)
    if show_centroids:
        for k in centroid_keys:
            keys_to_print.add(k)
    if show_all_reflection_data:
        for k in formats:
            keys_to_print.add(k)

    def format_column(key, data, format_strings=None):
        if isinstance(data, flex.vec3_double):
            c_strings = [
                c.as_string(format_strings[i].strip())
                for i, c in enumerate(data.parts())
            ]
        elif isinstance(data, flex.miller_index):
            c_strings = [
                c.as_string(format_strings[i].strip())
                for i, c in enumerate(data.as_vec3_double().parts())
            ]
        elif isinstance(data, flex.size_t):
            c_strings = [data.as_int().as_string(format_strings[0].strip())]
        elif isinstance(data, flex.shoebox):
            x1, x2, y1, y2, z1, z2 = data.bounding_boxes().parts()
            bbox_sizes = ((z2 - z1) * (y2 - y1) * (x2 - x1)).as_double()
            c_strings = [bbox_sizes.as_string(format_strings[0].strip())]
            key += " (N pix)"
        else:
            c_strings = [data.as_string(format_strings[0].strip())]

        column = flex.std_string()
        max_element_lengths = [c.max_element_length() for c in c_strings]
        for i in range(len(c_strings[0])):

            column.append(f"%{len(key)}s" % ", ".join(
                ("%%%is" % max_element_lengths[j]) % c_strings[j][i]
                for j in range(len(c_strings))))
        return column

    if keys_to_print:
        keys = [k for k in keys_to_print if k in rlist]
        if max_reflections is not None:
            max_reflections = min(len(rlist), max_reflections)
        else:
            max_reflections = len(rlist)

        columns = []

        for k in keys:
            columns.append(
                format_column(k,
                              rlist[k],
                              format_strings=formats[k].split(",")))

        text.append("")
        text.append("Printing %i of %i reflections:" %
                    (max_reflections, len(rlist)))
        line = []
        for j in range(len(columns)):
            key = keys[j]
            if key == "shoebox":
                key += " (N pix)"
            width = max(len(key), columns[j].max_element_length())
            line.append("%%%is" % width % key)
        text.append(" ".join(line))
        for i in range(max_reflections):
            line = (c[i] for c in columns)
            text.append(" ".join(line))

    return "\n".join(text)
Exemple #40
0
class SimpleBreadthFirstLearner(TemplateLearner):
    def __init__(self, solver_instance: Prolog, max_body_literals=4):
        super().__init__(solver_instance)
        self._max_body_literals = max_body_literals

    def initialise_pool(self):
        self._candidate_pool = OrderedSet()

    def put_into_pool(
            self, candidates: typing.Union[Clause, Procedure,
                                           typing.Sequence]) -> None:
        if isinstance(candidates, Clause):
            self._candidate_pool.add(candidates)
        else:
            self._candidate_pool |= candidates

    def get_from_pool(self) -> Clause:
        return self._candidate_pool.pop(0)

    def evaluate(self, examples: Task,
                 clause: Clause) -> typing.Union[int, float]:
        covered = self._execute_program(examples, clause)

        pos, neg = examples.get_examples()

        covered_pos = pos.intersection(covered)
        covered_neg = neg.intersection(covered)

        if len(covered_neg) > 0:
            return 0
        else:
            return len(covered_pos)

    def stop_inner_search(self, eval: typing.Union[int, float], examples: Task,
                          clause: Clause) -> bool:
        if eval > 0:
            return True
        else:
            return False

    def process_expansions(
            self, examples: Task, exps: typing.Sequence[Clause],
            hypothesis_space: TopDownHypothesisSpace
    ) -> typing.Sequence[Clause]:
        # eliminate every clause with more body literals than allowed
        exps = [cl for cl in exps if len(cl) <= self._max_body_literals]

        # check if every clause has solutions
        exps = [(cl, self._solver.has_solution(*cl.get_body().get_literals()))
                for cl in exps]
        new_exps = []

        for ind in range(len(exps)):
            if exps[ind][1]:
                # keep it if it has solutions
                new_exps.append(exps[ind][0])
            else:
                # remove from hypothesis space if it does not
                hypothesis_space.remove(exps[ind][0])

        return new_exps
 def keys(self):
     return OrderedSet(super().keys())
Exemple #42
0
    def __init__(self, filename):
        super().__init__()

        triggers = OrderedDict()
        queries = OrderedDict()
        actions = OrderedDict()
        functions = dict(trigger=triggers, query=queries, action=actions)
        self.functions = functions
        self.entities = OrderedSet()
        devices = []
        trigger_or_query_params = set()

        enum_types = OrderedDict()

        # Token order:
        # first the padding, go and end of sentence
        # then the begin tokens
        # then triggers - queries - actions
        # in this order
        # then parameters names
        # then operators
        # then values
        # then entity tokens
        #
        # This order is important as it affects the 3-part aligner
        # algorithm

        tokens = ['<<PAD>>', '<<EOS>>', '<<GO>>']
        self.num_control_tokens = 3
        tokens += BEGIN_TOKENS
        self.num_begin_tokens = len(BEGIN_TOKENS)

        # add the special functions
        functions['trigger']['tt:$builtin.now'] = []
        functions['query']['tt:$builtin.noop'] = []
        functions['action']['tt:$builtin.notify'] = []

        param_tokens = OrderedSet()
        param_tokens.add('tt-param:$event')
        trigger_or_query_params.add('tt-param:$event')

        with open(filename, 'r') as fp:
            for line in fp.readlines():
                line = line.strip().split()
                function_type = line[0]
                function = line[1]
                if function_type == 'device':
                    devices.append(function)
                    continue
                if function_type == 'entity':
                    self.entities.add(function)
                    continue

                parameters = line[2:]
                paramlist = []
                functions[function_type][function] = paramlist

                for i in range(len(parameters) // 2):
                    param = parameters[2 * i]
                    type = parameters[2 * i + 1]

                    paramlist.append((param, type))
                    param_tokens.add('tt-param:' + param)
                    if function_type != 'action':
                        trigger_or_query_params.add('tt-param:' + param)

                    if type.startswith('Array('):
                        elementtype = type[len('Array('):-1]
                    else:
                        elementtype = type
                    if elementtype.startswith('Enum('):
                        enums = elementtype[len('Enum('):-1].split(',')
                        if not elementtype in enum_types:
                            enum_types[elementtype] = enums

        for function_type in ('trigger', 'query', 'action'):
            for function in functions[function_type]:
                tokens.append(function)
        self.num_functions = len(
            tokens) - self.num_control_tokens - self.num_begin_tokens

        tokens += param_tokens
        self.num_params = len(param_tokens)

        tokens += OPERATORS
        tokens += VALUES
        tokens += SPECIAL_TOKENS
        tokens += BOOKKEEPING_TOKENS
        tokens += devices

        enumtokenset = set()
        for enum_type in enum_types.values():
            for enum in enum_type:
                if enum in enumtokenset:
                    continue
                enumtokenset.add(enum)
                tokens.append(enum)

        for unitlist in UNITS.values():
            tokens += unitlist

        for i in range(MAX_ARG_VALUES):
            for entity in ENTITIES:
                tokens.append(entity + "_" + str(i))
        for generic_entity in self.entities:
            for i in range(MAX_ARG_VALUES):
                tokens.append('GENERIC_ENTITY_' + generic_entity + "_" +
                              str(i))

        print('num functions', self.num_functions)
        print('num triggers', len(self.functions['trigger']))
        print('num queries', len(self.functions['query']))
        print('num actions', len(self.functions['action']))
        print('num params', self.num_params)
        first_value_token = self.num_functions + self.num_begin_tokens + self.num_control_tokens
        print('num value tokens', len(tokens) - first_value_token)

        self.tokens = tokens
        self.dictionary = dict()
        for i, token in enumerate(self.tokens):
            self.dictionary[token] = i

        # build a DFA that will parse the thingtalk-ish code

        states = []
        transitions = []
        state_names = []

        def to_ids(tokens, words):
            return list([words[x] for x in tokens])

        def add_allowed_tokens(state, tokens):
            state[to_ids(tokens, self.dictionary)] = 1

        def new_state(name):
            state = np.zeros((self.output_size, ))
            states.append(state)
            state_names.append(name)
            return len(states) - 1

        # start with one of the begin tokens
        self.start_state = new_state('start')

        # in the before end state we just wait for EOS
        self.before_end_state = new_state('before_end')

        # in the end state we are done
        self.end_state = new_state('end')
        transitions.append((self.before_end_state, self.end_state, '<<EOS>>'))
        transitions.append((self.end_state, self.end_state, '<<PAD>>'))

        # bookkeeping
        bookkeeping_id = new_state('bookkeeping')
        transitions.append((self.start_state, bookkeeping_id, 'bookkeeping'))
        transitions.append((bookkeeping_id, self.end_state, '<<EOS>>'))
        self.bookeeping_state_id = bookkeeping_id

        # special
        special_id = new_state('special')
        transitions.append((bookkeeping_id, special_id, 'special'))
        for t in SPECIAL_TOKENS:
            transitions.append((special_id, self.before_end_state, t))

        # command
        command_id = new_state('command')
        transitions.append((bookkeeping_id, command_id, 'command'))
        # help/configure/discover command
        help_id = new_state('device_or_generic')
        transitions.append((command_id, help_id, 'help'))
        transitions.append((help_id, self.before_end_state, 'generic'))
        for d in devices:
            transitions.append((help_id, self.before_end_state, d))

        # answers
        answer_id = new_state('answer')
        transitions.append((bookkeeping_id, answer_id, 'answer'))
        for v in VALUES:
            if v != '0' and v != '1':
                transitions.append((answer_id, self.before_end_state, v))
        for v in ENTITIES:
            if v != 'NUMBER':
                for i in range(MAX_ARG_VALUES):
                    transitions.append(
                        (answer_id, self.before_end_state, v + '_' + str(i)))
        before_unit = new_state('answer_before_unit')
        for i in range(MAX_ARG_VALUES):
            transitions.append((answer_id, before_unit, 'NUMBER_' + str(i)))
        transitions.append((answer_id, before_unit, '0'))
        transitions.append((answer_id, before_unit, '1'))
        transitions.append((before_unit, self.end_state, '<<EOS>>'))
        for base_unit in UNITS:
            for unit in UNITS[base_unit]:
                transitions.append((before_unit, self.before_end_state, unit))

        def do_invocation(invocation_name, params, for_action=False):
            state_id = new_state(invocation_name)

            # allow one USERNAME_ parameter to follow the invocation immediately
            for i in range(MAX_ARG_VALUES):
                transitions.append((state_id, state_id, 'USERNAME_' + str(i)))

            # go to each parameter
            for param_name, param_type in params:
                if param_type in ('Any'):
                    continue
                elementtype = param_type
                is_array = False
                is_measure = False
                if param_type.startswith('Array('):
                    is_array = True
                    elementtype = param_type[len('Array('):-1]
                if elementtype in TYPE_RENAMES:
                    elementtype = TYPE_RENAMES[elementtype]
                if elementtype.startswith('Measure('):
                    is_measure = True
                    operators = ['is', '<', '>']
                    base_unit = elementtype[len('Measure('):-1]
                    values = UNITS[base_unit]
                elif elementtype.startswith('Enum('):
                    operators = ['is']
                    values = enum_types[elementtype]
                elif elementtype == 'Entity(tt:device)':
                    operators = ['is']
                    values = devices
                elif elementtype in TYPES:
                    operators, values = TYPES[elementtype]
                elif elementtype.startswith('Entity('):
                    operators = ['is']
                    values = [
                        'GENERIC_ENTITY_' + elementtype[len('Entity('):-1],
                        'QUOTED_STRING'
                    ]
                else:
                    operators, values = TYPES[elementtype]
                if is_array:
                    if for_action:
                        continue
                    else:
                        operators = ['has']
                elif for_action:
                    operators = ['is']

                before_op = new_state(invocation_name + '_tt-param:.' +
                                      param_name)
                transitions.append(
                    (state_id, before_op, 'tt-param:' + param_name))
                before_value = new_state(invocation_name + '_tt-param:' +
                                         param_name + ':value')

                for op in operators:
                    transitions.append((before_op, before_value, op))
                if is_measure:
                    before_unit = new_state(invocation_name + '_tt-param:' +
                                            param_name + ':unit')
                    for i in range(MAX_ARG_VALUES):
                        transitions.append((before_value, before_unit, '0'))
                        transitions.append((before_value, before_unit, '1'))
                        transitions.append(
                            (before_value, before_unit, 'NUMBER_' + str(i)))
                    for unit in values:
                        transitions.append((before_unit, state_id, unit))
                else:
                    for v in values:
                        if v[0].isupper():
                            for i in range(MAX_ARG_VALUES):
                                transitions.append(
                                    (before_value, state_id, v + '_' + str(i)))
                        else:
                            transitions.append((before_value, state_id, v))
                if is_measure and base_unit == 'ms':
                    for i in range(MAX_ARG_VALUES):
                        transitions.append(
                            (before_value, state_id, 'DURATION_' + str(i)))
                for v in trigger_or_query_params:
                    transitions.append((before_value, state_id, v))

            return state_id

        # rules
        rule_id = new_state('rule')
        transitions.append((self.start_state, rule_id, 'rule'))
        trigger_ids = []
        query_ids = []

        for trigger_name, params in triggers.items():
            state_id = do_invocation(trigger_name, params, for_action=False)
            transitions.append((rule_id, state_id, trigger_name))
            trigger_ids.append(state_id)
        for query_name, params in queries.items():
            state_id = do_invocation(query_name, params, for_action=False)
            for trigger_id in trigger_ids:
                transitions.append((trigger_id, state_id, query_name))
            query_ids.append(state_id)
        for action_name, params in actions.items():
            state_id = do_invocation(action_name, params, for_action=True)
            for query_id in query_ids:
                transitions.append((query_id, state_id, action_name))
            transitions.append((state_id, self.end_state, '<<EOS>>'))

        # do a second copy of the transition matrix for split sequences
        self.function_states = np.zeros((self.num_functions, ), dtype=np.int32)
        self.function_states.fill(-1)
        for part in ('trigger', 'query', 'action'):
            for function_name, params in self.functions[part].items():
                token = self.dictionary[
                    function_name] - self.num_control_tokens - self.num_begin_tokens
                state_id = do_invocation(function_name,
                                         params,
                                         for_action=(part == 'action'))
                transitions.append((state_id, self.end_state, '<<EOS>>'))
                self.function_states[token] = state_id

        # now build the actual DFA
        num_states = len(states)
        self.num_states = num_states
        print("num states", num_states)
        print("num tokens", self.output_size)
        self.transition_matrix = np.zeros((num_states, self.output_size),
                                          dtype=np.int32)
        self.transition_matrix.fill(-1)
        self.allowed_token_matrix = np.zeros((num_states, self.output_size),
                                             dtype=np.bool8)

        for from_state, to_state, token in transitions:
            token_id = self.dictionary[token]
            if self.transition_matrix[from_state, token_id] != -1 and \
                self.transition_matrix[from_state, token_id] != to_state:
                raise ValueError("Ambiguous transition around token " + token +
                                 " in state " + state_names[from_state])
            self.transition_matrix[from_state, token_id] = to_state
            self.allowed_token_matrix[from_state, token_id] = True

        if True:
            visited = set()

            def dfs(state):
                visited.add(state)
                any_out = False
                for next_state in self.transition_matrix[state]:
                    if next_state == -1:
                        continue
                    any_out = True
                    if next_state in visited:
                        continue
                    dfs(next_state)
                if not any_out:
                    raise ValueError(
                        'Reachable state %d (%s) has no outgoing states' %
                        (state, state_names[state]))

            dfs(self.start_state)

        self.state_names = state_names
Exemple #43
0
 def local(self, groups):
     self._local = OrderedSet(groups)
Exemple #44
0
class ThingtalkGrammar(AbstractGrammar):
    def __init__(self, filename):
        super().__init__()

        triggers = OrderedDict()
        queries = OrderedDict()
        actions = OrderedDict()
        functions = dict(trigger=triggers, query=queries, action=actions)
        self.functions = functions
        self.entities = OrderedSet()
        devices = []
        trigger_or_query_params = set()

        enum_types = OrderedDict()

        # Token order:
        # first the padding, go and end of sentence
        # then the begin tokens
        # then triggers - queries - actions
        # in this order
        # then parameters names
        # then operators
        # then values
        # then entity tokens
        #
        # This order is important as it affects the 3-part aligner
        # algorithm

        tokens = ['<<PAD>>', '<<EOS>>', '<<GO>>']
        self.num_control_tokens = 3
        tokens += BEGIN_TOKENS
        self.num_begin_tokens = len(BEGIN_TOKENS)

        # add the special functions
        functions['trigger']['tt:$builtin.now'] = []
        functions['query']['tt:$builtin.noop'] = []
        functions['action']['tt:$builtin.notify'] = []

        param_tokens = OrderedSet()
        param_tokens.add('tt-param:$event')
        trigger_or_query_params.add('tt-param:$event')

        with open(filename, 'r') as fp:
            for line in fp.readlines():
                line = line.strip().split()
                function_type = line[0]
                function = line[1]
                if function_type == 'device':
                    devices.append(function)
                    continue
                if function_type == 'entity':
                    self.entities.add(function)
                    continue

                parameters = line[2:]
                paramlist = []
                functions[function_type][function] = paramlist

                for i in range(len(parameters) // 2):
                    param = parameters[2 * i]
                    type = parameters[2 * i + 1]

                    paramlist.append((param, type))
                    param_tokens.add('tt-param:' + param)
                    if function_type != 'action':
                        trigger_or_query_params.add('tt-param:' + param)

                    if type.startswith('Array('):
                        elementtype = type[len('Array('):-1]
                    else:
                        elementtype = type
                    if elementtype.startswith('Enum('):
                        enums = elementtype[len('Enum('):-1].split(',')
                        if not elementtype in enum_types:
                            enum_types[elementtype] = enums

        for function_type in ('trigger', 'query', 'action'):
            for function in functions[function_type]:
                tokens.append(function)
        self.num_functions = len(
            tokens) - self.num_control_tokens - self.num_begin_tokens

        tokens += param_tokens
        self.num_params = len(param_tokens)

        tokens += OPERATORS
        tokens += VALUES
        tokens += SPECIAL_TOKENS
        tokens += BOOKKEEPING_TOKENS
        tokens += devices

        enumtokenset = set()
        for enum_type in enum_types.values():
            for enum in enum_type:
                if enum in enumtokenset:
                    continue
                enumtokenset.add(enum)
                tokens.append(enum)

        for unitlist in UNITS.values():
            tokens += unitlist

        for i in range(MAX_ARG_VALUES):
            for entity in ENTITIES:
                tokens.append(entity + "_" + str(i))
        for generic_entity in self.entities:
            for i in range(MAX_ARG_VALUES):
                tokens.append('GENERIC_ENTITY_' + generic_entity + "_" +
                              str(i))

        print('num functions', self.num_functions)
        print('num triggers', len(self.functions['trigger']))
        print('num queries', len(self.functions['query']))
        print('num actions', len(self.functions['action']))
        print('num params', self.num_params)
        first_value_token = self.num_functions + self.num_begin_tokens + self.num_control_tokens
        print('num value tokens', len(tokens) - first_value_token)

        self.tokens = tokens
        self.dictionary = dict()
        for i, token in enumerate(self.tokens):
            self.dictionary[token] = i

        # build a DFA that will parse the thingtalk-ish code

        states = []
        transitions = []
        state_names = []

        def to_ids(tokens, words):
            return list([words[x] for x in tokens])

        def add_allowed_tokens(state, tokens):
            state[to_ids(tokens, self.dictionary)] = 1

        def new_state(name):
            state = np.zeros((self.output_size, ))
            states.append(state)
            state_names.append(name)
            return len(states) - 1

        # start with one of the begin tokens
        self.start_state = new_state('start')

        # in the before end state we just wait for EOS
        self.before_end_state = new_state('before_end')

        # in the end state we are done
        self.end_state = new_state('end')
        transitions.append((self.before_end_state, self.end_state, '<<EOS>>'))
        transitions.append((self.end_state, self.end_state, '<<PAD>>'))

        # bookkeeping
        bookkeeping_id = new_state('bookkeeping')
        transitions.append((self.start_state, bookkeeping_id, 'bookkeeping'))
        transitions.append((bookkeeping_id, self.end_state, '<<EOS>>'))
        self.bookeeping_state_id = bookkeeping_id

        # special
        special_id = new_state('special')
        transitions.append((bookkeeping_id, special_id, 'special'))
        for t in SPECIAL_TOKENS:
            transitions.append((special_id, self.before_end_state, t))

        # command
        command_id = new_state('command')
        transitions.append((bookkeeping_id, command_id, 'command'))
        # help/configure/discover command
        help_id = new_state('device_or_generic')
        transitions.append((command_id, help_id, 'help'))
        transitions.append((help_id, self.before_end_state, 'generic'))
        for d in devices:
            transitions.append((help_id, self.before_end_state, d))

        # answers
        answer_id = new_state('answer')
        transitions.append((bookkeeping_id, answer_id, 'answer'))
        for v in VALUES:
            if v != '0' and v != '1':
                transitions.append((answer_id, self.before_end_state, v))
        for v in ENTITIES:
            if v != 'NUMBER':
                for i in range(MAX_ARG_VALUES):
                    transitions.append(
                        (answer_id, self.before_end_state, v + '_' + str(i)))
        before_unit = new_state('answer_before_unit')
        for i in range(MAX_ARG_VALUES):
            transitions.append((answer_id, before_unit, 'NUMBER_' + str(i)))
        transitions.append((answer_id, before_unit, '0'))
        transitions.append((answer_id, before_unit, '1'))
        transitions.append((before_unit, self.end_state, '<<EOS>>'))
        for base_unit in UNITS:
            for unit in UNITS[base_unit]:
                transitions.append((before_unit, self.before_end_state, unit))

        def do_invocation(invocation_name, params, for_action=False):
            state_id = new_state(invocation_name)

            # allow one USERNAME_ parameter to follow the invocation immediately
            for i in range(MAX_ARG_VALUES):
                transitions.append((state_id, state_id, 'USERNAME_' + str(i)))

            # go to each parameter
            for param_name, param_type in params:
                if param_type in ('Any'):
                    continue
                elementtype = param_type
                is_array = False
                is_measure = False
                if param_type.startswith('Array('):
                    is_array = True
                    elementtype = param_type[len('Array('):-1]
                if elementtype in TYPE_RENAMES:
                    elementtype = TYPE_RENAMES[elementtype]
                if elementtype.startswith('Measure('):
                    is_measure = True
                    operators = ['is', '<', '>']
                    base_unit = elementtype[len('Measure('):-1]
                    values = UNITS[base_unit]
                elif elementtype.startswith('Enum('):
                    operators = ['is']
                    values = enum_types[elementtype]
                elif elementtype == 'Entity(tt:device)':
                    operators = ['is']
                    values = devices
                elif elementtype in TYPES:
                    operators, values = TYPES[elementtype]
                elif elementtype.startswith('Entity('):
                    operators = ['is']
                    values = [
                        'GENERIC_ENTITY_' + elementtype[len('Entity('):-1],
                        'QUOTED_STRING'
                    ]
                else:
                    operators, values = TYPES[elementtype]
                if is_array:
                    if for_action:
                        continue
                    else:
                        operators = ['has']
                elif for_action:
                    operators = ['is']

                before_op = new_state(invocation_name + '_tt-param:.' +
                                      param_name)
                transitions.append(
                    (state_id, before_op, 'tt-param:' + param_name))
                before_value = new_state(invocation_name + '_tt-param:' +
                                         param_name + ':value')

                for op in operators:
                    transitions.append((before_op, before_value, op))
                if is_measure:
                    before_unit = new_state(invocation_name + '_tt-param:' +
                                            param_name + ':unit')
                    for i in range(MAX_ARG_VALUES):
                        transitions.append((before_value, before_unit, '0'))
                        transitions.append((before_value, before_unit, '1'))
                        transitions.append(
                            (before_value, before_unit, 'NUMBER_' + str(i)))
                    for unit in values:
                        transitions.append((before_unit, state_id, unit))
                else:
                    for v in values:
                        if v[0].isupper():
                            for i in range(MAX_ARG_VALUES):
                                transitions.append(
                                    (before_value, state_id, v + '_' + str(i)))
                        else:
                            transitions.append((before_value, state_id, v))
                if is_measure and base_unit == 'ms':
                    for i in range(MAX_ARG_VALUES):
                        transitions.append(
                            (before_value, state_id, 'DURATION_' + str(i)))
                for v in trigger_or_query_params:
                    transitions.append((before_value, state_id, v))

            return state_id

        # rules
        rule_id = new_state('rule')
        transitions.append((self.start_state, rule_id, 'rule'))
        trigger_ids = []
        query_ids = []

        for trigger_name, params in triggers.items():
            state_id = do_invocation(trigger_name, params, for_action=False)
            transitions.append((rule_id, state_id, trigger_name))
            trigger_ids.append(state_id)
        for query_name, params in queries.items():
            state_id = do_invocation(query_name, params, for_action=False)
            for trigger_id in trigger_ids:
                transitions.append((trigger_id, state_id, query_name))
            query_ids.append(state_id)
        for action_name, params in actions.items():
            state_id = do_invocation(action_name, params, for_action=True)
            for query_id in query_ids:
                transitions.append((query_id, state_id, action_name))
            transitions.append((state_id, self.end_state, '<<EOS>>'))

        # do a second copy of the transition matrix for split sequences
        self.function_states = np.zeros((self.num_functions, ), dtype=np.int32)
        self.function_states.fill(-1)
        for part in ('trigger', 'query', 'action'):
            for function_name, params in self.functions[part].items():
                token = self.dictionary[
                    function_name] - self.num_control_tokens - self.num_begin_tokens
                state_id = do_invocation(function_name,
                                         params,
                                         for_action=(part == 'action'))
                transitions.append((state_id, self.end_state, '<<EOS>>'))
                self.function_states[token] = state_id

        # now build the actual DFA
        num_states = len(states)
        self.num_states = num_states
        print("num states", num_states)
        print("num tokens", self.output_size)
        self.transition_matrix = np.zeros((num_states, self.output_size),
                                          dtype=np.int32)
        self.transition_matrix.fill(-1)
        self.allowed_token_matrix = np.zeros((num_states, self.output_size),
                                             dtype=np.bool8)

        for from_state, to_state, token in transitions:
            token_id = self.dictionary[token]
            if self.transition_matrix[from_state, token_id] != -1 and \
                self.transition_matrix[from_state, token_id] != to_state:
                raise ValueError("Ambiguous transition around token " + token +
                                 " in state " + state_names[from_state])
            self.transition_matrix[from_state, token_id] = to_state
            self.allowed_token_matrix[from_state, token_id] = True

        if True:
            visited = set()

            def dfs(state):
                visited.add(state)
                any_out = False
                for next_state in self.transition_matrix[state]:
                    if next_state == -1:
                        continue
                    any_out = True
                    if next_state in visited:
                        continue
                    dfs(next_state)
                if not any_out:
                    raise ValueError(
                        'Reachable state %d (%s) has no outgoing states' %
                        (state, state_names[state]))

            dfs(self.start_state)

        self.state_names = state_names

    def get_embeddings(self, use_types=False):
        if not use_types:
            return np.identity(self.output_size, np.float32)

        num_entity_tokens = (len(ENTITIES) +
                             len(self.entities)) * MAX_ARG_VALUES
        num_other_tokens = len(self.tokens) - num_entity_tokens

        num_entities = len(ENTITIES) + len(self.entities)
        embed_size = num_other_tokens + num_entities + MAX_ARG_VALUES
        embedding = np.zeros((len(self.tokens), embed_size), dtype=np.float32)
        for token_id, token in enumerate(self.tokens):
            if '_' in token and token[0].isupper():
                continue
            embedding[token_id, token_id] = 1
        for i, entity in enumerate(ENTITIES):
            assert not np.any(embedding[:, num_other_tokens + i] > 0)
            for j in range(MAX_ARG_VALUES):
                token_id = self.dictionary[entity + '_' + str(j)]
                embedding[token_id, num_other_tokens + i] = 1
                embedding[token_id, num_other_tokens + num_entities + j] = 1
        for i, entity in enumerate(self.entities):
            assert not np.any(
                embedding[:, num_other_tokens + len(ENTITIES) + i] > 0)
            for j in range(MAX_ARG_VALUES):
                token_id = self.dictionary['GENERIC_ENTITY_' + entity + '_' +
                                           str(j)]
                embedding[token_id, num_other_tokens + len(ENTITIES) + i] = 1
                embedding[token_id, num_other_tokens + num_entities + j] = 1

        for i in range(len(embedding)):
            assert np.any(embedding[i] > 0)
        return embedding

    def dump_tokens(self):
        for token in self.tokens:
            print(token)

    def split_batch_in_parts(self, labels_batch):
        batch_size = len(labels_batch)
        top_batch = np.empty((batch_size, ), dtype=np.int32)
        special_label_batch = np.zeros((batch_size, MAX_SPECIAL_LENGTH),
                                       dtype=np.int32)
        part_function_batches = dict()
        part_sequence_batches = dict()
        part_sequence_length_batches = dict()
        for part in ('trigger', 'query', 'action'):
            part_function_batches[part] = np.zeros((batch_size, ),
                                                   dtype=np.int32)
            part_sequence_batches[part] = np.zeros(
                (batch_size, MAX_PRIMITIVE_LENGTH), dtype=np.int32)
            part_sequence_length_batches[part] = np.zeros((batch_size, ),
                                                          dtype=np.int32)

        rule_token = self.dictionary['rule']
        first_value_token = self.num_functions + self.num_begin_tokens + self.num_control_tokens
        for i, label in enumerate(labels_batch):
            top_batch[i] = label[0]
            if top_batch[i] != rule_token:
                special_label_batch[i] = label[1:1 + MAX_SPECIAL_LENGTH]
                for part in ('trigger', 'query', 'action'):
                    if part == 'trigger':
                        function_offset = self.num_begin_tokens + self.num_control_tokens
                    elif part == 'query':
                        function_offset = self.num_begin_tokens + self.num_control_tokens + len(
                            self.functions['trigger'])
                    else:
                        function_offset = self.num_begin_tokens + self.num_control_tokens + len(
                            self.functions['trigger']) + len(
                                self.functions['query'])
                    # add dummy values to the sequences to preserve the ability to compute
                    # losses and grammar constraints
                    part_function_batches[part][i] = function_offset
                    part_sequence_batches[part][i, 0] = self.end
            else:
                special_label_batch[i, 0] = self.end
                j = 1
                for part in ('trigger', 'query', 'action'):
                    if part == 'trigger':
                        function_offset = self.num_begin_tokens + self.num_control_tokens
                    elif part == 'query':
                        function_offset = self.num_begin_tokens + self.num_control_tokens + len(
                            self.functions['trigger'])
                    else:
                        function_offset = self.num_begin_tokens + self.num_control_tokens + len(
                            self.functions['trigger']) + len(
                                self.functions['query'])
                    function_max = len(self.functions[part])
                    assert function_offset <= label[
                        j] < function_offset + function_max, (
                            function_offset, function_max, label[j],
                            self.tokens[label[j]])

                    part_function_batches[part][i] = label[j]
                    j += 1
                    start = j
                    while label[j] >= first_value_token:
                        j += 1
                    end = j
                    assert end - start + 1 < MAX_PRIMITIVE_LENGTH
                    part_sequence_batches[part][i, 0:end -
                                                start] = label[start:end]
                    part_sequence_batches[part][i, end - start] = self.end
                    part_sequence_length_batches[part][i] = end - start + 1
        return top_batch, special_label_batch, part_function_batches, part_sequence_batches, part_sequence_length_batches

    def vectorize_program(self, program, max_length=60):
        vector, length = super().vectorize_program(program, max_length)
        self.normalize_sequence(vector)
        return vector, length

    def parse(self, program):
        curr_state = self.start_state
        for token_id in program:
            next = self.transition_matrix[curr_state, token_id]
            if next == -1:
                raise ValueError("Unexpected token " + self.tokens[token_id] +
                                 " in " + (' '.join(self.tokens[x]
                                                    for x in program)) +
                                 " (in state " + self.state_names[curr_state] +
                                 ")")
            #print("transition", self.state_names[curr_state], "->", self.state_names[next])
            curr_state = next

        if curr_state != self.end_state:
            raise ValueError("Premature end of program in " +
                             (' '.join(self.tokens[x]
                                       for x in program)) + " (in state " +
                             self.state_names[curr_state] + ")")
        #print(*(self.tokens[x] for x in program))

    def parse_all(self, fp):
        vectors = []
        for line in fp.readlines():
            try:
                program = line.strip().split()
                vector = self.vectorize_program(program)[0]
                self.parse(vector)
                vectors.append(vector)
            except ValueError as e:
                print(e)
        return np.array(vectors, dtype=np.int32)

    def get_function_init_state(self, function_tokens):
        next_state = tf.gather(
            self.function_states, function_tokens -
            (self.num_begin_tokens + self.num_control_tokens))
        assert2 = tf.Assert(tf.reduce_all(next_state >= 0), [function_tokens])
        with tf.control_dependencies([assert2]):
            return tf.identity(next_state)

    def get_init_state(self, batch_size):
        return tf.ones((batch_size, ), dtype=tf.int32) * self.start_state

    def constrain_value_logits(self, logits, curr_state):
        first_value_token = self.num_functions + self.num_begin_tokens + self.num_control_tokens
        num_value_tokens = self.output_size - first_value_token
        value_allowed_token_matrix = np.concatenate(
            (self.allowed_token_matrix[:, :self.num_control_tokens],
             self.allowed_token_matrix[:, first_value_token:]),
            axis=1)

        with tf.name_scope('constrain_logits'):
            allowed_tokens = tf.gather(tf.constant(value_allowed_token_matrix),
                                       curr_state)
            assert allowed_tokens.get_shape()[1:] == (self.num_control_tokens +
                                                      num_value_tokens, )

            constrained_logits = logits - tf.to_float(
                tf.logical_not(allowed_tokens)) * 1e+10
        return constrained_logits

    def constrain_logits(self, logits, curr_state):
        with tf.name_scope('constrain_logits'):
            allowed_tokens = tf.gather(tf.constant(self.allowed_token_matrix),
                                       curr_state)
            assert allowed_tokens.get_shape()[1:] == (self.output_size, )

            constrained_logits = logits - tf.to_float(
                tf.logical_not(allowed_tokens)) * 1e+10
        return constrained_logits

    def value_transition(self, curr_state, next_symbols, batch_size):
        first_value_token = self.num_functions + self.num_begin_tokens + self.num_control_tokens
        num_value_tokens = self.output_size - first_value_token
        with tf.name_scope('grammar_transition'):
            adjusted_next_symbols = tf.where(
                next_symbols >= self.num_control_tokens,
                next_symbols + (first_value_token - self.num_control_tokens),
                next_symbols)

            assert1 = tf.Assert(
                tf.reduce_all(
                    tf.logical_and(next_symbols < num_value_tokens,
                                   next_symbols >= 0)),
                [curr_state, next_symbols])
            with tf.control_dependencies([assert1]):
                transitions = tf.gather(tf.constant(self.transition_matrix),
                                        curr_state)
            assert transitions.get_shape()[1:] == (self.output_size, )

            indices = tf.stack(
                (tf.range(0, batch_size), adjusted_next_symbols), axis=1)
            next_state = tf.gather_nd(transitions, indices)

            assert2 = tf.Assert(
                tf.reduce_all(next_state >= 0),
                [curr_state, adjusted_next_symbols, next_state])
            with tf.control_dependencies([assert2]):
                return tf.identity(next_state)

    def transition(self, curr_state, next_symbols, batch_size):
        with tf.name_scope('grammar_transition'):
            transitions = tf.gather(tf.constant(self.transition_matrix),
                                    curr_state)
            assert transitions.get_shape()[1:] == (self.output_size, )

            indices = tf.stack((tf.range(0, batch_size), next_symbols), axis=1)
            next_state = tf.gather_nd(transitions, indices)
            return next_state

    def _normalize_invocation(self, seq, start):
        assert self.tokens[seq[start]].startswith('tt:')
        if self.tokens[seq[start]].startswith('USERNAME_'):
            start += 1
        end = start

        params = []
        while end < len(seq) and seq[end] != self.end and self.tokens[
                seq[end]].startswith('tt-param:'):
            param_id = seq[end]
            end += 1
            if end >= len(seq) or seq[end] == self.end:
                # truncated output
                return end
            operator = seq[end]
            end += 1
            if end >= len(seq) or seq[end] == self.end:
                # this can occur at training time, if the output is truncated
                #raise AssertionError("missing value for " + self.tokens[param_id])
                params.append((param_id, operator, []))
                continue
            param_value = [seq[end]]
            end += 1
            while end < len(seq) and seq[end] != self.end and not self.tokens[
                    seq[end]].startswith('tt:'):
                param_value.append(seq[end])
                end += 1
            params.append((param_id, operator, param_value))
        params.sort(key=lambda x: x[0])
        assert end <= len(seq)

        i = start
        for param_id, operator, param_value in params:
            seq[i] = param_id
            seq[i + 1] = operator
            seq[i + 2:i + 2 + len(param_value)] = param_value
            i += 2 + len(param_value)
            assert i <= end

        return end

    def normalize_sequence(self, seq):
        i = 0
        if seq[0] == self.dictionary['rule']:
            i += 1
            i = self._normalize_invocation(seq, i)
            if i < len(seq) and seq[i] != self.end:
                i = self._normalize_invocation(seq, i)
            if i < len(seq) and seq[i] != self.end:
                i = self._normalize_invocation(seq, i)

    def compare(self, gold, decoded):
        decoded = list(decoded)
        #self._normalize_sequence(decoded)
        return gold == decoded
Exemple #45
0
 def __init__(self, advance_func=None):
     self._player_queue = OrderedSet()
     self.advance_func = advance_func or self._default_advance_turn
Exemple #46
0
 def placeholders(self):
     return OrderedSet(
         Placeholder(body).name
         for body in re.findall(self.placeholder_pattern, self.content))
Exemple #47
0
def _deserialize_graph(graph_pb):
    """
    Will deserialize a graph and return the list of all ops in that graph. Does not bother
    filtering down to only the original set of ops the user passed in for serialization
    (if that's what the user desired upon serializing with the serialization
    only_return_handle_ops parameter).
    """
    # For safety we clear this registry
    GLOBAL_AXIS_REGISTRY.clear()

    ops = list(map(protobuf_to_op, graph_pb.ops))
    uuid_lookup = {op.uuid.bytes: op for op in ops}
    for edge in graph_pb.edges:
        head_op = uuid_lookup[edge.from_uuid.uuid]
        tail_op = uuid_lookup[edge.to_uuid.uuid]
        if edge.edge_type == ops_pb.Edge.DATA:  # args
            tail_op._args = tail_op._args + (head_op, )
        elif edge.edge_type == ops_pb.Edge.CONTROL:  # control_deps
            head_op._control_deps.add(tail_op)
        elif edge.edge_type == ops_pb.Edge.CONTAINER:
            if '_ngraph_forward' in edge.attrs:  # forward
                head_op._forward = tail_op
            else:  # ops
                if not hasattr(head_op, '_ops'):
                    head_op._ops = []
                head_op._ops.append(tail_op)
        elif edge.edge_type == ops_pb.Edge.OTHER:
            if '_ngraph_attribute' in edge.attrs:
                setattr(head_op,
                        edge.attrs['_ngraph_attribute'].scalar.string_val,
                        tail_op)
            elif '_ngraph_list_attribute' in edge.attrs:
                key = edge.attrs['_ngraph_list_attribute'].scalar.string_val
                # import pdb; pdb.set_trace()
                if hasattr(head_op, key):
                    getattr(head_op, key).add(tail_op)
                else:
                    setattr(head_op, key, OrderedSet([tail_op]))
        else:
            raise ValueError("Edge not mapped to op: {}".format(edge))

    # This must come after tensor has been set which occurs after edges
    # op.dtype
    for py_op, pb_op in zip(ops, graph_pb.ops):
        py_op.dtype = pb_to_dtype(pb_op.dtype)

    # Done with this and don't want it to bleed to subsequent serializations
    GLOBAL_AXIS_REGISTRY.clear()

    # Assemble list of nodes to return depending on if the user wanted to
    # return all ops or only those that were originally serialized (and
    # implicitly those upstream)
    final_ops = []
    for op in ops:
        if hasattr(op, '_ngraph_ser_handle'):
            del op._ngraph_ser_handle
            final_ops.append(op)
    if len(final_ops) > 0:
        return final_ops
    else:
        return ops
Exemple #48
0
class HetrComputation(Computation):
    """
    Lightweight wrapper class for handling runtime execution of child computations for Hetr
    """
    def __init__(self, hetr, computation_op):
        self.child_computations = dict()
        self.transformer = hetr
        self.send_nodes = hetr.send_nodes
        self.computation_op = computation_op

        # self.returns could be replaced by comp_op.returns if it were expressed as a set
        self.returns = OrderedSet()
        if isinstance(computation_op.returns, collections.Container):
            self.returns.update(list(computation_op.returns))
        elif isinstance(computation_op.returns, Op):
            self.returns.update(list([computation_op.returns]))

        # if one of the requested results is marked as distributed across devices,
        # wrap it in a ResultOp to facilitate DistributedPass inserting a gather operation
        new_returns = OrderedSet()
        for op in self.returns:
            if 'device_id' in op.metadata and \
                    isinstance(op.metadata['device_id'], (list, tuple)):
                op.metadata['is_split_op'] = True
                new_result = ResultOp(device_id=0, args=tuple([op]))
                op.metadata['hetr_replaced_by'] = new_result
                new_result.metadata['replaces_op'] = op
                new_returns.add(new_result)
            else:
                new_returns.add(op)

        # Do Hetr passes
        pass_ops = new_returns | OrderedSet(self.computation_op.parameters)
        for graph_pass in self.transformer.graph_passes:
            pass_ops = pass_ops | OrderedSet(hetr.send_nodes)
            graph_pass.do_pass(ops=pass_ops)

        # hack around new TensorValueOp that wraps AssignableTensorOp
        # autogenerated by creating a ComputationOp:
        for p in self.computation_op.parameters:
            if isinstance(p, TensorValueOp):
                p.metadata.update(p.states_read[0].metadata)

        # simplify by already having asynctrans made by passes
        for t_name, trans in iteritems(self.transformer.child_transformers):
            my_params = [
                (g_pos, p)
                for g_pos, p in enumerate(self.computation_op.parameters)
                if p.metadata['transformer'] == t_name
            ]
            my_ops = [
                op for op in self.send_nodes | new_returns
                if op.metadata['transformer'] == t_name
            ]
            transform_ops = [
                op.args[0] if isinstance(op, ResultOp) else op for op in my_ops
            ]

            comp = trans.computation(transform_ops,
                                     tuple([p for pos, p in my_params]))
            comp.param_idx = [g_pos for g_pos, p in my_params]

            # when there is a ResultOp, hack around it
            comp.returns = dict()
            for i, op in enumerate(my_ops):
                if op in self.returns and 'hetr_replaced_by' not in op.metadata:
                    comp.returns[op] = i
                elif 'replaces_op' in op.metadata and op.metadata[
                        'replaces_op'] in self.returns:
                    comp.returns[op.metadata['replaces_op']] = i
            self.child_computations[t_name] = comp

    def __call__(self, *args, **kwargs):
        """
        Executes child computations in parallel.

        :arg args: list of values to the placeholders specified in __init__ *args

        :return: tuple of return values, one per return specified in __init__ returns list.
        """
        args = self.unpack_args_or_feed_dict(args, kwargs)
        for child in itervalues(self.child_computations):
            child.feed_input([args[i] for i in child.param_idx])

        return_vals = dict()
        for child in itervalues(self.child_computations):
            return_vals.update(child.get_results())
        if isinstance(self.computation_op.returns, Op):
            return return_vals[self.computation_op.returns]
        elif isinstance(self.computation_op.returns, collections.Set):
            return return_vals
        elif isinstance(self.computation_op.returns, collections.Sequence):
            return tuple(return_vals[op] for op in self.computation_op.returns)
        else:
            return None
Exemple #49
0
    def __init__(self, hetr, computation_op):
        self.child_computations = dict()
        self.transformer = hetr
        self.send_nodes = hetr.send_nodes
        self.computation_op = computation_op

        # self.returns could be replaced by comp_op.returns if it were expressed as a set
        self.returns = OrderedSet()
        if isinstance(computation_op.returns, collections.Container):
            self.returns.update(list(computation_op.returns))
        elif isinstance(computation_op.returns, Op):
            self.returns.update(list([computation_op.returns]))

        # if one of the requested results is marked as distributed across devices,
        # wrap it in a ResultOp to facilitate DistributedPass inserting a gather operation
        new_returns = OrderedSet()
        for op in self.returns:
            if 'device_id' in op.metadata and \
                    isinstance(op.metadata['device_id'], (list, tuple)):
                op.metadata['is_split_op'] = True
                new_result = ResultOp(device_id=0, args=tuple([op]))
                op.metadata['hetr_replaced_by'] = new_result
                new_result.metadata['replaces_op'] = op
                new_returns.add(new_result)
            else:
                new_returns.add(op)

        # Do Hetr passes
        pass_ops = new_returns | OrderedSet(self.computation_op.parameters)
        for graph_pass in self.transformer.graph_passes:
            pass_ops = pass_ops | OrderedSet(hetr.send_nodes)
            graph_pass.do_pass(ops=pass_ops)

        # hack around new TensorValueOp that wraps AssignableTensorOp
        # autogenerated by creating a ComputationOp:
        for p in self.computation_op.parameters:
            if isinstance(p, TensorValueOp):
                p.metadata.update(p.states_read[0].metadata)

        # simplify by already having asynctrans made by passes
        for t_name, trans in iteritems(self.transformer.child_transformers):
            my_params = [
                (g_pos, p)
                for g_pos, p in enumerate(self.computation_op.parameters)
                if p.metadata['transformer'] == t_name
            ]
            my_ops = [
                op for op in self.send_nodes | new_returns
                if op.metadata['transformer'] == t_name
            ]
            transform_ops = [
                op.args[0] if isinstance(op, ResultOp) else op for op in my_ops
            ]

            comp = trans.computation(transform_ops,
                                     tuple([p for pos, p in my_params]))
            comp.param_idx = [g_pos for g_pos, p in my_params]

            # when there is a ResultOp, hack around it
            comp.returns = dict()
            for i, op in enumerate(my_ops):
                if op in self.returns and 'hetr_replaced_by' not in op.metadata:
                    comp.returns[op] = i
                elif 'replaces_op' in op.metadata and op.metadata[
                        'replaces_op'] in self.returns:
                    comp.returns[op.metadata['replaces_op']] = i
            self.child_computations[t_name] = comp
Exemple #50
0
 def calc_build(self,_seen=None):
     """decides if it needs to be built by recursively asking it's prerequisites
     the same question.
     _seen is an internal variable (a set) for optimising the search. I'll be relying 
     on the set being a mutable container in order to not have to pass it explicitly 
     back up the call stack."""
     #There is an oportunity to optimise calculations to occur only once for rules that are called multiple
     #times by using a shared (global) buildseq + _already_seen set, or by passing those structures into
     #the calc_build method call.
     #i.e. if (self in buildseq) or (self in _already_seen): return buildseq
     #Or we can memoize this method
     
     #updated_only should be calculated during build calculation time (rather than build time) for consistancy.
     self.updated_only #force evaluation of lazy property
     
     buildseq = OrderedSet()
     _seen = set() if not _seen else _seen
     _seen.add(self) # this will also solve any circular dependency issues!
     
     for req in self.order_only:
         if not os.path.exists(req):
             reqrule = Rule.get(req,None) #super(ExplicitRule,self).get(req,None)
             if reqrule:
                 if reqrule not in _seen:
                     buildseq.update(reqrule.calc_build())
                 else:
                     warnings.warn('rule for %r has already been processed' %req,stacklevel=2)
             else:
                 warnings.warn('%r has an order_only prerequisite with no rule' %self,stacklevel=2)
     
     for req in self.reqs:
         reqrule = Rule.get(req,None) #super(ExplicitRule,self).get(req,None)
         if reqrule:
             if reqrule not in _seen:
                 buildseq.update(reqrule.calc_build())
             else:
                 warnings.warn('rule for %r has already been processed' %req,stacklevel=2)
         else: #perform checks
             try:
                 self.get_mtime(req) #get_mtime is cached to reduce number of file accesses
             except OSError as e:
                 raise AssertionError("No rule or file found for %r for targets: %r" %(req,self.targets))
         
     if len(buildseq)==0:
         if self.PHONY or any([not os.path.exists(target) for target in self.targets]):
             buildseq.add(self)
         else:
             oldest_target = self._oldest_target
             
             #Since none of the prerequisites have rules that need to update, we can assume
             #that all prerequisites should be real files (phony rules always update which
             #should skip this section of code). Hence non-existing files imply an malformed build
             #file.
             for req in self.reqs:
                 try: 
                     req_mtime = self.get_mtime(req)
                     if req_mtime > oldest_target:
                         buildseq.add(self)
                         break
                         
                 except OSError as e: 
                     raise AssertionError("A non file prerequisite was found (%r) for targets %r in wrong code path" %(req,self.targets))
     else:
         buildseq.add(self)
     
     return buildseq
Exemple #51
0
class ComputationGraphTransformer(Transformer):
    def __init__(self, **kwargs):
        super(ComputationGraphTransformer, self).__init__(**kwargs)
        self.computations = OrderedSet()
        self.device_buffers = None
        self.op_tensors = None
        self.op_tensor_views = None
        self.initialize_allocations()
        self.finalized = False
        self.allocated = False
        self.initialized = False

    def run_registered_graph_passes(self, ops, **kwargs):
        for graph_pass in self.graph_passes:
            graph_pass.wrapped_do_pass(ops=ops, **kwargs)
        return ops

    def initialize_allocations(self):
        """
        Inititializes allocation caches.

        """
        self.op_tensors = dict()
        self.op_tensor_views = dict()
        self.device_buffers = OrderedSet()

    def get_op_tensor(self, op):
        """
        Returns the tensor allocated for this op.

        Args:
            op: A computation graph op.

        Returns:
            A device tensor (DeviceBuffer).

        """
        return self.op_tensors[
            op.forwarded.tensor.forwarded.tensor_description().base]

    def has_op_tensor(self, op):
        """
        Returns true if the op has a device tensor.

        Args:
            op: A computation graph op.

        Returns:
            True if the op has a device tensor.

        """
        return op.forwarded.tensor.forwarded.tensor_description(
        ).base in self.op_tensors

    def get_op_tensor_view(self, op):
        """
        Returns the tensor view for this op.

        Args:
            op: A computation graph op.

        Returns:
            A device tensor view.

        """
        return self.op_tensor_views[
            op.forwarded.tensor.forwarded.tensor_description()]

    def get_tensor_view_value(self, op, host_tensor=None):
        """
        Returns the contents of the tensor view for op.

        Args:
            op: The computation graph op.
            host_tensor: Optional tensor to copy value into.

        Returns:
            A NumPy tensor with the elements associated with op.

        """
        return self.get_op_tensor_view(op).get(host_tensor)

    def get_tensor_description_tensor(self, tensor_description):
        """
        Returns a tensor for a tensor description.

        Deprecated. Use op version.

        Args:
            tensor_description:

        Returns:
            A device tensor (DeviceBuffer).


        """
        return self.op_tensors[tensor_description.base]

    def has_tensor_description_tensor(self, tensor_description):
        """
        Tests if a tensor description has a device tensor.

        Deprecated, only used by flex.

        Args:
            tensor_description:

        Returns:
            True if the tensor description has a device tensor.

        """
        return tensor_description.base in self.op_tensors

    def get_tensor_description_tensor_view(self, tensor_description):
        """
        Returns a tensor for a tensor description.

        Deprecated, only used by flex.

        Args:
            tensor_description:

        Returns:
            A device tensor view.

        """
        return self.op_tensor_views[tensor_description]

    def host_to_device(self, computation, parameters, args):
        """
        Copy args to parameters in computation.

        Args:
            computation: The computation.
            parameters: Parameters of the computation.
            args: Values for the parameters.

        """
        for param, arg in zip(parameters, args):
            self.get_op_tensor_view(param)[...] = arg

    def device_to_host(self, computation, op, tensor=None):
        """
        Copy a computation result from the device back to the host.

        Args:
            computation: The computation.
            op: The op associated with the value.
            tensor: Optional tensor for returned value.

        Returns:
            The value of op.

        """
        if self.has_op_tensor(op):
            return self.get_op_tensor_view(op).get(tensor)

    @property
    def use_exop(self):
        """

        Returns: True if this transformer uses the execution graph.

        """
        return False

    @abc.abstractmethod
    def start_transform_allocate(self):
        """
        Called just before allocation code is transformed.
        """

    @abc.abstractmethod
    def finish_transform_allocate(self):
        """
        Called after last allocation is transformed.
        """

    @abc.abstractmethod
    def transform_ordered_ops(self, computation, ordered_ops, name):
        """
        Generate code to compute ordered_ops.

        Arguments:
            computation: The computation being compiled.
            ordered_ops: Ops to compute
            name: The name of the computation.

        Returns: Handle for generated code
        """

    @abc.abstractmethod
    def finish_transform(self):
        """
        Finish generating the model.
        """

    @abc.abstractmethod
    def allocate_storage(self):
        """
        Allocate storage on the device.
        """

    @abc.abstractmethod
    def device_buffer_storage(self, bytes, dtype, name):
        """
        Make a DeviceBuffer.

        Arguments:
            bytes: Size of buffer.
            dtype: dtype of buffer.
            name: Name of the storage variable

        returns: A DeviceBuffer.
        """

    def allocate(self):
        """
        Allocate storage and then initializes constants.

        Will finalize if not already done.
        """
        if self.allocated:
            return

        if not self.finalized:
            logging.info("Finalizing transformer.")
            self._transform_computations()

        self.allocate_storage()

        init_states = OrderedSet()
        for op in OrderedSet(self.ops):
            op = op.forwarded
            states = op.states_read | op.states_written
            for state in states:
                state = state.forwarded
                if state.initial_value is not None:
                    init_states.add(state)
        for state in init_states:
            tensor_description = state.tensor.tensor_description()
            self.get_tensor_description_tensor_view(tensor_description)[
                ...] = state.initial_value

        self.allocated = True

    def add_computation(self, computation):
        """
        Adds a computation to the transformer.

        Arguments:
            computation: A computation Op.

        Returns:
            Callable.
        """
        if self.finalized:
            raise ValueError(
                'Cannot create computations from a finalized transformer')
        computation = super(ComputationGraphTransformer,
                            self).add_computation(computation)
        self.computations.add(computation)
        return computation

    def initialize(self):
        """
        Initialize storage.  Will allocate if not already performed.
        """
        if self.initialized:
            return
        self.allocate()

        # Need to set initialized before we are done because the init computation will
        # try to initialize.
        self.initialized = True

    def _transform_computations(self):
        """
        Transform computation graphs to a form that can be run.
        """

        # Run passes on the computation graphs
        all_results = []
        for comp in self.computations:
            all_results.append(comp.computation_op)

        all_ops = self.run_registered_graph_passes(ops=all_results)

        # Collect up all ops from the graph and obtain the init graph
        all_ops = OrderedSet(Op.ordered_ops(all_ops))

        def ensure_tensor(op):
            op = op.forwarded
            tensor_description = op.tensor_description()
            base = tensor_description.base
            tensor = self.op_tensors.get(base, None)
            if tensor is None:
                tensor = self.device_buffer_storage(base.tensor_size,
                                                    base.dtype, base.name)
                self.op_tensors[base] = tensor
                self.device_buffers.add(tensor)
            tensor_view = tensor.device_tensor(tensor_description)
            self.op_tensor_views[tensor_description] = tensor_view

        self.ops = Op.ordered_ops(all_ops)
        for op in self.ops:
            if op.is_tensor_op:
                ensure_tensor(op)

        self.start_transform_allocate()
        for device_buffer in self.device_buffers:
            device_buffer.transform_allocate()
        self.finish_transform_allocate()

        # Compile the computations now that we know their storage
        for comp in self.computations:
            comp.computation_name = \
                self.transform_ordered_ops(comp,
                                           Op.ordered_ops([comp.computation_op]),
                                           name=comp.name)
        self.finish_transform()
        self.finalized = True
Exemple #52
0
    from construct import *
except ImportError as e:
    from modules.easydependencies import handle_importerror
    handle_importerror(e)

    from modules.easydependencies import setup_third_party
    setup_third_party()
    from modules.handyfunctions import get_frames_folder, os_adapt, get_modules_path, to_command, InfiniteTimer
    import sdl2.ext
    from sdl2 import *
    from sdl2.ext import colorpalettes
    from orderedset import OrderedSet
    from construct import *

kb_state = SDL_GetKeyboardState(None)
pressed_key = OrderedSet()


class DynamicBG:
    """Make a dynamic background that changes colors"""
    def __init__(self,
                 factory,
                 renderer,
                 ext_window,
                 on_top_sprite=None,
                 i=188,
                 fps=60):
        self._factory = factory
        self._renderer = renderer
        self._on_top_sprite = on_top_sprite
        self._switch_on = True
class Node(object):
    UNKNOWN_STATE = "_UNKNOWN_"

    def __init__(self, id_, master_graph):
        self.id_ = id_
        self.master_graph = master_graph
        self.edges = OrderedSet()
        self._state = None

        # determine ownership
        self.determined = False
        self.graph = None

    def __repr__(self):
        if self.determined:
            if self.graph:
                if self.state is not self.UNKNOWN_STATE:
                    return "<Node#%s: %s %r>" \
                           % (self.id_, self.graph.name, self.state)
                else:
                    return "<Node#%s: %s>" % (self.id_, self.graph.name)
            else:
                return "<!Node#%s: DETERMINE NONE!>" % self.id_
        elif self.determined is False:
            if self.graph is self.master_graph:
                return "<!Node#%s: GUESS MASTER!>" % self.id_
            elif self.graph:
                if self.state is not self.UNKNOWN_STATE:
                    return "<Node#%s: ?%s %r>" \
                           % (self.id_, self.graph.name, self.state)
                else:
                    return "<Node#%s: ?%s>" % (self.id_, self.graph.name)
            else:
                return "<!Node#%s: NEW!>" % self.id_

    @property
    def state(self):
        if self._state is None:
            return self.UNKNOWN_STATE
        else:
            return self._state

    @state.setter
    def state(self, val):
        self._state = val

    @property
    def correct(self):
        # Allowed combinations:
        # 1. determined = True, graph is master
        # 2. determined = True, graph is not master
        # 3. determined = False, graph is not master
        if self.determined and self.graph is not None:
            return True
        if self.determined is False and isinstance(self.graph, LeafGraph):
            return True
        print("%s is not correct!" % self)
        return False

    @property
    def guessed(self):
        return self.determined is False and self.graph is not None

    @property
    def determined_master(self):
        return self.determined and self.graph is self.master_graph

    @property
    def is_new(self):
        return self.determined is False and self.graph is None

    def add_edge(self, edge):
        # NOTE: Order matters
        self.edges.add(edge)

    def guess_graph(self, graph):
        assert graph is not self.master_graph
        if self.determined:
            pass
        elif self.graph is None:
            self.graph = graph
        elif self.graph is graph:
            pass
        else:
            self.determine_graph(self.master_graph)

    def determine_graph(self, graph):
        if not self.determined:
            if self.graph is graph:
                graph.determine_node(self)
            else:
                self.master_graph.determine_node(self)
        elif self.determined_master is False and self.graph is not graph:
            raise RuntimeError("Node#%s is determined %s, but assigned %s"
                               % (self, self.graph.name, graph.name))

    def decide_edge(self, log):
        for edge in self.edges:
            if edge.accept(log):
                return edge
        return None

    def accept_edge(self, edge):
        return edge in self.edges
 def __init__(self):
     self.state_initialization_ops = OrderedSet()
Exemple #55
0
 def get_players(self):
     return OrderedSet(self._player_queue)
Exemple #56
0
 def remote(self, groups):
     self._remote = OrderedSet(self._process_remote_groups(groups))
Exemple #57
0
# encoding: cinje

: from orderedset import OrderedSet
: from marrow.util.url import QueryString

: def render_video_block block
: """Render a YouTube video embed."""

: classes = OrderedSet(block.properties.get('cls', '').split() + ['block', 'video'])

: if 'width' in block.properties
	: classes.add('col-md-' + str(block.properties.width))
: end

<div&{id=block.properties.get('id', block.id), class_=classes}>
	<iframe width="560" height="315" src="https://www.youtube.com/embed/${block.video}" frameborder="0" allowfullscreen></iframe>
</div>

Exemple #58
0
 def __init__(self, src_dataset_name, grouping_cols, new_dataset_name):
     super(GroupByOperator, self).__init__(src_dataset_name)
     self.grouping_cols = OrderedSet(grouping_cols)
     self.new_dataset_name = new_dataset_name
     self.grouped_dataset = None
     self._id = self.create_id()
Exemple #59
0
class City(simpy.Environment):
    def __init__(self, env, n_people, rng, x_range, y_range, start_time,
                 init_percent_sick, Human):
        self.env = env
        self.rng = rng
        self.x_range = x_range
        self.y_range = y_range
        self.total_area = (x_range[1] - x_range[0]) * (y_range[1] - y_range[0])
        self.n_people = n_people
        self.start_time = start_time
        self.init_percent_sick = init_percent_sick
        self.last_date_to_check_tests = self.env.timestamp.date()
        self.test_count_today = defaultdict(int)
        self.test_type_preference = list(
            zip(*sorted(TEST_TYPES.items(),
                        key=lambda x: x[1]['preference'])))[0]
        print("Initializing locations ...")
        self.initialize_locations()

        self.humans = []
        self.households = OrderedSet()
        print("Initializing humans ...")
        self.initialize_humans(Human)

        self.log_static_info()

        print("Computing their preferences")
        self._compute_preferences()
        self.tracker = Tracker(env, self)
        # self.tracker.track_initialized_covid_params(self.humans)

        self.intervention = None

    def create_location(self, specs, type, name, area=None):
        _cls = Location
        if type in ['household', 'senior_residency']:
            _cls = Household
        if type == 'hospital':
            _cls = Hospital

        return _cls(
            env=self.env,
            rng=self.rng,
            name=f"{type}:{name}",
            location_type=type,
            lat=self.rng.randint(*self.x_range),
            lon=self.rng.randint(*self.y_range),
            area=area,
            social_contact_factor=specs['social_contact_factor'],
            capacity=None if not specs['rnd_capacity'] else self.rng.randint(
                *specs['rnd_capacity']),
            surface_prob=specs['surface_prob'])

    @property
    def tests_available(self):
        if self.last_date_to_check_tests != self.env.timestamp.date():
            self.last_date_to_check_tests = self.env.timestamp.date()
            for k in self.test_count_today.keys():
                self.test_count_today[k] = 0
        return any(self.test_count_today[test_type] < TEST_TYPES[test_type]
                   ['capacity'] for test_type in self.test_type_preference)

    def get_available_test(self):
        for test_type in self.test_type_preference:
            if self.test_count_today[test_type] < TEST_TYPES[test_type][
                    'capacity']:
                self.test_count_today[test_type] += 1
                return test_type

    def initialize_locations(self):
        for location, specs in LOCATION_DISTRIBUTION.items():
            if location in ['household']:
                continue

            n = math.ceil(self.n_people / specs["n"])
            area = _get_random_area(n, specs['area'] * self.total_area,
                                    self.rng)
            locs = [
                self.create_location(specs, location, i, area[i])
                for i in range(n)
            ]
            setattr(self, f"{location}s", locs)

    def initialize_humans(self, Human):
        # allocate humans to houses such that (unsolved)
        # 1. average number of residents in a house is (approx.) 2.6
        # 2. not all residents are below 15 years of age
        # 3. age occupancy distribution follows HUMAN_DSITRIBUTION.residence_preference.house_size

        # current implementation is an approximate heuristic

        # make humans
        count_humans = 0
        house_allocations = {2: [], 3: [], 4: [], 5: []}
        n_houses = 0
        for age_bin, specs in HUMAN_DISTRIBUTION.items():
            n = math.ceil(specs['p'] * self.n_people)
            ages = self.rng.randint(*age_bin, size=n)

            senior_residency_preference = specs['residence_preference'][
                'senior_residency']

            professions = ['healthcare', 'school', 'others', 'retired']
            p = [specs['profession_profile'][x] for x in professions]
            profession = self.rng.choice(professions, p=p, size=n)

            for i in range(n):
                count_humans += 1
                age = ages[i]

                # residence
                res = None
                if self.rng.random() < senior_residency_preference:
                    res = self.rng.choice(self.senior_residencys)
                # workplace
                if profession[i] == "healthcare":
                    workplace = self.rng.choice(self.hospitals +
                                                self.senior_residencys)
                elif profession[i] == 'school':
                    workplace = self.rng.choice(self.schools)
                elif profession[i] == 'others':
                    type_of_workplace = self.rng.choice(
                        [0, 1, 2], p=OTHERS_WORKPLACE_CHOICE, size=1).item()
                    type_of_workplace = [
                        self.workplaces, self.stores, self.miscs
                    ][type_of_workplace]
                    workplace = self.rng.choice(type_of_workplace)
                else:
                    workplace = res

                self.humans.append(
                    Human(
                        env=self.env,
                        city=self,
                        rng=self.rng,
                        name=count_humans,
                        age=age,
                        household=res,
                        workplace=workplace,
                        profession=profession[i],
                        rho=RHO,
                        gamma=GAMMA,
                        infection_timestamp=self.start_time if
                        self.rng.random() < self.init_percent_sick else None))

        # assign houses
        # stores tuples - (location, current number of residents, maximum number of residents allowed)
        remaining_houses = []
        for human in self.humans:
            if human.household is not None:
                continue
            if len(remaining_houses) == 0:
                cap = self.rng.choice(range(1, 6),
                                      p=HOUSE_SIZE_PREFERENCE,
                                      size=1)
                x = self.create_location(LOCATION_DISTRIBUTION['household'],
                                         'household', len(self.households))

                remaining_houses.append((x, cap))

            # get_best_match
            res = None
            for c, (house, n_vacancy) in enumerate(remaining_houses):
                new_avg_age = (human.age + sum(x.age
                                               for x in house.residents)) / (
                                                   len(house.residents) + 1)
                if new_avg_age > MIN_AVG_HOUSE_AGE:
                    res = house
                    n_vacancy -= 1
                    if n_vacancy == 0:
                        remaining_houses = remaining_houses[:
                                                            c] + remaining_houses[
                                                                c + 1:]
                    break

            if res is None:
                for i, (l, u) in enumerate(HUMAN_DISTRIBUTION.keys()):
                    if l <= human.age < u:
                        bin = (l, u)
                        break

                house_size_preference = HUMAN_DISTRIBUTION[(
                    l, u)]['residence_preference']['house_size']
                cap = self.rng.choice(range(1, 6),
                                      p=house_size_preference,
                                      size=1)
                res = self.create_location(LOCATION_DISTRIBUTION['household'],
                                           'household', len(self.households))
                if cap - 1 > 0:
                    remaining_houses.append((res, cap - 1))

            # FIXME: there is some circular reference here
            res.residents.append(human)
            human.assign_household(res)
            self.households.add(res)

        # assign area to house
        area = _get_random_area(
            len(self.households),
            LOCATION_DISTRIBUTION['household']['area'] * self.total_area,
            self.rng)
        for i, house in enumerate(self.households):
            house.area = area[i]

        # this allows for easy O(1) access of humans for message passing
        self.hd = {human.name: human for human in self.humans}

    def log_static_info(self):
        for h in self.humans:
            Event.log_static_info(self, h, self.env.timestamp)

    @property
    def events(self):
        return list(itertools.chain(*[h.events for h in self.humans]))

    def events_slice(self, begin, end):
        return list(
            itertools.chain(*[h.events_slice(begin, end)
                              for h in self.humans]))

    def pull_events_slice(self, end):
        return list(
            itertools.chain(*[h.pull_events_slice(end) for h in self.humans]))

    def _compute_preferences(self):
        """ compute preferred distribution of each human for park, stores, etc."""
        for h in self.humans:
            h.stores_preferences = [
                (compute_distance(h.household, s) + 1e-1)**-1
                for s in self.stores
            ]
            h.parks_preferences = [
                (compute_distance(h.household, s) + 1e-1)**-1
                for s in self.parks
            ]

    def run(self, duration, outfile, start_time, all_possible_symptoms, port,
            n_jobs):
        self.current_day = 0

        print(f"INTERVENTION_DAY: {INTERVENTION_DAY}")
        while True:

            #
            if INTERVENTION_DAY >= 0 and self.current_day == INTERVENTION_DAY:
                self.intervention = get_intervention(INTERVENTION)
                _ = [h.notify(self.intervention) for h in self.humans]
                print(self.intervention)

            # update risk every day
            if isinstance(self.intervention, Tracing):
                self.intervention.update_human_risks(
                    city=self,
                    symptoms=all_possible_symptoms,
                    port=port,
                    n_jobs=n_jobs,
                    data_path=outfile)

            #
            # if (COLLECT_TRAINING_DATA or GET_RISK_PREDICTOR_METRICS) and (self.current_day == 0 and INTERVENTION_DAY < 0):
            #     _ = [h.notify(collect_training_data=True) for h in self.humans]
            #     print("naive risk calculation without changing behavior... Humans notified!")

            self.tracker.increment_day()
            self.current_day += 1

            # Let the day pass
            yield self.env.timeout(duration / TICK_MINUTE)
Exemple #60
0
 def initialise_pool(self):
     self._candidate_pool = OrderedSet()