Example #1
    def _get_raw_data(self):
        Parses log file
        :return: tuple(
                       [ipaddress, lease end time, ...],
                       time to parse leases file
            with open(self.leases_path, 'rt') as dhcp_leases:
                time_start = time()
                part1 = filterfalse(find_lease, dhcp_leases)
                part2 = filterfalse(find_ends, dhcp_leases)
                raw_result = dict(zip(part1, part2))
                time_end = time()

                file_parse_time = round((time_end - time_start) * 1000)

        except Exception as e:
            self.error("Failed to parse leases file:", str(e))
            return None

            result = (raw_result, file_parse_time)
            return result
Example #2
def getPitches(pitchNames=('C', 'E', 'G'), bassPitch='C3', maxPitch='C8'):
    Given a list of pitchNames, a bassPitch, and a maxPitch, returns a sorted list of
    pitches between the two limits (inclusive) which correspond to items in pitchNames.

    >>> from music21.figuredBass import segment
    >>> from music21 import pitch

    >>> pitches = segment.getPitches()
    >>> print(', '.join([p.nameWithOctave for p in pitches]))
    C3, E3, G3, C4, E4, G4, C5, E5, G5, C6, E6, G6, C7, E7, G7, C8

    >>> pitches = segment.getPitches(['G', 'B', 'D', 'F'], bassPitch=pitch.Pitch('B2'))
    >>> print(', '.join([p.nameWithOctave for p in pitches]))
    B2, D3, F3, G3, B3, D4, F4, G4, B4, D5, F5, G5, B5, D6, F6, G6, B6, D7, F7, G7, B7

    >>> pitches = segment.getPitches(['F##', 'A#', 'C#'], bassPitch=pitch.Pitch('A#3'))
    >>> print(', '.join([p.nameWithOctave for p in pitches]))
    A#3, C#4, F##4, A#4, C#5, F##5, A#5, C#6, F##6, A#6, C#7, F##7, A#7
    if isinstance(bassPitch, str):
        bassPitch = pitch.Pitch(bassPitch)
    if isinstance(maxPitch, str):
        maxPitch = pitch.Pitch(maxPitch)

    iter1 = itertools.product(pitchNames, range(maxPitch.octave + 1))
    iter2 = map(lambda x: pitch.Pitch(x[0] + str(x[1])), iter1)
    iter3 = itertools.filterfalse(lambda samplePitch: bassPitch > samplePitch, iter2)
    iter4 = itertools.filterfalse(lambda samplePitch: samplePitch > maxPitch, iter3)
    allPitches = list(iter4)
    return allPitches
    def write(self, new_cache):
        # if new_cache is empty goa might not be running, avoid wiping contacts
        if not new_cache:

        # update contacts
        new_diffs = list(itertools.filterfalse(lambda x: x in self.cache, new_cache))

        for old_item in self.cache:
            old_num = ''.join(re.findall(r'\d+', old_item['number']))

            for new_item in new_diffs:
                new_num = ''.join(re.findall(r'\d+', new_item['number']))

                if old_item['name'] in ('', new_item['name']) and old_num == new_num:

        # remove old folks
        old_diffs = list(itertools.filterfalse(lambda x: x in new_cache, self.cache))

        for old_item in old_diffs:
            if old_item['origin'] != 'kdeconnect':

        # add new folks
        for new_item in new_diffs:

        with open(self.cache_path, 'w') as cache_file:
            json.dump(new_cache, cache_file)
Example #4
    def assertFiles(self, obj, field_path, fn, compression=None, filter=lambda _: False):  # pylint: disable=invalid-name
        """Compare output file of a processor to the given correct file.

        :param obj: Data object which includes file that we want to
        :type obj: :obj:`resolwe.flow.models.Data`

        :param field_path: Path to file name in Data object.
        :type field_path: :obj:`str`

        :param fn: File name (and relative path) of file to which we
            want to compare. Name/path is relative to ``tests/files``
            folder of a Django application.
        :type fn: :obj:`str`

        :param compression: If not None, files will be uncompressed with
            the appropriate compression library before comparison.
            Currently supported compression formats are "gzip" and
        :type compression: :obj:`str`

        :param filter: Function for filtering the contents of output files. It
            is used in :obj:`itertools.filterfalse` function and takes one
            parameter, a line of the output file. If it returns `True`, the
            line is excluded from comparison of the two files.
        :type filter: :obj:`function`

        open_kwargs = {}
        if compression is None:
            open_fn = open
            # by default, open() will open files as text and return str
            # objects, but we need bytes objects
            open_kwargs['mode'] = 'rb'
        elif compression == 'gzip':
            open_fn = gzip.open
        elif compression == 'zip':
            open_fn = zipfile.ZipFile.open
            raise ValueError("Unsupported compression format.")

        field = dict_dot(obj.output, field_path)
        output = os.path.join(settings.FLOW_EXECUTOR['DATA_PATH'], str(obj.pk), field['file'])
        with open_fn(output, **open_kwargs) as output_file:
            output_contents = b"".join([line for line in filterfalse(filter, output_file)])
        output_hash = hashlib.sha256(output_contents).hexdigest()

        wanted = os.path.join(self.files_path, fn)

        if not os.path.isfile(wanted):
            shutil.copyfile(output, wanted)
            self.fail(msg="Output file {} missing so it was created.".format(fn))

        with open_fn(wanted, **open_kwargs) as wanted_file:
            wanted_contents = b"".join([line for line in filterfalse(filter, wanted_file)])
        wanted_hash = hashlib.sha256(wanted_contents).hexdigest()
        self.assertEqual(wanted_hash, output_hash,
                         msg="File contents hash mismatch: {} != {}".format(
                             wanted_hash, output_hash) + self._debug_info(obj))
Example #5
  def _initial_update(self, files):
    oldfiles = {f[0] for f in self._db.execute('select filename from pkginfo where pkgrepo = ?', (self.name,))}
    oldfiles.update(f[0] for f in self._db.execute('select filename from sigfiles where pkgrepo = ?', (self.name,)))

    for f in sorted(filterfalse(filterPkg, files - oldfiles), key=pkgsortkey):
      self.dispatch(f, 'add')

    for f in sorted(filterfalse(filterPkg, oldfiles - files), key=pkgsortkey):
      self.dispatch(f, 'remove')
Example #6
    def check_files(self, **kwargs):
        """Check files listed with the 'localfiles' fields."""
        # Get configuration options
        ignore = kwargs.get('ignore', [])
        filters = kwargs.get('filter', [])

        # Sets for recording entries:
        # - ok: has 'localfile' field, file exists
        # - other: hardcopies or online articles
        # - missing: no 'localfile' field
        # - broken: 'localfile' field exists, but is wrong
        sets = {k: set() for k in ['ok', 'other', 'missing', 'broken']}

        # Get the set of files in the current directory
        r = re.compile('(' + ')|('.join(ignore) + ')')
        files = filterfalse(os.path.isdir, iglob('**', recursive=True))
        files = sorted(filterfalse(r.search, files))

        # Iterate through database entries
        for e in self.db.entries:
            if e.has_file:
                if e.file_exists:
                    for fn in e.file_rel_path():
                        except ValueError:
                            if os.path.exists(fn):
                                # File exists, but has perhaps been filtered or
                                # is outside the tree.
                    sets['broken'] |= {(e['ID'], lf) for lf in e['localfile']}
                # Apply user filters
                done = False
                for f in filters:
                    if f['field'] in e and f['value'] in e[f['field']]:
                        done = True
                if not done:

        # Output
        output_format = kwargs.get('format', 'plain')
        if output_format == 'plain':
            output = self._check_files_plain
        elif output_format == 'csv':
            output = self._check_files_csv
        output(sorted(sets['ok']), sorted(sets['other']),
               sorted(sets['missing']), sorted(sets['broken']),
Example #7
    def getPitches(self, bassPitch, notationString=None, maxPitch=None):
        Takes in a bassPitch, a notationString, and a maxPitch representing the highest
        possible pitch that can be returned. Returns a sorted list of pitches which
        correspond to the pitches of each specific pitch name found through getPitchNames
        that fall between the bassPitch and the maxPitch, inclusive of both.

        if maxPitch is None, then B5 s used instead.

        >>> from music21.figuredBass import realizerScale
        >>> fbScale = realizerScale.FiguredBassScale()
        Root position triad

        >>> [str(p) for p in fbScale.getPitches('C3') ]
        ['C3', 'E3', 'G3', 'C4', 'E4', 'G4', 'C5', 'E5', 'G5']

        First inversion triad

        >>> [str(p) for p in fbScale.getPitches('D3', '6') ]
        ['D3', 'F3', 'B3', 'D4', 'F4', 'B4', 'D5', 'F5', 'B5']
        Root position seventh chord, showing MaxPitch
        >>> fbScale.getPitches(pitch.Pitch('G3'), '7', 'F4')
        [<music21.pitch.Pitch G3>, <music21.pitch.Pitch B3>, 
         <music21.pitch.Pitch D4>, <music21.pitch.Pitch F4>]
        if maxPitch is None:
            maxPitch = pitch.Pitch('B5')
        bassPitch = convertToPitch(bassPitch)
        maxPitch = convertToPitch(maxPitch)
        pitchNames = self.getPitchNames(bassPitch, notationString)
        iter1 = itertools.product(pitchNames, range(maxPitch.octave + 1))
        if six.PY3:
            iter2 = map( # pylint: disable=bad-builtin
                        lambda x: pitch.Pitch(x[0] + str(x[1])), iter1) 
            iter3 = itertools.filterfalse( # @UndefinedVariable
                                          lambda samplePitch: bassPitch > samplePitch, iter2) 
            iter4 = itertools.filterfalse( # @UndefinedVariable
                                          lambda samplePitch: samplePitch > maxPitch, iter3) 
            iter2 = itertools.imap( # @UndefinedVariable
                            lambda x: pitch.Pitch(x[0] + str(x[1])), iter1)  
            iter3 = itertools.ifilterfalse(  # @UndefinedVariable
                            lambda samplePitch: bassPitch > samplePitch, iter2) 
            iter4 = itertools.ifilterfalse(  # @UndefinedVariable
                            lambda samplePitch: samplePitch > maxPitch, iter3)  
        allPitches = list(iter4)
        return allPitches
Example #8
def main():
    s = input()
    l = list(filterfalse(lambda x: not x.islower(), s))
    u = list(filterfalse(lambda x: not x.isupper(), s))
    d = list(filterfalse(lambda x: not x.isdigit(), s))
    o = list(filterfalse(lambda x: int(x) % 2 == 0, d))
    e = list(filterfalse(lambda x: int(x) % 2 != 0, d))


    print(*(l + u + o + e), sep="")
Example #9
    def read_data(self, filename):
        count = 0
        #fields = [['name', 'RA', 'type1','type3', 'mag1', 'mag3', 'T1', 'P_orb', 'P3', 'EB', 'Spectr2', 'M1_M2', 'Incl', 'M1', 'M2'],
                    #['Alt_name', 'DEC', 'type2','type4', 'mag2', 'mag4', 'T2', 'P2', 'P4', 'SB', 'Spectr1', 'e_M1_M2', 'e_Incl', 'e_M1', 'e_M2']]
        fields = ['name', 'alias', 'flag1', 'flag2', 'ra', 'dec', 
                  'type1', 'type2', 'type3', 'type4', 'mag1', 'mag2', 'mag3', 'mag4',
                  'T1', 'T2', 'P0', 'P2', 'P3', 'P4', 'EB', 'SB', 'spectr2', 'spectr1',
                  'q', 'q_E', 'Incl',  'Incl_E', 'M1', 'M1_E', 'M2', 'M2_E'] 
        data = []
        with open(filename,'r') as fp:
            datalines = itt.filterfalse( lambda s: s.startswith(('\n','-')), fp )
            header = [next(datalines), next(datalines)]#consume(datalines, 2)       #header
            splitter = lambda line: map( str.strip, line.strip('\n|\r').split('|') )
            for i, lpair in enumerate( grouper(datalines, 2) ):
                data.append( interleave( *map( splitter, lpair ) ) )
                #data.append( dat )

        #Interleaving data and fields (every 2 lines refer to a single catalogue object)
        #data = [interleave(*s) for s in zip(*data)]
        #fields = interleave(fields)

        print('Data for %i objects successfully read.\n\n' %(i+1))

        return np.array(fields), np.array(data).T
Example #10
    def update_candidates(self):
        pattern = ''
        sources = ''
        self.__selected_candidates = []
        self.__candidates = []
        for name, all, candidates in self.__denite.filter_candidates(
            self.__candidates += candidates
            sources += '{}({}/{}) '.format(name, len(candidates), len(all))

            if pattern == '':
                matchers = self.__denite.get_source(name).matchers
                pattern = next(filterfalse(
                    lambda x: x == '',
                        self.__context['input']) for x in matchers
                     if self.__denite.get_filter(x)]), '')
        self.__matched_pattern = pattern
        self.__candidates_len = len(self.__candidates)
        if self.__context['reversed']:

        if self.__denite.is_async():
            sources = '[async] ' + sources
        self.__statusline_sources = sources
Example #11
    def search_path(self, begword, endword):
        """Search shortest way from word to word
        (Поиск кратчайшего пути от узла begword до endword)

        graph.search_path(begword, endword) -> [str]
        if path is found raise KeyError
        if begword not in self:
        if endword not in self:
        if begword == endword:
            return [begword]

        tup = (begword,)
        queue, steps = deque((tup,)), set(tup)
        while queue:
            path = queue.popleft()            
            for link in filterfalse(steps.__contains__,  self[path[-1]]):
                new_path = path + (link,)
                if link == endword:
                    return new_path
        raise KeyError('Not Found: ' + endword)
 def test_itertools_filterfalse(self):
     Tests whether itertools.filterfalse is available.
     from itertools import filterfalse
     not_div_by_3 = filterfalse(lambda x: x % 3 == 0, range(8))
     self.assertEqual(list(not_div_by_3), [1, 2, 4, 5, 7])
    def add_batch(self, batch, state_hash=None, required=False):
        with self._condition:
            if self._final:
                raise SchedulerError("Scheduler is finalized. Cannot take"
                                     " new batches")
            preserve = required
            if not required:
                # If this is the first non-required batch, it is preserved for
                # the schedule to be completed (i.e. no empty schedules in the
                # event of unschedule_incomplete_batches being called before
                # the first batch is completed).
                preserve = _first(
                    filterfalse(lambda sb: sb.required,
                                self._batch_by_id.values())) is None

            batch_signature = batch.header_signature
            self._batch_by_id[batch_signature] = \
                _AnnotatedBatch(batch, required=required, preserve=preserve)

            if state_hash is not None:
                self._required_state_hashes[batch_signature] = state_hash
            batch_length = len(batch.transactions)
            for idx, txn in enumerate(batch.transactions):
                if idx == batch_length - 1:
                self._txn_to_batch[txn.header_signature] = batch_signature
Example #14
def partition(pred, iterable):
    Use a predicate to partition entries into false entries and true entries
    # partition(is_odd, range(10)) --> 0 2 4 6 8   and  1 3 5 7 9
    t1, t2 = itertools.tee(iterable)
    return itertools.filterfalse(pred, t1), filter(pred, t2)
Example #15
def unique_everseen(iterable, key=None):
    List unique elements, preserving order.

    Remember all elements ever seen.
    # unique_everseen('AAAABBBCCDAABBB') --> A B C D
    # unique_everseen('ABBCcAD', str.lower) --> A B C D
        from itertools import ifilterfalse
        from itertools import filterfalse

    seen = set()
    seen_add = seen.add
    if key is None:
            for element in ifilterfalse(seen.__contains__, iterable):
                yield element
            for element in filterfalse(seen.__contains__, iterable):
                yield element
        for element in iterable:
            k = key(element)
            if k not in seen:
                yield element
Example #16
  def partition(cls, iterable, pred):
    Use a predicate to partition items into false and true entries.

    t1, t2 = itertools.tee(iterable)
    return cls(itertools.filterfalse(pred, t1), filter(pred, t2))
Example #17
    def _get_data(self):
        Parse new log lines
        :return: dict
            raw = self._get_raw_data()
            if raw is None:
                return None
            elif not raw:
                return self.data
        except (ValueError, AttributeError):
            return None

        # Fail2ban logs looks like
        # 2016-12-25 12:36:04,711 fail2ban.actions[2455]: WARNING [ssh] Ban
        data = dict(
                [len(list(filterfalse(lambda line: (jail + '] Ban') not in line, raw))) for jail in self.jails_list]

        for jail in data:
            self.data[jail] += data[jail]

        return self.data
Example #18
 def update(self):
     """Update the nello lock properties."""
     # Location identifiers
     location_id = self._nello_lock.location_id
     short_id = self._nello_lock.short_id
     address = self._nello_lock.address
     self._name = 'Nello {}'.format(short_id)
     self._device_attrs = {
         ATTR_ADDRESS: address,
         ATTR_LOCATION_ID: location_id
     # Process recent activity
     activity = self._nello_lock.activity
     if self._activity:
         # Filter out old events
         new_activity = list(
             filterfalse(lambda x: x in self._activity, activity))
         if new_activity:
             for act in new_activity:
                 activity_type = act.get('type')
                 if activity_type == 'bell.ring.denied':
                     event_data = {
                         'address': address,
                         'date': act.get('date'),
                         'description': act.get('description'),
                         'location_id': location_id,
                         'short_id': short_id
                     self.hass.bus.fire(EVENT_DOOR_BELL, event_data)
     # Save the activity history so that we don't trigger an event twice
     self._activity = activity
Example #19
    def test_filterfalse_wrongargs(self):
        import itertools

        it = itertools.filterfalse(0, [1])
        raises(TypeError, next, it)

        raises(TypeError, itertools.filterfalse, bool, None)
Example #20
    def __eq__(self, r):
        res = True

        if self is r:
            return True
        elif not isinstance(r, self.__class__):
            res = NotImplemented
            keys_to_cmp = [
            ldict = self.__dict__
            rdict = r.__dict__

            results = list(filterfalse(lambda k: ldict.get(k, None) == rdict.get(k, None),

            # if results is empty, all keys evaluated to equal
            res = bool(not results)

            if DEBUG_INDEXERCONFIG and not res:
                for key in results:
                    print("%s failed: '%s' != '%s'" %
                                    (key, ldict.get(key, None), rdict.get(key, None)))

        return res
Example #21
    def compute_testing_installability(self):
        """Computes the installability of packages in testing

        This method computes the installability of all packages in
        testing and caches the result.  This has the advantage of
        making "is_installable" queries very fast for all packages
        in testing.

        check_inst = self._check_inst
        cbroken = self._cache_broken
        cache_inst = self._cache_inst
        eqv_table = self._eqv_table
        testing = self._testing
        tcopy = [x for x in testing]
        for t in filterfalse(cache_inst.__contains__, tcopy):
            if t in cbroken:
            res = check_inst(t)
            if t in eqv_table:
                eqv = (x for x in eqv_table[t] if x in testing)
                if res:
                    eqv_set = frozenset(eqv)
                    testing -= eqv_set
                    cbroken |= eqv_set
Example #22
    def compute_installability(self):
        """Computes the installability of all the packages in the suite

        This method computes the installability of all packages in
        the suite and caches the result.  This has the advantage of
        making "is_installable" queries very fast for all packages
        in the suite.

        universe = self._universe
        check_inst = self._check_inst
        cbroken = self._cache_broken
        cache_inst = self._cache_inst
        suite_contents = self._suite_contents
        tcopy = [x for x in suite_contents]
        for t in filterfalse(cache_inst.__contains__, tcopy):
            if t in cbroken:
            res = check_inst(t)
            if t in universe.equivalent_packages:
                eqv = (x for x in universe.packages_equivalent_to(t) if x in suite_contents)
                if res:
                    eqv_set = frozenset(eqv)
                    suite_contents -= eqv_set
                    cbroken |= eqv_set
Example #23
def apply_move(paths, force, dry_run):
    """ Core process of moving files. """
    assert isinstance(paths, list)
    assert len(paths)

    not_existed = [pair.dst for pair in itertools.filterfalse(lambda p:
        not p.dst.exists(), paths)]

    if len(not_existed) and not force:
        raise RuntimeError(
            'The following file(s) already exists:\n'
            'Use `--force` to override.'
            .format('\n  '.join(not_existed))

    for pair in paths:
        assert pair.src.is_absolute()
        assert pair.dst.is_absolute()

            dst_dir = pair.dst.parent
            if not dst_dir.exists():
                print('Creating directory: `{}`'.format(dst_dir))
                if not dry_run:
            print('Moving `{}` to `{}`'.format(pair.src, pair.dst))
            if not dry_run:
                shutil.move(str(pair.src), str(pair.dst))

        except OSError as ex:
            raise RuntimeError(ex)
Example #24
 def _parse_all_merged_entities(self):
     """set self._all_merged_entities to the longest possible(wrapping)
     tokens including non-entity tokens
     self._all_merged_entities = list(filterfalse(
         lambda token: self._is_wrapped(token, self.all_entities),
Example #25
def unique_everseen(iterable, key=None):
    The generator to list unique elements, preserving the order. Remember all
    elements ever seen. This was taken from the itertools recipes.

        iterable: An iterable to process.
        key: Optional function to run when checking elements (e.g., str.lower)

        Generator: Yields a generator object.

    seen = set()
    seen_add = seen.add

    if key is None:

        for element in filterfalse(seen.__contains__, iterable):

            yield element


        for element in iterable:

            k = key(element)

            if k not in seen:

                yield element
def filter_by_min_spt(dataset, item_count, min_spt, trans_count):
		(i) filterd dataset (remove items w/ freq < min_spt)
		(ii) filtered item counter
	pass in:
		(i) dataset (the raw dataset)
		(ii) dict w/ items for keys, values are item frequency,
			returned by call to item_counter, for queries this
			is probably an f-list, built by item_counter
		(iii) min_spt (float, eg, 0.03 means each item must appear in
			at least 3% of the dataset)
		(iv) total number of transactions
	removes any item from every transaction if that item's total freq
	is below min_spt
	to call this fn, bind the call to two variables, like so:
	filtered_trans, item_count_dict = filter_by_min_spt(...)
	excluded_items = get_items_below_min_spt(dataset, min_spt, trans_count)
	if not excluded_items:
		# if all items are above min_spt, ie, there are no items to exclude
		# so just return original args
		return dataset, item_count
		# there is at least one item to exclude
		# now build the expression required by 'IT.filterfalse' from the
		# list of excluded items
		filter_str = build_min_spt_filter_str(dataset, min_spt, trans_count)
		# remove those items below min_spt threshold items
		tx = [IT.filterfalse(lambda q: eval(filter_str), trans)
				for trans in dataset]
		ic = {k:v for k, v in item_count.items() if (v/trans_count) >= min_spt}
		return list(map(list, tx)), ic
Example #27
    def new(self):
        if len(self.sessions) > self.maxsession:
            raise MaxDatabaseConnection


        self.itersession = itertools.filterfalse(getfree, self.sessions)
Example #28
    def __init__(self, session=DbSession, uri='pq://localhost/postgres', maxconnect=10):

        # session to pooled
        if isinstance(session, DbSession):
            self.dbsession = session
            raise InvalidDatabaseSessionType

        # DB connect uri
        self.uri = uri

        # set max connection limits
        if maxconnect < 5:
            self.maxsession = 5
            self.maxsession = maxconnect

        # least db connection count
        self.minsession = int(self.maxsession / 5)

        # our dbsession
        self.sessions = set()

        # create initial dbsession #minsession
        for x in range(self.minsession):

        # init first session
        self.itersession = itertools.filterfalse(getfree, self.sessions)
Example #29
    def clean_article_ids(self):
        article_ids = self.cleaned_data["article_ids"].split("\n")
        article_ids = filter(bool, map(str.strip, article_ids))

        # Parse all article ids as integer
            article_ids = list(map(int, article_ids))
        except ValueError:
            offender = repr(next(filterfalse(str.isnumeric, article_ids)))
            raise ValidationError("{offender} is not an integer".format(**locals()))

        # Check if they can be chosen
        articlesets = self.cleaned_data["articlesets"]
        distinct_args = ["id"] if db_supports_distinct_on() else []
        all_articles = Article.objects.filter(articlesets_set__in=articlesets).distinct(*distinct_args)
        chosen_articles = Article.objects.filter(id__in=article_ids).distinct(*distinct_args)
        intersection = all_articles & chosen_articles

        if chosen_articles.count() != intersection.count():
            # Find offenders (skipping non-existing, which we can only find when
            # fetching all possible articles)
            existing = all_articles.values_list("id", flat=True)
            offenders = chosen_articles.exclude(id__in=existing).values_list("id", flat=True)
            raise ValidationError(
                ("Articles {offenders} not in chosen articlesets or some non-existent"
                 .format(**locals())), code="invalid")

        return article_ids
Example #30
def bat_graph(cave):
    nodes = cave.bats
    inter_bat_edges = [Edge(x, y) for x in nodes for y in nodes if not x is y]
    bat_alpha_edges = [Edge(x, cave.alpha) for x in nodes]
    edges = inter_bat_edges + bat_alpha_edges
    arcs_no_walls = filterfalse(lambda edge: any([wall.intersects(edge) for wall in cave.walls]), edges)
    return Graph(nodes, frozenset(arcs_no_walls))
Example #31
    def _run(self, env: Union[Dict[str, str], MergedMap] = None) -> None:
        logging.debug("\033[1;31m--------- %s [%s] ---------\033[0m" %
                      (self.name, self._compiled))

        if self._compiled:


        cwd = self.cwd
        if not isempty(self.sourcedir):
            src = '%s/%s/' % (self.sourcedir, self.name)
            if os.path.exists(src) and src != cwd and not os.path.exists(cwd):
                Command(['rsync', '-avH', '--delete', '--quiet', src,

        # in seq: 1. os env; 2. global environment; 3. self pre-environment; 4. local func env arguments
        self._build_env.update(os.environ.copy(), mode=BuildEnv_Mode.MERGE)
        if env is not None:
            self._build_env.update(env, mode=BuildEnv_Mode.MERGE)
        #use_env.merge(self._pre_environment, merge=MergeMapType.AsString)
        #use_env.merge(Environment.global_env, merge=MergeMapType.AsString)

        #use_env.merge(os.environ.copy(), merge=MergeMapType.AsString)

        def sticky_env() -> None:
            self._build_env.update({'PREFIX': Environment.working.prefix})

        def func(stage: Stage, item: Union[Command, Env]) -> None:
            use_env = self._build_env.curr_env
            # Each command is run in a clean isolated environment, and
            # their enviroment is only updated by explicitly user-defined env
            if isinstance(item, Command):
                if stage == Stage.BuildScript:
                    #logging.debug('> Current Directory: %s' % os.path.abspath(cwd))
                    #logging.debug('Command [shell=%s, dry_run=%s]: %s' % (c.shell, self.dry_run, str(c)))
                    if use_env is not None:
                        logging.debug('> CFLAGS: %s' % use_env.get('CFLAGS'))
                        logging.debug('> LDFLAGS: %s' % use_env.get('LDFLAGS'))
                item._run(env=use_env, dry_run=Environment._dry_run)
            elif type(item) is Env:
                val = item.parse(cwd=cwd, env=use_env)
                logging.debug('> Update Current Unit Env by: %s' % val)
                self._build_env.update(val, mode=BuildEnv_Mode.MERGE)


        for stage, item in itertools.filterfalse(lambda x: x[1] is None,
            if stage == Stage.BuildScript:
            logging.debug('"\033[1;34m [%s] \033[0m" PREFIX: %s' %
                          (stage, self._build_env.curr_env.get('PREFIX')))
            func(stage, item)

        for env in self._export_env:
            val = env.parse(cwd=cwd, env=self._build_env.curr_env)

        self._compiled = True

    def request_match(self, fls_comp_id, fd_comp_id, game_week, season, limit=None):
        Retrieve the match details
        :param fls_comp_id: FLS comp id
        :param fd_comp_id: footballdata comp id
        :param game_week: 1, 2, 3....n
        :param season: Identifier for the season e.g. 2018-2019 (for football-data API)
        :param limit: result set limiter
        :return: Matches over a given gameweek for a given competition and season
        :rtype: list
        # Placeholder
        # competition_id = 2002
        # In FLS, 2 is id for premier league, 2021 in football-data
        # fls_comp_id = 2
        # fd_comp_id = 2021

        if isinstance(season, str):
            season = int(season.split("-")[0])

        joint_matches = []
        fantasy_matches = self.fantasy.request_matches()
        fantasy_matches = list(filter(lambda x: x[Match.FANTASY_GAME_WEEK] == int(game_week), fantasy_matches))
        fd_matches = self.fd.request_competition_match(competition_id=fd_comp_id,
                                                       **{fdf.MATCHDAY: game_week, fdf.SEASON: season})

        game_week_start = dt.strptime(fd_matches[0][Match.MATCH_UTC_DATE], '%Y-%m-%dT%H:%M:%SZ')
        game_week_end = game_week_start + timedelta(days=4)  # 4 days per game week
        fls_matches = self.fls.request_matches(**{flsf.FROM_DATETIME: game_week_start,
                                                  flsf.TO_DATETIME: game_week_end})

        fls_matches = list(filterfalse(lambda x: x[Match.FLS_API_COMPETITION_ID] != fls_comp_id, fls_matches))

        for match in fd_matches:
            match_start_datetime = dt.strptime(match[Match.MATCH_UTC_DATE], '%Y-%m-%dT%H:%M:%SZ')
            match_start_datetime = dt.strftime(match_start_datetime, '%Y-%d-%mT%H:%M:%SZ')
            home_team = match[Match.HOME_TEAM]
            away_team = match[Match.AWAY_TEAM]
            home_score = match[Match.FULL_TIME_HOME_SCORE]
            away_score = match[Match.FULL_TIME_AWAY_SCORE]
            for fls_match in fls_matches:
                if fls_match[Match.HOME_TEAM] in home_team and fls_match[Match.AWAY_TEAM] in away_team and \
                        home_score == fls_match[Match.FULL_TIME_HOME_SCORE] and \
                        away_score == fls_match[Match.FULL_TIME_AWAY_SCORE]:
                    temp_dict = {**match, **fls_match}
                    adv_match_details = self.fls.request_match_details(match_id=fls_match[Match.FLS_MATCH_ID])
                    temp_dict2 = {**temp_dict, **adv_match_details}
                    for f_match in fantasy_matches:
                        f_home_team = self.fantasy.name_to_id(f_match[Match.FANTASY_HOME_TEAM_ID])
                        f_away_team = self.fantasy.name_to_id(f_match[Match.FANTASY_AWAY_TEAM_ID])
                        parsed_date = dt.strftime(dt.strptime(f_match[Match.START_TIME], '%Y-%m-%dT%H:%M:%SZ'),
                        if parsed_date == match_start_datetime:
                            if f_home_team in home_team and f_away_team in away_team and \
                                    home_score == f_match[Match.FULL_TIME_HOME_SCORE] and \
                                    away_score == f_match[Match.FULL_TIME_AWAY_SCORE]:
                                final_dict = {**f_match, **temp_dict2}

        if limit:
            return joint_matches[:limit]

        return joint_matches
Example #33
 def __iter__(self):
     return itertools.filterfalse(self.pred, self.data)
Example #34
def second_order_param_shift(tape,
                             shift=np.pi / 2,
    r"""Generate the second-order CV parameter-shift tapes and postprocessing methods required
    to compute the gradient of a gate parameter with respect to an
    expectation value.

    .. note::

        The 2nd order method can handle also first-order observables, but
        1st order method may be more efficient unless it's really easy to
        experimentally measure arbitrary 2nd order observables.

    .. warning::

        The 2nd order method can only be executed on devices that support the
        :class:`~.PolyXP` observable.

        tape (.QuantumTape): quantum tape to differentiate
        dev_wires (.Wires): wires on the device the parameter-shift method is computed on
        argnum (int or list[int] or None): Trainable parameter indices to differentiate
            with respect to. If not provided, the derivative with respect to all
            trainable indices are returned.
        shift (float): The shift value to use for the two-term parameter-shift formula.
            Only valid if the operation in question supports the two-term parameter-shift
            rule (that is, it has two distinct eigenvalues) and ``gradient_recipes``
            is ``None``.
        gradient_recipes (tuple(list[list[float]] or None)): List of gradient recipes
            for the parameter-shift method. One gradient recipe must be provided
            per trainable parameter.

        tuple[list[QuantumTape], function]: A tuple containing a
        list of generated tapes, in addition to a post-processing
        function to be applied to the evaluated tapes.
    argnum = argnum or list(tape.trainable_params)
    gradient_recipes = gradient_recipes or [None] * len(argnum)

    gradient_tapes = []
    shapes = []
    obs_indices = []
    gradient_values = []

    for idx, _ in enumerate(tape.trainable_params):
        t_idx = list(tape.trainable_params)[idx]
        op = tape._par_info[t_idx]["op"]

        if idx not in argnum:
            # parameter has zero gradient


        # get the gradient recipe for the trainable parameter
        recipe = gradient_recipes[argnum.index(idx)]
        recipe = recipe or _get_operation_recipe(tape, idx, shift=shift)
        recipe = _process_gradient_recipe(recipe)
        coeffs, multipliers, shifts = recipe

        if len(shifts) != 2:
            # The 2nd order CV parameter-shift rule only accepts two-term shifts
            raise NotImplementedError(
                "Taking the analytic gradient for order-2 operators is "
                f"unsupported for operation {op} which has a "
                "gradient recipe of more than two terms.")

        shifted_tapes = generate_shifted_tapes(tape, idx, shifts, multipliers)

        # evaluate transformed observables at the original parameter point
        # first build the Heisenberg picture transformation matrix Z
        Z0 = op.heisenberg_tr(dev_wires, inverse=True)
        Z2 = shifted_tapes[0]._par_info[t_idx]["op"].heisenberg_tr(dev_wires)
        Z1 = shifted_tapes[1]._par_info[t_idx]["op"].heisenberg_tr(dev_wires)

        # derivative of the operation
        Z = Z2 * coeffs[0] + Z1 * coeffs[1]
        Z = Z @ Z0

        # conjugate Z with all the descendant operations
        B = np.eye(1 + 2 * len(dev_wires))
        B_inv = B.copy()

        succ = tape.graph.descendants_in_order((op, ))
        operation_descendents = itertools.filterfalse(
            qml.circuit_graph._is_observable, succ)
        observable_descendents = filter(qml.circuit_graph._is_observable, succ)

        for BB in operation_descendents:
            if not BB.supports_heisenberg:
                # if the descendant gate is non-Gaussian in parameter-shift differentiation
                # mode, then there must be no observable following it.

            B = BB.heisenberg_tr(dev_wires) @ B
            B_inv = B_inv @ BB.heisenberg_tr(dev_wires, inverse=True)

        Z = B @ Z @ B_inv  # conjugation

        g_tape = tape.copy(copy_operations=True)
        constants = []

        # transform the descendant observables into their derivatives using Z
        transformed_obs_idx = []

        for obs in observable_descendents:
            # get the index of the descendent observable
            idx = tape.observables.index(obs)

            transformed_obs = _transform_observable(obs, Z, dev_wires)

            A = transformed_obs.parameters[0]
            constant = None

            # Check if the transformed observable corresponds to a constant term.
            if len(A.nonzero()[0]) == 1:
                if A.ndim == 2 and A[0, 0] != 0:
                    constant = A[0, 0]

                elif A.ndim == 1 and A[0] != 0:
                    constant = A[0]


            g_tape._measurements[idx] = qml.measure.MeasurementProcess(
                _transform_observable(obs, Z, dev_wires))

        if not any(i is None for i in constants):
            # Check if *all* transformed observables corresponds to a constant term.
            # term. If this is the case for all transformed observables on the tape,
            # then <psi|A|psi> = A<psi|psi> = A,
            # and we can avoid the device execution.
            shapes[-1] = 0


    def processing_fn(results):
        grads = []
        start = 0

        if not results:
            results = [np.zeros([tape.output_dim])]

        interface = qml.math.get_interface(results[0])
        iterator = enumerate(zip(shapes, gradient_values, obs_indices))

        for i, (shape, grad_value, obs_ind) in iterator:

            if shape == 0:
                # parameter has zero gradient
                g = qml.math.zeros_like(results[0], like=interface)

                if grad_value:
                    g = qml.math.scatter_element_add(g,


            obs_result = results[start:start + shape]
            start = start + shape

            # compute the linear combination of results and coefficients
            obs_result = qml.math.stack(obs_result[0])
            g = qml.math.zeros_like(obs_result, like=interface)

            if qml.math.get_interface(g) not in ("tensorflow", "autograd"):
                obs_ind = (obs_ind, )

            g = qml.math.scatter_element_add(g,

        # The following is for backwards compatibility; currently,
        # the device stacks multiple measurement arrays, even if not the same
        # size, resulting in a ragged array.
        # In the future, we might want to change this so that only tuples
        # of arrays are returned.
        for i, g in enumerate(grads):
            g = qml.math.convert_like(g, results[0])
            if hasattr(g, "dtype") and g.dtype is np.dtype("object"):
                grads[i] = qml.math.hstack(g)

        return qml.math.T(qml.math.stack(grads))

    return gradient_tapes, processing_fn
Example #35
def compute():
    ans = next(itertools.filterfalse(test_goldbach, itertools.count(9, 2)))
    return str(ans)
Example #36
File: cover.py Project: gvc14/sacad
  async def preProcessForComparison(results, target_size, size_tolerance_prct):
    """ Process results to prepare them for future comparison and sorting. """
    # find reference (=image most likely to match target cover ignoring factors like size and format)
    reference = None
    for result in results:
      if result.source_quality is CoverSourceQuality.REFERENCE:
        if ((reference is None) or
                                       size_tolerance_prct=size_tolerance_prct) > 0)):
          reference = result

    # remove results that are only refs
    results = list(itertools.filterfalse(operator.attrgetter("is_only_reference"), results))

    # remove duplicates
    no_dup_results = []
    for result in results:
      is_dup = False
      for result_comp in results:
        if ((result_comp is not result) and
            (result_comp.urls == result.urls) and
                               size_tolerance_prct=size_tolerance_prct) < 0)):
          is_dup = True
      if not is_dup:
    dup_count = len(results) - len(no_dup_results)
    if dup_count > 0:
      logging.getLogger("Cover").info("Removed %u duplicate results" % (dup_count))
      results = no_dup_results

    if reference is not None:
      logging.getLogger("Cover").info("Reference is: %s" % (reference))
      reference.is_similar_to_reference = True

      # calculate sigs
      futures = []
      for result in results:
        coroutine = result.updateSignature()
        future = asyncio.ensure_future(coroutine)
      if reference.is_only_reference:
        assert(reference not in results)
        coroutine = reference.updateSignature()
        future = asyncio.ensure_future(coroutine)
      if futures:
        await asyncio.wait(futures)
      for future in futures:
        future.result()  # raise pending exception if any

      # compare other results to reference
      for result in results:
        if ((result is not reference) and
                (result.thumbnail_sig is not None) and
                (reference.thumbnail_sig is not None)):
          result.is_similar_to_reference = __class__.areImageSigsSimilar(result.thumbnail_sig,
          if result.is_similar_to_reference:
            logging.getLogger("Cover").debug("%s is similar to reference" % (result))
            logging.getLogger("Cover").debug("%s is NOT similar to reference" % (result))
      logging.getLogger("Cover").warning("No reference result found")

    return results
Example #37
def _install_wheel(
    name,  # type: str
    wheel_zip,  # type: ZipFile
    wheel_path,  # type: str
    scheme,  # type: Scheme
    pycompile=True,  # type: bool
    warn_script_location=True,  # type: bool
    direct_url=None,  # type: Optional[DirectUrl]
    requested=False,  # type: bool
    # type: (...) -> None
    """Install a wheel.

    :param name: Name of the project to install
    :param wheel_zip: open ZipFile for wheel being installed
    :param scheme: Distutils scheme dictating the install directories
    :param req_description: String used in place of the requirement, for
    :param pycompile: Whether to byte-compile installed Python files
    :param warn_script_location: Whether to check that scripts are installed
        into a directory on PATH
    :raises UnsupportedWheel:
        * when the directory holds an unpacked wheel with incompatible
        * when the .dist-info dir does not match the wheel
    info_dir, metadata = parse_wheel(wheel_zip, name)

    if wheel_root_is_purelib(metadata):
        lib_dir = scheme.purelib
        lib_dir = scheme.platlib

    # Record details of the files moved
    #   installed = files copied from the wheel to the destination
    #   changed = files changed while installing (scripts #! line typically)
    #   generated = files newly generated during the install (script wrappers)
    installed = {}  # type: Dict[RecordPath, RecordPath]
    changed = set()  # type: Set[RecordPath]
    generated = []  # type: List[str]

    def record_installed(srcfile, destfile, modified=False):
        # type: (RecordPath, str, bool) -> None
        """Map archive RECORD paths to installation RECORD paths."""
        newpath = _fs_to_record_path(destfile, lib_dir)
        installed[srcfile] = newpath
        if modified:

    def all_paths():
        # type: () -> Iterable[RecordPath]
        names = wheel_zip.namelist()
        # If a flag is set, names may be unicode in Python 2. We convert to
        # text explicitly so these are valid for lookup in RECORD.
        decoded_names = map(ensure_text, names)
        for name in decoded_names:
            yield cast("RecordPath", name)

    def is_dir_path(path):
        # type: (RecordPath) -> bool
        return path.endswith("/")

    def assert_no_path_traversal(dest_dir_path, target_path):
        # type: (str, str) -> None
        if not is_within_directory(dest_dir_path, target_path):
            message = (
                "The wheel {!r} has a file {!r} trying to install"
                " outside the target directory {!r}"
            raise InstallationError(
                message.format(wheel_path, target_path, dest_dir_path)

    def root_scheme_file_maker(zip_file, dest):
        # type: (ZipFile, str) -> Callable[[RecordPath], File]
        def make_root_scheme_file(record_path):
            # type: (RecordPath) -> File
            normed_path = os.path.normpath(record_path)
            dest_path = os.path.join(dest, normed_path)
            assert_no_path_traversal(dest, dest_path)
            return ZipBackedFile(record_path, dest_path, zip_file)

        return make_root_scheme_file

    def data_scheme_file_maker(zip_file, scheme):
        # type: (ZipFile, Scheme) -> Callable[[RecordPath], File]
        scheme_paths = {}
        for key in SCHEME_KEYS:
            encoded_key = ensure_text(key)
            scheme_paths[encoded_key] = ensure_text(
                getattr(scheme, key), encoding=sys.getfilesystemencoding()

        def make_data_scheme_file(record_path):
            # type: (RecordPath) -> File
            normed_path = os.path.normpath(record_path)
                _, scheme_key, dest_subpath = normed_path.split(os.path.sep, 2)
            except ValueError:
                message = (
                    "Unexpected file in {}: {!r}. .data directory contents"
                    " should be named like: '<scheme key>/<path>'."
                ).format(wheel_path, record_path)
                raise InstallationError(message)

                scheme_path = scheme_paths[scheme_key]
            except KeyError:
                valid_scheme_keys = ", ".join(sorted(scheme_paths))
                message = (
                    "Unknown scheme key used in {}: {} (for file {!r}). .data"
                    " directory contents should be in subdirectories named"
                    " with a valid scheme key ({})"
                    wheel_path, scheme_key, record_path, valid_scheme_keys
                raise InstallationError(message)

            dest_path = os.path.join(scheme_path, dest_subpath)
            assert_no_path_traversal(scheme_path, dest_path)
            return ZipBackedFile(record_path, dest_path, zip_file)

        return make_data_scheme_file

    def is_data_scheme_path(path):
        # type: (RecordPath) -> bool
        return path.split("/", 1)[0].endswith(".data")

    paths = all_paths()
    file_paths = filterfalse(is_dir_path, paths)
    root_scheme_paths, data_scheme_paths = partition(
        is_data_scheme_path, file_paths

    make_root_scheme_file = root_scheme_file_maker(
        ensure_text(lib_dir, encoding=sys.getfilesystemencoding()),
    files = map(make_root_scheme_file, root_scheme_paths)

    def is_script_scheme_path(path):
        # type: (RecordPath) -> bool
        parts = path.split("/", 2)
        return (
            len(parts) > 2 and
            parts[0].endswith(".data") and
            parts[1] == "scripts"

    other_scheme_paths, script_scheme_paths = partition(
        is_script_scheme_path, data_scheme_paths

    make_data_scheme_file = data_scheme_file_maker(wheel_zip, scheme)
    other_scheme_files = map(make_data_scheme_file, other_scheme_paths)
    files = chain(files, other_scheme_files)

    # Get the defined entry points
    distribution = pkg_resources_distribution_for_wheel(
        wheel_zip, name, wheel_path
    console, gui = get_entrypoints(distribution)

    def is_entrypoint_wrapper(file):
        # type: (File) -> bool
        # EP, EP.exe and EP-script.py are scripts generated for
        # entry point EP by setuptools
        path = file.dest_path
        name = os.path.basename(path)
        if name.lower().endswith('.exe'):
            matchname = name[:-4]
        elif name.lower().endswith('-script.py'):
            matchname = name[:-10]
        elif name.lower().endswith(".pya"):
            matchname = name[:-4]
            matchname = name
        # Ignore setuptools-generated scripts
        return (matchname in console or matchname in gui)

    script_scheme_files = map(make_data_scheme_file, script_scheme_paths)
    script_scheme_files = filterfalse(
        is_entrypoint_wrapper, script_scheme_files
    script_scheme_files = map(ScriptFile, script_scheme_files)
    files = chain(files, script_scheme_files)

    for file in files:
        record_installed(file.src_record_path, file.dest_path, file.changed)

    def pyc_source_file_paths():
        # type: () -> Iterator[str]
        # We de-duplicate installation paths, since there can be overlap (e.g.
        # file in .data maps to same location as file in wheel root).
        # Sorting installation paths makes it easier to reproduce and debug
        # issues related to permissions on existing files.
        for installed_path in sorted(set(installed.values())):
            full_installed_path = os.path.join(lib_dir, installed_path)
            if not os.path.isfile(full_installed_path):
            if not full_installed_path.endswith('.py'):
            yield full_installed_path

    def pyc_output_path(path):
        # type: (str) -> str
        """Return the path the pyc file would have been written to.
        return importlib.util.cache_from_source(path)

    # Compile all of the pyc files for the installed files
    if pycompile:
        with captured_stdout() as stdout:
            with warnings.catch_warnings():
                for path in pyc_source_file_paths():
                    # Python 2's `compileall.compile_file` requires a str in
                    # error cases, so we must convert to the native type.
                    path_arg = ensure_str(
                        path, encoding=sys.getfilesystemencoding()
                    success = compileall.compile_file(
                        path_arg, force=True, quiet=True
                    if success:
                        pyc_path = pyc_output_path(path)
                        assert os.path.exists(pyc_path)
                        pyc_record_path = cast(
                            "RecordPath", pyc_path.replace(os.path.sep, "/")
                        record_installed(pyc_record_path, pyc_path)

    maker = PipScriptMaker(None, scheme.scripts)

    # Ensure old scripts are overwritten.
    # See https://github.com/pypa/pip/issues/1800
    maker.clobber = True

    # Ensure we don't generate any variants for scripts because this is almost
    # never what somebody wants.
    # See https://bitbucket.org/pypa/distlib/issue/35/
    maker.variants = {''}

    # This is required because otherwise distlib creates scripts that are not
    # executable.
    # See https://bitbucket.org/pypa/distlib/issue/32/
    maker.set_mode = True

    # Generate the console and GUI entry points specified in the wheel
    scripts_to_generate = get_console_script_specs(console)

    gui_scripts_to_generate = list(starmap('{} = {}'.format, gui.items()))

    generated_console_scripts = maker.make_multiple(scripts_to_generate)

        maker.make_multiple(gui_scripts_to_generate, {'gui': True})

    if warn_script_location:
        msg = message_about_scripts_not_on_PATH(generated_console_scripts)
        if msg is not None:

    generated_file_mode = 0o666 & ~current_umask()

    def _generate_file(path, **kwargs):
        # type: (str, **Any) -> Iterator[BinaryIO]
        with adjacent_tmp_file(path, **kwargs) as f:
            yield f
        os.chmod(f.name, generated_file_mode)
        replace(f.name, path)

    dest_info_dir = os.path.join(lib_dir, info_dir)

    # Record pip as the installer
    installer_path = os.path.join(dest_info_dir, 'INSTALLER')
    with _generate_file(installer_path) as installer_file:

    # Record the PEP 610 direct URL reference
    if direct_url is not None:
        direct_url_path = os.path.join(dest_info_dir, DIRECT_URL_METADATA_NAME)
        with _generate_file(direct_url_path) as direct_url_file:

    # Record the REQUESTED file
    if requested:
        requested_path = os.path.join(dest_info_dir, 'REQUESTED')
        with open(requested_path, "w"):

    record_text = distribution.get_metadata('RECORD')
    record_rows = list(csv.reader(record_text.splitlines()))

    rows = get_csv_rows_for_installed(

    # Record details of all files installed
    record_path = os.path.join(dest_info_dir, 'RECORD')

    with _generate_file(record_path, **csv_io_kwargs('w')) as record_file:
        # The type mypy infers for record_file is different for Python 3
        # (typing.IO[Any]) and Python 2 (typing.BinaryIO). We explicitly
        # cast to typing.IO[str] as a workaround.
        writer = csv.writer(cast('IO[str]', record_file))
def filter_finished_tasks(tasks_info, finished_task_ids):
    # Allow for O(1) lookups on average to speed up the process
    finished_task_ids = set(finished_task_ids)

    return list(
        itertools.filterfalse(lambda x: x[0] in finished_task_ids, tasks_info))
Example #39
def partition(gen, pred):
    t1, t2 = tee(gen)

    return filter(pred, t2), filterfalse(pred, t1)
Example #40
    def migrate(self, expression, name_migration_map=None):
        """Migrate an expression created for a different constraint set to self.
        Returns an expression that can be used with this constraintSet

        All the foreign variables used in the expression are replaced by
        variables of this constraint set. If the variable was replaced before
        the replacement is taken from the provided migration map.

        The migration mapping is updated with new replacements.

        :param expression: the potentially foreign expression
        :param name_migration_map: mapping of already migrated variables. maps from string name of foreign variable to its currently existing migrated string name. this is updated during this migration.
        :return: a migrated expression where all the variables are local. name_migration_map is updated

        if name_migration_map is None:
            name_migration_map = {}

        #  name_migration_map -> object_migration_map
        #  Based on the name mapping in name_migration_map build an object to
        #  object mapping to be used in the replacing of variables
        #  inv: object_migration_map's keys should ALWAYS be external/foreign
        #  expressions, and its values should ALWAYS be internal/local expressions
        object_migration_map = {}

        # List of foreign vars used in expression
        foreign_vars = itertools.filterfalse(self.is_declared,
        for foreign_var in foreign_vars:
            # If a variable with the same name was previously migrated
            if foreign_var.name in name_migration_map:
                migrated_name = name_migration_map[foreign_var.name]
                native_var = self.get_variable(migrated_name)
                assert (
                    native_var is not None
                ), "name_migration_map contains a variable that does not exist in this ConstraintSet"
                object_migration_map[foreign_var] = native_var
                # foreign_var was not found in the local declared variables nor
                # any variable with the same name was previously migrated
                # let's make a new unique internal name for it
                migrated_name = foreign_var.name
                if migrated_name in self._declarations:
                    migrated_name = self._make_unique_name(
                # Create and declare a new variable of given type
                if isinstance(foreign_var, Bool):
                    new_var = self.new_bool(name=migrated_name)
                elif isinstance(foreign_var, BitVec):
                    new_var = self.new_bitvec(foreign_var.size,
                elif isinstance(foreign_var, Array):
                    # Note that we are discarding the ArrayProxy encapsulation
                    new_var = self.new_array(
                    raise NotImplementedError(
                        f"Unknown expression type {type(foreign_var)} encountered during expression migration"
                # Update the var to var mapping
                object_migration_map[foreign_var] = new_var
                # Update the name to name mapping
                name_migration_map[foreign_var.name] = new_var.name

        #  Actually replace each appearance of migrated variables by the new ones
        migrated_expression = replace(expression, object_migration_map)
        return migrated_expression
Example #41
 def partition(pred, iterable):
     """From itertools documentation"""
     t1, t2 = tee(iterable)
     return list(filterfalse(pred, t1)), list(filter(pred, t2))
Example #42
def _filter(flist, skip):
    return list(filterfalse(skip.__contains__, flist))
Example #43
def partition(pred: Callable, iterable: Iterable):
    """Use a predicate to partition entries into false entries and true entries"""
    iter_1, iter_2 = tee(iterable)
    return filterfalse(pred, iter_1), filter(pred, iter_2)
Example #44
def iter_2(lst):
    return list(itertools.filterfalse(lambda i: len(i) < 5, lst))
from itertools import filterfalse, chain

# False, 0, [], None
f = filterfalse(None, chain([None, 0, []], range(10)))

for i in f:
Example #46

## dropwhile - skips items from an iterator until a function returns True
values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
less_than_seven = lambda x: x < 7
it = itertools.dropwhile(less_than_seven, values)

## filterfalse - returns all items which are False
values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
evens = lambda x: x % 2 == 0

filter_result = filter(evens, values)
print('Filter:      ', list(filter_result))

filter_false_result = itertools.filterfalse(evens, values)
print('Filter false:', list(filter_false_result))

#Producing combinations of Items from iterators
## accumulate - folds an item from the iterator into a running value
values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
sum_reduce = itertools.accumulate(values)
print('sum:   ', list(sum_reduce))

def sum_modulo_20(first, second):
    output = first + second
    return output % 20

modulo_reduce = itertools.accumulate(values, sum_modulo_20)
Example #47
def partition(pred, list_):
    # Partition the items in lists / non-lists
    list_ = [x for x in list_ if x]
    lists = list(filter(pred, list_))
    not_lists = itertools.filterfalse(pred, list_)
    return (lists, not_lists)
def PartitionIterator(pred: typing.Callable[[object], bool],
                      stream: typing.Iterable[object]):

    (t1, t2) = itertools.tee(stream)

    return (itertools.filterfalse(pred, t1), filter(pred, t2))
Example #49
def get_assigned_names_excluding_class_attributes(tree):
    assigned_items = get_assigned_vars(tree, names_only=False)
    return {
        getattr(node, 'id', None)
        for node in filterfalse(is_class_attribute, assigned_items)
Example #50
def _partition(pred, iterable):
    t1, t2 = itertools.tee(iterable)
    return (list(itertools.filterfalse(pred, t1)), list(filter(pred, t2)))
Example #51
def list_subtract(l1, l2):
    return list(filterfalse(lambda x: x in l2, l1))
Example #52
# ============
# Main script.
# ============
REGEX1 = re.compile(
REGEX2 = re.compile(r"\bRECYCLE\b", re.IGNORECASE)
TEMPDIR = Path(os.path.expandvars("%_TMPDIR%"))

# -----
collection = csv.reader(arguments.collection,
                        dialect=CustomDialect)  # type: Any
collection = [compress(item, [0, 0, 1, 0, 0]) for item in collection]
collection = list(
    filterfalse(match_(REGEX2.search), chain.from_iterable(collection)))

# -----
ape = [
     Path(file).parent.parent / "1.Monkey's Audio" / f"{Path(file).stem}.ape")
    for file in collection
]  # type: List[Tuple[str, Path]]
dsf = [(file, Path(file).parent.parent / "1.DSD 64" / f"{Path(file).stem}.dsf")
       for file in collection]  # type: List[Tuple[str, Path]]
flac = [(file, Path(file).parent.parent / "1.Free Lossless Audio Codec" /
        for file in collection]  # type: List[Tuple[str, Path]]

# -----
collection = chain.from_iterable(
Example #53
def vowel(c):
    return c.lower() in 'aeiou'

print(list(filter(vowel, 'Aardvark')))

import itertools
print(list(itertools.filterfalse(vowel, 'Aardvark')))
print(list(itertools.dropwhile(vowel, 'Aardvark')))
print(list(itertools.takewhile(vowel, 'Aardvark')))
print(list(itertools.compress('Aardvark', (1, 0, 1, 1, 0, 1))))
print(list(itertools.islice('Aardvark', 4)))
print(list(itertools.islice('Aardvark', 4, 7)))
print(list(itertools.islice('Aardvark', 1, 7, 2)))
Example #54
def export_e_emus(session, settings, nodes, emus_file):
    fieldnames = [
    manager_rows = []
    engagement_rows = []

    settings["username-itsystem-uuid"] = (session.query(
        ItSystem.uuid).filter(ItSystem.navn == "Active Directory").scalar())

    for node in tqdm(PreOrderIter(nodes["root"]),
                     desc="export e"):
        ou = node.unit

        # normal engagements - original export
        engagements = (session.query(Engagement).filter(
            Engagement.enhed_uuid == ou.uuid).all())
        engagements = filterfalse(hourly_paid, engagements)
        engagements = filterfalse(partial(discarded, settings), engagements)
        for engagement in engagements:
            logger.info("adding engagement %s", engagement.uuid)
                build_engagement_row(session, settings, ou, engagement))
            engagement_count(nodes, ou)

        # manager engagements - above mentioned musskema adaptation
        for manager in session.query(Leder).filter(
                Leder.enhed_uuid == ou.uuid).all():
            if manager.bruger_uuid is None:
                logger.info("skipping vacant manager %s", manager.uuid)
                continue  # vacant manager
                # extend, as there can be zero or one
                    build_manager_rows(session, settings, ou, manager))

    if not len(manager_rows):
        logger.error("no managers found - did You forget to"
                     " specify correct EMUS_RESPONSIBILITY_CLASS")
    rows = engagement_rows + manager_rows
        "writing %d engagement rows and %d manager rows to file",

    last_changed = datetime.datetime.now().strftime("%Y-%m-%d")
    for r in rows:
        eng_uuid = r.get("engagementUUID") or r["personUUID"]
            '<employee id="%s" uuid="%s" client="%s" lastChanged="%s">\n' % (
        for fn in fieldnames:
            emus_file.write("<%s>%s</%s>\n" % (fn, escape(r.get(fn, "")), fn))
Example #55
from itertools import chain
print(list(chain([1, 2, 3], [4, 5], [6, 7])))

from itertools import filterfalse
        filterfalse(lambda x: len(x) < 5,
                    ['hello', 'i', 'write', 'cool', 'code'])))

def count_digits(num):
    x = str(num)
    res = 0
    for i in x:
        res += int(i)
    return res


def count_words(list_of_words):
    res = 0
    for i in range(len(list_of_words)):
        sign = True
        for j in range(i + 1, len(list_of_words)):
            if list_of_words[i] == list_of_words[j]:
                sign = False
        if sign:
            res += 1
Example #56
    def filter_candidates(self, context):
        for source in self._current_sources:
            ctx = source.context
            ctx['matchers'] = context['matchers']
            ctx['input'] = context['input']
            if context['smartcase']:
                ctx['ignorecase'] = re.search(r'[A-Z]', ctx['input']) is None
            ctx['mode'] = context['mode']
            ctx['async_timeout'] = 0.03 if ctx['mode'] != 'insert' else 0.02
            if ctx['prev_input'] != ctx['input']:
                ctx['prev_time'] = time.time()
                if ctx['is_interactive']:
                    ctx['event'] = 'interactive'
                    ctx['all_candidates'] = self._gather_source_candidates(
                        ctx, source)
            ctx['prev_input'] = ctx['input']
            entire = ctx['all_candidates']
            if ctx['is_async']:
                ctx['event'] = 'async'
                entire += self._gather_source_candidates(ctx, source)
            if len(entire) > 20000 and (time.time() - ctx['prev_time'] <
                                        int(context['skiptime']) / 1000.0):
                ctx['is_skipped'] = True
                yield self._get_source_status(
                    ctx, source, entire, []), [], []
            if not entire:
                yield self._get_source_status(
                    ctx, source, entire, []), [], []

            ctx['is_skipped'] = False
            partial = []
            ctx['candidates'] = entire
            for i in range(0, len(entire), 1000):
                ctx['candidates'] = entire[i:i+1000]
                matchers = [self._filters[x] for x in
                            (ctx['matchers'].split(',') if ctx['matchers']
                             else source.matchers)
                            if x in self._filters]
                self.match_candidates(ctx, matchers)
                partial += ctx['candidates']
                if len(partial) >= source.max_candidates:
            ctx['candidates'] = partial
            for f in [self._filters[x]
                      for x in source.sorters + source.converters
                      if x in self._filters]:
                ctx['candidates'] = f.filter(ctx)
            partial = ctx['candidates'][: source.max_candidates]
            for c in partial:
                c['source_name'] = source.name
                c['source_index'] = source.index
            ctx['candidates'] = []

            patterns = filterfalse(lambda x: x == '', (
                for x in source.matchers if self._filters[x]))

            yield self._get_source_status(
                ctx, source, entire, partial), partial, patterns
Example #57
 def incomplete_files(self):
     return list(chain(*(job.output
                         for job in
                                filterfalse(self.needrun, self.jobs)))))
Example #58
def main(args):
    """Start device identification based on command line parameters."""
    if sys.version_info[0] < 3:
        raise Exception("This tool must be run using Python 3!")
    parser = argparse.ArgumentParser(
        "Tool for classifying IoT devices based on captured network traffic")
    parser = argparser_add_verbose(parser)
        help=("Packet capture file (in PCAP or PCAPNG format) "
              "with recorded traffic for device identification"),
        help="minimum number of packets to classify device "
        "(devices with smaller number will not be classified) (default: 3)",
        "maximum number of packets used to classify device (default: 1000)",
    parser.add_argument("--ip", "-I", help="use IP filter to identify device")
        help="display only short result of classification",

    options = parser.parse_args(args)
    test_name = "device identification"

    if options.verbose:
        print("options: {}".format(options))
        os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"

    packets = load_packets(options.pcap)

    if options.pcap and not options.ip:
        list_ips = ips_from_pcap(packets)
        list_ips = prepare_ips(options.ip)

    list_ips = list(filterfalse(lambda x: x.endswith(".0"), list_ips))
    list_ips = list(filterfalse(lambda x: x.endswith(".1"), list_ips))
    list_ips = list(filterfalse(lambda x: x.endswith(".255"), list_ips))
    list_ips = list(filterfalse(lambda x: x.startswith("224.0."), list_ips))
    list_ips = list(filterfalse(lambda x: x.startswith("232."), list_ips))
    list_ips = list(filterfalse(lambda x: x.startswith("233."), list_ips))
    list_ips = list(filterfalse(lambda x: x.startswith("234."), list_ips))
    list_ips = list(filterfalse(lambda x: x.startswith("239."), list_ips))
    list_ips = list(filterfalse(lambda x: x.startswith("ff0"), list_ips))

    if options.verbose:
        print("list_ips: {}".format(list_ips))

    print(f"[.] Started {test_name}")
        for test_ip in list_ips:
            print(f"[.] Started classification for IP: {test_ip}")
            start_time = time.time()
                show_probability=not options.short_result,
            classify_time = time.time() - start_time
            print(f"[.] Classification time: {classify_time:.2f} sec")
        print(f"[.] Finished {test_name} (for all IPs)")
    except KeyboardInterrupt:
def diff_states(a, b):
    '''Diff 2 arbitrary conftool states'''

    return list(map(str, filterfalse(lambda x: x in b, a)))
Example #60
def curves_iter(curves: Iterable[CurveOrRange]
) -> Iterator[Curve]:
    return cast(Iterator[Curve], 
        filterfalse(lambda expr: expr[1] is None, curves))