Пример #1
0
def get_content_snippet2(content, keyword, max_lines=10):
    max_lines = int(max_lines)
    p = re.compile(
        r'(?P<before>.*)%s(?P<after>.*)' %
        re.escape(keyword),
        re.MULTILINE | re.IGNORECASE | re.DOTALL)
    m = p.search(content)
    html = ""
    if m:
        words = list(filter(
            lambda x: x != "",
            striptags(
                m.group("before")).split("\n")))
        before_lines = words[-max_lines // 2:]
        words = list(filter(
            lambda x: x != "",
            striptags(
                m.group("after")).split("\n")))
        after = "<br/>".join(words[:max_lines - len(before_lines)])
        before = "<br/>".join(before_lines)
        html = "%s %s %s" % (before, striptags(keyword), after)
        kw_p = re.compile(r'(%s)' % keyword, re.IGNORECASE)
        html = kw_p.sub(r"<strong>\1</strong>", html)
        html = mark_safe(html)
    else:
        html = " ".join(
            list(filter(
                lambda x: x != "",
                striptags(content).replace(
                    "\n",
                    " ").split(" ")))[
                :max_lines])
    return html
Пример #2
0
def check(process_output, judge_output, split_on='lines', **kwargs):
    split_pattern = {
        'lines': b'[\r\n]',
        'whitespace': b'[\s]',
    }.get(split_on)

    if not split_pattern:
        raise InternalError('invalid `split_on` mode')

    process_lines = list(filter(None, resplit(split_pattern, utf8bytes(process_output))))
    judge_lines = list(filter(None, resplit(split_pattern, utf8bytes(judge_output))))

    if len(process_lines) != len(judge_lines):
        return False

    if split_on == 'lines':
        process_lines = list(map(six.binary_type.split, process_lines))
        judge_lines = list(map(six.binary_type.split, judge_lines))

    process_lines.sort()
    judge_lines.sort()

    for process_line, judge_line in zip(process_lines, judge_lines):
        if process_line != judge_line:
            return False

    return True
Пример #3
0
    def startup(self, group):
        """ Prepare for a new run.

        Args
        ----
        group : `Group`
            Group that owns this recorder.
        """

        myparams = myunknowns = myresids = set()

        if MPI:
            rank = group.comm.rank
            owned = group._owning_ranks

        # Compute the inclusion lists for recording
        if self.options['record_params']:
            myparams = set(filter(self._check_path, group.params))
        if self.options['record_unknowns']:
            myunknowns = set(filter(self._check_path, group.unknowns))
        if self.options['record_resids']:
            myresids = set(filter(self._check_path, group.resids))

        self._filtered[group.pathname] = {
            'p': myparams,
            'u': myunknowns,
            'r': myresids
        }
Пример #4
0
 def label_and_sentence(line, clean_fn):
     label_text = re.split(TSVSeqLabelReader.SPLIT_ON, line)
     label = label_text[0]
     text = label_text[1:]
     text = ' '.join(list(filter(lambda s: len(s) != 0, [clean_fn(w) for w in text])))
     text = list(filter(lambda s: len(s) != 0, re.split('\s+', text)))
     return label, text
Пример #5
0
def parse_celery_workers(celery_workers):
    """
    Parses the response from the flower get workers api into a list of hosts
    we expect to be running and a list of hosts we expect to be stopped
    """
    expect_stopped = []
    expect_running = list(filter(
        lambda hostname: not hostname.endswith('_timestamp'),
        celery_workers,
    ))

    timestamped_workers = list(filter(
        lambda hostname: hostname.endswith('_timestamp'),
        celery_workers,
    ))

    def _strip_timestamp(hostname):
        return '.'.join(hostname.split('.')[:-1])

    timestamped_workers = sorted(timestamped_workers, key=_strip_timestamp)

    for hostname, group in groupby(timestamped_workers, _strip_timestamp):

        sorted_workers = sorted(list(group), reverse=True)
        expect_running.append(sorted_workers.pop(0))
        expect_stopped.extend(sorted_workers)
    return expect_running, expect_stopped
Пример #6
0
    def process_trade(self, trade_event):

        if trade_event.sid not in self.open_orders:
            return

        if trade_event.volume < 1:
            # there are zero volume trade_events bc some stocks trade
            # less frequently than once per minute.
            return

        orders = self.open_orders[trade_event.sid]
        orders.sort(key=lambda o: o.dt)
        # Only use orders for the current day or before
        current_orders = filter(
            lambda o: o.dt <= trade_event.dt,
            orders)

        processed_orders = []
        for txn, order in self.process_transactions(trade_event,
                                                    current_orders):
            processed_orders.append(order)
            yield txn, order

        # remove closed orders. we should only have to check
        # processed orders
        def not_open(order):
            return not order.open
        closed_orders = filter(not_open, processed_orders)
        for order in closed_orders:
            orders.remove(order)

        if len(orders) == 0:
            del self.open_orders[trade_event.sid]
Пример #7
0
    def process(self):
        """
        Process the file upload and add products to the range
        """
        all_ids = set(self.extract_ids())
        products = self.range.included_products.all()
        existing_skus = products.values_list('stockrecord__partner_sku',
                                             flat=True)
        existing_skus = set(filter(bool, existing_skus))
        existing_upcs = products.values_list('upc', flat=True)
        existing_upcs = set(filter(bool, existing_upcs))
        existing_ids = existing_skus.union(existing_upcs)
        new_ids = all_ids - existing_ids

        products = Product._default_manager.filter(
            models.Q(stockrecord__partner_sku__in=new_ids) |
            models.Q(upc__in=new_ids))
        for product in products:
            self.range.add_product(product)

        # Processing stats
        found_skus = products.values_list('stockrecord__partner_sku',
                                          flat=True)
        found_skus = set(filter(bool, found_skus))
        found_upcs = set(filter(bool, products.values_list('upc', flat=True)))
        found_ids = found_skus.union(found_upcs)
        missing_ids = new_ids - found_ids
        dupes = set(all_ids).intersection(existing_ids)

        self.mark_as_processed(products.count(), len(missing_ids), len(dupes))
Пример #8
0
    def _get_eligible_broker_pair(self, under_loaded_rg, eligible_partition):
        """Evaluate and return source and destination broker-pair from over-loaded
        and under-loaded replication-group if possible, return None otherwise.

        Return source broker with maximum partitions and destination broker with
        minimum partitions based on following conditions:-
        1) At-least one broker in under-loaded group which does not have
        victim-partition. This is because a broker cannot have duplicate replica.
        2) At-least one broker in over-loaded group which has victim-partition
        """
        under_brokers = list(filter(
            lambda b: eligible_partition not in b.partitions,
            under_loaded_rg.brokers,
        ))
        over_brokers = list(filter(
            lambda b: eligible_partition in b.partitions,
            self.brokers,
        ))

        # Get source and destination broker
        source_broker, dest_broker = None, None
        if over_brokers:
            source_broker = max(
                over_brokers,
                key=lambda broker: len(broker.partitions),
            )
        if under_brokers:
            dest_broker = min(
                under_brokers,
                key=lambda broker: len(broker.partitions),
            )
        return (source_broker, dest_broker)
Пример #9
0
    def test_get_questions_with_repeats(self):
        """
        This test ensures that questions that start with the repeat group id
        do not get marked as repeats. For example:

            /data/repeat_name <-- repeat group path
            /data/repeat_name_count <-- question path

        Before /data/repeat_name_count would be tagged as a repeat incorrectly.
        See http://manage.dimagi.com/default.asp?234108 for context
        """
        form = self.app.get_form(self.form_with_repeats_unique_id)
        questions = form.wrapped_xform().get_questions(
            ['en'],
            include_groups=True,
        )

        repeat_name_count = list(filter(
            lambda question: question['value'] == '/data/repeat_name_count',
            questions,
        ))[0]
        self.assertIsNone(repeat_name_count['repeat'])

        repeat_question = list(filter(
            lambda question: question['value'] == '/data/repeat_name/question5',
            questions,
        ))[0]
        self.assertEqual(repeat_question['repeat'], '/data/repeat_name')
Пример #10
0
def _leaf_versions(tree, rc):
    '''
    Recursively traverse the versions tree in a depth-first fashion,
    and collect the last node of each branch, i.e. leaf versions.
    '''
    versions = []
    if _is_iterable(tree):
        for subtree in tree:
            versions.extend(_leaf_versions(subtree, rc))
        if not versions:
            if rc:
                last_rc = next(filter(lambda v: v.is_rc, reversed(tree)), None)
                last_prod = next(
                    filter(lambda v: not v.is_rc, reversed(tree)), None)
                if last_rc and last_prod and (last_prod < last_rc):
                    versions.extend([last_prod, last_rc])
                elif not last_prod:
                    versions.append(last_rc)
                else:
                    # Either there is no RC, or we ignore the RC as older than
                    # the latest production version:
                    versions.append(last_prod)
            else:
                versions.append(tree[-1])
    return versions
Пример #11
0
def removePyc(folder, only_excess=True, show_logs=True):

    folder = sp(folder)

    for root, dirs, files in os.walk(folder):

        pyc_files = filter(lambda filename: filename.endswith(".pyc"), files)
        py_files = set(filter(lambda filename: filename.endswith(".py"), files))
        excess_pyc_files = (
            filter(lambda pyc_filename: pyc_filename[:-1] not in py_files, pyc_files) if only_excess else pyc_files
        )

        for excess_pyc_file in excess_pyc_files:
            full_path = os.path.join(root, excess_pyc_file)
            if show_logs:
                log.debug("Removing old PYC file: %s", full_path)
            try:
                os.remove(full_path)
            except:
                log.error("Couldn't remove %s: %s", (full_path, traceback.format_exc()))

        for dir_name in dirs:
            full_path = os.path.join(root, dir_name)
            if len(os.listdir(full_path)) == 0:
                try:
                    os.rmdir(full_path)
                except:
                    log.error("Couldn't remove empty directory %s: %s", (full_path, traceback.format_exc()))
Пример #12
0
    def test_simple_plan_add_on_creation(self):
        # add a sample plan to the plans backend
        mocurly.backend.plans_backend.add_object(self.base_backed_plan_data['plan_code'], self.base_backed_plan_data)

        self.assertEqual(len(mocurly.backend.plan_add_ons_backend.datastore), 0)

        # now create some addons
        plan = recurly.Plan.get(self.base_backed_plan_data['plan_code'])
        for add_on in self.base_add_on_data:
            add_on['name'] = add_on['add_on_code'].upper()
            add_on['unit_amount_in_cents'] = recurly.Money(**add_on['unit_amount_in_cents'])
            plan.create_add_on(recurly.AddOn(**add_on))

        self.assertEqual(len(mocurly.backend.plan_add_ons_backend.datastore), 2)
        foo_add_on_backed = mocurly.backend.plan_add_ons_backend.get_object(self.base_backed_plan_data['plan_code'] + '__foo')
        add_ons = filter(lambda add_on: add_on['add_on_code'] == 'foo', self.base_add_on_data)
        foo_add_on = next(add_ons)
        for k, v in foo_add_on.items():
            if k == 'unit_amount_in_cents':
                self.assertEqual(foo_add_on_backed[k], dict((curr, str(amt)) for curr, amt in v.currencies.items()))
            else:
                self.assertEqual(foo_add_on_backed[k], v)

        bar_add_on_backed = mocurly.backend.plan_add_ons_backend.get_object(self.base_backed_plan_data['plan_code'] + '__bar')
        add_ons = filter(lambda add_on: add_on['add_on_code'] == 'bar', self.base_add_on_data)
        bar_add_on = next(add_ons)
        for k, v in bar_add_on.items():
            if k == 'unit_amount_in_cents':
                self.assertEqual(bar_add_on_backed[k], dict((curr, str(amt)) for curr, amt in v.currencies.items()))
            else:
                self.assertEqual(bar_add_on_backed[k], v)

        # make sure foreign keys are linked properly
        self.assertEqual(len(plan.add_ons()), 2)
Пример #13
0
 def test_in(self):
     values = ['a', 'b', 'c']
     filter = self.get_filter('in', values)
     for value in values:
         self.assertTrue(filter({'foo': value}))
     for value in ['d', 'e', 'f']:
         self.assertFalse(filter({'foo': value}))
Пример #14
0
    def retrieve_keys(bucket, key, prefix='', postfix='', delim='/',
                      directories=False, recursive=False):
        """
        Retrieve keys from a bucket
        """
        if key and prefix:
            assert key.endswith(delim)

        key += prefix
        # check whether key is a directory
        if not key.endswith(delim) and key:
            # check for matching prefix
            if BotoClient.check_prefix(bucket, key + delim, delim=delim):
                # found a directory
                key += delim

        listdelim = delim if not recursive else None
        results = bucket.list(prefix=key, delimiter=listdelim)
        if postfix:
            func = lambda k_: BotoClient.filter_predicate(k_, postfix, inclusive=True)
            return filter(func, results)
        elif not directories:
            func = lambda k_: BotoClient.filter_predicate(k_, delim, inclusive=False)
            return filter(func, results)
        else:
            return results
Пример #15
0
    def get_revert_migrations(self, current_migrations, backup_migrations):
        current_migrations, all_migrations = itertools.tee(reversed(list(map(
            Migration,
            filter(None, current_migrations.splitlines()),
        ))))
        all_migrations = utils.OrderedSet(all_migrations)

        backup_migrations = reversed(list(map(
            Migration,
            filter(None, backup_migrations.splitlines()),
        )))

        revert_migrations = collections.OrderedDict()

        while True:
            while True:
                backup_migration = next(backup_migrations, None)
                if not backup_migration or backup_migration in all_migrations:
                    break
            for current_migration in current_migrations:
                if current_migration == backup_migration:
                    break
                revert_migration = self._get_parent_migration(
                    current_migration,
                    migrations=all_migrations,
                )
                revert_migrations[current_migration.app] = revert_migration

            if backup_migration is None:
                return revert_migrations.values()
Пример #16
0
 def DelayedFcnCall(*args, **kwargs):
   # Check to see if any args or kw are Later. If not, return normal fcn.
   checkIfLater = lambda x: type(x) == Later
   if (len(list(filter(checkIfLater, args))) == 0 and 
       len(list(filter(checkIfLater, list(kwargs.values())))) == 0):
     return fcn(*args, **kwargs)
   else:
     return CreateLaterFunction(fcn, *args, **kwargs)
Пример #17
0
    def startup(self, group):
        """ Prepare for new run. """

        # Compute the inclusion lists for recording
        params = list(filter(self._check_path, group.params))
        unknowns = list(filter(self._check_path, group.unknowns))
        resids = list(filter(self._check_path, group.resids))

        self._filtered[group.pathname] = (params, unknowns, resids)
Пример #18
0
    def __init__(self, namespaces=None, pollster_list=None):
        namespaces = namespaces or ['compute', 'central']
        pollster_list = pollster_list or []
        group_prefix = cfg.CONF.polling.partitioning_group_prefix

        # features of using coordination and pollster-list are exclusive, and
        # cannot be used at one moment to avoid both samples duplication and
        # samples being lost
        if pollster_list and cfg.CONF.coordination.backend_url:
            raise PollsterListForbidden()

        super(AgentManager, self).__init__()

        def _match(pollster):
            """Find out if pollster name matches to one of the list."""
            return any(utils.match(pollster.name, pattern) for
                       pattern in pollster_list)

        if type(namespaces) is not list:
            namespaces = [namespaces]

        # we'll have default ['compute', 'central'] here if no namespaces will
        # be passed
        extensions = (self._extensions('poll', namespace).extensions
                      for namespace in namespaces)
        # get the extensions from pollster builder
        extensions_fb = (self._extensions_from_builder('poll', namespace)
                         for namespace in namespaces)
        if pollster_list:
            extensions = (moves.filter(_match, exts)
                          for exts in extensions)
            extensions_fb = (moves.filter(_match, exts)
                             for exts in extensions_fb)

        self.extensions = list(itertools.chain(*list(extensions))) + list(
            itertools.chain(*list(extensions_fb)))

        if self.extensions == []:
            raise EmptyPollstersList()

        self.discovery_manager = self._extensions('discover')
        self.context = context.RequestContext('admin', 'admin', is_admin=True)
        self.partition_coordinator = coordination.PartitionCoordinator()

        # Compose coordination group prefix.
        # We'll use namespaces as the basement for this partitioning.
        namespace_prefix = '-'.join(sorted(namespaces))
        self.group_prefix = ('%s-%s' % (namespace_prefix, group_prefix)
                             if group_prefix else namespace_prefix)

        self.notifier = oslo_messaging.Notifier(
            messaging.get_transport(),
            driver=cfg.CONF.publisher_notifier.telemetry_driver,
            publisher_id="ceilometer.polling")

        self._keystone = None
        self._keystone_last_exception = None
Пример #19
0
 def purge_old(self):
     cur_ts = time.time()
     prev_ts = cur_ts - self.purge_elapsed
     self.failed_log = list(filter(lambda x: x > prev_ts, self.failed_log))
     self.succeeded_log = list(filter(lambda x: x > prev_ts,
                                      self.succeeded_log))
     self.failed_cnt = len(self.failed_log)
     self.succeed_cnt = len(self.succeeded_log)
     self.start_point = self._begin_log_ts()
Пример #20
0
def qsort(l, cmp_fn=cmp_num):
    i = len(l)
    if i == 0:
        return l
    pivot = i // 2
    elt = l[pivot]
    lt = list(filter(lambda x: cmp_fn(x, elt) == -1, l))
    eq = list(filter(lambda x: cmp_fn(x, elt) == 0, l))
    gt = list(filter(lambda x: cmp_fn(x, elt) == 1, l))

    return qsort(lt, cmp_fn=cmp_fn) + eq + qsort(gt, cmp_fn=cmp_fn)
Пример #21
0
def load_numpy(max_pos=-1, max_neg=-1, *, unpack = False, scale_to_largest_image=False, scale_to_size=None):
    # If you specify the maximum number of positive examples but not the maximum
    # number of negative examples, it tries to return a 50/50 split. That may not
    # actually make sense
    if max_neg==-1 and max_pos>-1:
        max_neg = max_pos

    if unpack:
        raise Exception('Unpacking to local FS is not supported yet')

    if scale_to_size is not None and scale_to_largest_image:
        raise Exception('Specify scaling to the largest image OR a size to scale to.')

    # Debatable whether this is actually needed.
    if scale_to_size is None and scale_to_largest_image:
        with zipfile.ZipFile(train_zip_path) as z:
            filelist = filter(jpg_re.match, z.namelist())
            scale_to_size = util.find_largest_image_in_zip(z, filelist)

    if scale_to_size is None and not scale_to_largest_image:
        # silently changing to a hardcoded value, beware!
        scale_to_size = [768,1024]

    n_pos = 0
    n_neg = 0
    X = list()
    y = list()
    # Actually do the load
    with zipfile.ZipFile(train_zip_path) as z:
        filelist = filter(jpg_re.match, z.namelist())
        for f in filelist:
            if cat_re.match(f) and n_pos != max_pos:
                y.append(1)
                n_pos += 1
                X.append(
                    util.process_img_from_file(
                        BytesIO(z.read(f)),
                        resize_dims = scale_to_size
                    )
                )
            elif (not cat_re.match(f)) and n_neg != max_neg:
                y.append(0)
                n_neg += 1
                X.append(
                    util.process_img_from_file(
                        BytesIO(z.read(f)),
                        resize_dims = scale_to_size
                    )
                )
            if n_pos == max_pos and n_neg == max_neg:
                break

    return (np.vstack(X), np.array(y
    ))
def quickinstall(plone, products):
    logger.info("Quick installing: %s", products)
    qit = plone.portal_quickinstaller
    not_installed_ids = [
        x['id'] for x in qit.listInstallableProducts(skipInstalled=1)]
    installed_ids = [x['id'] for x in qit.listInstalledProducts()]
    installed_products = list(filter(installed_ids.count, products))
    not_installed = list(filter(not_installed_ids.count, products))
    if installed_products:
        qit.reinstallProducts(installed_products)
    if not_installed_ids:
        qit.installProducts(not_installed)
Пример #23
0
 def _get_nodes(self, account=None, type=None):
     """Return affected nodes matching the filters."""
     result = self.affected_nodes
     if account:
         # Allow to filter by multiple accounts
         if not isinstance(account, list):
             account = [account]
         result = filter(
             lambda n: all([n.affects_account(a) for a in account]), result)
     if type:
         result = filter(lambda n: n.type == type, result)
     return list(result)
Пример #24
0
    def getAndAssertTableRowAction(self, response, table_name, action_name, row_id):
        table = response.context[table_name + "_table"]
        rows = list(moves.filter(lambda x: x.id == row_id, table.data))
        self.assertEqual(1, len(rows), "Did not find a row matching id '%s'" % row_id)
        row_actions = table.get_row_actions(rows[0])
        actions = list(moves.filter(lambda x: x.name == action_name, row_actions))

        msg_args = (action_name, table_name, row_id)
        self.assertTrue(len(actions) > 0, "No action named '%s' found in '%s' table for id '%s'" % msg_args)

        self.assertEqual(1, len(actions), "Multiple actions named '%s' found in '%s' table for id '%s'" % msg_args)

        return actions[0]
Пример #25
0
def count_instances(inst=None, group=None, referrers=False, referents=False, prev=0):
    gc.collect()

    if group:
        #         def t(x):
        #             try:
        #                 return group in x.__class__.__name__
        #             except:
        #                 pass
        t = lambda x: group in str(type(x))
        n = group
        objs = list(filter(t, gc.get_objects()))

        s = sum(sys.getsizeof(o) for o in objs) * 1024 ** -2
        nn = len(objs)
        print('{:<50s}:{} {} {} {}'.format(n, nn, s, prev, nn - prev))
        return nn

    elif inst:
        t = lambda x: isinstance(x, inst)
        n = str(inst)
        objs = list(filter(t, gc.get_objects()))
        s = sum(sys.getsizeof(o) for o in objs) * 1024 ** -2
        print('{:<50s}:{} {}'.format(n, len(objs), s))

    else:
        objs = gc.get_objects()
        key = lambda x: type(x)
        objs = sorted(objs, key=key)
        #         for g, aa in groupby(objs, key=key):
        #             d = [sys.getsizeof(ai) for ai in aa]
        #             s = sum(d)
        #             print '{} {} {}'.format(g, len(d), s)

        xx = [(g, [ai for ai in aa])
              for g, aa in groupby(objs, key=key)]

        xx = [(g, sum([sys.getsizeof(ai) for ai in aa]), len(aa))
              for g, aa in xx]

        for g, s, n in sorted(xx, key=lambda x: x[1]):
            if s > 1000:
                print('{:<50s} {} {}'.format(g, s, n))

    if referrers:
        for obj in objs:
            show_referrers(obj)
    if referents:
        for obj in objs:
            show_referents(obj)
Пример #26
0
    def getImages(self, files):

        def test(s):
            return getExt(s.lower()) in ['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tbn']
        files = set(filter(test, files))

        images = {
            'backdrop': set(filter(lambda s: re.search('(^|[\W_])fanart|backdrop\d*[\W_]', s.lower()) and self.filesizeBetween(s, self.file_sizes['backdrop']), files))
        }

        # Rest
        images['rest'] = files - images['backdrop']

        return images
Пример #27
0
    def group_structures(self, s_list, anonymous=False):
        """
        Given a list of structures, use fit to group
        them by structural equality.

        Args:
            s_list ([Structure]): List of structures to be grouped
            anonymous (bool): Wheher to use anonymous mode.

        Returns:
            A list of lists of matched structures
            Assumption: if s1 == s2 but s1 != s3, than s2 and s3 will be put
            in different groups without comparison.
        """
        if self._subset:
            raise ValueError("allow_subset cannot be used with"
                             " group_structures")

        original_s_list = list(s_list)
        s_list = self._process_species(s_list)

        # Use structure hash to pre-group structures
        if anonymous:
            c_hash = lambda c: c.anonymized_formula
        else:
            c_hash = self._comparator.get_hash
        s_hash = lambda s: c_hash(s[1].composition)
        sorted_s_list = sorted(enumerate(s_list), key=s_hash)
        all_groups = []

        # For each pre-grouped list of structures, perform actual matching.
        for k, g in itertools.groupby(sorted_s_list, key=s_hash):
            unmatched = list(g)
            while len(unmatched) > 0:
                i, refs = unmatched.pop(0)
                matches = [i]
                if anonymous:
                    inds = filter(lambda i: self.fit_anonymous(refs,
                            unmatched[i][1]), list(range(len(unmatched))))
                else:
                    inds = filter(lambda i: self.fit(refs, unmatched[i][1]),
                                  list(range(len(unmatched))))
                inds = list(inds)
                matches.extend([unmatched[i][0] for i in inds])
                unmatched = [unmatched[i] for i in range(len(unmatched))
                             if i not in inds]
                all_groups.append([original_s_list[i] for i in matches])

        return all_groups
Пример #28
0
def get_recognizer(args):
    """Get the file recognizer object from the configured options."""
    # Make sure we have empty sets when we have empty strings.
    skip_dirs = set(filter(None, args.skip_dirs.split(",")))
    skip_exts = set(filter(None, args.skip_exts.split(",")))
    fr = FileRecognizer(
        skip_hidden_files=args.skip_hidden_files,
        skip_backup_files=args.skip_backup_files,
        skip_hidden_dirs=args.skip_hidden_dirs,
        skip_dirs=skip_dirs,
        skip_exts=skip_exts,
        skip_symlink_files=not args.follow_symlinks,
        skip_symlink_dirs=not args.follow_symlinks,
    )
    return fr
Пример #29
0
 def inner(function):
     if isinstance(ambiguous_keyname, list):
         keynames = map(self._get_precise_keyname, filter(bool, ambiguous_keyname))
         for keyname in keynames:
             self._statusbar_dispatch_table[keyname] = function
     elif isinstance(ambiguous_keyname, binary_type):
         keynames = map(self._get_precise_keyname, filter(bool, BLANK.split(ambiguous_keyname)))
         for keyname in keynames:
             self._statusbar_dispatch_table[keyname] = function
     elif isinstance(ambiguous_keyname, text_type):
         keynames = map(self._get_precise_keyname, filter(bool, BLANK.split(b(ambiguous_keyname))))
         for keyname in keynames:
             self._statusbar_dispatch_table[keyname] = function
     else:
         raise(Exception("bad argument error"))
Пример #30
0
def _merge_options_wrapper(merge_options, first, second):
    """Send the vectors to merge_options with the right order.
    Assuming that merge_options will apply second options on first, send the
    less significant vector first. First checking the depths where the vectors
    were found. If equal, fallback to check which vector has more categories,
    and eventually check who has the most significant category configured.
    Most significant category is considered the right-most category.

    :param func merge_options:  Function that merges options.
    :param tuple first:         A vector to merge.
    :param tuple second:        A vector to merge.

    :returns: The output from merge_options.
    """
    first_opts = first[-1]
    second_opts = second[-1]
    first_depth = first_opts[DepthLevel]
    second_depth = second_opts[DepthLevel]
    logger.debug(
        'Vector depths (first, second) = (%s, %s)',
        first_depth, second_depth
    )
    if first_depth > second_depth:
        return merge_options(second_opts, first_opts)
    elif second_depth > first_depth:
        return merge_options(first_opts, second_opts)
    # Depth level is equal
    # Check who have more categories configured
    first_weight = len(tuple(filter(None, first[:-1])))
    second_weight = len(tuple(filter(None, second[:-1])))
    logger.debug(
        'Vector weights (first, second) = (%s, %s)',
        first_weight, second_weight
    )
    if first_weight > second_weight:
        return merge_options(second_opts, first_opts)
    elif second_weight > first_weight:
        return merge_options(first_opts, second_opts)
    # Both have same amount of categories configured
    # Check who has most significant category
    for first_c, second_c in reversed(list(zip(first[:-1], second[:-1]))):
        if first_c and not second_c:
            logger.debug('First has most significant category: %s', first_c)
            return merge_options(second_opts, first_opts)
        if second_c and not first_c:
            logger.debug('Second has most significant category: %s', second_c)
            return merge_options(first_opts, second_opts)
    return merge_options(first_opts, second_opts)
Пример #31
0
 def test_greater_than_equal(self):
     filter = self.get_filter('gte', 3)
     for match in (3, 11, '2'):
         self.assertTrue(filter({'foo': match}))
     for non_match in (-10, 0, 2):
         self.assertFalse(filter({'foo': non_match}))
Пример #32
0
    def _parse_varnishadm(self, output, tags):
        """ Parse out service checks from varnishadm.

        Example output:

            Backend b0 is Sick
            Current states  good:  2 threshold:  3 window:  5
            Average responsetime of good probes: 0.000000
            Oldest                                                    Newest
            ================================================================
            -------------------------------------------------------------444 Good IPv4
            -------------------------------------------------------------XXX Good Xmit
            -------------------------------------------------------------RRR Good Recv
            ----------------------------------------------------------HHH--- Happy
            Backend b1 is Sick
            Current states  good:  2 threshold:  3 window:  5
            Average responsetime of good probes: 0.000000
            Oldest                                                    Newest
            ================================================================
            ----------------------------------------------------------HHH--- Happy

        Example output (new output format):

            Backend name                   Admin      Probe
            boot.default                   probe      Healthy (no probe)
            boot.backend2                  probe      Healthy 4/4
              Current states  good:  4 threshold:  3 window:  4
              Average response time of good probes: 0.002504
              Oldest ================================================== Newest
              --------------------------------------------------------------44 Good IPv4
              --------------------------------------------------------------XX Good Xmit
              --------------------------------------------------------------RR Good Recv
              ------------------------------------------------------------HHHH Happy

        """
        # Process status by backend.
        backends_by_status = defaultdict(list)
        for line in output.split("\n"):
            backend, status, message = None, None, None
            # split string and remove all empty fields
            tokens = filter(None, line.strip().split(' '))
            tokens = [t for t in tokens]
            if len(tokens):
                if tokens == ['Backend', 'name', 'Admin', 'Probe']:
                    # skip the column headers that exist in new output format
                    continue
                # parse new output format
                # the backend name will include the vcl name
                # so split on first . to remove prefix
                elif len(tokens) >= 4 and tokens[1] in ['healthy', 'sick']:
                    # If the backend health was overriden, lets grab the
                    # overriden value instead of the probed health
                    backend = tokens[0].split('.', 1)[-1]
                    status = tokens[1].lower()
                elif len(tokens) >= 4 and tokens[1] == 'probe':
                    backend = tokens[0].split('.', 1)[-1]
                    status = tokens[2].lower()
                # Parse older Varnish backend output
                elif tokens[0] == 'Backend':
                    backend = tokens[1]
                    status = tokens[-1].lower()

                if tokens[0] == 'Current' and backend is not None:
                    try:
                        message = ' '.join(tokens[2:]).strip()
                    except Exception:
                        # If we can't parse a message still send a status.
                        self.log.exception(
                            'Error when parsing message from varnishadm')
                        message = ''

                if backend is not None:
                    backends_by_status[status].append((backend, message))

        for status, backends in iteritems(backends_by_status):
            check_status = BackendStatus.to_check_status(status)
            for backend, message in backends:
                service_checks_tags = ['backend:%s' % backend] + tags
                self.service_check(self.SERVICE_CHECK_NAME,
                                   check_status,
                                   tags=service_checks_tags,
                                   message=message)
Пример #33
0
 def test_equal(self):
     match = 'match'
     filter = self.get_filter('eq', match)
     self.assertTrue(filter({'foo': match}))
     self.assertFalse(filter({'foo': 'non-match'}))
     self.assertFalse(filter({'foo': None}))
Пример #34
0
 def __get_best_for_criteria(journeys, criteria):
     return min_from_criteria(
         filter(has_pt, journeys),
         [criteria, duration_crit, transfers_crit, nonTC_crit])
Пример #35
0
def curl(i_cube, j_cube, k_cube=None, ignore=None):
    r'''
    Calculate the 3d curl of the given vector of cubes.

    Args:

    * i_cube
        The i cube of the vector to operate on
    * j_cube
        The j cube of the vector to operate on

    Kwargs:

    * k_cube
        The k cube of the vector to operate on
    * ignore
        This argument is not used.
        .. deprecated:: 0.8
            The coordinates to ignore are determined automatically.

    Return (i_cmpt_curl_cube, j_cmpt_curl_cube, k_cmpt_curl_cube)

    The calculation of curl is dependent on the type of
    :func:`iris.coord_systems.CoordSystem` in the cube:

        Cartesian curl

            The Cartesian curl is defined as:

            .. math::

                \nabla\times \vec u =
                (\frac{\delta w}{\delta y} - \frac{\delta v}{\delta z})\vec a_i
                -
                (\frac{\delta w}{\delta x} - \frac{\delta u}{\delta z})\vec a_j
                +
                (\frac{\delta v}{\delta x} - \frac{\delta u}{\delta y})\vec a_k

        Spherical curl

            When spherical calculus is used, i_cube is the phi vector
            component (e.g. eastward), j_cube is the theta component
            (e.g. northward) and k_cube is the radial component.

            The spherical curl is defined as:

            .. math::

                \nabla\times \vec A = \frac{1}{r cos \theta}
                (\frac{\delta}{\delta \theta}
                (\vec A_\phi cos \theta) -
                \frac{\delta \vec A_\theta}{\delta \phi}) \vec r +
                \frac{1}{r}(\frac{1}{cos \theta}
                \frac{\delta \vec A_r}{\delta \phi} -
                \frac{\delta}{\delta r} (r \vec A_\phi))\vec \theta +
                \frac{1}{r}
                (\frac{\delta}{\delta r}(r \vec A_\theta) -
                \frac{\delta \vec A_r}{\delta \theta}) \vec \phi

            where phi is longitude, theta is latitude.

    '''
    if ignore is not None:
        ignore = None
        warnings.warn('The ignore keyword to iris.analysis.calculus.curl '
                      'is deprecated, ignoring is now done automatically.')

    # Get the vector quantity names.
    # (i.e. ['easterly', 'northerly', 'vertical'])
    vector_quantity_names, phenomenon_name = \
        spatial_vectors_with_phenom_name(i_cube, j_cube, k_cube)

    cubes = filter(None, [i_cube, j_cube, k_cube])

    # get the names of all coords binned into useful comparison groups
    coord_comparison = iris.analysis.coord_comparison(*cubes)

    bad_coords = coord_comparison['ungroupable_and_dimensioned']
    if bad_coords:
        raise ValueError("Coordinates found in one cube that describe "
                         "a data dimension which weren't in the other "
                         "cube ({}), try removing this coordinate.".format(
                             ', '.join(group.name() for group in bad_coords)))

    bad_coords = coord_comparison['resamplable']
    if bad_coords:
        raise ValueError('Some coordinates are different ({}), consider '
                         'resampling.'.format(
                             ', '.join(group.name() for group in bad_coords)))

    ignore_string = ''
    if coord_comparison['ignorable']:
        ignore_string = ' (ignoring {})'.format(
            ', '.join(group.name() for group in bad_coords))

    # Get the dim_coord, or None if none exist, for the xyz dimensions
    x_coord = i_cube.coord(axis='X')
    y_coord = i_cube.coord(axis='Y')
    z_coord = i_cube.coord(axis='Z')

    y_dim = i_cube.coord_dims(y_coord)[0]

    horiz_cs = i_cube.coord_system('CoordSystem')

    # Planar (non spherical) coords?
    ellipsoidal = isinstance(horiz_cs, (iris.coord_systems.GeogCS,
                                        iris.coord_systems.RotatedGeogCS))
    if not ellipsoidal:

        # TODO Implement some mechanism for conforming to a common grid
        dj_dx = _curl_differentiate(j_cube, x_coord)
        prototype_diff = dj_dx

        # i curl component (dk_dy - dj_dz)
        dk_dy = _curl_differentiate(k_cube, y_coord)
        dk_dy = _curl_regrid(dk_dy, prototype_diff)
        dj_dz = _curl_differentiate(j_cube, z_coord)
        dj_dz = _curl_regrid(dj_dz, prototype_diff)

        # TODO Implement resampling in the vertical (which regridding
        # does not support).
        if dj_dz is not None and dj_dz.data.shape != prototype_diff.data.shape:
            dj_dz = _curl_change_z(dj_dz, z_coord, prototype_diff)

        i_cmpt = _curl_subtract(dk_dy, dj_dz)
        dj_dz = dk_dy = None

        # j curl component (di_dz - dk_dx)
        di_dz = _curl_differentiate(i_cube, z_coord)
        di_dz = _curl_regrid(di_dz, prototype_diff)

        # TODO Implement resampling in the vertical (which regridding
        # does not support).
        if di_dz is not None and di_dz.data.shape != prototype_diff.data.shape:
            di_dz = _curl_change_z(di_dz, z_coord, prototype_diff)

        dk_dx = _curl_differentiate(k_cube, x_coord)
        dk_dx = _curl_regrid(dk_dx, prototype_diff)
        j_cmpt = _curl_subtract(di_dz, dk_dx)
        di_dz = dk_dx = None

        # k curl component ( dj_dx - di_dy)
        di_dy = _curl_differentiate(i_cube, y_coord)
        di_dy = _curl_regrid(di_dy, prototype_diff)
        # Since prototype_diff == dj_dx we don't need to recalculate dj_dx
#        dj_dx = _curl_differentiate(j_cube, x_coord)
#        dj_dx = _curl_regrid(dj_dx, prototype_diff)
        k_cmpt = _curl_subtract(dj_dx, di_dy)
        di_dy = dj_dx = None

        result = [i_cmpt, j_cmpt, k_cmpt]

    # Spherical coords (GeogCS or RotatedGeogCS).
    else:
        # A_\phi = i ; A_\theta = j ; A_\r = k
        # theta = lat ; phi = long ;
        # r_cmpt = 1 / (r * cos(lat)) *
        #    (d/dtheta (i_cube * sin(lat)) - d_j_cube_dphi)
        # phi_cmpt = 1/r * ( d/dr (r * j_cube) - d_k_cube_dtheta)
        # theta_cmpt = 1/r * ( 1/cos(lat) * d_k_cube_dphi - d/dr (r * i_cube)
        if y_coord.name() not in ['latitude', 'grid_latitude'] \
                or x_coord.name() not in ['longitude', 'grid_longitude']:
            raise ValueError('Expecting latitude as the y coord and '
                             'longitude as the x coord for spherical curl.')

        # Get the radius of the earth - and check for sphericity
        ellipsoid = horiz_cs
        if isinstance(horiz_cs, iris.coord_systems.RotatedGeogCS):
            ellipsoid = horiz_cs.ellipsoid
        if ellipsoid:
            # TODO: Add a test for this
            r = ellipsoid.semi_major_axis
            r_unit = cf_units.Unit("m")
            spherical = (ellipsoid.inverse_flattening == 0.0)
        else:
            r = DEFAULT_SPHERICAL_EARTH_RADIUS
            r_unit = DEFAULT_SPHERICAL_EARTH_RADIUS_UNIT
            spherical = True

        if not spherical:
            raise ValueError('Cannot take the curl over a non-spherical '
                             'ellipsoid.')

        lon_coord = x_coord.copy()
        lat_coord = y_coord.copy()
        lon_coord.convert_units('radians')
        lat_coord.convert_units('radians')
        lat_cos_coord = _coord_cos(lat_coord)

        # TODO Implement some mechanism for conforming to a common grid
        temp = iris.analysis.maths.multiply(i_cube, lat_cos_coord, y_dim)
        dicos_dtheta = _curl_differentiate(temp, lat_coord)
        prototype_diff = dicos_dtheta

        # r curl component: 1 / (r * cos(lat)) * (d_j_cube_dphi - dicos_dtheta)
        # Since prototype_diff == dicos_dtheta we don't need to
        # recalculate dicos_dtheta.
        d_j_cube_dphi = _curl_differentiate(j_cube, lon_coord)
        d_j_cube_dphi = _curl_regrid(d_j_cube_dphi, prototype_diff)
        new_lat_coord = d_j_cube_dphi.coord(axis='Y')
        new_lat_cos_coord = _coord_cos(new_lat_coord)
        lat_dim = d_j_cube_dphi.coord_dims(new_lat_coord)[0]
        r_cmpt = iris.analysis.maths.divide(_curl_subtract(d_j_cube_dphi,
                                                           dicos_dtheta),
                                            r * new_lat_cos_coord, dim=lat_dim)
        r_cmpt.units = r_cmpt.units / r_unit
        d_j_cube_dphi = dicos_dtheta = None

        # phi curl component: 1/r * ( drj_dr - d_k_cube_dtheta)
        drj_dr = _curl_differentiate(r * j_cube, z_coord)
        if drj_dr is not None:
            drj_dr.units = drj_dr.units * r_unit
        drj_dr = _curl_regrid(drj_dr, prototype_diff)
        d_k_cube_dtheta = _curl_differentiate(k_cube, lat_coord)
        d_k_cube_dtheta = _curl_regrid(d_k_cube_dtheta, prototype_diff)
        if drj_dr is None and d_k_cube_dtheta is None:
            phi_cmpt = None
        else:
            phi_cmpt = 1/r * _curl_subtract(drj_dr, d_k_cube_dtheta)
            phi_cmpt.units = phi_cmpt.units / r_unit

        drj_dr = d_k_cube_dtheta = None

        # theta curl component: 1/r * ( 1/cos(lat) * d_k_cube_dphi - dri_dr )
        d_k_cube_dphi = _curl_differentiate(k_cube, lon_coord)
        d_k_cube_dphi = _curl_regrid(d_k_cube_dphi, prototype_diff)
        if d_k_cube_dphi is not None:
            d_k_cube_dphi = iris.analysis.maths.divide(d_k_cube_dphi,
                                                       lat_cos_coord)
        dri_dr = _curl_differentiate(r * i_cube, z_coord)
        if dri_dr is not None:
            dri_dr.units = dri_dr.units * r_unit
        dri_dr = _curl_regrid(dri_dr, prototype_diff)
        if d_k_cube_dphi is None and dri_dr is None:
            theta_cmpt = None
        else:
            theta_cmpt = 1/r * _curl_subtract(d_k_cube_dphi, dri_dr)
            theta_cmpt.units = theta_cmpt.units / r_unit
        d_k_cube_dphi = dri_dr = None

        result = [phi_cmpt, theta_cmpt, r_cmpt]

    for direction, cube in zip(vector_quantity_names, result):
        if cube is not None:
            cube.rename('%s curl of %s' % (direction, phenomenon_name))

    return result
Пример #36
0
def parent_module(item):
    return next(filter(lambda x: isinstance(x, Module), parents_of(item)))
Пример #37
0
    def wrapper(request, *args, **kwargs):
        from . import models

        path = kwargs.pop('path', None)
        article_id = kwargs.pop('article_id', None)

        urlpath = None

        # fetch by urlpath.path
        if not path is None:
            try:
                urlpath = models.URLPath.get_by_path(path, select_related=True)
            except NoRootURL:
                return redirect('wiki:root_create')
            except models.URLPath.DoesNotExist:
                try:
                    pathlist = list(
                        filter(
                            lambda x: x != "",
                            path.split("/"),
                        ))
                    path = "/".join(pathlist[:-1])
                    parent = models.URLPath.get_by_path(path)
                    return HttpResponseRedirect(
                        reverse("wiki:create", kwargs={
                            'path': parent.path,
                        }) + "?slug=%s" % pathlist[-1])
                except models.URLPath.DoesNotExist:
                    return HttpResponseNotFound(
                        render_to_string(
                            "wiki/error.html",
                            context={'error_type': 'ancestors_missing'},
                            request=request))
            if urlpath.article:
                # urlpath is already smart about prefetching items on article
                # (like current_revision), so we don't have to
                article = urlpath.article
            else:
                # Be robust: Somehow article is gone but urlpath exists...
                # clean up
                return_url = reverse('wiki:get',
                                     kwargs={'path': urlpath.parent.path})
                urlpath.delete()
                return HttpResponseRedirect(return_url)

        # fetch by article.id
        elif article_id:
            # TODO We should try to grab the article form URLPath so the
            # caching is good, and fall back to grabbing it from
            # Article.objects if not
            articles = models.Article.objects

            article = get_object_or_404(articles, id=article_id)
            try:
                urlpath = models.URLPath.objects.get(articles__article=article)
            except models.URLPath.DoesNotExist as noarticle:
                models.URLPath.MultipleObjectsReturned = noarticle
                urlpath = None

        else:
            raise TypeError('You should specify either article_id or path')

        if not deleted_contents:
            # If the article has been deleted, show a special page.
            if urlpath:
                if urlpath.is_deleted():  # This also checks all ancestors
                    return redirect('wiki:deleted', path=urlpath.path)
            else:
                if article.current_revision and article.current_revision.deleted:
                    return redirect('wiki:deleted', article_id=article.id)

        if article.current_revision.locked and not_locked:
            return response_forbidden(request, article, urlpath)

        if can_read and not article.can_read(request.user):
            return response_forbidden(request, article, urlpath)

        if (can_write or can_create) and not article.can_write(request.user):
            return response_forbidden(request, article, urlpath)

        if can_create and not (request.user.is_authenticated()
                               or settings.ANONYMOUS_CREATE):
            return response_forbidden(request, article, urlpath)

        if can_delete and not article.can_delete(request.user):
            return response_forbidden(request, article, urlpath)

        if can_moderate and not article.can_moderate(request.user):
            return response_forbidden(request, article, urlpath)

        kwargs['urlpath'] = urlpath

        return func(request, article, *args, **kwargs)
Пример #38
0
def backward_function_tester(rng, func, inputs=None,
                             func_args=[], func_kwargs={},
                             rtol_accum=1e-5,
                             atol_f=1e-4, atol_b=1e-3, atol_accum=5e-2,
                             dstep=1e-3, backward=None, backward_b=None,
                             ctx=None, non_accum_check=False, skip_backward_check=False, insert_identity=[],
                             auto_forward=False):
    """ Automatic testing of backward function and backward pass of `func` by comparing it.
    The backward pass of `func` is the reference; therefore,
    the backward pass of `func` must be tested first!

    Syntax of `ref_func`: inputs, parameters
    """

    if ctx is None:
        ctx = nn.Context()
    if backward is None:
        backward = [True for _ in inputs]
    if pf.machine() == "aarch64":
        rtol_accum = 1e-3

    def create_variables(inputs, backward):
        vinputs = []
        for i, b in zip(inputs, backward):
            if i is None:
                vinputs += [None]
                continue
            vinp = nn.Variable(i.shape, need_grad=b)
            vinp.grad.zero()  # grads always not accumulation
            vinputs += [vinp]
            vinputs[-1].data.cast(i.dtype)[...] = i
        return vinputs

    vinputs = create_variables(inputs, backward)
    vinputs_for_clear_buffer = create_variables(inputs, backward)
    vinputs_for_nn_grad = create_variables(inputs, backward)

    vinputs_identity = []
    vinputs_identity_for_clear_buffer = []
    vinputs_identity_for_nn_grad = []
    if not insert_identity:
        insert_identity = [True] * len(vinputs)

    for idx, i in enumerate(zip(vinputs, vinputs_for_clear_buffer, vinputs_for_nn_grad)):
        with nn.auto_forward(auto_forward):
            i0, i1, i2 = i
            if i0 is None:
                vinputs_identity += [None]
                vinputs_identity_for_clear_buffer += [None]
                vinputs_identity_for_nn_grad += [None]
            elif insert_identity[idx]:
                vinputs_identity += [F.identity(i0)]
                vinputs_identity_for_clear_buffer += [F.identity(i1)]
                vinputs_identity_for_nn_grad += [F.identity(i2)]
            else:
                vinputs_identity += [i0]
                vinputs_identity_for_clear_buffer += [i1]
                vinputs_identity_for_nn_grad += [i2]

    # Forward and backward of the forward function with no buffer clear
    with nn.context_scope(ctx), nn.auto_forward(auto_forward):
        outputs0 = func(*(vinputs_identity + func_args), **func_kwargs)
        outputs0 = force_list(outputs0)
        F.sink(*outputs0).forward(clear_no_need_grad=False)
    grad_voutputs = []
    for output in outputs0:
        ograd = rng.randn(*output.shape)
        grad_voutputs.append(nn.Variable.from_numpy_array(
            ograd).apply(need_grad=True))
        output.g = ograd
    F.sink(*outputs0, one_input_grad=False).backward()
    vinputs = list(filter(lambda x: x is not None, vinputs))
    vinputs_identity = list(filter(lambda x: x is not None, vinputs_identity))
    vinputs_for_clear_buffer = list(
        filter(lambda x: x is not None, vinputs_for_clear_buffer))
    grad_inputs0 = [inp.g.copy() for inp in vinputs]

    # Forward and backward of the forward function with clear redundant buffer
    with nn.context_scope(ctx), nn.auto_forward(auto_forward):
        outputs_for_clear_buffer = func(
            *(vinputs_identity_for_clear_buffer + func_args), **func_kwargs)
        outputs_for_clear_buffer = force_list(outputs_for_clear_buffer)
        outputs_for_clear_buffer = list(map(lambda x: F.identity(
            x) if x is not None else None, outputs_for_clear_buffer))
        F.sink(*outputs_for_clear_buffer).forward(clear_no_need_grad=True)

    for o, ref_o in zip(outputs_for_clear_buffer, outputs0):
        o.g = ref_o.g

    # Check backward
    F.sink(*outputs_for_clear_buffer,
           one_input_grad=False).backward(clear_buffer=True)

    grad_inputs_for_clear_buffer = [inp.g.copy()
                                    for inp in vinputs_for_clear_buffer]
    for grad_ref, grad_res in zip(grad_inputs0, grad_inputs_for_clear_buffer):
        if grad_ref is None or grad_res is None:
            continue
        assert_allclose(grad_ref, grad_res, atol=atol_f,
                        err_msg="backward(clear_buffer=True) and backward(clear_buffer=False) results differ.")

    # Forward of the backward function
    from nnabla.backward_functions import registry
    func_name = output.parent.info.type_name
    func_backward = registry[func_name]
    grad_vinputs = grad_voutputs + vinputs
    grad_vinputs_identity = grad_voutputs + vinputs_identity
    func_info_args = output.parent.info.args
    with nn.context_scope(ctx), nn.auto_forward(auto_forward):
        ograds0 = func_backward(grad_vinputs_identity, **func_info_args)
        ograds0 = force_list(ograds0)
        ograds0_ = list(filter(lambda o: o is not None, ograds0))
        F.sink(*ograds0_).forward(clear_no_need_grad=True)
    outputs1 = []
    for i, ograd in enumerate(ograds0):
        outputs1.append(ograd.d.copy()) if ograd is not None else \
          outputs1.append(None)

    # Check num of returned elements
    assert_allclose(len(vinputs), len(outputs1),
                    err_msg="Length of the outputs ({}) does not match "
                    "the length of the inputs ({}) to the backward function".format(len(outputs1), len(vinputs)))

    # Check forward
    for i, elm in enumerate(zip(grad_inputs0, outputs1)):
        grad_ref, grad_res = elm
        if grad_ref is None or grad_res is None:
            continue
        assert_allclose(grad_ref, grad_res, atol=atol_f,
                        err_msg="Forward of the backward function ({}) fails at {}-th output.".format(
                            func_backward.__name__, i))

    # Check the same results between backward_function and nn.grad
    vinputs = [v for b, v in zip(backward, vinputs) if b]
    vinputs = list(filter(lambda x: x is not None, vinputs))

    with nn.context_scope(ctx), nn.auto_forward(auto_forward):
        outputs0_for_nn_grad = func(
            *(vinputs_identity_for_nn_grad + func_args), **func_kwargs)
        outputs0_for_nn_grad = force_list(outputs0_for_nn_grad)
        vinputs_identity_for_nn_grad = [v for b, v in zip(
            backward, vinputs_identity_for_nn_grad) if b]
        vinputs_identity_for_nn_grad = list(
            filter(lambda x: x is not None, vinputs_identity_for_nn_grad))

        ograds1 = nn.grad(outputs0_for_nn_grad, vinputs_identity_for_nn_grad,
                          grad_outputs=[g.d.copy() for g in grad_voutputs])
        F.sink(*ograds1).forward(clear_no_need_grad=True)
    ograds0 = list(filter(lambda o: o is not None, ograds0))
    ograds1 = list(filter(lambda o: o is not None, ograds1))
    for i in range(len(ograds0)):
        if ograds0[i].parent is None:
            continue
        assert_allclose(ograds0[i].d, ograds1[i].d, atol=atol_f,
                        err_msg="nn.grad and backward_functon results differ.")

    # Check backward
    # needed since we sometimes do need_grad=False for optimization, e.g., mask.
    def set_inputs(inputs0, vinputs):
        begin = 0
        for i in vinputs:
            end = begin + i.size
            i.d = inputs0[begin:end].reshape(i.shape)
            begin = end

    def obj_func(inputs0, voutput, vinputs):
        set_inputs(inputs0, vinputs)
        voutput.forward()
        y = voutput.d.copy()
        return y

    initial_grads = []
    for grad_vinput in grad_vinputs:
        if grad_vinput is None:
            continue
        g = np.asarray(rng.randn(*grad_vinput.shape))
        initial_grads.append(g)
    grad_inputs1 = np.concatenate([v.d.flatten()
                                   for v in grad_vinputs if v is not None])

    for i, ograd in enumerate(ograds0):
        # We can skip if the backward is the functions composite.
        # If the backward is of functions composite,
        # the numerical difference is really different from the analytical one for some functions.
        if skip_backward_check:
            continue

        if ograd is None or not backward[i]:
            continue
        for ig, v in zip(initial_grads, grad_vinputs):
            v.g = ig

        # This must be first since approx_fprime destroys the input values
        # analytical grad.
        rgrad = rng.randn()
        with nn.auto_forward(auto_forward):
            sum_ograd = F.sum(ograd) * rgrad
        sum_ograd.forward(clear_no_need_grad=True)
        sum_ograd.backward()
        analytical_grads = np.concatenate(
            [v.g.flatten() for v in grad_vinputs])
        analytical_grads -= np.concatenate([g.flatten()
                                            for g in initial_grads])
        # numerical grad
        from scipy.optimize import approx_fprime
        numerical_grads = approx_fprime(
            grad_inputs1, obj_func, dstep, sum_ograd, grad_vinputs)

        # grad_vinputs: dy_1, ..., dy_n, x_1, ..., x_n
        # grad_voutputs: dy_1, ..., dy_n
        seps = [0] + np.cumsum([int(np.prod(v.shape))
                                for v in grad_vinputs]).tolist()
        ngrads = len(grad_voutputs)
        ninputs = len(grad_vinputs)
        backward_b = [True] * ninputs if backward_b is None else backward_b
        for k, sep in enumerate(zip(seps[:-1], seps[1:])):
            if k >= ngrads and not backward[k - ngrads] or not backward_b[k]:
                continue
            s0, s1 = sep
            analytical_grad = analytical_grads[s0:s1]
            numerical_grad = numerical_grads[s0:s1]
            assert_allclose(analytical_grad, numerical_grad, rtol=rtol_accum, atol=atol_accum,
                            err_msg="Backward (accum) of the backward function ({}) wrt {}-th / {} input fails.".format(
                                func_backward.__name__, k, ninputs))

    # Some functions backward like AffineDataGrad and AffineFilterGrad does not check non-accum anywhere
    # so check those non-accum backward method here.
    if non_accum_check:
        # for any outputs, parents are the same function.
        parent = outputs0[0].parent
        inputs = parent.inputs
        # Accum
        initial_grads = np.concatenate(
            [inp.g.flatten() for inp, b in zip(inputs, backward) if b])
        accum = [True] * len(inputs)
        parent.backward(inputs, outputs0, accum=accum)
        accum_grads = np.concatenate([inp.g.flatten()
                                      for inp, b in zip(inputs, backward) if b])
        non_accum_grads0 = accum_grads - initial_grads
        # Non-accum
        accum = [False] * len(inputs)
        parent.backward(inputs, outputs0, accum=accum)
        non_accum_grads1 = np.concatenate(
            [inp.g.flatten() for inp, b in zip(inputs, backward) if b])
        # Check
        assert_allclose(non_accum_grads0, non_accum_grads1, atol=atol_b,
                        err_msg="Backward (non-accum) of the backward function ({}) fails.".format(
                            func_backward.__name__))
Пример #39
0
def function_tester(rng, func, ref_func, inputs,
                    func_args=[], func_kwargs={},
                    atol_f=1e-6, atol_b=1e-3, atol_accum=1e-6, dstep=1e-3, backward=None,
                    ctx=None, func_name=None, ref_grad=None, disable_half_test=False, atol_half=1e-1,
                    insert_identity=[], disable_clear_no_need_grad_test=False, auto_forward=False):
    """ Automatic testing of forward/backward pass of `func` by comparing it
    to the reference implementation in `ref_func`.

    Syntax of `ref_func`: inputs, parameters
    Syntax of `ref_grad`: inputs, output grads, parameters
    """

    if ctx is None:
        ctx = nn.Context()
    if backward is None:
        backward = [True for _ in inputs]

    # Create Variables
    # print('create_variable')

    def create_variables(inputs, backward):
        vinputs = []
        for i, b in zip(inputs, backward):
            if i is None:
                vinputs += [None]
                continue
            vinputs += [nn.Variable(i.shape, need_grad=b)]
            vinputs[-1].data.cast(i.dtype)[...] = i
        return vinputs

    # Half test
    if not disable_half_test:
        finputs = create_variables(inputs, backward)
        hinputs = create_variables(inputs, backward)
        half_test(rng, func, finputs, hinputs, func_args,
                  func_kwargs, backward, ctx, func_name, atol=atol_half)

    vinputs = create_variables(inputs, backward)
    # Checking forward
    # print('checking forward')
    with nn.context_scope(ctx), nn.auto_forward():
        o = func(*(vinputs + func_args), **func_kwargs)
    rinputs = copy.deepcopy(inputs)  # inputs for ref_func
    refs = ref_func(*(rinputs + func_args), **func_kwargs)

    refs = force_tuple(refs)
    o = force_tuple(o)
    assert len(o) == len(refs)
    for i, ref in enumerate(refs):
        res = o[i].d
        assert_allclose(ref, res, atol=atol_f,
                        err_msg="{} forward test fails".format(func_name))

    # Checking recomputation
    vinputs = create_variables(inputs, backward)
    recomputation_test(rng, func, vinputs, func_args,
                       func_kwargs, ctx)

    # Checking forward(clear_no_need_grad=True)
    if not disable_clear_no_need_grad_test:
        clear_no_need_grad_tester(rng, func, inputs, func_args, func_kwargs,
                                  backward, atol_f, ctx, func_name, insert_identity,
                                  auto_forward)

    # Checking function name
    try:
        import function_test_callback
        result = create_function_nnp(
            vinputs, o, func_name, func_args, func_kwargs)
        if result is not None:
            function_test_callback.callback(func_name, *result)
    except UnboundLocalError:
        pass
    except IndexError:
        pass
    except ImportError:
        pass

    # print('checking function name')
    if func_name is not None:
        assert o[0].parent.name == func_name

    # Checking backward
    # print('checking backward')
    if not True in backward:
        return

    # NNabla backward
    for v in vinputs:
        if v is None:
            continue
        if len(v.shape) == 0:
            v.g = randn(rng)
            continue
        v.g = randn(rng, *v.shape)
    # Verify grad
    vinputs = create_variables(inputs, backward)
    rinputs = copy.deepcopy(inputs)
    rinputs = [rinput if test else None for rinput,
               test in zip(rinputs, backward)]
    vgrads = [randn(rng, *o_.shape) for o_ in o]

    def reset_ograds():
        '''
        Reset output grads everytime we call backward.
        This is required because the output grad might
        be inplaced and modified during backward operation.
        '''
        for ovar, g in zip(o, vgrads):
            ovar.g = g

    agrads, ngrads = compute_analytical_and_numerical_grad(
        o[0].parent, vinputs, o, rinputs, vgrads, epsilon=dstep,
        rng=rng, ref_grad=ref_grad)
    if ref_grad is not None:
        rinputs = copy.deepcopy(inputs)
        doutputs = copy.deepcopy(vgrads)
        ngrads = ref_grad(*(rinputs + doutputs + func_args),
                          **func_kwargs, need_grad_flags=backward)

    assert_allclose(ngrads, agrads, atol=atol_b,
                    err_msg="{} backward w/o accumulation test fails".format(func_name))

    # Check if need_grad works
    for v, b in zip(vinputs, backward):
        if not b or v is None:
            continue
        v.grad.zero()
        v.need_grad = False
        reset_ograds()
        try:
            o[0].parent.forward(
                list(filter(lambda x: x is not None, vinputs)), o)
            o[0].parent.backward(
                list(filter(lambda x: x is not None, vinputs)), o)
        except RuntimeError as e:
            continue  # TODO
        assert np.all(v.g == 0)

    # test accum=False
    for i in range(len(vinputs)):
        if vinputs[i] is None:
            continue
        v = vinputs[i]
        v.need_grad = backward[i]

    for i in range(len(vinputs)):
        if vinputs[i] is None:
            continue
        v = vinputs[i]

        if not backward[i]:
            continue
        f = o[0].parent

        # Prepare function inputs
        finputs = list(filter(lambda x: x is not None, vinputs))

        # Save accum gradient result
        g = randn(rng, *v.shape)
        v.g = g
        reset_ograds()
        f.forward(finputs, o)
        f.backward(finputs, o)
        true_g = v.g - g

        # Check accum=False
        accum = [j != i for j, vv in enumerate(vinputs) if vv is not None]
        v.g = randn(rng, *v.shape)
        reset_ograds()
        f.forward(finputs, o)
        f.backward(finputs, o, accum)
        assert_allclose(
            v.g, true_g, atol=atol_accum,
            err_msg="{} backward w/ accumulation test fails.".format(func_name))

        # Check accum=False with NaN gradient
        v.g = np.float32('nan')
        reset_ograds()
        f.forward(finputs, o)
        f.backward(finputs, o, accum)
        assert not np.any(np.isnan(v.g))
Пример #40
0
def clear_no_need_grad_tester(rng, func, inputs, func_args=[], func_kwargs={}, backward=None, atol_f=1e-6, ctx=None,
                              func_name=None, insert_identity=[], auto_forward=False):
    if ctx is None:
        ctx = nn.Context()
    if backward is None:
        backward = [True for _ in inputs]
    if not True in backward:
        return

    state_rng = None
    if rng is not None:
        state_rng = rng.get_state()
    else:
        rng = rng = np.random.RandomState(313)

    def create_variables(inputs, backward):
        vinputs = []
        for i, b in zip(inputs, backward):
            if i is None:
                vinputs += [None]
                continue
            vinputs += [nn.Variable(i.shape, need_grad=b)]
            vinputs[-1].data.cast(i.dtype)[...] = i
        return vinputs

    vinputs = create_variables(inputs, backward)
    vinputs_clear_buffer = create_variables(inputs, backward)
    vinputs_identity_clear_buffer = []
    if not insert_identity:
        insert_identity = [True] * len(vinputs)

    with nn.context_scope(ctx), nn.auto_forward(auto_forward):
        for idx, i in enumerate(vinputs_clear_buffer):
            if i is None:
                vinputs_identity_clear_buffer += [None]
            elif insert_identity[idx]:
                vinputs_identity_clear_buffer += [F.identity(i)]
            else:
                vinputs_identity_clear_buffer += [i]

    # Checking forward(clear_no_need_grad=True)
    with nn.context_scope(ctx), nn.auto_forward(auto_forward):
        o = func(*(vinputs + func_args), **func_kwargs)
        o = force_tuple(o)
        F.sink(*o).forward(clear_no_need_grad=False)

        o_clear_buffer = func(
            *(vinputs_identity_clear_buffer + func_args), **func_kwargs)
        o_clear_buffer = force_tuple(o_clear_buffer)
        o_identity_clear_buffer = list(map(lambda x: F.identity(
            x) if x is not None else None, o_clear_buffer))
        o_identity_clear_buffer = list(
            filter(lambda x: x is not None, o_identity_clear_buffer))

        F.sink(*o_identity_clear_buffer).forward(clear_no_need_grad=True)

    for i in range(len(o)):
        if o[i] is None:
            continue
        ref = o[i].d
        res = o_identity_clear_buffer[i].d
        assert_allclose(ref, res, atol=atol_f,
                        err_msg="{} forward(clear_no_need_grad=True) test fails".format(func_name))

    vinputs = list(filter(lambda x: x is not None, vinputs))
    vinputs_clear_buffer = list(
        filter(lambda x: x is not None, vinputs_clear_buffer))

    for i in range(len(vinputs)):
        vinputs[i].grad.zero()
        vinputs_clear_buffer[i].grad.zero()

    for i in range(len(o)):
        if o[i] is None:
            continue
        o[i].g = randn(rng, *o[i].shape)
        o_identity_clear_buffer[i].g = o[i].g

    F.sink(*o).backward()
    F.sink(*o_identity_clear_buffer).backward(clear_buffer=True)

    for i in range(len(vinputs)):
        ref = vinputs[i].g
        res = vinputs_clear_buffer[i].g
        assert_allclose(ref, res, atol=atol_f,
                        err_msg="{} forward(clear_no_need_grad=True) and backward test fails".format(func_name))

    if state_rng:
        rng.set_state(state_rng)
Пример #41
0
    def _nn(self, d, n=1):
        """
        Internal method to be implemented by sub-classes to return the nearest
        `N` neighbors to the given descriptor element.

        When this internal method is called, we have already checked that there
        is a vector in ``d`` and our index is not empty.

        :param d: Descriptor element to compute the neighbors of.
        :type d: smqtk.representation.DescriptorElement

        :param n: Number of nearest neighbors to find.
        :type n: int

        :return: Tuple of nearest N DescriptorElement instances, and a tuple of
            the distance values to those neighbors.
        :rtype: (tuple[smqtk.representation.DescriptorElement], tuple[float])

        """
        log = self._log
        q = d.vector()[np.newaxis, :].astype(np.float32)
        log.debug("Received query for %d nearest neighbors", n)

        with self._model_lock:
            # Attempt to set n-probe of an IVF index
            self._set_index_nprobe()

            # noinspection PyArgumentList
            s_dists, s_ids = self._faiss_index.search(
                q, k=min(n, self._faiss_index.ntotal))
            s_dists, s_ids = np.sqrt(s_dists[0, :]), s_ids[0, :]
            s_ids = s_ids.astype(object)
            # s_id (the FAISS index indices) can equal -1 if fewer than the
            # requested number of nearest neighbors is returned. In this case,
            # eliminate the -1 entries
            self._log.debug("Getting descriptor UIDs from idx2uid mapping.")
            uuids = list(
                self._idx2uid_kvs.get_many(
                    filter(lambda s_id_: s_id_ >= 0, s_ids)))
            if len(uuids) < n:
                warnings.warn(
                    "Less than n={} neighbors were retrieved from "
                    "the FAISS index instance. Maybe increase "
                    "nprobe if this is an IVF index?".format(n),
                    RuntimeWarning)

            descriptors = tuple(
                self._descriptor_set.get_many_descriptors(uuids))

        log.debug("Min and max FAISS distances: %g, %g", min(s_dists),
                  max(s_dists))

        d_vectors = np.vstack(DescriptorElement.get_many_vectors(descriptors))
        d_dists = metrics.euclidean_distance(d_vectors, q)

        log.debug("Min and max descriptor distances: %g, %g", min(d_dists),
                  max(d_dists))

        order = d_dists.argsort()
        uuids, d_dists = zip(*((uuids[oidx], d_dists[oidx]) for oidx in order))

        log.debug("Returning query result of size %g", len(uuids))

        return descriptors, tuple(d_dists)
Пример #42
0
 def get_images_info(self):
     command = 'docker stack services --format {{.Image}} %s' % self.name
     images = filter(None, fabricio.run(command).splitlines())
     bucket = json.dumps(self._get_digests(images), sort_keys=True)
     return b64encode(bucket.encode()).decode()
Пример #43
0
    def __init__(
        self,
        worker_id,
        conf,
        namespaces=None,
        pollster_list=None,
    ):

        namespaces = namespaces or ['compute', 'central']
        pollster_list = pollster_list or []
        group_prefix = conf.polling.partitioning_group_prefix

        # features of using coordination and pollster-list are exclusive, and
        # cannot be used at one moment to avoid both samples duplication and
        # samples being lost
        if pollster_list and conf.coordination.backend_url:
            raise PollsterListForbidden()

        super(AgentManager, self).__init__(worker_id, conf)

        def _match(pollster):
            """Find out if pollster name matches to one of the list."""
            return any(
                fnmatch.fnmatch(pollster.name, pattern)
                for pattern in pollster_list)

        if type(namespaces) is not list:
            namespaces = [namespaces]

        # we'll have default ['compute', 'central'] here if no namespaces will
        # be passed
        extensions = (self._extensions('poll', namespace, self.conf).extensions
                      for namespace in namespaces)
        # get the extensions from pollster builder
        extensions_fb = (self._extensions_from_builder('poll', namespace)
                         for namespace in namespaces)
        if pollster_list:
            extensions = (moves.filter(_match, exts) for exts in extensions)
            extensions_fb = (moves.filter(_match, exts)
                             for exts in extensions_fb)

        self.extensions = list(itertools.chain(*list(extensions))) + list(
            itertools.chain(*list(extensions_fb)))

        if self.extensions == []:
            raise EmptyPollstersList()

        discoveries = (self._extensions('discover', namespace,
                                        self.conf).extensions
                       for namespace in namespaces)
        self.discoveries = list(itertools.chain(*list(discoveries)))
        self.polling_periodics = None

        self.partition_coordinator = coordination.PartitionCoordinator(
            self.conf)
        self.heartbeat_timer = utils.create_periodic(
            target=self.partition_coordinator.heartbeat,
            spacing=self.conf.coordination.heartbeat,
            run_immediately=True)

        # Compose coordination group prefix.
        # We'll use namespaces as the basement for this partitioning.
        namespace_prefix = '-'.join(sorted(namespaces))
        self.group_prefix = ('%s-%s' % (namespace_prefix, group_prefix)
                             if group_prefix else namespace_prefix)

        self.notifier = oslo_messaging.Notifier(
            messaging.get_transport(self.conf),
            driver=self.conf.publisher_notifier.telemetry_driver,
            publisher_id="ceilometer.polling")

        self._keystone = None
        self._keystone_last_exception = None
Пример #44
0
 def __str__(self):
     return '/'.join(filter(bool, [self.service, self.version, self.id]))
Пример #45
0
    def gettile(self,
                base_url=None,
                layer=None,
                style=None,
                format=None,
                tilematrixset=None,
                tilematrix=None,
                row=None,
                column=None,
                **kwargs):
        """Return a tile from the WMTS.

        Returns the tile image as a file-like object.

        Parameters
        ----------
        base_url : string
            Optional URL for request submission. Defaults to the URL of
            the GetTile operation as declared in the GetCapabilities
            response.
        layer : string
            Content layer name.
        style : string
            Optional style name. Defaults to the first style defined for
            the relevant layer in the GetCapabilities response.
        format : string
            Optional output image format,  such as 'image/jpeg'.
            Defaults to the first format defined for the relevant layer
            in the GetCapabilities response.
        tilematrixset : string
            Optional name of tile matrix set to use.
            Defaults to the first tile matrix set defined for the
            relevant layer in the GetCapabilities response.
        tilematrix : string
            Name of the tile matrix to use.
        row : integer
            Row index of tile to request.
        column : integer
            Column index of tile to request.
        **kwargs : extra arguments
            anything else e.g. vendor specific parameters

        Example
        -------
            >>> url = 'http://map1c.vis.earthdata.nasa.gov/wmts-geo/wmts.cgi'
            >>> wmts = WebMapTileService(url)
            >>> img = wmts.gettile(layer='VIIRS_CityLights_2012',\
                                   tilematrixset='EPSG4326_500m',\
                                   tilematrix='6',\
                                   row=4, column=4)
            >>> out = open('tile.jpg', 'wb')
            >>> bytes_written = out.write(img.read())
            >>> out.close()

        """
        vendor_kwargs = self.vendor_kwargs or {}
        vendor_kwargs.update(kwargs)

        # REST only WMTS
        if self.restonly:
            resurl = self.buildTileResource(layer, style, format,
                                            tilematrixset, tilematrix, row,
                                            column, **vendor_kwargs)
            u = openURL(resurl, username=self.username, password=self.password)
            return u

        # KVP implemetation
        data = self.buildTileRequest(layer, style, format, tilematrixset,
                                     tilematrix, row, column, **vendor_kwargs)

        if base_url is None:
            base_url = self.url
            try:
                methods = self.getOperationByName('GetTile').methods
                get_verbs = [
                    x for x in methods if x.get('type').lower() == 'get'
                ]
                if len(get_verbs) > 1:
                    # Filter by constraints
                    base_url = next(x for x in filter(list, ([
                        pv.get('url') for const in pv.get('constraints')
                        if 'kvp' in [x.lower() for x in const.values]
                    ] for pv in get_verbs if pv.get('constraints'))))[0]
                elif len(get_verbs) == 1:
                    base_url = get_verbs[0].get('url')
            except StopIteration:
                pass
        u = openURL(base_url,
                    data,
                    username=self.username,
                    password=self.password)

        # check for service exceptions, and return
        if u.info()['Content-Type'] == 'application/vnd.ogc.se_xml':
            se_xml = u.read()
            se_tree = etree.fromstring(se_xml)
            err_message = six.text_type(se_tree.find('ServiceException').text)
            raise ServiceException(err_message.strip(), se_xml)
        return u
Пример #46
0
def test_filter():
    from six.moves import filter
    f = filter(lambda x: x % 2, range(10))
    assert six.advance_iterator(f) == 1
Пример #47
0
 def splits(text):
     return list(filter(lambda s: len(s) != 0, re.split('\s+', text)))
Пример #48
0
    def test_complex_structure(self):
        # in slightly more compact format:
        # ((foo=bar) or (foo1=bar1 and foo2=bar2 and (foo3=bar3 or foo4=bar4)))
        filter = FilterFactory.from_spec({
            "type":
            "or",
            "filters": [
                {
                    "type": "property_match",
                    "property_name": "foo",
                    "property_value": "bar"
                },
                {
                    "type":
                    "and",
                    "filters": [
                        {
                            "type": "property_match",
                            "property_name": "foo1",
                            "property_value": "bar1"
                        },
                        {
                            "type": "property_match",
                            "property_name": "foo2",
                            "property_value": "bar2"
                        },
                        {
                            "type":
                            "or",
                            "filters": [{
                                "type": "property_match",
                                "property_name": "foo3",
                                "property_value": "bar3"
                            }, {
                                "type": "property_match",
                                "property_name": "foo4",
                                "property_value": "bar4"
                            }]
                        },
                    ]
                },
            ]
        })
        # first level or
        self.assertTrue(filter(dict(foo='bar')))
        # first level and with both or's
        self.assertTrue(filter(dict(foo1='bar1', foo2='bar2', foo3='bar3')))
        self.assertTrue(filter(dict(foo1='bar1', foo2='bar2', foo4='bar4')))

        # first and not right
        self.assertFalse(
            filter(dict(foo1='not bar1', foo2='bar2', foo3='bar3')))
        # second and not right
        self.assertFalse(
            filter(dict(foo1='bar1', foo2='not bar2', foo3='bar3')))
        # last and not right
        self.assertFalse(
            filter(
                dict(foo1='bar1',
                     foo2='bar2',
                     foo3='not bar3',
                     foo4='not bar4')))
Пример #49
0
def type_journeys(resp, req):
    """
    Set the type of the journeys
    """
    best_crit = arrival_crit if req["clockwise"] else departure_crit
    # first, we want a type for every journey. Just pick "rapid" by default.
    for j in resp.journeys:
        j.type = "rapid"

    # Then, we want something like the old types
    trip_caracs = [
        # comfort tends to limit the number of transfers and fallback
        ("comfort",
         trip_carac([has_no_car],
                    [transfers_crit, nonTC_crit, best_crit, duration_crit])),
        # for car we want at most one journey, the earliest one
        (
            "car",
            trip_carac(
                [has_car, has_pt
                 ],  # We don't want car only solution, we MUST have PT
                [best_crit, transfers_crit, nonTC_crit, duration_crit],
            ),
        ),
        # less_fallback tends to limit the fallback while walking
        (
            "less_fallback_walk",
            trip_carac([has_no_car, has_no_bike],
                       [nonTC_crit, transfers_crit, duration_crit, best_crit]),
        ),
        # less_fallback tends to limit the fallback for biking and bss
        (
            "less_fallback_bike",
            trip_carac([has_no_car, has_bike, has_no_bss],
                       [nonTC_crit, transfers_crit, duration_crit, best_crit]),
        ),
        # less_fallback tends to limit the fallback for biking and bss
        (
            "less_fallback_bss",
            trip_carac([has_no_car, has_bss],
                       [nonTC_crit, transfers_crit, duration_crit, best_crit]),
        ),
        # the fastest is quite explicit
        ("fastest",
         trip_carac([has_no_car],
                    [duration_crit, transfers_crit, nonTC_crit, best_crit])),
        # the non_pt journeys is the earliest journey without any public transport
        ("non_pt_walk",
         trip_carac([non_pt_journey, has_no_car, has_walk], [best_crit])),
        # the non_pt journey is the earliest journey without any public transport
        # only walking, biking or driving
        ("non_pt_bike",
         trip_carac([non_pt_journey, has_no_car, has_bike], [best_crit])),
        ("non_pt_bss",
         trip_carac([non_pt_journey, has_no_car, has_bss], [best_crit])),
        ("non_pt_car", trip_carac([non_pt_journey, has_car], [best_crit])),
    ]

    for name, carac in trip_caracs:
        sublist = list(filter(and_filters(carac.constraints), resp.journeys))
        best = min_from_criteria(sublist, carac.criteria)
        if best is not None:
            best.type = name

    # Finally, we want exactly one best, the ASAP one
    best = min_from_criteria(
        resp.journeys, [best_crit, duration_crit, transfers_crit, nonTC_crit])
    if best is not None:
        best.type = "best"
Пример #50
0
def walk_tree_depth_first(tree, show_levels=False,
                          preprocess=None, filter=None, start=None):
    """
    A non-recursive generator function that walks through a tree (and all
    children) yielding results.
    
    The tree should be of the form:
      [(NodeOne, None),
       (NodeTwo,
         [(Node2A, None),
          (Node2B, None),
          (Node2C,
            [(Node2C1, None), etc]
         ]
       (NodeThree, None),
        etc...]
    
    If show_levels is True, the values returned are (level, value) where level
    is zero for the top level items in the tree. Otherwise, just "value" is
    returned.
    
    If a callable "preprocess" is supplied, it is applied BEFORE the filter,
    as each element is encountered.
    
    If a callable "filter" is supplied, it is applied to whatever "preprocess"
    returned, and if it returns False for an item, the item and its children
    will be skipped.
    
    If "start" is given, the walk will be applied only to that node and its
    children. No preprocessing or filtering will be applied to other elements.
    """
    annotated_tree = [(0, element) for element in tree]
    
    to_process = deque(annotated_tree)
    
    # If we're not given a starting point, the top is the start
    found_starting_point = not start
    
    while to_process:
        (level, (node, children)) = to_process.popleft()
        
        if not found_starting_point and (node == start):
            # If we're given a starting point and we've found it, clear the list
            # and start from here
            found_starting_point = True
            level = 0
            to_process.clear()

        # This should NOT be an else case, since we may have just set this flag
        # to "True" above.
        if found_starting_point:
            if preprocess:
                value = preprocess(node)
            else:
                value = node
            
            if filter and not filter(value):
                continue
            
            if show_levels:
                yield (level, value)
            else:
                yield value

        if children:
            annotated_children = [(level+1, child) for child in children]
            annotated_children.reverse()
            to_process.extendleft(annotated_children)
Пример #51
0
 def test_less_than_equal(self):
     filter = self.get_filter('lte', 3)
     for match in (-10, 0, 2, 3):
         self.assertTrue(filter({'foo': match}))
     for non_match in (4, 11, '2'):
         self.assertFalse(filter({'foo': non_match}))
Пример #52
0
def _filter_similarity(mols, distance, generator, query_fps, cutoff):
    """Filter molecules by a certain distance to the reference fingerprints.
    User must supply distance funtion, FP generator, query FPs and cutoff."""
    return list(filter(
        lambda q: any(distance(generator(q), q_fp) >= float(cutoff)
                      for q_fp in query_fps), mols))
Пример #53
0
def ParseResults(ycsb_result_string, data_type='histogram'):
    """Parse YCSB results.

  Example input for histogram datatype:

    YCSB Client 0.1
    Command line: -db com.yahoo.ycsb.db.HBaseClient -P /tmp/pkb/workloada
    [OVERALL], RunTime(ms), 1800413.0
    [OVERALL], Throughput(ops/sec), 2740.503428935472
    [UPDATE], Operations, 2468054
    [UPDATE], AverageLatency(us), 2218.8513395574005
    [UPDATE], MinLatency(us), 554
    [UPDATE], MaxLatency(us), 352634
    [UPDATE], 95thPercentileLatency(ms), 4
    [UPDATE], 99thPercentileLatency(ms), 7
    [UPDATE], Return=0, 2468054
    [UPDATE], 0, 398998
    [UPDATE], 1, 1015682
    [UPDATE], 2, 532078
    ...

  Example input for hdrhistogram datatype:

    YCSB Client 0.12.0
    Command line: -db com.yahoo.ycsb.db.RedisClient -P /opt/pkb/workloadb
    [OVERALL], RunTime(ms), 29770.0
    [OVERALL], Throughput(ops/sec), 33590.86328518643
    [UPDATE], Operations, 49856.0
    [UPDATE], AverageLatency(us), 1478.0115532734276
    [UPDATE], MinLatency(us), 312.0
    [UPDATE], MaxLatency(us), 24623.0
    [UPDATE], 95thPercentileLatency(us), 3501.0
    [UPDATE], 99thPercentileLatency(us), 6747.0
    [UPDATE], Return=OK, 49856
    ...

  Example input for ycsb version after 0.13.0:

    ...
    Command line: -db com.yahoo.ycsb.db.HBaseClient10 ... -load
    YCSB Client 0.14.0

    Loading workload...
    Starting test.
    ...
    [OVERALL], RunTime(ms), 11411
    [OVERALL], Throughput(ops/sec), 8763.473841030585
    [INSERT], Operations, 100000
    [INSERT], AverageLatency(us), 74.92
    [INSERT], MinLatency(us), 5
    [INSERT], MaxLatency(us), 98495
    [INSERT], 95thPercentileLatency(us), 42
    [INSERT], 99thPercentileLatency(us), 1411
    [INSERT], Return=OK, 100000
    ...

  Example input for timeseries datatype:

    ...
    [OVERALL], RunTime(ms), 240007.0
    [OVERALL], Throughput(ops/sec), 10664.605615669543
    ...
    [READ], Operations, 1279253
    [READ], AverageLatency(us), 3002.7057071587874
    [READ], MinLatency(us), 63
    [READ], MaxLatency(us), 93584
    [READ], Return=OK, 1279281
    [READ], 0, 528.6142757498257
    [READ], 500, 360.95347448674966
    [READ], 1000, 667.7379547689283
    [READ], 1500, 731.5389357265888
    [READ], 2000, 778.7992281717318
    ...

  Args:
    ycsb_result_string: str. Text output from YCSB.
    data_type: Either 'histogram' or 'timeseries' or 'hdrhistogram'.
      'histogram' and 'hdrhistogram' datasets are in the same format, with the
      difference being lacking the (millisec, count) histogram component. Hence
      are parsed similarly.

  Returns:
    A dictionary with keys:
      client: containing YCSB version information.
      command_line: Command line executed.
      groups: list of operation group descriptions, each with schema:
        group: group name (e.g., update, insert, overall)
        statistics: dict mapping from statistic name to value
        histogram: list of (ms_lower_bound, count) tuples, e.g.:
          [(0, 530), (19, 1)]
        indicates that 530 ops took between 0ms and 1ms, and 1 took between
        19ms and 20ms. Empty bins are not reported.
  Raises:
    IOError: If the results contained unexpected lines.
  """
    # TODO: YCSB 0.9.0 output client and command line string to stderr, so
    # we need to support it in the future.
    lines = []
    client_string = 'YCSB'
    command_line = 'unknown'
    fp = six.StringIO(ycsb_result_string)
    result_string = next(fp).strip()

    def IsHeadOfResults(line):
        return line.startswith('[OVERALL]')

    while not IsHeadOfResults(result_string):
        if result_string.startswith('YCSB Client 0.'):
            client_string = result_string
        if result_string.startswith('Command line:'):
            command_line = result_string
        try:
            result_string = next(fp).strip()
        except StopIteration:
            raise IOError(
                'Could not parse YCSB output: {}'.format(ycsb_result_string))

    if result_string.startswith('[OVERALL]'):  # YCSB > 0.7.0.
        lines.append(result_string)
    else:
        # Received unexpected header
        raise IOError('Unexpected header: {0}'.format(client_string))

    # Some databases print additional output to stdout.
    # YCSB results start with [<OPERATION_NAME>];
    # filter to just those lines.
    def LineFilter(line):
        return re.search(r'^\[[A-Z]+\]', line) is not None

    lines = itertools.chain(lines, filter(LineFilter, fp))

    r = csv.reader(lines)

    by_operation = itertools.groupby(r, operator.itemgetter(0))

    result = collections.OrderedDict([('client', client_string),
                                      ('command_line', command_line),
                                      ('groups', collections.OrderedDict())])

    for operation, lines in by_operation:
        operation = operation[1:-1].lower()

        if operation == 'cleanup':
            continue

        op_result = {'group': operation, data_type: [], 'statistics': {}}
        latency_unit = 'ms'
        for _, name, val in lines:
            name = name.strip()
            val = val.strip()
            # Drop ">" from ">1000"
            if name.startswith('>'):
                name = name[1:]
            val = float(val) if '.' in val or 'nan' in val.lower() else int(
                val)
            if name.isdigit():
                if val:
                    if data_type == TIMESERIES and latency_unit == 'us':
                        val /= 1000.0
                    op_result[data_type].append((int(name), val))
            else:
                if '(us)' in name:
                    name = name.replace('(us)', '(ms)')
                    val /= 1000.0
                    latency_unit = 'us'
                op_result['statistics'][name] = val

        result['groups'][operation] = op_result
    return result
Пример #54
0
    def process_tokens(self, tokens):
        """process tokens and search for :

         _ non strict indentation (i.e. not always using the <indent> parameter as
           indent unit)
         _ too long lines (i.e. longer than <max_chars>)
         _ optionally bad construct (if given, bad_construct must be a compiled
           regular expression).
        """
        self._bracket_stack = [None]
        indents = [0]
        check_equal = False
        line_num = 0
        self._lines = {}
        self._visited_lines = {}
        token_handlers = self._prepare_token_dispatcher()
        self._last_line_ending = None
        last_blank_line_num = 0

        self._current_line = ContinuedLineState(tokens, self.config)
        for idx, (tok_type, token, start, _, line) in enumerate(tokens):
            if start[0] != line_num:
                line_num = start[0]
                # A tokenizer oddity: if an indented line contains a multi-line
                # docstring, the line member of the INDENT token does not contain
                # the full line; therefore we check the next token on the line.
                if tok_type == tokenize.INDENT:
                    self.new_line(TokenWrapper(tokens), idx - 1, idx + 1)
                else:
                    self.new_line(TokenWrapper(tokens), idx - 1, idx)

            if tok_type == tokenize.NEWLINE:
                # a program statement, or ENDMARKER, will eventually follow,
                # after some (possibly empty) run of tokens of the form
                #     (NL | COMMENT)* (INDENT | DEDENT+)?
                # If an INDENT appears, setting check_equal is wrong, and will
                # be undone when we see the INDENT.
                check_equal = True
                self._process_retained_warnings(TokenWrapper(tokens), idx)
                self._current_line.next_logical_line()
                self._check_line_ending(token, line_num)
            elif tok_type == tokenize.INDENT:
                check_equal = False
                self.check_indent_level(token, indents[-1] + 1, line_num)
                indents.append(indents[-1] + 1)
            elif tok_type == tokenize.DEDENT:
                # there's nothing we need to check here!  what's important is
                # that when the run of DEDENTs ends, the indentation of the
                # program statement (or ENDMARKER) that triggered the run is
                # equal to what's left at the top of the indents stack
                check_equal = True
                if len(indents) > 1:
                    del indents[-1]
            elif tok_type == tokenize.NL:
                if not line.strip('\r\n'):
                    last_blank_line_num = line_num
                self._check_continued_indentation(TokenWrapper(tokens),
                                                  idx + 1)
                self._current_line.next_physical_line()
            elif tok_type != tokenize.COMMENT:
                self._current_line.handle_line_start(idx)
                # This is the first concrete token following a NEWLINE, so it
                # must be the first token of the next program statement, or an
                # ENDMARKER; the "line" argument exposes the leading whitespace
                # for this statement; in the case of ENDMARKER, line is an empty
                # string, so will properly match the empty string with which the
                # "indents" stack was seeded
                if check_equal:
                    check_equal = False
                    self.check_indent_level(line, indents[-1], line_num)

            if tok_type == tokenize.NUMBER and token.endswith('l'):
                self.add_message('lowercase-l-suffix', line=line_num)

            try:
                handler = token_handlers[token]
            except KeyError:
                pass
            else:
                handler(tokens, idx)

        line_num -= 1  # to be ok with "wc -l"
        if line_num > self.config.max_module_lines:
            # Get the line where the too-many-lines (or its message id)
            # was disabled or default to 1.
            symbol = self.linter.msgs_store.check_message_id('too-many-lines')
            names = (symbol.msgid, 'too-many-lines')
            line = next(
                filter(None, map(self.linter._pragma_lineno.get, names)), 1)
            self.add_message('too-many-lines',
                             args=(line_num, self.config.max_module_lines),
                             line=line)

        # See if there are any trailing lines.  Do not complain about empty
        # files like __init__.py markers.
        if line_num == last_blank_line_num and line_num > 0:
            self.add_message('trailing-newlines', line=line_num)
Пример #55
0
def enhance_input_model(input_model):
    """
    Uses the input model schema to enhance the input model data structure,
    in order to make it easier to navigate:
      - replaces sub-element lists with key-indexed sub-element maps (to
      simplify lookup operations)
      - replaces attributes representing foreign keys with direct references to
      foreign elements (to simplify lookup and dereference operations)
      - deletes unused elements from the input models (e.g. server roles that
      aren't referenced by any control plane)

    Processes neutron configuration data, network group tags and routes and
    extends the input model data structure with information pertaining to
    identified neutron networks.

    :param input_model: original input model, as loaded from disk
    :return: enhanced input model data structure
    """

    input_model = deepcopy(input_model)

    map_list_attrs(input_model, input_model_schema)

    map_foreign_keys(input_model, 'input-model', input_model,
                     input_model_schema)

    prune_input_model(input_model, input_model_schema)

    # Assume there is at most one neutron configuration data
    neutron_config_data = list(
        filter(lambda config_data: 'neutron' in config_data['services'],
               input_model['configuration-data'].values()))
    neutron_config_data = \
        input_model['neutron-config-data'] = \
        neutron_config_data[0] if neutron_config_data else None

    # Collect all network group tags in a single map,
    # indexed by neutron group name
    neutron_network_tags = dict()
    # Collect all neutron provider/external networks in a single map,
    # indexed by network name
    neutron_networks = input_model['neutron-networks'] = dict()

    def add_neutron_network_tags(network_group_name, tags):
        tag = neutron_network_tags.setdefault(
            network_group_name, {'network-group': network_group_name})
        link_elements_by_foreign_key(
            tag,
            'network-group',
            input_model['network-groups'],
            ref_list_attr='neutron-tags',
            # Use a null key_attr value
            # to create a list of references
            element_key=None)
        tag.setdefault('tags', []).extend(tags)

    if neutron_config_data:
        # Starting in SUSE OpenStack Cloud 8, network tags may be defined
        # as part of a Neutron configuration-data object rather than as part
        # of a network-group object.
        for network_tag in neutron_config_data.get('network-tags', []):
            add_neutron_network_tags(network_tag['network-group'],
                                     network_tag['tags'])

        external_networks = convert_element_list_to_map(
            neutron_config_data['data'], 'neutron_external_networks')
        provider_networks = convert_element_list_to_map(
            neutron_config_data['data'], 'neutron_provider_networks')
        neutron_networks.update(external_networks)
        neutron_networks.update(provider_networks)
        for network in external_networks.itervalues():
            network['external'] = True
        for network in provider_networks.itervalues():
            network['external'] = False

    for network_group in input_model['network-groups'].itervalues():
        if neutron_config_data and 'tags' in network_group:
            add_neutron_network_tags(network_group['name'],
                                     network_group['tags'])
        link_elements_by_foreign_key_list(network_group,
                                          'routes',
                                          input_model['network-groups'],
                                          ref_list_attr='network-group-routes')
        link_elements_by_foreign_key_list(network_group,
                                          'routes',
                                          neutron_networks,
                                          ref_list_attr='network-group-routes')

        # Network groups may contain references to control plane load
        # balancers, which we have to transform into object references
        # explicitly here
        for cp in input_model['control-planes'].values():
            link_elements_by_foreign_key_list(network_group,
                                              'load-balancers',
                                              cp['load-balancers'],
                                              ref_list_attr=None)

    # Based on the collected neutron networks and network tags, identify
    # which network group is linked to which neutron network, by looking
    # at the provider physical network settings
    neutron_physnets = dict()
    for neutron_network in neutron_networks.itervalues():
        # The only neutron network without a provider is the external
        # "bridge" network.
        # Assume a default 'external' physnet value for this network.
        if 'provider' not in neutron_network:
            if neutron_network['external']:
                physnet = 'external'
            else:
                continue
        else:
            physnet = neutron_network['provider'][0]['physical_network']
        neutron_physnets[physnet] = neutron_network
    for network_tag in neutron_network_tags.itervalues():
        for tag in network_tag['tags']:
            if isinstance(tag, dict):
                tag = tag.values()[0]
            # The only relevant tag without a provider is the external
            # "bridge" network.
            # Assume a default 'external' physnet value for this network.
            if 'provider-physical-network' not in tag:
                if tag == 'neutron.l3_agent.external_network_bridge':
                    physnet = 'external'
                else:
                    continue
            else:
                physnet = tag['provider-physical-network']
            if physnet not in neutron_physnets:
                continue

            # Create a 'neutron-networks' attribute in the network group
            # element as a map of neutron networks indexed by physical
            # network name
            network_tag['network-group'].setdefault(
                'neutron-networks',
                dict())[physnet] = neutron_physnets[physnet]
            # Create a 'network-groups' attribute in the neutron network
            # element as a map of neutron groups indexed by network group name
            neutron_physnets[physnet].setdefault(
                'network-groups',
                dict())[network_tag['network-group']['name']] = \
                network_tag['network-group']

    return input_model
Пример #56
0
def handle_compressed_file(
    filename,
    datatypes_registry,
    ext='auto',
    tmp_prefix='sniff_uncompress_',
    tmp_dir=None,
    in_place=False,
    check_content=True,
    auto_decompress=True,
):
    """
    Check uploaded files for compression, check compressed file contents, and uncompress if necessary.

    Supports GZip, BZip2, and the first file in a Zip file.

    For performance reasons, the temporary file used for uncompression is located in the same directory as the
    input/output file. This behavior can be changed with the `tmp_dir` param.

    ``ext`` as returned will only be changed from the ``ext`` input param if the param was an autodetect type (``auto``)
    and the file was sniffed as a keep-compressed datatype.

    ``is_valid`` as returned will only be set if the file is compressed and contains invalid contents (or the first file
    in the case of a zip file), this is so lengthy decompression can be bypassed if there is invalid content in the
    first 32KB. Otherwise the caller should be checking content.
    """
    CHUNK_SIZE = 2**20  # 1Mb
    is_compressed = False
    compressed_type = None
    keep_compressed = False
    is_valid = False
    uncompressed = filename
    tmp_dir = tmp_dir or os.path.dirname(filename)
    for compressed_type, check_compressed_function in COMPRESSION_CHECK_FUNCTIONS:
        is_compressed, is_valid = check_compressed_function(
            filename, check_content=check_content)
        if is_compressed:
            break  # found compression type
    if is_compressed and is_valid:
        if ext in AUTO_DETECT_EXTENSIONS:
            # attempt to sniff for a keep-compressed datatype (observing the sniff order)
            sniff_datatypes = filter(lambda d: getattr(d, 'compressed', False),
                                     datatypes_registry.sniff_order)
            sniffed_ext = run_sniffers_raw(filename, sniff_datatypes)
            if sniffed_ext:
                ext = sniffed_ext
                keep_compressed = True
        else:
            datatype = datatypes_registry.get_datatype_by_extension(ext)
            keep_compressed = getattr(datatype, 'compressed', False)
    # don't waste time decompressing if we sniff invalid contents
    if is_compressed and is_valid and auto_decompress and not keep_compressed:
        fd, uncompressed = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir)
        compressed_file = DECOMPRESSION_FUNCTIONS[compressed_type](filename)
        # TODO: it'd be ideal to convert to posix newlines and space-to-tab here as well
        while True:
            try:
                chunk = compressed_file.read(CHUNK_SIZE)
            except IOError as e:
                os.close(fd)
                os.remove(uncompressed)
                compressed_file.close()
                raise IOError(
                    'Problem uncompressing %s data, please try retrieving the data uncompressed: %s'
                    % (compressed_type, util.unicodify(e)))
            if not chunk:
                break
            os.write(fd, chunk)
        os.close(fd)
        compressed_file.close()
        if in_place:
            # Replace the compressed file with the uncompressed file
            shutil.move(uncompressed, filename)
            uncompressed = filename
    elif not is_compressed or not check_content:
        is_valid = True
    return is_valid, ext, uncompressed, compressed_type
Пример #57
0
def generate_heat_model(input_model, virt_config):
    """
    Create a data structure that more or less describes the heat resources
    required to deploy the input model. The data structure can then later
    be used to generate a heat orchestration template.

    :param input_model: enhanced input model data structure
    :param virt_config: additional information regarding the virtual setup
    (images, flavors, disk sizes)
    :return: dictionary describing heat resources
    """
    heat_template = dict(description='Template for deploying Ardana {}'.format(
        input_model['cloud']['name']))

    clm_cidr = IPNetwork(input_model['baremetal']['subnet'],
                         input_model['baremetal']['netmask'])
    clm_network = None
    heat_networks = heat_template['networks'] = dict()

    # First, add L2 neutron provider networks defined in the input
    # model's neutron configuration
    for neutron_network in input_model['neutron-networks'].itervalues():
        heat_network = dict(name=neutron_network['name'],
                            is_conf=False,
                            is_mgmt=False,
                            external=neutron_network['external'])
        if neutron_network.get('cidr'):
            heat_network['cidr'] = neutron_network['cidr']
        if neutron_network.get('gateway'):
            heat_network['gateway'] = neutron_network['gateway']
        if neutron_network.get('provider'):
            provider = neutron_network['provider'][0]
            if provider['network_type'] == 'vlan':
                if not provider.get('segmentation_id'):
                    # Neutron network is incompletely defined (VLAN tag is
                    # dynamically allocated), so it cannot be defined as an
                    # individual heat network
                    continue
                heat_network['vlan'] = provider['segmentation_id']
            elif provider['network_type'] not in ['flat', 'vlan']:
                # Only layer 2 neutron provider networks are considered
                continue
        heat_networks[heat_network['name']] = heat_network

    # Collect all the routers required by routes configured in the input model,
    # as pairs of networks
    routers = set()

    # Next, add global networks
    for network in input_model['networks'].itervalues():
        cidr = None
        vlan = network['vlanid'] if network.get('tagged-vlan', True) else None
        gateway = IPAddress(
            network['gateway-ip']) if network.get('gateway-ip') else None
        if network.get('cidr'):
            cidr = IPNetwork(network['cidr'])

        heat_network = dict(name=network['name'],
                            is_conf=False,
                            is_mgmt=False,
                            external=False)
        if cidr:
            heat_network['cidr'] = str(cidr)
        if gateway:
            heat_network['gateway'] = str(gateway)

        # There is the special case of global networks being used to implement
        # flat neutron provider networks. For these networks, we need to
        # create a heat network based on the global network parameters
        # (i.e. VLAN) and a heat subnet based on the neutron network
        # parameters
        for neutron_network in network['network-group'].get(
                'neutron-networks', {}).itervalues():
            heat_neutron_network = heat_networks.get(neutron_network['name'])
            if not heat_neutron_network or heat_neutron_network.get('vlan'):
                # Ignore neutron networks that:
                #   - were not already considered at the previous step (i.e.
                #   are not fully defined or are not layer 2 based)
                #   - have a vlan (i.e. are not flat)
                continue

            # Replace the heat neutron network with this global network
            # This is the same as updating the heat global network with subnet
            # attributes taken from the neutron network
            del heat_networks[neutron_network['name']]
            heat_network = heat_neutron_network
            heat_network['name'] = network['name']

            # Only one flat neutron provider network can be associated with a
            # global network
            break

        if vlan:
            heat_network['vlan'] = vlan

        # For each route, track down the target network
        for route in network['network-group']['routes']:
            if route == 'default':
                # The default route is satisfied by adding the network to the
                # external router
                heat_network['external'] = True
            elif 'networks' in route:
                # If the route points to a network group, track down the
                # networks associated with it
                for network_name in route['networks'].keys():
                    routers.add((
                        heat_network['name'],
                        network_name,
                    ))
            else:
                routers.add((
                    heat_network['name'],
                    route['name'],
                ))

        if cidr and cidr in clm_cidr:
            clm_network = heat_network
            heat_network['external'] = heat_network['is_conf'] = True

            # Create an address pool range that excludes the list of server
            # static IP addresses
            fixed_ip_addr_list = \
                [IPAddress(server['ip-addr'])
                 for server in input_model['servers'].itervalues()]
            if gateway:
                fixed_ip_addr_list.append(gateway)
            start_addr = cidr[1]
            end_addr = cidr[-2]
            for fixed_ip_addr in sorted(list(set(fixed_ip_addr_list))):
                if start_addr <= fixed_ip_addr <= end_addr:
                    if fixed_ip_addr - start_addr < end_addr - fixed_ip_addr:
                        start_addr = fixed_ip_addr + 1
                    else:
                        end_addr = fixed_ip_addr - 1
            heat_network['allocation_pools'] = \
                [[str(start_addr), str(end_addr)]]

        elif ('component-endpoints' in network['network-group'] and 'default'
              in network['network-group']['component-endpoints']):
            heat_network['external'] = heat_network['is_mgmt'] = True

            # Create an address pool range that is outside of the range
            # of IP addresses allocated by Ardana
            mgmt_net_last = IPAddress(IPNetwork(network['cidr']).last)
            heat_network['allocation_pools'] = \
                [[str(mgmt_net_last - EXTERNAL_MGMT_ADDR_RANGE),
                  str(mgmt_net_last - 1)]]
        elif True in [
            ('public' in lb['roles'])
                for lb in network['network-group'].get('load-balancers', [])
        ]:
            heat_network['external'] = True

        heat_networks[network['name']] = heat_network

    heat_template['routers'] = []
    for network1, network2 in routers:
        if network1 not in heat_template['networks'] or \
           network2 not in heat_template['networks']:
            continue
        network1 = heat_template['networks'][network1]
        network2 = heat_template['networks'][network2]
        # Re-use the external router, if at least one of the networks is
        # already attached to it
        if network1['external'] or network2['external']:
            network1['external'] = network2['external'] = True
        else:
            heat_template['routers'].append(
                [network1['name'], network2['name']])

    heat_interface_models = heat_template['interface_models'] = dict()

    for interface_model in input_model['interface-models'].itervalues():
        heat_interface_model = \
            heat_interface_models[interface_model['name']] = \
            dict(
                name=interface_model['name'],
                ports=[]
            )
        ports = dict()
        clm_ports = dict()
        for interface in interface_model['network-interfaces'].itervalues():
            devices = interface['bond-data']['devices'] \
                if 'bond-data' in interface \
                else [interface['device']]
            for device in devices:
                port_list = ports
                port = dict(name=device['name'], networks=[])
                if 'bond-data' in interface:
                    port['bond'] = interface['device']['name']
                    port['primary'] = \
                        (device['name'] ==
                         interface['bond-data']['options'].get('primary',
                                                               device['name']))

                for network_group in \
                    interface.get('network-groups', []) + \
                        interface.get('forced-network-groups', []):

                    port['networks'].extend([
                        network['name']
                        for network in network_group['networks'].itervalues()
                    ])

                    # Attach the port only to those neutron networks that have
                    # been validated during the previous steps
                    port['networks'].extend([
                        network['name'] for network in network_group.get(
                            'neutron-networks', dict()).itervalues()
                        if network['name'] in heat_networks
                    ])

                    if clm_network['name'] in network_group['networks']:
                        # if the CLM port is a bond port, then only the
                        # primary is considered if configured
                        if not clm_ports and port.get('primary', True):
                            # Collect the CLM port separately, to put it at
                            # the top of the list and to mark it as the
                            # "management" port - the port to which the
                            # server's management IP address is assigned
                            port_list = clm_ports

                port_list[device['name']] = port

        # Add a port for each device, starting with those ports attached to
        # the CLM network while at the same time preserving the order of the
        # original ports. Ultimately, the port names will be re-aligned to
        # those in the input model by an updated NIC mappings input model
        # configuration
        heat_interface_model['ports'] = [
            p[1] for _, p in enumerate(
                sorted(clm_ports.items()) + sorted(ports.items()))
        ]

    # Generate storage setup (volumes)
    #
    # General strategy:
    #  - one volume for each physical volume specified in the disk model
    #  - the size of each volume cannot be determined from the input model,
    #  so this information needs to be supplied separately (TBD)

    heat_disk_models = heat_template['disk_models'] = dict()
    disks = virt_config['disks']

    for disk_model in input_model['disk-models'].itervalues():
        heat_disk_model = heat_disk_models[disk_model['name']] = dict(
            name=disk_model['name'], volumes=[])
        devices = []
        for volume_group in disk_model.get('volume-groups', []):
            devices += volume_group['physical-volumes']
        for device_group in disk_model.get('device-groups', []):
            devices += [device['name'] for device in device_group['devices']]
        for device in sorted(list(set(devices))):
            if device.endswith('da_root'):
                continue
            device = device.replace('/dev/sd', '/dev/vd')
            volume_name = device.replace('/dev/', '')

            size = virt_config['disk_size']
            # Check if disk size is configured explicitly for the disk model
            if disk_model['name'] in disks:
                size = disks[disk_model['name']]
                if isinstance(size, dict):
                    # Use the disk size specified for the volume name, or
                    # the disk model default, or the global default
                    size = size.get(volume_name) or \
                        size.get('default') or \
                        virt_config['disk_size']
            heat_disk_model['volumes'].append(
                dict(name=volume_name, mountpoint=device, size=size))

    # Generate VM setup (servers)
    #
    # General strategy:
    #  - one server for each server specified in the disk model
    #  - the CLM server is special:
    #    - identification: server hosting the lifecycle-manager
    #    service component
    #    - the floating IP is associated with the "CLM" port attached to it
    #  - the image and flavor used for the server cannot be determined from
    #  the input model so this information needs to be supplied separately

    heat_servers = heat_template['servers'] = []
    images = virt_config['images']
    flavors = virt_config['flavors']

    clm_server = None
    for server in input_model['servers'].itervalues():
        distro_id = server.get('distro-id', virt_config['sles_distro_id'])

        image = None
        # Check if image is configured explicitly
        # for the server or for the role
        if server['id'] in images:
            image = images[server['id']]
        elif server['role']['name'] in images:
            image = images[server['role']['name']]
        if isinstance(image, dict):
            # Use the image specified for the distribution, or
            # the global default
            image = image.get(distro_id)
        if not image:
            image = virt_config['sles_image']
            if distro_id == virt_config['rhel_distro_id']:
                image = virt_config['rhel_image']

        flavor = None
        # Check if image is configured explicitly
        # for the server or for the role
        if server['id'] in flavors:
            flavor = flavors[server['id']]
        elif server['role']['name'] in flavors:
            flavor = flavors[server['role']['name']]

        heat_server = dict(
            name=server['id'],
            ip_addr=server['ip-addr'],
            role=server['role']['name'],
            interface_model=server['role']['interface-model']['name'],
            disk_model=server['role']['disk-model']['name'],
            image=image,
            flavor=flavor,
            is_admin=False,
            is_controller=False,
            is_compute=False)
        # Figure out which server is the CLM host, which are controllers
        # and which are computes. This information is used e.g. to determine
        # the reboot order during the MU workflow and to identify flavors
        # unless explicitly specified for each server or server role
        service_groups = list(server['role'].get('clusters', {}).values())
        service_groups += list(server['role'].get('resources', {}).values())
        for service_group in service_groups:
            # The CLM server is the first server hosting the lifecycle-manager
            # service component.
            # Compute nodes host the nova-compute service component
            if 'nova-compute' in service_group['service-components']:
                heat_server['is_compute'] = True
                if not heat_server['flavor']:
                    heat_server['flavor'] = virt_config['compute_flavor']
            # Every server that is not a compute node and hosts service
            # components other than those required by the CLM is considered
            # a controller node
            else:
                ctrl_service_components = filter(
                    lambda sc: sc not in virt_config['clm_service_components'],
                    service_group['service-components'])
                if ctrl_service_components:
                    heat_server['is_controller'] = True
                    if not heat_server['flavor']:
                        heat_server['flavor'] = \
                            virt_config['controller_flavor']
            if not clm_server and \
                    'lifecycle-manager' in service_group['service-components']:
                clm_server = heat_server
                heat_server['is_admin'] = True
                if not heat_server['flavor']:
                    heat_server['flavor'] = virt_config['clm_flavor']

        heat_servers.append(heat_server)

    return heat_template
def GetAppHostname(app=None, app_id=None, service=None, version=None,
                   use_ssl=appinfo.SECURE_HTTP, deploy=True):
  """Returns the hostname of the given version of the deployed app.

  Args:
    app: Application resource. One of {app, app_id} must be given.
    app_id: str, project ID. One of {app, app_id} must be given. If both are
      provided, the hostname from app is preferred.
    service: str, the (optional) service being deployed
    version: str, the deployed version ID (omit to get the default version URL).
    use_ssl: bool, whether to construct an HTTPS URL.
    deploy: bool, if this is called during a deployment.

  Returns:
    str. Constructed URL.

  Raises:
    TypeError: if neither an app nor an app_id is provided
  """
  if not app and not app_id:
    raise TypeError('Must provide an application resource or application ID.')
  version = version or ''
  service_name = service or ''
  if service == DEFAULT_SERVICE:
    service_name = ''

  domain = DEFAULT_DOMAIN
  if not app and ':' in app_id:
    api_client = appengine_api_client.AppengineApiClient.GetApiClient()
    app = api_client.GetApplication()
  if app:
    app_id, domain = app.defaultHostname.split('.', 1)

  # Normally, AppEngine URLs are of the form
  # 'http[s]://version.service.app.appspot.com'. However, the SSL certificate
  # for appspot.com is not valid for subdomains of subdomains of appspot.com
  # (e.g. 'https://app.appspot.com/' is okay; 'https://service.app.appspot.com/'
  # is not). To deal with this, AppEngine recognizes URLs like
  # 'http[s]://version-dot-service-dot-app.appspot.com/'.
  #
  # This works well as long as the domain name part constructed in this fashion
  # is less than 63 characters long, as per the DNS spec. If the domain name
  # part is longer than that, we are forced to use the URL with an invalid
  # certificate.
  #
  # We've tried to do the best possible thing in every case here.
  subdomain_parts = list(filter(bool, [version, service_name, app_id]))
  scheme = 'http'
  if use_ssl == appinfo.SECURE_HTTP:
    subdomain = '.'.join(subdomain_parts)
    scheme = 'http'
  else:
    subdomain = ALT_SEPARATOR.join(subdomain_parts)
    if len(subdomain) <= MAX_DNS_LABEL_LENGTH:
      scheme = 'https'
    else:
      if deploy:
        format_parts = ['$VERSION_ID', '$SERVICE_ID', '$APP_ID']
        subdomain_format = ALT_SEPARATOR.join(
            [j for (i, j) in zip([version, service_name, app_id], format_parts)
             if i])
        msg = ('This deployment will result in an invalid SSL certificate for '
               'service [{0}]. The total length of your subdomain in the '
               'format {1} should not exceed {2} characters. Please verify '
               'that the certificate corresponds to the parent domain of your '
               'application when you connect.').format(service,
                                                       subdomain_format,
                                                       MAX_DNS_LABEL_LENGTH)
        log.warning(msg)
      subdomain = '.'.join(subdomain_parts)
      if use_ssl == appinfo.SECURE_HTTP_OR_HTTPS:
        scheme = 'http'
      elif use_ssl == appinfo.SECURE_HTTPS:
        if not deploy:
          msg = ('Most browsers will reject the SSL certificate for '
                 'service [{0}].').format(service)
          log.warning(msg)
        scheme = 'https'

  return '{0}://{1}.{2}'.format(scheme, subdomain, domain)
Пример #59
0
def apply_event(state, event, user_profile, include_subscribers):
    # type: (Dict[str, Any], Dict[str, Any], UserProfile, bool) -> None
    if event['type'] == "message":
        state['max_message_id'] = max(state['max_message_id'], event['message']['id'])
        if 'raw_unread_msgs' in state:
            apply_unread_message_event(
                user_profile,
                state['raw_unread_msgs'],
                event['message'],
                event['flags'],
            )

    elif event['type'] == "hotspots":
        state['hotspots'] = event['hotspots']
    elif event['type'] == "custom_profile_fields":
        state['custom_profile_fields'] = event['fields']
    elif event['type'] == "pointer":
        state['pointer'] = max(state['pointer'], event['pointer'])
    elif event['type'] == "realm_user":
        person = event['person']
        person_user_id = person['user_id']

        if event['op'] == "add":
            person = copy.deepcopy(person)
            person['is_active'] = True
            state['raw_users'][person_user_id] = person
        elif event['op'] == "remove":
            state['raw_users'][person_user_id]['is_active'] = False
        elif event['op'] == 'update':
            is_me = (person_user_id == user_profile.id)

            if is_me:
                if ('avatar_url' in person and 'avatar_url' in state):
                    state['avatar_source'] = person['avatar_source']
                    state['avatar_url'] = person['avatar_url']
                    state['avatar_url_medium'] = person['avatar_url_medium']

                for field in ['is_admin', 'email', 'full_name']:
                    if field in person and field in state:
                        state[field] = person[field]

                # In the unlikely event that the current user
                # just changed to/from being an admin, we need
                # to add/remove the data on all bots in the
                # realm.  This is ugly and probably better
                # solved by removing the all-realm-bots data
                # given to admin users from this flow.
                if ('is_admin' in person and 'realm_bots' in state):
                    prev_state = state['raw_users'][user_profile.id]
                    was_admin = prev_state['is_admin']
                    now_admin = person['is_admin']

                    if was_admin and not now_admin:
                        state['realm_bots'] = []
                    if not was_admin and now_admin:
                        state['realm_bots'] = get_owned_bot_dicts(user_profile)

            if person_user_id in state['raw_users']:
                p = state['raw_users'][person_user_id]
                for field in p:
                    if field in person:
                        p[field] = person[field]

    elif event['type'] == 'realm_bot':
        if event['op'] == 'add':
            state['realm_bots'].append(event['bot'])

        if event['op'] == 'remove':
            email = event['bot']['email']
            for bot in state['realm_bots']:
                if bot['email'] == email:
                    bot['is_active'] = False

        if event['op'] == 'update':
            for bot in state['realm_bots']:
                if bot['email'] == event['bot']['email']:
                    if 'owner_id' in event['bot']:
                        bot['owner'] = get_user_profile_by_id(event['bot']['owner_id']).email
                    else:
                        bot.update(event['bot'])

    elif event['type'] == 'stream':
        if event['op'] == 'create':
            for stream in event['streams']:
                if not stream['invite_only']:
                    stream_data = copy.deepcopy(stream)
                    if include_subscribers:
                        stream_data['subscribers'] = []
                    # Add stream to never_subscribed (if not invite_only)
                    state['never_subscribed'].append(stream_data)
                state['streams'].append(stream)
            state['streams'].sort(key=lambda elt: elt["name"])

        if event['op'] == 'delete':
            deleted_stream_ids = {stream['stream_id'] for stream in event['streams']}
            state['streams'] = [s for s in state['streams'] if s['stream_id'] not in deleted_stream_ids]
            state['never_subscribed'] = [stream for stream in state['never_subscribed'] if
                                         stream['stream_id'] not in deleted_stream_ids]

        if event['op'] == 'update':
            # For legacy reasons, we call stream data 'subscriptions' in
            # the state var here, for the benefit of the JS code.
            for obj in state['subscriptions']:
                if obj['name'].lower() == event['name'].lower():
                    obj[event['property']] = event['value']
            # Also update the pure streams data
            for stream in state['streams']:
                if stream['name'].lower() == event['name'].lower():
                    prop = event['property']
                    if prop in stream:
                        stream[prop] = event['value']
        elif event['op'] == "occupy":
            state['streams'] += event['streams']
        elif event['op'] == "vacate":
            stream_ids = [s["stream_id"] for s in event['streams']]
            state['streams'] = [s for s in state['streams'] if s["stream_id"] not in stream_ids]
    elif event['type'] == 'default_streams':
        state['realm_default_streams'] = event['default_streams']
    elif event['type'] == 'realm':
        if event['op'] == "update":
            field = 'realm_' + event['property']
            state[field] = event['value']

            # Tricky interaction: Whether we can create streams can get changed here.
            if (field in ['realm_create_stream_by_admins_only',
                          'realm_waiting_period_threshold']) and 'can_create_streams' in state:
                state['can_create_streams'] = user_profile.can_create_streams()
        elif event['op'] == "update_dict":
            for key, value in event['data'].items():
                state['realm_' + key] = value
                # It's a bit messy, but this is where we need to
                # update the state for whether password authentication
                # is enabled on this server.
                if key == 'authentication_methods':
                    state['realm_password_auth_enabled'] = (value['Email'] or value['LDAP'])
                    state['realm_email_auth_enabled'] = value['Email']
    elif event['type'] == "subscription":
        if not include_subscribers and event['op'] in ['peer_add', 'peer_remove']:
            return

        if event['op'] in ["add"]:
            if not include_subscribers:
                # Avoid letting 'subscribers' entries end up in the list
                for i, sub in enumerate(event['subscriptions']):
                    event['subscriptions'][i] = copy.deepcopy(event['subscriptions'][i])
                    del event['subscriptions'][i]['subscribers']

        def name(sub):
            # type: (Dict[str, Any]) -> Text
            return sub['name'].lower()

        if event['op'] == "add":
            added_names = set(map(name, event["subscriptions"]))
            was_added = lambda s: name(s) in added_names

            # add the new subscriptions
            state['subscriptions'] += event['subscriptions']

            # remove them from unsubscribed if they had been there
            state['unsubscribed'] = [s for s in state['unsubscribed'] if not was_added(s)]

            # remove them from never_subscribed if they had been there
            state['never_subscribed'] = [s for s in state['never_subscribed'] if not was_added(s)]

        elif event['op'] == "remove":
            removed_names = set(map(name, event["subscriptions"]))
            was_removed = lambda s: name(s) in removed_names

            # Find the subs we are affecting.
            removed_subs = list(filter(was_removed, state['subscriptions']))

            # Remove our user from the subscribers of the removed subscriptions.
            if include_subscribers:
                for sub in removed_subs:
                    sub['subscribers'] = [id for id in sub['subscribers'] if id != user_profile.id]

            # We must effectively copy the removed subscriptions from subscriptions to
            # unsubscribe, since we only have the name in our data structure.
            state['unsubscribed'] += removed_subs

            # Now filter out the removed subscriptions from subscriptions.
            state['subscriptions'] = [s for s in state['subscriptions'] if not was_removed(s)]

        elif event['op'] == 'update':
            for sub in state['subscriptions']:
                if sub['name'].lower() == event['name'].lower():
                    sub[event['property']] = event['value']
        elif event['op'] == 'peer_add':
            user_id = event['user_id']
            for sub in state['subscriptions']:
                if (sub['name'] in event['subscriptions'] and
                        user_id not in sub['subscribers']):
                    sub['subscribers'].append(user_id)
            for sub in state['never_subscribed']:
                if (sub['name'] in event['subscriptions'] and
                        user_id not in sub['subscribers']):
                    sub['subscribers'].append(user_id)
        elif event['op'] == 'peer_remove':
            user_id = event['user_id']
            for sub in state['subscriptions']:
                if (sub['name'] in event['subscriptions'] and
                        user_id in sub['subscribers']):
                    sub['subscribers'].remove(user_id)
    elif event['type'] == "presence":
        # TODO: Add user_id to presence update events / state format!
        presence_user_profile = get_user(event['email'], user_profile.realm)
        state['presences'][event['email']] = UserPresence.get_status_dict_by_user(
            presence_user_profile)[event['email']]
    elif event['type'] == "update_message":
        # We don't return messages in /register, so we don't need to
        # do anything for content updates, but we may need to update
        # the unread_msgs data if the topic of an unread message changed.
        if 'subject' in event:
            stream_dict = state['raw_unread_msgs']['stream_dict']
            topic = event['subject']
            for message_id in event['message_ids']:
                if message_id in stream_dict:
                    stream_dict[message_id]['topic'] = topic
    elif event['type'] == "delete_message":
        max_message = Message.objects.filter(
            usermessage__user_profile=user_profile).order_by('-id').first()
        if max_message:
            state['max_message_id'] = max_message.id
        else:
            state['max_message_id'] = -1

        remove_id = event['message_id']
        remove_message_id_from_unread_mgs(state, remove_id)
    elif event['type'] == "reaction":
        # The client will get the message with the reactions directly
        pass
    elif event['type'] == 'typing':
        # Typing notification events are transient and thus ignored
        pass
    elif event['type'] == "update_message_flags":
        # We don't return messages in `/register`, so most flags we
        # can ignore, but we do need to update the unread_msgs data if
        # unread state is changed.
        if event['flag'] == 'read' and event['operation'] == 'add':
            for remove_id in event['messages']:
                remove_message_id_from_unread_mgs(state, remove_id)
    elif event['type'] == "realm_domains":
        if event['op'] == 'add':
            state['realm_domains'].append(event['realm_domain'])
        elif event['op'] == 'change':
            for realm_domain in state['realm_domains']:
                if realm_domain['domain'] == event['realm_domain']['domain']:
                    realm_domain['allow_subdomains'] = event['realm_domain']['allow_subdomains']
        elif event['op'] == 'remove':
            state['realm_domains'] = [realm_domain for realm_domain in state['realm_domains']
                                      if realm_domain['domain'] != event['domain']]
    elif event['type'] == "realm_emoji":
        state['realm_emoji'] = event['realm_emoji']
    elif event['type'] == "alert_words":
        state['alert_words'] = event['alert_words']
    elif event['type'] == "muted_topics":
        state['muted_topics'] = event["muted_topics"]
    elif event['type'] == "realm_filters":
        state['realm_filters'] = event["realm_filters"]
    elif event['type'] == "update_display_settings":
        assert event['setting_name'] in UserProfile.property_types
        state[event['setting_name']] = event['setting']
    elif event['type'] == "update_global_notifications":
        assert event['notification_name'] in UserProfile.notification_setting_types
        state[event['notification_name']] = event['setting']
    else:
        raise AssertionError("Unexpected event type %s" % (event['type'],))
Пример #60
0
def split_by_synapse_domain(bandwidth, locations, arbors, treenode_connector,
                            minis):
    """ locations: dictionary of treenode ID vs tuple with x,y,z
        arbors: dictionary of skeleton ID vs list of DiGraph (that were, or not, split by confidence)
        treenode_connectors: dictionary of treenode ID vs list of tuples of connector_id, string of 'presynaptic_to' or 'postsynaptic_to'
    """
    arbors2 = {}  # Some arbors will be split further
    for skeleton_id, graphs in six.iteritems(arbors):
        subdomains = []
        arbors2[skeleton_id] = subdomains
        for graph in graphs:
            treenode_ids = []
            connector_ids = []
            relation_ids = []
            for treenode_id in filter(treenode_connector.has_key,
                                      graph.nodes_iter()):
                for c in treenode_connector.get(treenode_id):
                    connector_id, relation = c
                    treenode_ids.append(treenode_id)
                    connector_ids.append(connector_id)
                    relation_ids.append(relation)

            if not connector_ids:
                subdomains.append(graph)
                continue

            for parent_id, treenode_id in graph.edges_iter():
                loc0 = locations[treenode_id]
                loc1 = locations[parent_id]
                graph[parent_id][treenode_id]['weight'] = norm(
                    subtract(loc0, loc1))

            # Invoke Casey's magic
            max_density = tree_max_density(graph.to_undirected(), treenode_ids,
                                           connector_ids, relation_ids,
                                           [bandwidth])
            synapse_group = next(six.itervalues(max_density))
            # The list of nodes of each synapse_group contains only nodes that have connectors
            # A local_max is the skeleton node most central to a synapse_group
            anchors = {}
            for domain in six.itervalues(synapse_group):
                g = nx.DiGraph()
                g.add_nodes_from(
                    domain.node_ids
                )  # bogus graph, containing treenodes that point to connectors
                subdomains.append(g)
                anchors[domain.local_max] = g
            # Define edges between domains: create a simplified graph
            mini = simplify(graph, anchors.keys())
            # Replace each node by the corresponding graph, or a graph of a single node
            for node in mini.nodes_iter():
                g = anchors.get(node)
                if not g:
                    # A branch node that was not an anchor, i.e. did not represent a synapse group
                    g = nx.Graph()
                    g.add_node(node, {'branch': True})
                    subdomains.append(g)
                # Associate the Graph with treenodes that have connectors
                # with the node in the minified tree
                mini.node[node]['g'] = g
            # Put the mini into a map of skeleton_id and list of minis,
            # to be used later for defining intra-neuron edges in the circuit graph
            minis[skeleton_id].append(mini)

    return arbors2, minis