def test_added(self): """ total desired, pending and actual are added to cloud metrics """ td = 10 ta = 20 tp = 3 tt = 7 tg = 13 m = {'collectionTime': 100000, 'ttlInSeconds': 5 * 24 * 60 * 60} md = merge(m, {'metricValue': td, 'metricName': 'ord.desired'}) ma = merge(m, {'metricValue': ta, 'metricName': 'ord.actual'}) mp = merge(m, {'metricValue': tp, 'metricName': 'ord.pending'}) mt = merge(m, {'metricValue': tt, 'metricName': 'ord.tenants'}) mg = merge(m, {'metricValue': tg, 'metricName': 'ord.groups'}) req_data = [md, ma, mp, mt, mg] log = object() seq = [ (Func(time.time), const(100)), (service_request( ServiceType.CLOUD_METRICS_INGEST, "POST", "ingest", data=req_data, log=log).intent, noop) ] eff = add_to_cloud_metrics( m['ttlInSeconds'], 'ord', td, ta, tp, tt, tg, log=log) self.assertIsNone(perform_sequence(seq, eff))
def __init__(self, **kwds): from toolz import dicttoolz as dtz from . import crypto, project, report, dicer, tstamp kwds = dtz.merge( kwds, { 'conf_classes': [ crypto.GitAuthSpec, crypto.StamperAuthSpec, crypto.EncrypterSpec, project.Project, project.ProjectsDB, report.ReporterSpec, dicer.DicerSpec, tstamp.WstampSpec ], 'cmd_aliases': dtz.merge( base.reports_keeper_alias_kwd, { ('i', 'inp'): ('DicerCmd.inp', DicerCmd.inp.help), ('o', 'out'): ('DicerCmd.out', DicerCmd.out.help), }), 'cmd_flags': { 'with-inputs': ({ 'ReporterSpec': { 'include_input_in_dice': True }, }, report.ReporterSpec.include_input_in_dice.help ), # @UndefinedVariable **base.shrink_flags_kwd, }, }) super().__init__(**kwds)
def _inherit_parent_cmd(self, change): """ Inherit config-related stuff from up the cmd-chain. """ if self.parent: ## Collect parents, ordered like that: # subapp, self, parent1, ... # cmd_chain = self.my_cmd_chain() ## Collect separately and merge SPECs separately, # to prepend them before SPECs at the end. # conf_classes = list( itz.concat(cmd.conf_classes for cmd in cmd_chain)) ## Merge aliases/flags reversed. # cmd_aliases = dtz.merge(cmd.cmd_aliases for cmd in cmd_chain[::-1]) cmd_flags = dtz.merge(cmd.cmd_flags for cmd in cmd_chain[::-1]) else: ## We are root. cmd_chain = [self] conf_classes = list(self.conf_classes) cmd_aliases = self.cmd_aliases cmd_flags = self.cmd_flags cmd_classes = [type(cmd) for cmd in cmd_chain] self.classes = list(iset(cmd_classes + conf_classes)) self.aliases.update(cmd_aliases) self.flags.update(cmd_flags)
def test_factory(): assert merge(defaultdict(int, {1: 2}), {2: 3}) == {1: 2, 2: 3} assert (merge(defaultdict(int, {1: 2}), {2: 3}, factory=lambda: defaultdict(int)) == defaultdict(int, {1: 2, 2: 3})) assert not (merge(defaultdict(int, {1: 2}), {2: 3}, factory=lambda: defaultdict(int)) == {1: 2, 2: 3}) assert raises(TypeError, lambda: merge({1: 2}, {2: 3}, factoryy=dict))
def gen_slices(k, svy, qn, resp, m_vars, m_filt): loc = {'svy_id': k, 'dset_id': 'yrbss'} slices = [ merge(loc, s) for s in svy.generate_slices(qn, True, m_vars, m_filt) ] slices += [ merge(loc, s) for s in svy.generate_slices(qn, False, m_vars, m_filt) ] return slices
def test_factory(self): D, kw = self.D, self.kw assert merge(defaultdict(int, D({1: 2})), D({2: 3})) == {1: 2, 2: 3} assert (merge(defaultdict(int, D({1: 2})), D({2: 3}), factory=lambda: defaultdict(int)) == defaultdict(int, D({1: 2, 2: 3}))) assert not (merge(defaultdict(int, D({1: 2})), D({2: 3}), factory=lambda: defaultdict(int)) == {1: 2, 2: 3}) assert raises(TypeError, lambda: merge(D({1: 2}), D({2: 3}), factoryy=dict))
def get_article(self, output='json'): title = self.extract_title(self.context) context_with_title = dicttoolz.merge(self.context, {'title': title}) published_date = self.extract_published_date(context_with_title) context_with_date = dicttoolz.merge(context_with_title, {'published_date': published_date}) content = self.extract_html_content(context_with_date) context = dicttoolz.merge(context_with_date, {'content': content}) if output == 'json': return json.dumps(context) elif output == 'dict': return context return context
def test_factory(): assert merge(defaultdict(int, {1: 2}), {2: 3}) == {1: 2, 2: 3} assert (merge(defaultdict(int, {1: 2}), {2: 3}, factory=lambda: defaultdict(int)) == defaultdict( int, { 1: 2, 2: 3 })) assert not (merge(defaultdict(int, {1: 2}), {2: 3}, factory=lambda: defaultdict(int)) == { 1: 2, 2: 3 }) assert raises(TypeError, lambda: merge({1: 2}, {2: 3}, factoryy=dict))
def parse_transforms(transforms_dict_seq): """Return dictionary of transforms based on `transforms` in config_dict. All this really does is merge the transforms defined in multiple configuration dictionaries. """ return dicttoolz.merge(transforms_dict_seq)
def write_entry(obj, lang): try: key = client.key('String', obj["id"], namespace=lang) del obj["id"] entity = client.get(key) or datastore.Entity(key, exclude_from_indexes=string_exclude_from_indexes) # Update translations if obj["is_translated"] or obj["is_approved"]: # Update everything entity["source"] = obj["source"] entity["target"] = obj["target"] entity["translation_source"] = obj["translation_source"] entity["is_translated"] = obj["is_translated"] entity["is_approved"] = obj["is_approved"] elif entity.get("translation_source", "") == "BEAST": # Pretranslated # only update if string changed # unsure if this can happen with the same string ID if entity["source"] != obj["source"]: entity["source"] = obj["source"] entity["target"] = obj["target"] entity["translation_source"] = obj["translation_source"] else: # Not pretranslated, but not translated # Update "just in case it changed" entity["source"] = obj["source"] entity["target"] = obj["target"] entity["translation_source"] = obj["translation_source"] # Update keys that were previously not present entity.update(merge(obj, entity)) string_update_rules(lang, entity) client.put(entity) except Exception as ex: traceback.print_exc()
def parse_colorizers(colorizers_dict_seq): """Return dictionary of transforms based on `colorizers` in config_dict. This converts `colorizers` field in a configuration dict into color transforms. For example, take the following configuration:: 'colorizers': { 'warn_color': { 'match_pattern': ' WARN ', 'on_color': 'on_yellow', }, }, That configuration is parsed to return the transform:: from linecook.transforms.core import color_text color_text(' WARN ', on_color='on_yellow') """ colorizers_dict = dicttoolz.merge(colorizers_dict_seq) return { key: color_text(color_kwargs) for key, color_kwargs in colorizers_dict.items() }
def covid_archetype_structure( clinical_author_name: str, clinical_author_id: str, document_date: datetime.date, form_values: Mapping, ) -> dict: return dict( clinicalAuthorName=clinical_author_name, clinicalAuthorId=clinical_author_id, documentTime=document_date.strftime('%Y-%m-%d'), bodyTemp=dict( magnitude=form_values['body_temperature_degrees_C'], units='Cel', ), symptoms=dicttoolz.merge( {'dateOfOnset': form_values['date_of_onset'].strftime('%Y-%m-%d')}, archetype_values_from_presence_field( prefix='firstSymptomsPresence', presence=form_values['first_symptoms_presence']), archetype_values_from_presence_field( prefix='coughPresence', presence=form_values['cough_presence']), archetype_values_from_presence_field( prefix='feverPresence', presence=form_values['fever_presence']), archetype_values_from_presence_field( prefix='difficultyBreathingPresence', presence=form_values['difficulty_breathing_presence']), archetype_values_from_presence_field( prefix='soreThroatPresence', presence=form_values['sore_throat_presence']), ), )
def get_ipo(): # 大量股票上市日期为空 db = get_db('wy') collection = db['IPO资料'] docs = collection.find({}, projection={ '_id': 0, '股票代码': 1, '上市日期': 1, }) df = pd.DataFrame.from_records(docs) df['上市日期'] = pd.to_datetime(df['上市日期'], errors='coerce') wy_dates = { code: pd.to_datetime(dt, errors='coerce') for code, dt in zip(df['股票代码'], df['上市日期']) } ipo_dates = _listing_date() dates = merge(wy_dates, ipo_dates) def f(code): try: return dates[code] except KeyError: return pd.NaT df['上市日期'] = df['股票代码'].map(f) df.dropna(inplace=True) df.rename(columns={'股票代码': 'sid'}, inplace=True) df['sid'] = df['sid'].astype('int64') df[AD_FIELD_NAME] = df['上市日期'] - pd.Timedelta(days=1) return df
def get_train(train_id): conn = getattr(g, 'db_conn') query = r.table('train_movements').filter(r.row['train_id'] == train_id).order_by(r.desc('actual_timestamp')) mvs = list(query.run(conn)) info = get_train_info(mvs[0]) train = merge(info, {'movements': mvs}) return json.dumps(train, default=json_formats.date_handler)
def rolling_fit_opt_weights(df, opt_weights_func, look_ahead_per): """applies opt_weights_func to rolling window on pandas df""" num_rows = df.shape[0] p = pipe(xrange(num_rows), filter(lambda x: x + look_ahead_per < num_rows), map(lambda x: {df.index[x]: opt_weights_func(df.iloc[x:x+look_ahead_per+1])})) return pd.DataFrame(merge(p)).T
def _server(group, state): server = sample_servers()[0] return merge(server, { 'status': state, 'metadata': { 'rax:auto_scaling_group_id': group } })
def make_env(self, env_id): params = merge(params_default, { 'client_pool': self.client_pool }) print("{} Minecraft clients available".format(len(self.client_pool))) join_token = marlo.make(env_id, params) return marlo.init(join_token[0])
def __init__(self, name, hist_return=None, industry_weight=None, property_dict=defaultdict(str), **kwargs): self.name = name self.property = merge(_REQUIRED_BENCHMARK_PROPERTY, property_dict) self.production_data_format = kwargs.get('production_data_format', OutputDataFormat.MULTI_INDEX_DF) self.hist_return = hist_return self.industry_weight = industry_weight # 对标指数的行业成分比例 self._validate_data_format() self._validate_date_format()
def test_merge_with_non_dict_mappings(): class Foo(Mapping): def __init__(self, d): self.d = d def __iter__(self): return iter(self.d) def __getitem__(self, key): return self.d[key] def __len__(self): return len(self.d) d = Foo({1: 1}) assert merge(d) is d or merge(d) == {1: 1} assert merge_with(sum, d) == {1: 1}
def test_merge_iterable_arg(self): D, kw = self.D, self.kw assert merge([D({ 1: 1, 2: 2 }), D({3: 4})], **kw) == D({ 1: 1, 2: 2, 3: 4 })
def __init__(self, N, base_model, **kwargs): self.ensemble_size = N self.base_model_params = base_model.__dict__ self.model_params = [ dicttoolz.merge(self.base_model_params, {key: kwargs[key][i] for key in kwargs}) for i in range(self.ensemble_size) ] self.models = [Model(**params) for params in self.model_params]
def main(): print(header) stream = open(args.config, 'r') default = open('./configs/default.yaml', 'r') parameters = load(stream) default_parameters = load(default) if (args.command == 'train'): parameters = merge(default_parameters, parameters) print("Training parameters\n-------") print_dic(parameters) runner = Runner(**parameters) runner.run() else: parameters = merge(merge(default_parameters, parameters), { 'deterministic_evaluation': args.det, 'load_dir': args.load_dir }) evaluator = Evaluator(**parameters) evaluator.evaluate()
def test_dicttoolz(): d1 = {'foo': 'bar'} d2 = {'baz': 'quux'} assert_that(merge(d1, d2)).is_equal_to({'foo': 'bar', 'baz': 'quux'}) assert_that(d1).is_equal_to({'foo': 'bar'}) assert_that(assoc(d1, 'a', 1)).is_equal_to({'foo': 'bar', 'a': 1}) assert_that(dissoc(d2, 'baz')).is_equal_to({}) struct = {'a': [{'c': 'hello'}]} assert_that(get_in(['a', 0, 'c'], struct)).is_equal_to(struct['a'][0]['c']) assert_that(get_in(['a', 0, 'd'], struct, 'not found')).is_equal_to('not found')
def add_to_cloud_metrics(ttl, region, group_metrics, num_tenants, config, log=None, _print=False): """ Add total number of desired, actual and pending servers of a region to Cloud metrics. :param str region: which region's metric is collected :param group_metrics: List of :obj:`GroupMetric` :param int num_tenants: total number of tenants :param dict config: Config json dict containing convergence tenants info :param log: Optional logger :param bool _print: Should it print activity on stdout? Useful when running as a script :return: `Effect` with None """ epoch = yield Effect(Func(time.time)) metric_part = {'collectionTime': int(epoch * 1000), 'ttlInSeconds': ttl} tenanted_metrics, total = calc_total(group_metrics) if log is not None: log.msg( 'total desired: {td}, total_actual: {ta}, total pending: {tp}', td=total.desired, ta=total.actual, tp=total.pending) if _print: print('total desired: {}, total actual: {}, total pending: {}'.format( total.desired, total.actual, total.pending)) metrics = [('desired', total.desired), ('actual', total.actual), ('pending', total.pending), ('tenants', num_tenants), ('groups', len(group_metrics))] for tenant_id, metric in sorted(tenanted_metrics.items()): metrics.append(("{}.desired".format(tenant_id), metric.desired)) metrics.append(("{}.actual".format(tenant_id), metric.actual)) metrics.append(("{}.pending".format(tenant_id), metric.pending)) # convergence tenants desired and actual conv_tenants = keyfilter( partial(tenant_is_enabled, get_config_value=lambda k: get_in([k], config)), tenanted_metrics) conv_desired = sum(m.desired for m in conv_tenants.itervalues()) conv_actual = sum(m.actual for m in conv_tenants.itervalues()) metrics.extend( [("conv_desired", conv_desired), ("conv_actual", conv_actual), ("conv_divergence", conv_desired - conv_actual)]) data = [merge(metric_part, {'metricValue': value, 'metricName': '{}.{}'.format(region, metric)}) for metric, value in metrics] yield service_request(ServiceType.CLOUD_METRICS_INGEST, 'POST', 'ingest', data=data, log=log)
def clean_movement_message(msg, msg_type, conn): extras = {'type': msg_type} body = msg['body'] for key in body.keys(): if key.endswith('_stanox'): logger.debug('Train {}: Lookup stanox {} for field {}'.format(body['train_id'], body[key], key)) extras = merge(extras, get_geo(body[key], key[:-len('_stanox')], conn)) if key.endswith('_timestamp'): try: logger.debug('Converting timestamp for field {}'.format(key)) intval = int(body[key]) extras[key] = r.epoch_time(intval / 1000.0) except: pass if body[key] == 'true' or body[key] == 'false': extras[key] = bool(body[key] == 'true') return merge(body, extras)
def get_step_limits_from_conf(limit_conf): """ Get step limits along with defaults for steps not in limit_conf :param dict limit_conf: step name -> limit mapping :return: `dict` of step class -> limit """ step_limits = { step_conf_to_class[step_conf]: limit for step_conf, limit in limit_conf.items()} return merge(_DEFAULT_STEP_LIMITS, step_limits)
def parse_rethinkdb_url(url: str) -> dict: parse_ret = urlparse(url) config = { "host": parse_ret.hostname, "port": parse_ret.port, "user": parse_ret.username, "password": parse_ret.password, "db": parse_ret.path[1:] if len(parse_ret) > 1 else None } config = valfilter(lambda x: x is not None, config) return merge(_default_config, config)
def _listing_date(): """上市日期""" sh_df = ak.stock_info_sh_name_code(indicator="主板A股") sh = { code: pd.to_datetime(dt, errors='coerce') for code, dt in zip(sh_df['SECURITY_CODE_A'], sh_df['LISTING_DATE']) } sz_df = ak.stock_info_sz_name_code(indicator="A股列表") sz = { code: pd.to_datetime(dt, errors='coerce') for code, dt in zip(sz_df['A股代码'], sz_df['A股上市日期']) } return merge(sh, sz)
def _renderDirectory(self, ruleHits, ruleStats, directory, filename): # Generate output HTML for each rule for rule, hits in ruleHits.items(): # Render hits for individual rule outfilePathJSON = os.path.join(directory, rule.machine_name + ".json") if len(hits) > 0: # Render hits # Generate JSON API jsonAPI = { "timestamp": self.timestamp, "downloadTimestamp": self.downloadTimestamp, "rule": rule.meta_dict, "hits": [valfilter(bool, {"msgstr": entry.msgstr, # valfilter: remove empty values for smaller JSON "msgid": entry.msgid, "tcomment": entry.tcomment, "hit": hit, "origImages": origImages, "translatedImages": translatedImages, "crowdinLink": "{0}#q={1}".format(self.translationURLs[filename], genCrowdinSearchString(entry)) }) for entry, hit, filename, origImages, translatedImages in hits] } writeJSONToFile(outfilePathJSON, jsonAPI) else: # Remove file (redirects to 404 file) if there are no exportHitsAsJSON if os.path.isfile(outfilePathJSON): os.remove(outfilePathJSON) # Render file index page (no filelist) ruleInfos = [merge(rule.meta_dict, {"num_hits": ruleStats[rule]}) for rule in self.rules if ruleStats[rule] > 0] ruleInfos.sort(key=lambda o: -o["severity"]) # Invert sort order js = { "pageTimestamp": self.timestamp, "downloadTimestamp": self.downloadTimestamp, "stats": ruleInfos, "files": [merge(self.statsByFile[filename], {"filename": filename}) for filename in self.files if self.statsByFile[filename]["notices"] > 0] } writeJSONToFile(os.path.join(directory, "index.json"), js)
def __init__(self, name, hist_return=None, industry_weight=None, property_dict=defaultdict(str), **kwargs): self.name = name self.property = merge(_REQUIRED_BENCHMARK_PROPERTY, property_dict) self.production_data_format = kwargs.get( 'production_data_format', OutputDataFormat.MULTI_INDEX_DF) self.hist_return = hist_return self.industry_weight = industry_weight # 对标指数的行业成分比例 self._validate_data_format() self._validate_date_format()
def get_stock_status(): """股票状态词典 键:股票代码 值:退市日期。在市交易的股票代码其值为空 """ df = ak.stock_info_a_code_name() p1 = set(df['code'].to_list()) p2 = set(get_recent_trading_stocks()) codes = p1 | p2 d1 = {code: None for code in codes} d2 = get_delist_stock_dates() # 注意,退市字典要放在次位置 # 如有交叉键,则以次位置的值替代 return merge(d1, d2)
def collect_recipes(config_dicts, transforms_registry): """Return recipe dictionary from a list of configuration dictionaries. Args: config_dicts (list(dict)): Unparsed configuration dictionaries. For each dictionary, only use the 'recipes' value, which itself is a dictionary, where the keys are recipe names and values are lists of transform functions or transform names. transforms_registry (dict): Dictionary containing named transform functions. See also `collect_tranforms`, which build this registry. """ recipe_dict = dicttoolz.merge(get_value_from_each("recipes", config_dicts)) return { name: list(resolve_recipe(r, transforms_registry)) for name, r in recipe_dict.items() }
def computeRuleHitsForFileSet(self, xliffs): """ For each file in the given filename -> PO object dictionary, compute the Rule -> Hits dictonary. Stores the information in the current instance. Does not return anything """ # Compute dict with sorted & prettified filenames self.files = sorted(xliffs.keys()) # Add all futures to the executor futures = [ self.executor.submit(self.computeRuleHits, filename) for filename in xliffs.keys() ] # Process the results in first-received order. Also keep track of rule performance self.fileRuleHits = collections.defaultdict(dict) n_finished = 0 # Intermediate result storage raw_results = collections.defaultdict( dict) # filename -> {rule: result} for future in concurrent.futures.as_completed(futures): # Extract result for filename, rule, result in future.result(): self.fileRuleHits[filename][rule] = result # Track progress n_finished += 1 if n_finished % 1000 == 0: percent_finished = n_finished * 100. / len(futures) print("Rule computation finished {0:.2f} %".format( percent_finished)) # Compute total stats by file self.statsByFile = { filename: merge(self.ruleHitsToSeverityCountMap(ruleHits), {"translation_url": self.translationURLs[filename]}) for filename, ruleHits in self.fileRuleHits.items() } # Compute map filename -> {rule: numHits for rule} self.statsByFileAndRule = { filename: valmap(len, ruleHits) for filename, ruleHits in self.fileRuleHits.items() } # Compute map rule -> numHits for rule self.totalStatsByRule = merge_with(sum, *(self.statsByFileAndRule.values()))
def prepare_server_launch_config(group_id, server_config, lb_descriptions): """ Prepare a server config (the server part of the Group's launch config) with any necessary dynamic data. :param str group_id: The group ID :param PMap server_config: The server part of the Group's launch config, as per :obj:`otter.json_schema.group_schemas.server` except as the value of a one-element PMap with key "server". :param iterable lb_descriptions: iterable of :class:`ILBDescription` providers """ updated_metadata = merge( get_in(('server', 'metadata'), server_config, {}), generate_metadata(group_id, lb_descriptions)) return set_in(server_config, ('server', 'metadata'), updated_metadata)
def mark_deleted_servers(old, new): """ Given dictionaries containing old and new servers, return a list of all servers, with the deleted ones annotated with a status of DELETED. :param list old: List of old servers :param list new: List of latest servers :return: List of updated servers """ def sdict(servers): return {s['id']: s for s in servers} old = sdict(old) new = sdict(new) deleted_ids = set(old.keys()) - set(new.keys()) for sid in deleted_ids: old[sid] = assoc(old[sid], "status", "DELETED") return merge(old, new).values()
def blast2summary_dict(db, blastpath): # (Path, Path) -> list[dict] """Reading in a blast output file, lookup all seqids to get taxids with a single blastdbcmd. Then, lookup the taxonomy using ETE2 via the taxid, and add that info to the blast info.""" rows = csv.DictReader(open(blastpath), delimiter='\t', fieldnames=[ 'qseqid', 'sseqid', 'pid', 'alnlen', 'gapopen', 'qstart', 'qend', 'sstart', 'send', 'evalue', 'bitscore' ]) rows = list(rows) seqids = map(get('sseqid'), rows) taxids = get_taxid(db, seqids) gis = (s.split('|')[1] for s in seqids) matches = dict( (taxids[gi], row) for gi, row in zip(gis, rows) if gi in taxids) ncbi = NCBITaxa( ) # downloads database and creates SQLite database if needed return dictmap(lambda tid, row: merge(row, taxonomy(ncbi, tid)), matches)
def visualize(node_list, depth): graph = graph_init() neighbourhood = node_populate_neighbours(graph, set(node_list), depth) subgraph = nx.subgraph(graph, neighbourhood) layout = nx.spring_layout(subgraph) nx.draw_networkx_nodes( subgraph, layout, with_labels=True, ) nx.draw_networkx_labels( subgraph, layout, labels={ node: data.get("name", data.get("label", node)) for node, data in subgraph.nodes(True) }, ) nx.draw_networkx_edges( subgraph, layout, ) nx.draw_networkx_edge_labels( subgraph, layout, edge_labels=reduce( lambda current, incoming: merge( current, { (incoming[0], incoming[1]): arrow_get_type( subgraph.get_edge_data(*incoming).get( "uri", incoming[-1])) }, ), subgraph.edges, {}, ), ) plt.show()
def computeRuleHitsForFileSet(self, poFiles): """ For each file in the given filename -> PO object dictionary, compute the Rule -> Hits dictonary. Stores the information in the current instance. Does not return anything """ # Compute dict with sorted & prettified filenames self.files = sorted(poFiles.keys()) # Add all futures to the executor futures = list(itertools.chain(*(self.computeRuleHits(po, filename) for filename, po in poFiles.items()))) # Process the results in first-received order. Also keep track of rule performance self.fileRuleHits = collections.defaultdict(dict) n_finished = 0 # Intermediate result storage raw_results = collections.defaultdict(dict) # filename -> {rule: result} for future in concurrent.futures.as_completed(futures): # Extract result filename, rule, result = future.result() self.fileRuleHits[filename][rule] = result # Track progress n_finished += 1 if n_finished % 1000 == 0: percent_finished = n_finished * 100. / len(futures) print("Rule computation finished {0:.2f} %".format(percent_finished)) # Compute total stats by file self.statsByFile = { filename: merge(self.ruleHitsToSeverityCountMap(ruleHits), { "translation_url": self.translationURLs[filename]}) for filename, ruleHits in self.fileRuleHits.items() } # Compute map filename -> {rule: numHits for rule} self.statsByFileAndRule = { filename: valmap(len, ruleHits) for filename, ruleHits in self.fileRuleHits.items() } # Compute map rule -> numHits for rule self.totalStatsByRule = merge_with(sum, *(self.statsByFileAndRule.values()))
def unchanged_divergent_groups(clock, current, timeout, group_metrics): """ Return list of GroupMetrics that have been divergent and unchanged for timeout seconds :param IReactorTime clock: Twisted time used to track :param dict current: Currently tracked divergent groups :param float timeout: Timeout in seconds :param list group_metrics: List of group metrics :return: (updated current, List of (group, divergent_time) tuples) """ converged, diverged = partition_bool( lambda gm: gm.actual + gm.pending == gm.desired, group_metrics) # stop tracking all converged and deleted groups deleted = set(current.keys()) - metrics_set(group_metrics) updated = current.copy() for g in metrics_set(converged) | deleted: updated.pop(g, None) # Start tracking divergent groups depending on whether they've changed now = clock.seconds() to_log, new = [], {} for gm in diverged: pair = (gm.tenant_id, gm.group_id) if pair in updated: last_time, values = updated[pair] if values != hash((gm.desired, gm.actual, gm.pending)): del updated[pair] continue time_diff = now - last_time if time_diff > timeout and time_diff % timeout <= 60: # log on intervals of timeout. For example, if timeout is 1 hr # then log every hour it remains diverged to_log.append((gm, time_diff)) else: new[pair] = now, hash((gm.desired, gm.actual, gm.pending)) return merge(updated, new), to_log
def got_additional_headers(additional_headers): return request_func(*args, headers=merge(headers, additional_headers), **kwargs)
def _server(group, state): server = sample_servers()[0] return merge(server, {"status": state, "metadata": {"rax:auto_scaling_group_id": group}})
def test_added(self): """ total desired, pending and actual are added to cloud metrics """ metrics = [ GroupMetrics("t1", "g1", 3, 2, 0), GroupMetrics("t2", "g1", 4, 4, 1), GroupMetrics("t2", "g", 100, 20, 0), GroupMetrics("t3", "g3", 5, 3, 0), ] config = {"non-convergence-tenants": ["t1"]} m = {"collectionTime": 100000, "ttlInSeconds": 5 * 24 * 60 * 60} md = merge(m, {"metricValue": 112, "metricName": "ord.desired"}) ma = merge(m, {"metricValue": 29, "metricName": "ord.actual"}) mp = merge(m, {"metricValue": 1, "metricName": "ord.pending"}) mt = merge(m, {"metricValue": 3, "metricName": "ord.tenants"}) mg = merge(m, {"metricValue": 4, "metricName": "ord.groups"}) mt1d = merge(m, {"metricValue": 3, "metricName": "ord.t1.desired"}) mt1a = merge(m, {"metricValue": 2, "metricName": "ord.t1.actual"}) mt1p = merge(m, {"metricValue": 0, "metricName": "ord.t1.pending"}) mt2d = merge(m, {"metricValue": 104, "metricName": "ord.t2.desired"}) mt2a = merge(m, {"metricValue": 24, "metricName": "ord.t2.actual"}) mt2p = merge(m, {"metricValue": 1, "metricName": "ord.t2.pending"}) mt3d = merge(m, {"metricValue": 5, "metricName": "ord.t3.desired"}) mt3a = merge(m, {"metricValue": 3, "metricName": "ord.t3.actual"}) mt3p = merge(m, {"metricValue": 0, "metricName": "ord.t3.pending"}) cd = merge(m, {"metricValue": 109, "metricName": "ord.conv_desired"}) ca = merge(m, {"metricValue": 27, "metricName": "ord.conv_actual"}) cdiv = merge(m, {"metricValue": 82, "metricName": "ord.conv_divergence"}) req_data = [md, ma, mp, mt, mg, mt1d, mt1a, mt1p, mt2d, mt2a, mt2p, mt3d, mt3a, mt3p, cd, ca, cdiv] log = mock_log() seq = [ (Func(time.time), const(100)), (service_request(ServiceType.CLOUD_METRICS_INGEST, "POST", "ingest", data=req_data, log=log).intent, noop), ] eff = add_to_cloud_metrics(m["ttlInSeconds"], "ord", metrics, 3, config, log) # number of tenants self.assertIsNone(perform_sequence(seq, eff)) log.msg.assert_called_once_with( "total desired: {td}, total_actual: {ta}, total pending: {tp}", td=112, ta=29, tp=1 )
def merge(self, other: 'Map[A, B]'): return Map(dicttoolz.merge(self, other))
def update_factor_property(factor_property): ret = merge(_REQUIRED_FACTOR_PROPERTY, factor_property) return ret
def searchEntry(self, request, params): return SearchEntryResource( store=self.store, params=merge(self.params, params))
def _server(group, state): server = sample_servers()[0] return merge( server, {'status': state, 'metadata': {'rax:auto_scaling_group_id': group}})
str : None, object : None, Union : lambda xs: xs[0], List : lambda x : [examples[x]]*3, } simple = lambda t: dict(name=dict(type=t,value=None,example=examples.get(t))) primitives = [int, bool, str, float] primdict = dict(zip(primitives, [simple]*len(primitives))) enum = compose(simple, lambda x: x.__name__) def handle_union(xs): xs = list(map(lambda x: x['name']['type'], xs)) return dict(name=dict(choices=xs, example=xs[0],value=None)) def handle_list(t): t = next(t) return dict(name=dict(type=t,value=None,example=examples[List](t))), tfuncs= merge(primdict, { object : enum, NamedTuple : enum, Optional : lambda x: merge(simple(x), {'optional' : True}), List : handle_list, #Union : lambda xs: dict(name=dict(choices=xs, example=xs[0],value=None), Union : handle_union }) #{n : t for n,t in res = traverse_type(TrimOpts, tfuncs) from functools import reduce from itertools import starmap #res = reduce(merge, map(lambda x: traverse_type(x, tfuncs), TrimOpts._field_types), {}) print(TrimOpts.__dict__) res = {k : traverse_type(t, tfuncs) for k,t in TrimOpts._field_types.items()} print(res)
def test_added(self): """ total desired, pending and actual are added to cloud metrics """ metrics = [GroupMetrics('t1', 'g1', 3, 2, 0), GroupMetrics('t2', 'g1', 4, 4, 1), GroupMetrics('t2', 'g', 100, 20, 0), GroupMetrics('t3', 'g3', 5, 3, 0)] config = {"non-convergence-tenants": ["t1"]} m = {'collectionTime': 100000, 'ttlInSeconds': 5 * 24 * 60 * 60} md = merge(m, {'metricValue': 112, 'metricName': 'ord.desired'}) ma = merge(m, {'metricValue': 29, 'metricName': 'ord.actual'}) mp = merge(m, {'metricValue': 1, 'metricName': 'ord.pending'}) mt = merge(m, {'metricValue': 3, 'metricName': 'ord.tenants'}) mg = merge(m, {'metricValue': 4, 'metricName': 'ord.groups'}) mt1d = merge(m, {'metricValue': 3, 'metricName': 'ord.t1.desired'}) mt1a = merge(m, {'metricValue': 2, 'metricName': 'ord.t1.actual'}) mt1p = merge(m, {'metricValue': 0, 'metricName': 'ord.t1.pending'}) mt2d = merge(m, {'metricValue': 104, 'metricName': 'ord.t2.desired'}) mt2a = merge(m, {'metricValue': 24, 'metricName': 'ord.t2.actual'}) mt2p = merge(m, {'metricValue': 1, 'metricName': 'ord.t2.pending'}) mt3d = merge(m, {'metricValue': 5, 'metricName': 'ord.t3.desired'}) mt3a = merge(m, {'metricValue': 3, 'metricName': 'ord.t3.actual'}) mt3p = merge(m, {'metricValue': 0, 'metricName': 'ord.t3.pending'}) cd = merge(m, {'metricValue': 109, 'metricName': 'ord.conv_desired'}) ca = merge(m, {'metricValue': 27, 'metricName': 'ord.conv_actual'}) cdiv = merge(m, {'metricValue': 82, 'metricName': 'ord.conv_divergence'}) req_data = [md, ma, mp, mt, mg, mt1d, mt1a, mt1p, mt2d, mt2a, mt2p, mt3d, mt3a, mt3p, cd, ca, cdiv] log = mock_log() seq = [ (Func(time.time), const(100)), (service_request( ServiceType.CLOUD_METRICS_INGEST, "POST", "ingest", data=req_data, log=log).intent, noop) ] eff = add_to_cloud_metrics(m['ttlInSeconds'], 'ord', metrics, 3, # number of tenants config, log) self.assertIsNone(perform_sequence(seq, eff)) log.msg.assert_called_once_with( 'total desired: {td}, total_actual: {ta}, total pending: {tp}', td=112, ta=29, tp=1)
def searchNoType(self, request, params): return SearchResultResource( store=self.store, params=merge(self.params, params, {'searchType': None}))
def test_merge_iterable_arg(): assert merge([{1: 1, 2: 2}, {3: 4}]) == {1: 1, 2: 2, 3: 4}
def test_merge(self): D, kw = self.D, self.kw assert merge(D({1: 1, 2: 2}), D({3: 4}), **kw) == D({1: 1, 2: 2, 3: 4})
def test_merge_iterable_arg(self): D, kw = self.D, self.kw assert merge([D({1: 1, 2: 2}), D({3: 4})], **kw) == D({1: 1, 2: 2, 3: 4})
def test_merge(): assert merge({1: 1, 2: 2}, {3: 4}) == {1: 1, 2: 2, 3: 4}
def request(*args, **kwargs): headers = kwargs.pop('headers', {}) headers = headers if headers is not None else {} return request_func(*args, headers=merge(fixed_headers, headers), **kwargs)
def searchWithType(self, request, params): return SearchResultResource( store=self.store, params=merge(self.params, params))