def encode_task(task): if not task.deps: result = assoc(task.spec, 'inputs', task.inputs) output_sha = sha(bencode.bencode(result)) return assoc(result, 'output', output_sha) first_dep = task.deps[0] inputs = assoc(task.inputs, first_dep.name, encode_task(first_dep)['output']) return encode_task(Task(task.name, task.spec, task.deps[1:], inputs))
def test_assoc(): assert assoc({}, "a", 1) == {"a": 1} assert assoc({"a": 1}, "a", 3) == {"a": 3} assert assoc({"a": 1}, "b", 3) == {"a": 1, "b": 3} # Verify immutability: d = {'x': 1} oldd = d assoc(d, 'x', 2) assert d is oldd
def fit(self, X, y=None, sample_weight=None, exposure=None): fit_args = self._process_args(X=X, y=y, sample_weight=sample_weight, exposure=exposure) # Create internal cross-validating estimators self.cross_validating_estimators_ = OrderedDict( (k, CrossValidatingEstimator(v, cv=self.cv, n_jobs=self.n_jobs, verbose=self.verbose, pre_dispatch=self.pre_dispatch)) for k, v in self.ordered_regressors.items()) # frozendict(valmap(lambda x: # CrossValidatingEstimator(x, cv=self.cv, n_jobs=self.n_jobs, # verbose=self.verbose, # pre_dispatch=self.pre_dispatch), self.regressors).items()) # Fit the inner regressors using cross-validation for est_name, est in self.cross_validating_estimators_.items(): if self.verbose > 0: print('Super learner is fitting %s...' % est_name) est.fit(**fit_args) if self.verbose > 0: print('Super learner finished fitting %s.' % est_name) # Fit the outer meta-regressor. Cross validation is not used here. Instead, # users of the SuperLearner are free to wrap the SuperLearner in a # CrossValidatingEstimator. meta_fit_args = assoc( fit_args, 'X', np.concatenate(tuple( map(growd(2), [ est.cv_predictions_ for est in self.cross_validating_estimators_.values() ])), axis=1)) if self.y_transformer is not None: self.y_transformer_ = clone(self.y_transformer).fit(**fit_args) meta_fit_args = assoc( meta_fit_args, 'y', self.y_transformer_.transform( **dissoc(fit_args, 'sample_weight', 'y'))) if self.verbose > 0: print('Super learner fitting meta-regressor...') self.meta_regressor_ = clone(self.meta_regressor).fit(**meta_fit_args) if self.verbose > 0: print('Super learner meta-regressor fitting complete.') # All scikit-learn compatible estimators must return self from fit return self
def test_split_only_var_key(self): """ Only the values for the specified key will be split, no matter how long the other keys are. """ message = 'x: {x}, y: {y}' event = {'x': '123', 'y': '12345'} result = split_cf_messages(message, 'x', max_length=14)(event) self.assertEqual(result, [(assoc(event, 'x', '1'), message), (assoc(event, 'x', '2'), message), (assoc(event, 'x', '3'), message)])
def test_assoc(self): D, kw = self.D, self.kw assert assoc(D({}), "a", 1, **kw) == D({"a": 1}) assert assoc(D({"a": 1}), "a", 3, **kw) == D({"a": 3}) assert assoc(D({"a": 1}), "b", 3, **kw) == D({"a": 1, "b": 3}) # Verify immutability: d = D({'x': 1}) oldd = d assoc(d, 'x', 2, **kw) assert d is oldd
def test_split_only_var_key(self): """ Only the values for the specified key will be split, no matter how long the other keys are. """ message = "x: {x}, y: {y}" event = {"x": "123", "y": "12345"} result = split_cf_messages(message, "x", max_length=14)(event) self.assertEqual( result, [(assoc(event, "x", "1"), message), (assoc(event, "x", "2"), message), (assoc(event, "x", "3"), message)], )
def test_split_only_var_key(self): """ Only the values for the specified key will be split, no matter how long the other keys are. """ message = 'x: {x}, y: {y}' event = {'x': '123', 'y': '12345'} result = split_cf_messages(message, 'x', max_length=14)(event) self.assertEqual( result, [(assoc(event, 'x', '1'), message), (assoc(event, 'x', '2'), message), (assoc(event, 'x', '3'), message)])
def test_no_split_on_empty_field(self): """ Do not split the event the field is an empty list, even if it is too long. """ message = 'Hello {there} human being {punctuation}' event = {'there': [], 'punctuation': '!', 'extra': 'unused'} result = split_cf_messages(message, 'there', max_length=5)(event) self.assertEqual(result, [(assoc(event, 'there', ''), message)])
def test_no_split_on_empty_field(self): """ Do not split the event the field is an empty list, even if it is too long. """ message = "Hello {there} human being {punctuation}" event = {"there": [], "punctuation": "!", "extra": "unused"} result = split_cf_messages(message, "there", max_length=5)(event) self.assertEqual(result, [(assoc(event, "there", ""), message)])
def test_no_need_to_split_if_below_length(self): """ Do not split the event if the message is sufficiently short. However, do format it so that the list becomes a comma-separated string. """ message = "Hello {there} human being {punctuation}" event = {"there": [1, 2, 3, 4], "punctuation": "!", "extra": "unused"} result = split_cf_messages(message, "there")(event) self.assertEqual(result, [(assoc(event, "there", "1, 2, 3, 4"), message)])
def test_no_need_to_split_if_below_length(self): """ Do not split the event if the message is sufficiently short. However, do format it so that the list becomes a comma-separated string. """ message = 'Hello {there} human being {punctuation}' event = {'there': [1, 2, 3, 4], 'punctuation': '!', 'extra': 'unused'} result = split_cf_messages(message, 'there')(event) self.assertEqual(result, [(assoc(event, 'there', '1, 2, 3, 4'), message)])
async def create(self, dto: CreateTodoItemDto): self.items, new_item = pipe( self.items.keys(), last, lambda key: key + 1, lambda new_key: TodoItem( id=new_key, msg=dto.msg, is_done=dto.is_done), lambda item: (assoc(self.items, item.id, item), item), ) return new_item
def get_credentials(client_secret_file: str, token_storage_file: str, user_id: str) -> Credentials: tokens = _get_tokens(token_storage_file) credentials = option.cata( Credentials, lambda: _get_new_credentials(client_secret_file) )(tokens.get(user_id)) _save_tokens(token_storage_file, assoc(tokens, user_id, credentials.token)) return credentials
def on_message(self, headers, message): self._logger.info(headers) decoded_messages = json.loads(message) self._logger.info('Received a total of {} message(s)' .format(len(decoded_messages))) for decoded_message in decoded_messages: doc = assoc(decoded_message, 'type', headers['subscription']) r.table('raw_messages').insert(doc).run(conn) tm.process_message(doc, conn) self._mq.ack(id=headers['message-id'], subscription=headers['subscription'])
def test_dicttoolz(): d1 = {'foo': 'bar'} d2 = {'baz': 'quux'} assert_that(merge(d1, d2)).is_equal_to({'foo': 'bar', 'baz': 'quux'}) assert_that(d1).is_equal_to({'foo': 'bar'}) assert_that(assoc(d1, 'a', 1)).is_equal_to({'foo': 'bar', 'a': 1}) assert_that(dissoc(d2, 'baz')).is_equal_to({}) struct = {'a': [{'c': 'hello'}]} assert_that(get_in(['a', 0, 'c'], struct)).is_equal_to(struct['a'][0]['c']) assert_that(get_in(['a', 0, 'd'], struct, 'not found')).is_equal_to('not found')
def _update_state(state, action): new_worker_pos = _move_projection_tile(state['worker']['pos'], action) if new_worker_pos is not None: state = tzd.assoc_in(state, ['worker', 'pos'], new_worker_pos) # without joining on example 2, 500 iters: 22.36s # with joining on example 2, 500 iters: 3.87s new_wrapped_shells = state['wrapped_shells'] + [_pt2shell(state['worker']['pos'])] wrapped = shapely.ops.unary_union([shapely.geometry.Polygon(sh) for sh in new_wrapped_shells]) wrapped_simple = wrapped.simplify(0.01) return tzd.assoc(state, 'wrapped_shells', _polygon2shells(wrapped_simple))
def _predict_action(state): mine = shapely.geometry.Polygon(state['desc']['mine_shell']) obstacles = [shapely.geometry.Polygon(sh) for sh in state['desc']['obstacle_shells']] obstacle = shapely.ops.unary_union(obstacles) situable = mine.difference(obstacle) wrappeds = [shapely.geometry.Polygon(sh) for sh in state['wrapped_shells']] wrapped = shapely.ops.unary_union(wrappeds) not_wrapped = situable.difference(wrapped) if not_wrapped.area < 1.0: return None, state last_move = state.get('last_move', 'W') for move in [last_move, 'W', 'S', 'A', 'D']: proj = _move_projection_center(state['worker']['pos'], move) if not_wrapped.contains(proj): return move, tzd.dissoc(state, 'path_pts_to_not_wrapped') if not state.get('path_pts_to_not_wrapped'): target_tile = tzf.thread_first(not_wrapped.representative_point(), _shapely_point2pt, _snap_to_tile) print('Finding shortest path from tile {} to {}'.format(state['worker']['pos'], target_tile)) if tzd.get_in(['cache', 'incidence_m'], state) is None: incidence_m = _incidence_matrix(situable) state = tzd.assoc_in(state, ['cache', 'incidence_m'], incidence_m) else: incidence_m = state['cache']['incidence_m'] target_vertex_ind = _incidence_ind(target_tile[0], target_tile[1], x_size=math.ceil(situable.bounds[2])) path_dists, path_predecessors = sp.sparse.csgraph.shortest_path(csgraph=incidence_m, directed=False, return_predecessors=True, unweighted=True, indices=target_vertex_ind) start_vertex_ind = _incidence_ind(state['worker']['pos'][0], state['worker']['pos'][1], x_size=math.ceil(situable.bounds[2])) path_inds = _path_inds(path_predecessors, start_vertex_ind) path_pts = [_incidence_pt(ind, x_size=math.ceil(situable.bounds[2])) for ind in path_inds] print('Found path: {}'.format(path_pts)) state = tzd.assoc(state, 'path_pts_to_not_wrapped', path_pts) path_move = _projection_pt_move(state['worker']['pos'], state['path_pts_to_not_wrapped'][0]) if path_move is not None: return path_move, tzd.update_in(state, ['path_pts_to_not_wrapped'], lambda p: p[1:]) return 'Z', state
def filterFeatures(match): usableKeys = ['players', 'radiant_win', 'hero_id', 'player_slot'] isUsable = lambda k: k in usableKeys toplvlFiltered = keyfilter(isUsable, match) filteredPlayers = [] for player in toplvlFiltered['players']: side = decideSide(player['player_slot']) playerData = assoc(keyfilter(isUsable, player), 'team', side) playerData = dissoc(playerData, 'player_slot') filteredPlayers.append(playerData) toplvlFiltered['players'] = filteredPlayers return toplvlFiltered
def test_assoc(): assert assoc({}, "a", 1) == {"a": 1} assert assoc({"a": 1}, "a", 3) == {"a": 3} assert assoc({"a": 1}, "b", 3) == {"a": 1, "b": 3} # Verify immutability: d = {'x': 1} oldd = d assoc(d, 'x', 2) assert d is oldd # Test object support: c = C() assert assoc(c, "a", 1).__dict__ == {"a": 1} c.a = 1 assert assoc(c, "a", 3).__dict__ == {"a": 3} assert assoc(c, "b", 3).__dict__ == {"a": 1, "b": 3} # Verify immutability: o = C() o.x = 1 assoc(o, 'x', 2) assert o.x == 1
async def update(self, dto: UpdateTodoItemDto, id_: int): item = get_in([id_], self.items) if not item: return None self.items, new_item = pipe( (item, dto), lambda items: { **items[0].dict(), **items[1].dict(exclude_defaults=True) }, lambda data: TodoItem(**data), lambda todo: (assoc(self.items, id_, todo), todo), ) return new_item
def test_success(self, mock_gsgr): rows = [ {'created_at': '0', 'desired': 'some', 'status': 'ACTIVE'}, {'desired': 'some', 'status': 'ACTIVE'}, # no created_at {'created_at': '0', 'status': 'ACTIVE'}, # no desired {'created_at': '0', 'desired': 'some'}, # no status {'created_at': '0', 'desired': 'some', 'status': 'DISABLED'}, {'created_at': '0', 'desired': 'some', 'deleting': 'True', }, {'created_at': '0', 'desired': 'some', 'status': 'ERROR'}] rows = [assoc(row, "tenantId", "t1") for row in rows] mock_gsgr.return_value = succeed(rows) results = self.successResultOf(get_scaling_groups("client")) self.assertEqual(results, {"t1": [rows[0], rows[3]]}) mock_gsgr.assert_called_once_with( "client", props=["status", "deleting", "created_at"])
def set_desired_to_actual_group(dispatcher, cass_client, group): """ Set group's desired to current number of servers in the group """ try: res_eff = get_all_launch_server_data( group["tenantId"], group["groupId"], datetime.utcnow()) eff = Effect(TenantScope(res_eff, group["tenantId"])) resources = yield perform(dispatcher, eff) actual = active_servers_count(resources["servers"]) print("group", group, "setting desired to ", actual) yield cass_client.execute( ('UPDATE scaling_group SET desired=:desired WHERE ' '"tenantId"=:tenantId AND "groupId"=:groupId'), assoc(group, "desired", actual), DEFAULT_CONSISTENCY) except Exception as e: print("Couldn't set group {} to {} due to {}".format(group, actual, e))
def mark_deleted_servers(old, new): """ Given dictionaries containing old and new servers, return a list of all servers, with the deleted ones annotated with a status of DELETED. :param list old: List of old servers :param list new: List of latest servers :return: List of updated servers """ def sdict(servers): return {s['id']: s for s in servers} old = sdict(old) new = sdict(new) deleted_ids = set(old.keys()) - set(new.keys()) for sid in deleted_ids: old[sid] = assoc(old[sid], "status", "DELETED") return merge(old, new).values()
def test_trezor_sign_transaction(mock_trezor, mock_account): trezor_signer = TrezorSigner() transaction_dict = assoc(TRANSACTION_DICT, key='from', value=mock_account.address) signed_transaction = trezor_signer.sign_transaction( transaction_dict=transaction_dict) assert isinstance(signed_transaction, HexBytes) # assert valid deserializable transaction transaction = Transaction.from_bytes(signed_transaction) # Confirm the integrity of the sender and recipient address failure_message = 'WARNING: transaction "to" field was mutated' sender_checksum_address = to_checksum_address(transaction.to) assert sender_checksum_address != mock_account.address, failure_message assert sender_checksum_address == TRANSACTION_DICT['to'], failure_message assert sender_checksum_address == transaction_dict['to'] # positive assert sender_checksum_address != mock_account.address # negative
def blast2summary_dict(db, blastpath, ete2_db): # (Path, Path) -> list[dict] """Reading in a blast output file, lookup all seqids to get taxids with a single blastdbcmd. Then, lookup the taxonomy using ETE2 via the taxid, and add that info to the blast info.""" # rows = csv.DictReader(open(blastpath), delimiter='\t',fieldnames=[SEQID, 'sseqid','pid', 'alnlen','gapopen','qstart','qend','sstart','send','evalue','bitscore']) rows = csv.DictReader(open(blastpath), delimiter='\t', fieldnames=blast_columns) rows = list(rows) seqids = map(get('sseqid'), rows) taxids = get_taxid(db, seqids) def get_gi(s): fields = s.split('|') if len(fields) > 1: return fields[1] else: raise ValueError("Seq ID %s is missing GI fields and '|'" % s) gis = imap(get_gi, seqids) #TODO: change matches to use something unique--not the TAXID! actually, why is it a dict # in the first place? it should be a list of dictionaries, and then map over # the dictionaries to merge them with the taxonomy info # this will replace the lines: # matches = . . . # items = . . . #matches = dict((taxids[gi], row) for gi, row in zip(gis,rows) if gi in taxids) ncbi = NCBITaxa( ete2_db) # downloads database and creates SQLite database if needed # items = dictmap(lambda tid,row: merge(row, taxonomy(ncbi, tid)), matches) matches = [ assoc(row, 'taxid', taxids[gi]) for gi, row in zip(gis, rows) if gi in taxids ] items = [merge(row1, taxonomy(ncbi, row1['taxid'])) for row1 in matches] res = imap(partial(keyfilter, csv_fields.__contains__), items) return res
def get_metric(name, k): return {name: assoc(METRICS[name], 'k', k)}
def cat(self, item: Tuple[A, B]): return Map(dicttoolz.assoc(self, *item))
items = ujson.load(args.input_metadata_file) print(f'loading stats file..', file=sys.stderr) stats = ujson.load(args.input_stat_file) # create dictionary of stats zero_vector = return_zero_vector(stats, args.label) # composing from the right will make this code more readable compute_labels = compose(partial(compute_label, zero_vector, args.label)) print('[') prev = None for x in map(lambda item: assoc(item, 'label', compute_labels(item)), items): if not x['label']: continue if prev is not None: print(',') print(json.dumps(x, indent=2, separators=(',', ': ')), end='') prev = x print('\n]')
def run(f, *args, **kwargs): result = f(*args, **kwargs) result.addCallback(lambda x: (x[0], assoc(x[1], 'locked', True))) return result
def run(f, *args, **kwargs): result = f(*args, **kwargs) result.addCallback( lambda x: (x[0], assoc(x[1], 'locked', True))) return result
async def replace(self, dto: CreateTodoItemDto, id_: int): if not get_in([id_], self.items): return None self.items = assoc(self.items, id_, TodoItem(id=id_, msg=dto.msg, is_done=dto.is_done)) return get_in([id_], self.items)
def http_json_rpc(token=None, uri=None, verify_ssl=None, json_data=None, workspace=None, proxies=None, fire_and_forget=False, check_allow_transmit=None): """ Sends a json_rpc request over http. Returns: dict: The decoded response from the server. Args: token (str): oauth2 token uri (str): the server uri to connect to verify_ssl (bool): passed to requests. flag to check the server's certs, or not. json_data (json-encodable object): the payload to send workspace (int): workspace to connect to. If None, let the server connect to the default workspace for your user or token proxies (dict): Dictionary mapping protocol or protocol and hostname to the URL of the proxy. fire_and_forget (bool,optional): return from the method after the request is sent (not wait for response) check_allow_transmit (callable, optional): For use in retry, callable method to see if retries are still valid to send """ def auth_header(): if workspace: return "Bearer_{}_ws{}".format(token, workspace) else: return "Bearer_{}".format(token) def streamable(): if json_data and json_data.get('method') in STREAM_ENDPOINTS: return True return False if token: headers = { "Authorization": auth_header(), "Content-Type": "application/json", } else: headers = { "Content-Type": "application/json", } payload = json.dumps(assoc(json_data, 'id', 0), default=unsupported_object_json_encoder, option=json.OPT_NAIVE_UTC | json.OPT_NON_STR_KEYS) def get_session(): if fire_and_forget: return FuturesSession() return requests.sessions.Session() with get_session() as session: if streamable(): retry = 0 else: retry = RPCRetry(check_allow_transmit=check_allow_transmit) adapter = HTTPAdapter(max_retries=retry) session.mount('http://', adapter) session.mount('https://', adapter) if streamable(): handle, file_name = tempfile.mkstemp(dir=download_folder, prefix="download_", suffix=".tmp") os.close( handle ) # Can't control the access mode, so close this one and open another. with open(file_name, 'wb') as tmp_file: with session.post(uri, headers=headers, data=payload, verify=verify_ssl, proxies=proxies, allow_redirects=False, stream=True) as response: response.raise_for_status() try: result = response.json() return result except Exception: # JSONDecodeError: Should be this, but which library? json or simplejson - depends what is installed pass for chunk in response.iter_content(chunk_size=None): tmp_file.write(chunk) return file_name elif fire_and_forget: r_future = session.post(uri, headers=headers, data=payload, verify=verify_ssl, proxies=proxies, allow_redirects=False) # Adding a callback that will raise an exception if there was a problem with the request def on_request_complete(request_future): try: response = request_future.result() response.raise_for_status() except: print(f'Exception for method {json_data.get("method")}') raise r_future.add_done_callback(on_request_complete) else: try: response = session.post(uri, headers=headers, data=payload, verify=verify_ssl, proxies=proxies, allow_redirects=False) response.raise_for_status() result = response.json() return result except Exception as e: print(f'Exception for method {json_data.get("method")}') raise