def _eval(self, row: typing.Dict): ret_val = 0 for coeff_name in self._coeff_map.keys(): # case where coefficient is a normal feature if coeff_name in row.keys(): try: ret_val += self._coeff_map[coeff_name] * row[coeff_name] except Exception as err: print(f"Got {type(err)} error when trying to add {coeff_name} term. Value is {row[coeff_name]}. Type is {type(row[coeff_name])}") raise err # enum case, where we have coefficient = feature_name.enum_val # e.g. maybe we've got a feature that returns a classification as either Foo or Bar, and we want Foo's to count as 0.5 towards the model. # then we'll have coefficient 0.5 = classification.Foo. # in the code here, we get logit += 0.5 * 1.0 for Foo, and 0.5 * 0.0 for Bar. # I'm mildly confused as to why I put this case in here, must have found some case where it could occur? elif re.search("\w+\.\w+", coeff_name): pieces = coeff_name.split(".") if pieces[0] in row.keys(): ret_val += self._coeff_map[coeff_name] * (1.0 if row[pieces[0]] == pieces[1] else 0.0) else: print(f"Found an element of model that is not a feature: {coeff_name}") # specific cases, where we just hardcode a thing that must be consistent across all models. elif coeff_name == "Intercept": ret_val += self._coeff_map[coeff_name] elif coeff_name == "display_name": pass # default case, print a line. else: print(f"Found an element of model that is not a feature: {coeff_name}") return ret_val
def _dict_to_sorted_items( mapping: tp.Dict ) -> tp.Generator[tp.Tuple[tp.Hashable, tp.Any], None, None]: ''' Convert a dict into two arrays. Note that sorting is only necessary in Python 3.5, and should not be done if an ordered dict ''' if isinstance(mapping, OrderedDict) or _DICT_STABLE: # cannot use fromiter as do not know type keys = mapping.keys() else: keys = sorted(mapping.keys()) for k in keys: yield k, mapping[k]
def validate_simple_dict(dict_: typing.Dict) -> None: """ A simple dict validator with: key as 0-9a-zA-Z-_. based string value as either: string, int, bool, float types """ for key in dict_.keys(): if not isinstance(key, str): raise ValidationError( 'Dictionary key "{}" is not a string'.format(key)) regex_validator = Regexp(regex=(re.compile("^[0-9a-zA-Z-_.]+$"))) try: regex_validator(key) except ValidationError as exc: raise ValidationError('Dictionary key "{}" incorrect : {}'.format( key, str(exc))) from exc # INFO - G.M - We assume float is the type used for float conversion, # this may change depending # on how the json parser is configured. float_type = float invalid_key_value_pairs = [ (key, value) for key, value in dict_.items() if not isinstance(value, (str, int, bool, float_type, type(None))) ] if invalid_key_value_pairs: raise ValidationError( "Only string/number/null values are allowed as dictionary value. Invalid values: {}" .format(invalid_key_value_pairs))
def deck_parser(parsed_dict: typing.Dict): """ try to convert a dictionary into a deck :param parsed_dict: dictionary to parse :return: a new CardInDeck object """ # CardInDeck (or Side) if set(parsed_dict.keys()) == {"name", "edition", "number", "ed_number"}: return CardInDeck(**parsed_dict) # this is the list of decks elif set(parsed_dict.keys()) == {"main", "side", "name"}: return parsed_dict else: raise JSONDecodeError("Unknown set of keys : {}".format( parsed_dict.keys()))
def readobj(key: str, val: typing.Dict, denylist: typing.List, allowlist: typing.List): if not isinstance(val, dict): return str(val) if "id" not in val: raise ValueError("Missing field id in entry " + str(val)) itemid = val["id"] del val["id"] if "deny" in val: # val["deny"] is an object that contains one or more entries per entry. # the entries are matched against the matrix new_denial = { mapping.get(key, key): [itemid], **extract_mapped_list(val["deny"]) } for blkey, v in new_denial.items(): for vi in v[1:]: denylist.append({**new_denial, blkey: vi}) new_denial[blkey] = v[0] denylist.append(new_denial) del val["deny"] if "allow" in val: allowlist.append({ "key": mapping.get(key, key), "value": itemid, "allowed": extract_mapped_list(val["allow"]) }) del val["allow"] if len(val.keys()) == 0: return itemid else: return itemid, val
def __call__(self, kvs: typing.Dict, nupdates: int) -> None: assert self.file is not None, "Call info first" # Add our current row to the history extra_keys = list(kvs.keys() - self.keys) extra_keys.sort() if extra_keys: self.keys.extend(extra_keys) self.file.seek(0) lines = self.file.readlines() self.file.seek(0) for (i, k) in enumerate(self.keys): if i > 0: self.file.write(',') self.file.write(k) self.file.write('\n') for line in lines[1:]: self.file.write(line[:-1]) self.file.write(self.sep * len(extra_keys)) self.file.write('\n') for (i, k) in enumerate(self.keys): if i > 0: self.file.write(',') v = kvs.get(k) if v is not None: self.file.write(str(v)) self.file.write('\n') self.file.flush()
def __call__(self, sample: typing.Dict) -> typing.Dict: if isinstance(sample["image"], list): tensor_images = [] for i in sample["image"]: tensor_images.append(self.convert_to_tensor(i)) sample["image"] = tensor_images else: sample["image"] = self.convert_to_tensor(sample["image"]) if "label" in sample.keys(): if isinstance(sample["label"], list): label = [] for i in sample["label"]: i[np.where(i == 4)] = 3 i = torch.from_numpy(i).float() label.append(i) sample["label"] = label else: # TODO: Check whether this label renaming is necessary sample["label"][np.where(sample["label"] == 4)] = 3 label = torch.from_numpy(sample["label"]).float() one_hot_tensor = torch.nn.functional.one_hot(label.long(), num_classes=4) sample["label"] = one_hot_tensor.permute( 2, 0, 1).to(dtype=torch.float32) return sample
def compute_binary_profile(self, user_items_dict: t.Dict): user_features = {} # partial = 1/len(user_items_dict) for item in user_items_dict.keys(): for feature in self._data.side_information_data.feature_map.get(item, []): # user_features[feature] = user_features.get(feature, 0) + partial user_features[feature] = user_features.get(feature, 1) return user_features
def _dump_sign(self, data: typing.Dict): l = list(data.keys()) l.sort() s = '' for i in l: s += i + '=' + str(data[i]) + '&' s = s[:-1] return hashlib.md5(s.encode('utf-8')).hexdigest()
def _dump_sign(self, data: typing.Dict): l = list(data.keys()) l.sort() s = '' for i in l: s += i+'='+str(data[i])+'&' s = s[:-1] return hashlib.md5(s.encode('utf-8')).hexdigest()
def collection_json_parser(parsed_dict: typing.Dict): """ Try to parse the user's collection :param parsed_dict: dictionary to parse :return: user's collection """ # CardInCollection if set(parsed_dict.keys()) == { "number", "name", "edition", "normal", "foil" }: return CardInCollection(**parsed_dict) # the deck is parsed elif set(parsed_dict.keys()) == {"collection", "decks"}: return parsed_dict else: return deck_parser(parsed_dict)
def compute_incentive(self, data: t.Dict) \ -> t.Dict: P_incentive = None D_incentive = None min_zone_id = self.valid_zones.min() max_zone_id = self.valid_zones.max() d = max_zone_id - min_zone_id if 'P_zone' in data.keys(): P_zone = int(data['P_zone']) P_zone_sugg = int(data['P_zone_sugg']) if P_zone not in self.valid_zones: return {'error': 'Invalid pick-up zone Id'} if P_zone_sugg not in self.valid_zones: return {'error': 'Invalid suggested pick-up zone Id'} P_incentive = 50. * abs(P_zone_sugg - P_zone) / d if 'D_zone' in data.keys(): D_zone = int(data['D_zone']) D_zone_sugg = int(data['D_zone_sugg']) if D_zone not in self.valid_zones: return {'error': 'Invalid drop-off zone Id'} if D_zone_sugg not in self.valid_zones: return {'error': 'Invalid suggested drop-off zone Id'} D_incentive = 50. * abs(D_zone_sugg - D_zone) / d response = {'currency': 'EUR'} if P_incentive is not None: response['P_incentive'] = \ round(P_incentive, 3) if D_incentive is not None: response['D_incentive'] = \ round(D_incentive, 3) return response
async def mset(self, mapping: ty.Dict = {}, expire: int = 0, *args): assert len(mapping.keys()) + len(args) < 20, "Max keys for mset is 20!" if expire > 0: for k, v in mapping.items(): await self.set(key=k, value=v, expire=expire) else: for k, v in mapping.items(): args = args + (k, v) await self._rclient.mset(*args)
def _validate(cls, data: typing.Dict, schema: typing.Dict): for field, validator in schema.items(): if field not in data.keys(): raise KeyError(f'missing configuration field "{field:s}"') if not validator(data[field]): raise ValueError(f'invalid value in field "{field:s}"') return True
def _build_model( distance_in_cents_between_melodic_pitches: typing.Tuple[int, ...], cents_to_pitch: typing.Dict, melodic_pitches: typing.Tuple[abc.Pitch, ...], ) -> typing.Tuple[cp_model.CpModel, typing.List]: """Build ortools model for finding best imitation in given harmony.""" model = cp_model.CpModel() domain = cp_model.Domain.FromValues(cents_to_pitch.keys()) variables = [ model.NewIntVarFromDomain(domain, f"pitch{nth_pitch}") for nth_pitch in range(len(melodic_pitches)) ] max_step = max(cents_to_pitch.keys()) - min(cents_to_pitch.keys()) absolute_differences = [] for nth_step, variable0, variable1, distance in zip( range(len(distance_in_cents_between_melodic_pitches)), variables, variables[1:], distance_in_cents_between_melodic_pitches, ): difference_between_desired_and_real_distance = model.NewIntVar( -max_step, max_step, f"difference_between_desired_and_real_distance{nth_step}_0", ) abs_difference_between_desired_and_real_distance = model.NewIntVar( 0, max_step, f"abs_difference_between_desired_and_real_distance{nth_step}_1") model.Add(difference_between_desired_and_real_distance == distance - (variable1 - variable0)) model.AddAbsEquality( abs_difference_between_desired_and_real_distance, difference_between_desired_and_real_distance, ) absolute_differences.append( abs_difference_between_desired_and_real_distance) model.Minimize(sum(absolute_differences)) return model, variables
def __call__(self, sample: typing.Dict) -> typing.Dict: sample["image"] = self.resize(sample["image"]) if "label" in sample.keys(): mask = sample["label"] new_mask = self.resize(mask) sample["label"] = new_mask return sample
def validate_value(self, data: _t.Dict): if not isinstance(data, dict): self.fail('not a JSON') invalid_keys = list(k for k in data.keys() if k not in self.__valid_keys) if invalid_keys: self.fail(f'invalid key {invalid_keys[0]}') for key in self.__valid_keys: if key not in data: self.fail(f'"{key}" is not set')
def __check_item_entry(self, meta_config_entry: typing.Dict): if 'type' not in meta_config_entry: raise MissingKeys({'type'}) if meta_config_entry['type'] == 'node': for key in set(meta_config_entry.keys()) - {'type'}: self.__check_item_entry_with_type(meta_config_entry, key) else: self.__check_entry_type(meta_config_entry) if meta_config_entry['type'] not in self._callbacks: raise InvalidConfigType()
def validate_value(self, data: _t.Dict): if not isinstance(data, dict): self.fail('not a JSON') invalid_keys = [k for k in data.keys() if k not in self.__valid_keys] if invalid_keys: self.fail('invalid key', invalid_key=invalid_keys[0]) for key in self.__valid_keys: if key not in data: self.fail('missing key', missing_key=key)
def assertCheckDict(self, first: _t.Dict, second: _t.Dict, msg: _t.Text = None): """ Fail if the two fields in dicts are unequal as determined by the '==' operator. Checks if first not contains or not equal field in second """ for field_name in first.keys(): self.assertEqual(first[field_name], second.get(field_name, None), msg or [first, second])
def make_namedtuple(self, data: typing.Dict, **kwargs): """ converts the loaded data dict into a namedtuple :param data: loaded data :param kwargs: marshmallow kwargs :return: namedtuple """ name = self.__class__.__name__.replace('Schema', '') Object = BaseSchema.namedtuple_factory(name, data.keys()) return Object(**data)
def compute_metrics(trues_df: pd.DataFrame, preds: t.Dict, top_n_ranks: t.Tuple = (1, 5)): true_labels = [] pred_labels = [] for name in preds.keys(): true_labels.append( trues_df[trues_df['names'] == name]['labels'].values.tolist()[0]) pred_labels.append(preds[name]) return (top_n_accuracy(preds=pred_labels, truths=true_labels, n=rank) for rank in top_n_ranks)
def fill_table_fields(self, fields_dict: typing.Dict) -> typing.NoReturn: fields_list = list(fields_dict.keys()) types_list = list(fields_dict.values()) if len(types_list) != len(fields_list): raise exception.DifferentCount() self.types = types_list self.fields = fields_list for index, type_name in enumerate(self.types): if self.__check_type_name(type_name): self.types[index] = self.types_dict[type_name[:3]] else: raise exception.TypeNotExists(type_name) self.fields_count = len(self.fields)
def process_sampling(ns: SimpleNamespace, public_users: t.Dict, public_items: t.Dict, i_train: sp.csr_matrix, test: t.Dict, validation=False) -> sp.csr_matrix: i_test = [(public_users[user], public_items[i]) for user, items in test.items() if user in public_users.keys() for i in items.keys() if i in public_items.keys()] rows = [u for u, _ in i_test] cols = [i for _, i in i_test] i_test = sp.csr_matrix((np.ones_like(rows), (rows, cols)), dtype='float32', shape=(len(public_users.keys()), len(public_items.keys()))) candidate_negatives = ((i_test + i_train).astype('bool') != True) ns = ns.negative_sampling strategy = getattr(ns, "strategy", None) if strategy == "random": num_items = getattr(ns, "num_items", None) if num_items is not None: if str(num_items).isdigit(): negative_items = NegativeSampler.sample_by_random_uniform(candidate_negatives, num_items) pass else: raise Exception("Number of negative items value not recognized") else: raise Exception("Number of negative items option is missing") elif strategy == "fixed": files = getattr(ns, "files", None) if files is not None: if not isinstance(files, list): files = [files] file_ = files[0] if validation == False else files[1] negative_items = NegativeSampler.read_from_files(public_users, public_items, file_) pass else: raise Exception("Missing strategy") return negative_items
async def named_execute(self, query: str, args: typing.Dict, timeout: typing.Optional[float] = None) -> str: """Extended versions of `execute` with support of the named parameters.""" # Convert the received query in the parameter to all lower-case query = query.lower() # Fetch the column string that is passed as part of INTO clause query_columns = query.split('into')[1].split('values')[0].strip() # Convert the column into list for further detailed logging columns = [ column.strip() for column in query_columns[query_columns.index('(') + 1:-1].split(',') ] number_of_columns = len(columns) number_of_values = len(list(args.keys())) missing_columns = set(columns) - set(args.keys()) # Scenario: When the value passed in parameter do not match the number of columns if number_of_values < number_of_columns: raise KeyError( 'Values missing. Columns that are missing values: {0}'.format( missing_columns)) # Scenario: When the expected column id is not received if 'id' not in args: raise KeyError( 'Expected column name-> id :: Provided column name-> {}'. format(list(args.keys())[0])) converted_query, asyncpg_args = self._prepare_asyncpg_parameters( query, args, query_module.QueryParamsDictConverter()) return await super().execute(converted_query, *asyncpg_args, timeout=timeout)
def touch_cond(self, info: typing.Dict, value: typing.Union[StrictInt, float]): trend = info.pop("trend") if len(info) == 1: data = info[list(info.keys())[0]] if trend == Trend.Up: if data <= value: return True elif trend == Trend.Down: if data >= value: return True elif trend == Trend.Equal: if data == value: return True
def could_be(cls, message: t.Dict) -> bool: """ Check if the required fields for this class are in the message Note: it checks if the value exists not if it is valid. validation is handled in try_from_message """ ALWAYS_REQUIRED_FIELDS = {"content_id", "content_type"} fields = message.keys() if not ALWAYS_REQUIRED_FIELDS.issubset(fields): return False if not cls.get_required_subfields().issubset(fields): return False return True
def select_max_books(mapper_customer_books: typing.Dict): customer_hold_count_books: typing.List = list() max_books = len(mapper_customer_books[list( mapper_customer_books.keys())[0]]) # find the max of books for customer in mapper_customer_books: if max_books < len(mapper_customer_books[customer]): max_books = len(mapper_customer_books[customer]) # find all customers who hold maximum of books for customer in mapper_customer_books: if max_books == len(mapper_customer_books[customer]): customer_hold_count_books.append(customer) return customer_hold_count_books
def remove_empty_fields(d: typing.Dict) -> typing.Dict: """ Recursively remove empty collections and Nones from dict """ if d is None: return None # stash keys because we can't change a dict while iterating # using keys() keys = list(d.keys()) for k in keys: v = d[k] if isinstance(v, dict): remove_empty_fields(v) if v is None or v == {} or v == []: d.pop(k) return d
def read_from_files(public_users: t.Dict, public_items: t.Dict, filepath: str) -> sp.csr_matrix: map_ = {} with open(filepath) as file: for line in file: line = line.rstrip("\n").split('\t') int_set = {public_items[int(i)] for i in line[1:] if int(i) in public_items.keys()} map_[public_users[int(make_tuple(line[0])[0])]] = int_set rows_cols = [(u, i) for u, items in map_.items() for i in items] rows, cols = zip(*rows_cols) # rows = [u for u, _ in rows_cols] # cols = [i for _, i in rows_cols] negative_samples = sp.csr_matrix((np.ones_like(rows), (rows, cols)), dtype='bool', shape=(len(public_users), len(public_items))) return negative_samples
def is_passport_valid(passport: typing.Dict) -> bool: """Is the passport valid according to the rules?""" # check keys if not passport.keys() >= constants.REQUIRED_KEYS: return False # specific values byr = int(passport["byr"]) if byr < 1920 or byr > 2002: return False # byr (Birth Year) - four digits; at least 1920 and at most 2002. # iyr (Issue Year) - four digits; at least 2010 and at most 2020. # eyr (Expiration Year) - four digits; at least 2020 and at most 2030. ranges = constants.YEAR_RANGES for k, (min_, max_) in ranges.items(): if int(passport[k]) < min_ or int(passport[k]) > max_: return False # hgt (Height) - a number followed by either cm or in: # If cm, the number must be at least 150 and at most 193. # If in, the number must be at least 59 and at most 76. hgt = int(passport["hgt"][:-2]) unit = passport["hgt"][-2:] ranges = { "cm": (150, 193), "in": (59, 76), } if unit not in ranges or hgt < ranges[unit][0] or hgt > ranges[unit][1]: return False # hcl (Hair Color) - a # followed by exactly six characters 0-9 or a-f. hcl = passport["hcl"] hexes = set(string.hexdigits.lower()) if not hcl.startswith("#") or set(hcl[1:]) > hexes: return False ecl = passport["ecl"] if ecl not in constants.VALID_EYE_COLOURS: return False # pid (Passport ID) - a nine-digit number, including leading zeroes. pid = passport["pid"] if len(pid) != 9 or set(pid) > set(string.digits): return False return True
def k(self): return List(*Dict.keys(self))
def keys_view(self): return Dict.keys(self)