def _eval(self, row: typing.Dict):
ret_val = 0
for coeff_name in self._coeff_map.keys():
# case where coefficient is a normal feature
if coeff_name in row.keys():
try:
ret_val += self._coeff_map[coeff_name] * row[coeff_name]
except Exception as err:
print(f"Got {type(err)} error when trying to add {coeff_name} term. Value is {row[coeff_name]}. Type is {type(row[coeff_name])}")
raise err
# enum case, where we have coefficient = feature_name.enum_val
# e.g. maybe we've got a feature that returns a classification as either Foo or Bar, and we want Foo's to count as 0.5 towards the model.
# then we'll have coefficient 0.5 = classification.Foo.
# in the code here, we get logit += 0.5 * 1.0 for Foo, and 0.5 * 0.0 for Bar.
# I'm mildly confused as to why I put this case in here, must have found some case where it could occur?
elif re.search("\w+\.\w+", coeff_name):
pieces = coeff_name.split(".")
if pieces[0] in row.keys():
ret_val += self._coeff_map[coeff_name] * (1.0 if row[pieces[0]] == pieces[1] else 0.0)
else:
print(f"Found an element of model that is not a feature: {coeff_name}")
# specific cases, where we just hardcode a thing that must be consistent across all models.
elif coeff_name == "Intercept":
ret_val += self._coeff_map[coeff_name]
elif coeff_name == "display_name":
pass
# default case, print a line.
else:
print(f"Found an element of model that is not a feature: {coeff_name}")
return ret_val
def _dict_to_sorted_items(
mapping: tp.Dict
) -> tp.Generator[tp.Tuple[tp.Hashable, tp.Any], None, None]:
'''
Convert a dict into two arrays. Note that sorting is only necessary in Python 3.5, and should not be done if an ordered dict
'''
if isinstance(mapping, OrderedDict) or _DICT_STABLE:
# cannot use fromiter as do not know type
keys = mapping.keys()
else:
keys = sorted(mapping.keys())
for k in keys:
yield k, mapping[k]
def validate_simple_dict(dict_: typing.Dict) -> None:
"""
A simple dict validator with:
key as 0-9a-zA-Z-_. based string
value as either: string, int, bool, float types
"""
for key in dict_.keys():
if not isinstance(key, str):
raise ValidationError(
'Dictionary key "{}" is not a string'.format(key))
regex_validator = Regexp(regex=(re.compile("^[0-9a-zA-Z-_.]+$")))
try:
regex_validator(key)
except ValidationError as exc:
raise ValidationError('Dictionary key "{}" incorrect : {}'.format(
key, str(exc))) from exc
# INFO - G.M - We assume float is the type used for float conversion,
# this may change depending
# on how the json parser is configured.
float_type = float
invalid_key_value_pairs = [
(key, value) for key, value in dict_.items()
if not isinstance(value, (str, int, bool, float_type, type(None)))
]
if invalid_key_value_pairs:
raise ValidationError(
"Only string/number/null values are allowed as dictionary value. Invalid values: {}"
.format(invalid_key_value_pairs))
def deck_parser(parsed_dict: typing.Dict):
"""
try to convert a dictionary into a deck
:param parsed_dict: dictionary to parse
:return: a new CardInDeck object
"""
# CardInDeck (or Side)
if set(parsed_dict.keys()) == {"name", "edition", "number", "ed_number"}:
return CardInDeck(**parsed_dict)
# this is the list of decks
elif set(parsed_dict.keys()) == {"main", "side", "name"}:
return parsed_dict
else:
raise JSONDecodeError("Unknown set of keys : {}".format(
parsed_dict.keys()))
def readobj(key: str, val: typing.Dict, denylist: typing.List,
allowlist: typing.List):
if not isinstance(val, dict):
return str(val)
if "id" not in val:
raise ValueError("Missing field id in entry " + str(val))
itemid = val["id"]
del val["id"]
if "deny" in val:
# val["deny"] is an object that contains one or more entries per entry.
# the entries are matched against the matrix
new_denial = {
mapping.get(key, key): [itemid],
**extract_mapped_list(val["deny"])
}
for blkey, v in new_denial.items():
for vi in v[1:]:
denylist.append({**new_denial, blkey: vi})
new_denial[blkey] = v[0]
denylist.append(new_denial)
del val["deny"]
if "allow" in val:
allowlist.append({
"key": mapping.get(key, key),
"value": itemid,
"allowed": extract_mapped_list(val["allow"])
})
del val["allow"]
if len(val.keys()) == 0:
return itemid
else:
return itemid, val
def __call__(self, kvs: typing.Dict, nupdates: int) -> None:
assert self.file is not None, "Call info first"
# Add our current row to the history
extra_keys = list(kvs.keys() - self.keys)
extra_keys.sort()
if extra_keys:
self.keys.extend(extra_keys)
self.file.seek(0)
lines = self.file.readlines()
self.file.seek(0)
for (i, k) in enumerate(self.keys):
if i > 0:
self.file.write(',')
self.file.write(k)
self.file.write('\n')
for line in lines[1:]:
self.file.write(line[:-1])
self.file.write(self.sep * len(extra_keys))
self.file.write('\n')
for (i, k) in enumerate(self.keys):
if i > 0:
self.file.write(',')
v = kvs.get(k)
if v is not None:
self.file.write(str(v))
self.file.write('\n')
self.file.flush()
def __call__(self, sample: typing.Dict) -> typing.Dict:
if isinstance(sample["image"], list):
tensor_images = []
for i in sample["image"]:
tensor_images.append(self.convert_to_tensor(i))
sample["image"] = tensor_images
else:
sample["image"] = self.convert_to_tensor(sample["image"])
if "label" in sample.keys():
if isinstance(sample["label"], list):
label = []
for i in sample["label"]:
i[np.where(i == 4)] = 3
i = torch.from_numpy(i).float()
label.append(i)
sample["label"] = label
else:
# TODO: Check whether this label renaming is necessary
sample["label"][np.where(sample["label"] == 4)] = 3
label = torch.from_numpy(sample["label"]).float()
one_hot_tensor = torch.nn.functional.one_hot(label.long(),
num_classes=4)
sample["label"] = one_hot_tensor.permute(
2, 0, 1).to(dtype=torch.float32)
return sample
def compute_binary_profile(self, user_items_dict: t.Dict):
user_features = {}
# partial = 1/len(user_items_dict)
for item in user_items_dict.keys():
for feature in self._data.side_information_data.feature_map.get(item, []):
# user_features[feature] = user_features.get(feature, 0) + partial
user_features[feature] = user_features.get(feature, 1)
return user_features
def _dump_sign(self, data: typing.Dict):
l = list(data.keys())
l.sort()
s = ''
for i in l:
s += i + '=' + str(data[i]) + '&'
s = s[:-1]
return hashlib.md5(s.encode('utf-8')).hexdigest()
def _dump_sign(self, data: typing.Dict):
l = list(data.keys())
l.sort()
s = ''
for i in l:
s += i+'='+str(data[i])+'&'
s = s[:-1]
return hashlib.md5(s.encode('utf-8')).hexdigest()
def collection_json_parser(parsed_dict: typing.Dict):
"""
Try to parse the user's collection
:param parsed_dict: dictionary to parse
:return: user's collection
"""
# CardInCollection
if set(parsed_dict.keys()) == {
"number", "name", "edition", "normal", "foil"
}:
return CardInCollection(**parsed_dict)
# the deck is parsed
elif set(parsed_dict.keys()) == {"collection", "decks"}:
return parsed_dict
else:
return deck_parser(parsed_dict)
def compute_incentive(self, data: t.Dict) \
-> t.Dict:
P_incentive = None
D_incentive = None
min_zone_id = self.valid_zones.min()
max_zone_id = self.valid_zones.max()
d = max_zone_id - min_zone_id
if 'P_zone' in data.keys():
P_zone = int(data['P_zone'])
P_zone_sugg = int(data['P_zone_sugg'])
if P_zone not in self.valid_zones:
return {'error': 'Invalid pick-up zone Id'}
if P_zone_sugg not in self.valid_zones:
return {'error': 'Invalid suggested pick-up zone Id'}
P_incentive = 50. * abs(P_zone_sugg - P_zone) / d
if 'D_zone' in data.keys():
D_zone = int(data['D_zone'])
D_zone_sugg = int(data['D_zone_sugg'])
if D_zone not in self.valid_zones:
return {'error': 'Invalid drop-off zone Id'}
if D_zone_sugg not in self.valid_zones:
return {'error': 'Invalid suggested drop-off zone Id'}
D_incentive = 50. * abs(D_zone_sugg - D_zone) / d
response = {'currency': 'EUR'}
if P_incentive is not None:
response['P_incentive'] = \
round(P_incentive, 3)
if D_incentive is not None:
response['D_incentive'] = \
round(D_incentive, 3)
return response
async def mset(self, mapping: ty.Dict = {}, expire: int = 0, *args):
assert len(mapping.keys()) + len(args) < 20, "Max keys for mset is 20!"
if expire > 0:
for k, v in mapping.items():
await self.set(key=k, value=v, expire=expire)
else:
for k, v in mapping.items():
args = args + (k, v)
await self._rclient.mset(*args)
def _validate(cls, data: typing.Dict, schema: typing.Dict):
for field, validator in schema.items():
if field not in data.keys():
raise KeyError(f'missing configuration field "{field:s}"')
if not validator(data[field]):
raise ValueError(f'invalid value in field "{field:s}"')
return True
def _build_model(
distance_in_cents_between_melodic_pitches: typing.Tuple[int, ...],
cents_to_pitch: typing.Dict,
melodic_pitches: typing.Tuple[abc.Pitch, ...],
) -> typing.Tuple[cp_model.CpModel, typing.List]:
"""Build ortools model for finding best imitation in given harmony."""
model = cp_model.CpModel()
domain = cp_model.Domain.FromValues(cents_to_pitch.keys())
variables = [
model.NewIntVarFromDomain(domain, f"pitch{nth_pitch}")
for nth_pitch in range(len(melodic_pitches))
]
max_step = max(cents_to_pitch.keys()) - min(cents_to_pitch.keys())
absolute_differences = []
for nth_step, variable0, variable1, distance in zip(
range(len(distance_in_cents_between_melodic_pitches)),
variables,
variables[1:],
distance_in_cents_between_melodic_pitches,
):
difference_between_desired_and_real_distance = model.NewIntVar(
-max_step,
max_step,
f"difference_between_desired_and_real_distance{nth_step}_0",
)
abs_difference_between_desired_and_real_distance = model.NewIntVar(
0, max_step,
f"abs_difference_between_desired_and_real_distance{nth_step}_1")
model.Add(difference_between_desired_and_real_distance == distance -
(variable1 - variable0))
model.AddAbsEquality(
abs_difference_between_desired_and_real_distance,
difference_between_desired_and_real_distance,
)
absolute_differences.append(
abs_difference_between_desired_and_real_distance)
model.Minimize(sum(absolute_differences))
return model, variables
def __call__(self, sample: typing.Dict) -> typing.Dict:
sample["image"] = self.resize(sample["image"])
if "label" in sample.keys():
mask = sample["label"]
new_mask = self.resize(mask)
sample["label"] = new_mask
return sample
def validate_value(self, data: _t.Dict):
if not isinstance(data, dict):
self.fail('not a JSON')
invalid_keys = list(k for k in data.keys()
if k not in self.__valid_keys)
if invalid_keys:
self.fail(f'invalid key {invalid_keys[0]}')
for key in self.__valid_keys:
if key not in data:
self.fail(f'"{key}" is not set')
def __check_item_entry(self, meta_config_entry: typing.Dict):
if 'type' not in meta_config_entry:
raise MissingKeys({'type'})
if meta_config_entry['type'] == 'node':
for key in set(meta_config_entry.keys()) - {'type'}:
self.__check_item_entry_with_type(meta_config_entry, key)
else:
self.__check_entry_type(meta_config_entry)
if meta_config_entry['type'] not in self._callbacks:
raise InvalidConfigType()
def validate_value(self, data: _t.Dict):
if not isinstance(data, dict):
self.fail('not a JSON')
invalid_keys = [k for k in data.keys() if k not in self.__valid_keys]
if invalid_keys:
self.fail('invalid key', invalid_key=invalid_keys[0])
for key in self.__valid_keys:
if key not in data:
self.fail('missing key', missing_key=key)
def assertCheckDict(self,
first: _t.Dict,
second: _t.Dict,
msg: _t.Text = None):
"""
Fail if the two fields in dicts are unequal as determined by the '==' operator.
Checks if first not contains or not equal field in second
"""
for field_name in first.keys():
self.assertEqual(first[field_name], second.get(field_name, None),
msg or [first, second])
def make_namedtuple(self, data: typing.Dict, **kwargs):
"""
converts the loaded data dict into a namedtuple
:param data: loaded data
:param kwargs: marshmallow kwargs
:return: namedtuple
"""
name = self.__class__.__name__.replace('Schema', '')
Object = BaseSchema.namedtuple_factory(name, data.keys())
return Object(**data)
def compute_metrics(trues_df: pd.DataFrame,
preds: t.Dict,
top_n_ranks: t.Tuple = (1, 5)):
true_labels = []
pred_labels = []
for name in preds.keys():
true_labels.append(
trues_df[trues_df['names'] == name]['labels'].values.tolist()[0])
pred_labels.append(preds[name])
return (top_n_accuracy(preds=pred_labels, truths=true_labels, n=rank)
for rank in top_n_ranks)
def fill_table_fields(self, fields_dict: typing.Dict) -> typing.NoReturn:
fields_list = list(fields_dict.keys())
types_list = list(fields_dict.values())
if len(types_list) != len(fields_list):
raise exception.DifferentCount()
self.types = types_list
self.fields = fields_list
for index, type_name in enumerate(self.types):
if self.__check_type_name(type_name):
self.types[index] = self.types_dict[type_name[:3]]
else:
raise exception.TypeNotExists(type_name)
self.fields_count = len(self.fields)
def process_sampling(ns: SimpleNamespace, public_users: t.Dict, public_items: t.Dict, i_train: sp.csr_matrix,
test: t.Dict, validation=False) -> sp.csr_matrix:
i_test = [(public_users[user], public_items[i])
for user, items in test.items() if user in public_users.keys()
for i in items.keys() if i in public_items.keys()]
rows = [u for u, _ in i_test]
cols = [i for _, i in i_test]
i_test = sp.csr_matrix((np.ones_like(rows), (rows, cols)), dtype='float32',
shape=(len(public_users.keys()), len(public_items.keys())))
candidate_negatives = ((i_test + i_train).astype('bool') != True)
ns = ns.negative_sampling
strategy = getattr(ns, "strategy", None)
if strategy == "random":
num_items = getattr(ns, "num_items", None)
if num_items is not None:
if str(num_items).isdigit():
negative_items = NegativeSampler.sample_by_random_uniform(candidate_negatives, num_items)
pass
else:
raise Exception("Number of negative items value not recognized")
else:
raise Exception("Number of negative items option is missing")
elif strategy == "fixed":
files = getattr(ns, "files", None)
if files is not None:
if not isinstance(files, list):
files = [files]
file_ = files[0] if validation == False else files[1]
negative_items = NegativeSampler.read_from_files(public_users, public_items, file_)
pass
else:
raise Exception("Missing strategy")
return negative_items
async def named_execute(self,
query: str,
args: typing.Dict,
timeout: typing.Optional[float] = None) -> str:
"""Extended versions of `execute` with support of the named
parameters."""
# Convert the received query in the parameter to all lower-case
query = query.lower()
# Fetch the column string that is passed as part of INTO clause
query_columns = query.split('into')[1].split('values')[0].strip()
# Convert the column into list for further detailed logging
columns = [
column.strip()
for column in query_columns[query_columns.index('(') +
1:-1].split(',')
]
number_of_columns = len(columns)
number_of_values = len(list(args.keys()))
missing_columns = set(columns) - set(args.keys())
# Scenario: When the value passed in parameter do not match the number of columns
if number_of_values < number_of_columns:
raise KeyError(
'Values missing. Columns that are missing values: {0}'.format(
missing_columns))
# Scenario: When the expected column id is not received
if 'id' not in args:
raise KeyError(
'Expected column name-> id :: Provided column name-> {}'.
format(list(args.keys())[0]))
converted_query, asyncpg_args = self._prepare_asyncpg_parameters(
query, args, query_module.QueryParamsDictConverter())
return await super().execute(converted_query,
*asyncpg_args,
timeout=timeout)
def touch_cond(self, info: typing.Dict, value: typing.Union[StrictInt,
float]):
trend = info.pop("trend")
if len(info) == 1:
data = info[list(info.keys())[0]]
if trend == Trend.Up:
if data <= value:
return True
elif trend == Trend.Down:
if data >= value:
return True
elif trend == Trend.Equal:
if data == value:
return True
def could_be(cls, message: t.Dict) -> bool:
"""
Check if the required fields for this class are in the message
Note: it checks if the value exists not if it is valid.
validation is handled in try_from_message
"""
ALWAYS_REQUIRED_FIELDS = {"content_id", "content_type"}
fields = message.keys()
if not ALWAYS_REQUIRED_FIELDS.issubset(fields):
return False
if not cls.get_required_subfields().issubset(fields):
return False
return True
def select_max_books(mapper_customer_books: typing.Dict):
customer_hold_count_books: typing.List = list()
max_books = len(mapper_customer_books[list(
mapper_customer_books.keys())[0]])
# find the max of books
for customer in mapper_customer_books:
if max_books < len(mapper_customer_books[customer]):
max_books = len(mapper_customer_books[customer])
# find all customers who hold maximum of books
for customer in mapper_customer_books:
if max_books == len(mapper_customer_books[customer]):
customer_hold_count_books.append(customer)
return customer_hold_count_books
def remove_empty_fields(d: typing.Dict) -> typing.Dict:
"""
Recursively remove empty collections and Nones from dict
"""
if d is None:
return None
# stash keys because we can't change a dict while iterating
# using keys()
keys = list(d.keys())
for k in keys:
v = d[k]
if isinstance(v, dict):
remove_empty_fields(v)
if v is None or v == {} or v == []:
d.pop(k)
return d
def read_from_files(public_users: t.Dict, public_items: t.Dict, filepath: str) -> sp.csr_matrix:
map_ = {}
with open(filepath) as file:
for line in file:
line = line.rstrip("\n").split('\t')
int_set = {public_items[int(i)] for i in line[1:] if int(i) in public_items.keys()}
map_[public_users[int(make_tuple(line[0])[0])]] = int_set
rows_cols = [(u, i) for u, items in map_.items() for i in items]
rows, cols = zip(*rows_cols)
# rows = [u for u, _ in rows_cols]
# cols = [i for _, i in rows_cols]
negative_samples = sp.csr_matrix((np.ones_like(rows), (rows, cols)), dtype='bool',
shape=(len(public_users), len(public_items)))
return negative_samples
def is_passport_valid(passport: typing.Dict) -> bool:
"""Is the passport valid according to the rules?"""
# check keys
if not passport.keys() >= constants.REQUIRED_KEYS:
return False
# specific values
byr = int(passport["byr"])
if byr < 1920 or byr > 2002:
return False
# byr (Birth Year) - four digits; at least 1920 and at most 2002.
# iyr (Issue Year) - four digits; at least 2010 and at most 2020.
# eyr (Expiration Year) - four digits; at least 2020 and at most 2030.
ranges = constants.YEAR_RANGES
for k, (min_, max_) in ranges.items():
if int(passport[k]) < min_ or int(passport[k]) > max_:
return False
# hgt (Height) - a number followed by either cm or in:
# If cm, the number must be at least 150 and at most 193.
# If in, the number must be at least 59 and at most 76.
hgt = int(passport["hgt"][:-2])
unit = passport["hgt"][-2:]
ranges = {
"cm": (150, 193),
"in": (59, 76),
}
if unit not in ranges or hgt < ranges[unit][0] or hgt > ranges[unit][1]:
return False
# hcl (Hair Color) - a # followed by exactly six characters 0-9 or a-f.
hcl = passport["hcl"]
hexes = set(string.hexdigits.lower())
if not hcl.startswith("#") or set(hcl[1:]) > hexes:
return False
ecl = passport["ecl"]
if ecl not in constants.VALID_EYE_COLOURS:
return False
# pid (Passport ID) - a nine-digit number, including leading zeroes.
pid = passport["pid"]
if len(pid) != 9 or set(pid) > set(string.digits):
return False
return True
def k(self):
return List(*Dict.keys(self))
def keys_view(self):
return Dict.keys(self)
