def get_variable_counts(df, var_lookup, domain_code):
"""
Parameters:
df (pandas.Dataframe) - form of `pivot_counts_df` output
var_lookup (dict) - mapping of Variable ids to labels
domain_code (str) - column accessor name for aggregation
Returns:
pandas.Dataframe or None
"""
if domain_code not in df.columns:
return None
df2 = df[domain_code].reset_index()
grouped = df2.groupby(['study', 'study_label', domain_code],
as_index=False).max()
if len(grouped['count'].dropna()) == 0:
return None
grouped['var_code'] = grouped[domain_code].map(
valmap(lambda x: x[0]['code'], var_lookup))
grouped['var_label'] = grouped[domain_code].map(
valmap(lambda x: x[0]['label'], var_lookup))
return grouped
def pretty_json(item):
if 'feature_maps' in item:
item['feature_maps'] = valmap(__wrap_to_prevent_indentation,
item['feature_maps'])
if 'activations' in item:
item['activations'] = valmap(__wrap_to_prevent_indentation,
item['activations'])
return json.dumps(item, indent=4, cls=NoIndentEncoder)
def parse(raw_form: str):
form_data = valmap(first, parse_qs(raw_form, strict_parsing=True))
return RsvpFormData(guest_id=form_data[RsvpFormData.GUEST_ID_FIELD],
party_id=form_data[RsvpFormData.PARTY_ID_FIELD],
attending=valmap(
_parse_bool,
dissoc(form_data, RsvpFormData.GUEST_ID_FIELD,
RsvpFormData.PARTY_ID_FIELD)))
def __init__(self, stage):
self.stage = stage
elements = stage.stage_elements()
def rotation_for_element(e):
data = stage.element_data(e)
return data['rotation']
element_pairs = list(zip(elements, elements[1:]+[elements[0]]))
pair_dict = dict(zip(elements, element_pairs))
rotation_dict = valmap(lambda es: (rotation_for_element(es[0]), rotation_for_element(es[1])), pair_dict)
difference_dict = valmap(lambda ps: (ps[1] - ps[0]) % 360, rotation_dict)
self.distance = difference_dict
def decision_function(self, X, exposure=None):
if not hasattr(self, 'estimator_'):
raise NotFittedError()
pred_args = valmap(growd(2),
valfilter(notnone, dict(X=X, exposure=exposure)))
score = self.estimator_.predict(**pred_args)
return score
def _parse_metrics(metrics):
# skipped metrics can sometimes be in unicode, replace unicode with NA if it exists
metrics = dtz.valmap(lambda x: 'nan' if isinstance(x, unicode) else x, metrics)
# missing = set(["Genes Detected", "Transcripts Detected", "Mean Per Base Cov."])
correct = set(["rRNA", "rRNA_rate"])
percentages = set(["Intergenic pct", "Intronic pct", "Exonic pct"])
to_change = dict({"5'-3' bias": 1,
"Intergenic pct": "Intergenic Rate",
"Intronic pct": "Intronic Rate",
"Exonic pct": "Exonic Rate",
"Duplication Rate of Mapped": 1,
"Average_insert_size": 1,
})
total = ["Not aligned", "Aligned to genes", "No feature assigned"]
out = {}
total_reads = sum([int(metrics[name]) for name in total])
out.update({key: val for key, val in metrics.iteritems() if key in correct})
[metrics.update({name: 1.0 * float(metrics[name]) / 100}) for name in
percentages]
for name in to_change:
if not to_change[name]:
continue
try:
if to_change[name] == 1:
out.update({name: float(metrics[name])})
else:
out.update({to_change[name]: float(metrics[name])})
# if we can't convert metrics[name] to float (?'s or other non-floats)
except ValueError:
continue
return out
def _to_pandas_code(reverse_categories, codes, missing_value):
"""更新字典
确保missing_value其代码为-1
原始代码 0 ~ len(categories)
修改为 -1 ~ len(categories) - 1
"""
zero_d = valfilter(lambda x: x == 0, reverse_categories)
zero_d_key = list(zero_d.keys())[0]
# 互换 zero <-> missing_value
none_first_d = reverse_categories.copy()
none_first_d[zero_d_key] = reverse_categories[missing_value]
none_first_d[missing_value] = reverse_categories[zero_d_key]
def dec_one(x):
return x - 1
none_first_d = valmap(dec_one, none_first_d)
new_codes = codes.copy()
# 将missing_value直接更改为 -1
new_codes = np.where(new_codes == reverse_categories[missing_value], -1,
new_codes)
# 将0键code更改为 missing_value code
new_codes = np.where(new_codes == reverse_categories[zero_d_key],
reverse_categories[missing_value], new_codes)
return new_codes, none_first_d
def fetch_exercise_info(lang, exercise):
url = "https://www.khanacademy.org/api/internal/translations/exercise_strings/{}?lang={}".format(exercise, lang)
response = requests.get(url)
exerciseInfo = response.json()
# Assemble string lists
aidToStrings = defaultdict(list)
stringidRe = re.compile("crwdns(\d+):")
for ti in exerciseInfo["translationItems"]:
stringid = int(stringidRe.match(ti["jiptString"]).group(1))
aidToStrings[ti["assessmentItem"]].append(stringid)
# Assemble exercise type map
aidToPtype = {}
for ptype, aids in exerciseInfo["problemTypes"].items():
for aid in aids:
aidToPtype[aid] = ptype
# Extract structure map
aidToStructure = valmap(json.loads, exerciseInfo["translatedAssessmentItems"])
# Assemble into one list of objects
result = {}
aids = set(itertools.chain(aidToStrings.keys(), aidToPtype.keys(), aidToStructure.keys()))
for aid in aids:
result[aid] = {
"type": aidToPtype.get(aid, None),
"strings": aidToStrings.get(aid, []),
"structure": aidToStructure.get(aid, []),
}
return result
def explore(self):
"""
Function to conduct the exploration process
"""
# Conduct max_iter iterations
for iter_num in tqdm.tqdm(range(self.max_iter)):
# If stat_collection_freq, run an exploit
if (iter_num + 1) % self.stat_collection_freq == 0:
logger.info("Collecting stats...")
self.collect_stats(iter_num)
self.single_iteration()
self.collect_stats(iter_num)
# Dump the results to file
if self.fname:
json.dump(
{
"policy":
dicttoolz.keymap(
str,
dicttoolz.valmap(
lambda d: dicttoolz.keymap(str, d),
self.policy.policy,
),
),
"stats":
self.stats,
},
open(self.fname, "w"),
)
logger.info(msg=self.stats)
def _parse_metrics(metrics):
# skipped metrics can sometimes be in unicode, replace unicode with NA if it exists
metrics = dtz.valmap(lambda x: 'nan' if isinstance(x, unicode) else x, metrics)
# missing = set(["Genes Detected", "Transcripts Detected", "Mean Per Base Cov."])
correct = set(["rRNA", "rRNA_rate"])
percentages = set(["Intergenic pct", "Intronic pct", "Exonic pct"])
to_change = dict({"5'-3' bias": 1, "Intergenic pct": "Intergenic Rate",
"Intronic pct": "Intronic Rate",
"Exonic pct": "Exonic Rate",
"Duplication Rate of Mapped": 1,
"Average insert size": 1,
})
total = ["Not aligned", "Aligned to genes", "No feature assigned"]
out = {}
total_reads = sum([int(metrics[name]) for name in total])
out.update({key: val for key, val in metrics.iteritems() if key in correct})
[metrics.update({name: 1.0 * float(metrics[name]) / 100}) for name in
percentages]
for name in to_change:
if not to_change[name]:
continue
try:
if to_change[name] == 1:
out.update({name: float(metrics[name])})
else:
out.update({to_change[name]: float(metrics[name])})
# if we can't convert metrics[name] to float (?'s or other non-floats)
except ValueError:
continue
return out
def __init__(self, config: Dict):
self._config_serializers: Dict[str, Type[Serializer]] = config["serializers"]
self._config_bindings: Dict[Type, str] = config["serializer_bindings"]
self._serializer_map: Dict[Type, Type[Serializer]] = dicttoolz.valmap(
lambda v: self._config_serializers[v], self._config_bindings
)
# The bindings are sorted by specifity (meaning the lower the types
# in the hierarchy occupy the first positions)
self._bindings = list(sort((k, v) for k, v in self._serializer_map.items()))
self._default_serializer: Optional[Type[Serializer]]
try:
self._default_serializer = self._config_serializers[config["default"]]
except Exception:
self._default_serializer = None
self._serializers_by_id: Dict[int, Type[Serializer]] = {
v.identifier(): v for k, v in self._config_serializers.items() # type: ignore
}
self._quickserializer_by_identity: List[Optional[Type[Serializer]]] = [
None
] * 1024
for k, v in self._serializers_by_id.items():
self._quickserializer_by_identity[k] = v
def _parse_metrics(metrics):
# skipped metrics can sometimes be in unicode, replace unicode with NA if it exists
metrics = dtz.valmap(lambda x: 'nan' if isinstance(x, unicode) else x, metrics)
missing = set(["Genes Detected", "Transcripts Detected",
"Mean Per Base Cov."])
correct = set(["Intergenic pct", "Intronic pct", "Exonic pct"])
to_change = dict({"5'-3' bias": 1, "Intergenic pct": "Intergenic Rate",
"Intronic pct": "Intronic Rate", "Exonic pct": "Exonic Rate",
"Not aligned": 0, 'Aligned to genes': 0, 'Non-unique alignment': 0,
"No feature assigned": 0, "Duplication Rate of Mapped": 1,
"Fragment Length Mean": 1,
"rRNA": 1, "Ambiguou alignment": 0})
total = ["Not aligned", "Aligned to genes", "No feature assigned"]
out = {}
total_reads = sum([int(metrics[name]) for name in total])
out['rRNA rate'] = 1.0 * int(metrics["rRNA"]) / total_reads
out['Mapped'] = sum([int(metrics[name]) for name in total[1:]])
out['Mapping Rate'] = 1.0 * int(out['Mapped']) / total_reads
[out.update({name: 0}) for name in missing]
[metrics.update({name: 1.0 * float(metrics[name]) / 100}) for name in correct]
for name in to_change:
if not to_change[name]:
continue
if to_change[name] == 1:
out.update({name: float(metrics[name])})
else:
out.update({to_change[name]: float(metrics[name])})
return out
def binarySearchParamsParallel(X, y, model, params, paramRanges):
with multiprocessing.Pool(multiprocessing.cpu_count()) as p:
args_generator = ((X,
y,
model,
params,
*x) for x in paramRanges)
results = p.starmap(compareValsBaseCase, args_generator)
name_result_tuples = zip((x[0] for x in paramRanges),
results)
valsAndScores = dict(name_result_tuples)
topVals = valmap(getTopVals, valsAndScores)
score = model(X,
y,
{**params,
**topVals},
)
return {"values": topVals,
"score": score,
"valsAndScores": valsAndScores,
"n_iterations": sum(x.shape[0]
for x in valsAndScores.values())}
def _parse_metrics(metrics):
# skipped metrics can sometimes be in unicode, replace unicode with NA if it exists
metrics = dtz.valmap(lambda x: 'nan' if isinstance(x, unicode) else x, metrics)
missing = set(["Genes Detected", "Transcripts Detected",
"Mean Per Base Cov."])
correct = set(["Intergenic pct", "Intronic pct", "Exonic pct"])
to_change = dict({"5'-3' bias": 1, "Intergenic pct": "Intergenic Rate",
"Intronic pct": "Intronic Rate", "Exonic pct": "Exonic Rate",
"Not aligned": 0, 'Aligned to genes': 0, 'Non-unique alignment': 0,
"No feature assigned": 0, "Duplication Rate of Mapped": 1,
"Fragment Length Mean": 1,
"rRNA": 1, "Ambiguou alignment": 0})
total = ["Not aligned", "Aligned to genes", "No feature assigned"]
out = {}
total_reads = sum([int(metrics[name]) for name in total])
out['rRNA rate'] = 1.0 * int(metrics["rRNA"]) / total_reads
out['Mapped'] = sum([int(metrics[name]) for name in total[1:]])
out['Mapping Rate'] = 1.0 * int(out['Mapped']) / total_reads
[out.update({name: 0}) for name in missing]
[metrics.update({name: 1.0 * float(metrics[name]) / 100}) for name in correct]
for name in to_change:
if not to_change[name]:
continue
try:
if to_change[name] == 1:
out.update({name: float(metrics[name])})
else:
out.update({to_change[name]: float(metrics[name])})
# if we can't convert metrics[name] to float (?'s or other non-floats)
except ValueError:
continue
return out
def parse(raw_form: str):
form_data = valmap(first, parse_qs(raw_form, strict_parsing=True))
return RideShareFormData(
guest_id=form_data[RideShareFormData.GUEST_ID_FIELD],
party_id=form_data[RideShareFormData.PARTY_ID_FIELD],
rideshare=_parse_bool(
form_data[RideShareFormData.RIDESHARE_FIELD]))
def histogram(results):
from toolz import recipes, dicttoolz
import math
counts = recipes.countby(lambda r: r.upstream_status, results.values())
bars = dicttoolz.valmap(lambda v: "#" * int(math.ceil(float(v) / len(results) * 100)), counts)
for k in bars:
print("%-20s %s (%d)" % (k.capitalize() if k else "No status", bars[k], counts[k]))
def chunk_maps(datamaps, factory=dict):
if isinstance(datamaps, dict):
return datamaps
keys = datamaps[0].keys()
datamap = {k: list(toolz.pluck(k, datamaps)) for k in keys}
return dicttoolz.valmap(np.array, datamap, factory=factory)
def score(self, X, y, sample_weight=None, exposure=None):
partial_arguments = self._process_args(y=y, sample_weight=sample_weight, exposure=exposure)
predict_arguments = self._process_args(X=X, exposure=exposure)
loss_function = lambda pred: self.loss_function(pred=shrinkd(1, pred), **valmap(shrinkd(1), partial_arguments))
prediction = shrinkd(1, self.predict(**predict_arguments))
loss = loss_function(prediction)
initial_prediction = shrinkd(1, self.coefficients_[0] * self.estimators_[0].predict(**predict_arguments))
initial_loss = loss_function(initial_prediction)
return (initial_loss - loss) / initial_loss
def compare_results(compa, cols, vals):
vals = tuple(map(none_to_minus_inf, vals))
res = set([row['id'] for row in engine.execute(select(table.columns).where(compa))])
all_ = [valmap(none_to_minus_inf, row) for row in engine.execute(select(table.columns))]
cor = set()
for row in all_:
if tuple(row[col.name] for col in cols) > vals:
cor.add(row['id'])
assert_equal(res, cor)
def sample(stream, key, limit):
items = ijson.items(sys.stdin, 'item')
classes = groupby(key, items)
samples = valmap(
lambda xs: random.sample(xs, limit
if len(xs) >= limit else len(xs)), classes)
sample = list(concat(samples.values()))
return sample
def histogram(results):
from toolz import recipes, dicttoolz
import math
counts = recipes.countby(lambda r: r.upstream_status, results.values())
bars = dicttoolz.valmap(
lambda v: "#" * int(math.ceil(float(v) / len(results) * 100)), counts)
for k in bars:
print("%-20s %s (%d)" %
(k.capitalize() if k else "No status", bars[k], counts[k]))
def emptystr_to_none(item):
"""
Replace empty strings with `None` recursively inside a JSON-like object.
"""
if isinstance(item, list):
return list(map(emptystr_to_none, item))
if isinstance(item, dict):
return valmap(emptystr_to_none, item)
else:
return None if item == "" else item
def consolidate(group):
"""
[List] group => [Dictionary] consolidated position
group is a list of cash entries of the same currency, here we add up
their amount
"""
p = group[0].copy()
p['balance'] = sum(map(lambda p: p['balance'], group))
return valmap(lambda v: removeBOM(v) if isinstance(v, str) else v, p)
def load_from_file(self, filename):
"""
Function to load policy from file
"""
from_file = json.load(open(filename))
self.policy.policy = dicttoolz.valmap(
lambda d: dicttoolz.keymap(eval, d),
dicttoolz.keymap(eval, from_file.get("policy")),
)
self.stats = from_file.get("stats")
def getTotaledImportances(labels, forest):
featDict = dict(zip(labels, forest.feature_importances_))
uniqueLabels = set(x.split("Is")[0] for x in featDict.keys())
featsAndOneHots = {y: [x for x in labels if y in x] for y in uniqueLabels}
summedFeats = valmap(lambda x: sum(featDict[y] for y in x),
featsAndOneHots)
return sorted(summedFeats.items(), key=lambda x: x[1], reverse=True)
def rotationMatrix(path):
viewVectors = getTranslationVectorPerCamera(path)
viewVectors = valmap(lambda arr: arr[:3], viewVectors)
camera3 = viewVectors['card3JPG']
camera5 = viewVectors['card5JPG']
angle = angleBetween(np.array(camera3), np.array(camera5))
rotationMatrix = createYRotationMatric(np.deg2rad(180))
print(np.rad2deg(-angle))
return rotationMatrix
def pofilesAPI():
"""Get a sorted list of all PO files"""
response.content_type = "application/json"
# Use the DE translation map as master (for no good reason, really)
with open("cache/translation-filemap-de.json") as infile:
filemap = json.load(infile)
# Extract only paths
pathdict = valmap(operator.itemgetter("path"), filemap)
# Sort lexicographically
pathlist = list(pathdict.values())
pathlist = sorted(pathlist)
return json.dumps(pathlist)
def pofilesAPI():
"""Get a sorted list of all PO files"""
response.content_type = 'application/json'
# Use the DE translation map as master (for no good reason, really)
with open("cache/translation-filemap-de.json") as infile:
filemap = json.load(infile)
# Extract only paths
pathdict = valmap(operator.itemgetter("path"), filemap)
# Sort lexicographically
pathlist = list(pathdict.values())
pathlist = sorted(pathlist)
return json.dumps(pathlist)
def getCameraParameters(pathToBlockExchangeXML, pathToAgisoftXML):
tree = ET.parse(pathToBlockExchangeXML)
root = tree.getroot()
block = root.find('Block')
photoGroups = block.find('Photogroups').findall('Photogroup')
counter = 1
cardToPhotoGroup = {}
for group in photoGroups:
photos = group.findall('Photo')
photos = map(getCameraName, photos)
for photo in photos:
cardToPhotoGroup[photo] = "photogoup{}".format(counter)
counter += 1
counter = 1
photogoupToCameraParameters = {}
for group in photoGroups:
params = {}
imageDimensions = map(lambda dim: dim.text,
list(group.find('ImageDimensions')))
distortion = group.find('Distortion')
distortions = "{} {}".format(
distortion.find('K2').text,
distortion.find('K3').text)
params['ViewportPx'] = "{} {}".format(imageDimensions[0],
imageDimensions[1])
params['LensDistortion'] = distortions
params['CenterPx'] = "{} {}".format(
int(imageDimensions[0]) / 2,
int(imageDimensions[1]) / 2)
params['CameraType'] = '0'
params['PixelSizeMm'] = "1 1"
photogoupToCameraParameters["photogoup{}".format(counter)] = params
counter += 1
focalmmPerPhoto = getFocalFromAgisoftXml(pathToAgisoftXML)
cardParams = valmap(
lambda photogroup: photogoupToCameraParameters[photogroup],
cardToPhotoGroup)
for card in cardParams:
focalLength = focalmmPerPhoto[card]
cardParams[card]['FocalMm'] = focalLength
return cardParams
def as_samplerate(self, samplerate):
"""
Returns a copy of the current filter bank with a different samplerate.
All filters are recomputed when calling this function.
"""
# Fast path if samplerate has not changed
if samplerate == self.samplerate:
return self
# Create new bank with recomputed filters.
ret = FilterBank(samplerate)
ret.filters = valmap(operator.methodcaller("as_samplerate", samplerate), self.filters)
return ret
def __init__(self, stage):
self.stage = stage
elements = stage.stage_elements()
def point_for_element(e):
data = stage.element_data(e)
# XXX: Y coordinate grows downwards on canvas
return Vect2D(data['center_x'], -data['center_y'])
element_pairs = list(zip(elements, elements[1:]+[elements[0]]))
pair_dict = dict(zip(elements, element_pairs))
#pair_dict = dict([(x[0], x) for x in element_pairs])
point_dict = valmap(lambda es: (point_for_element(es[0]), point_for_element(es[1])), pair_dict)
vector_dict = valmap(lambda ps: ps[1] - ps[0], point_dict)
#vector_pairs_dict = valmap(lambda es: (vector_dict[es[0]], vector_dict[es[1]]), vector_dict)
other_pairs = list(zip([elements[-1]] + elements[:-1], elements))
other_dict = dict(zip(elements, other_pairs))
other_pairs_dict = valmap(lambda es: (vector_dict[es[0]], vector_dict[es[1]]), other_dict)
self.data = {
'vector_angles': valmap(lambda vs: (-vs[0]).angleBetween(vs[1]), other_pairs_dict),
'angles': valmap(lambda v: v.angle(), vector_dict),
'length': valmap(lambda v: v.length(), vector_dict)
}
def find_matching_fn_abi(abi, fn_identifier=None, args=None, kwargs=None):
args = args or tuple()
kwargs = kwargs or dict()
filters = []
num_arguments = len(args) + len(kwargs)
if fn_identifier is FallbackFn:
return get_fallback_func_abi(abi)
if not is_text(fn_identifier):
raise TypeError("Unsupported function identifier")
name_filter = functools.partial(filter_by_name, fn_identifier)
arg_count_filter = functools.partial(filter_by_argument_count, num_arguments)
encoding_filter = functools.partial(filter_by_encodability, args, kwargs)
filters.extend([
name_filter,
arg_count_filter,
encoding_filter,
])
function_candidates = pipe(abi, *filters)
if len(function_candidates) == 1:
return function_candidates[0]
else:
matching_identifiers = name_filter(abi)
matching_function_signatures = [abi_to_signature(func) for func in matching_identifiers]
arg_count_matches = len(arg_count_filter(matching_identifiers))
encoding_matches = len(encoding_filter(matching_identifiers))
if arg_count_matches == 0:
diagnosis = "\nFunction invocation failed due to improper number of arguments."
elif encoding_matches == 0:
diagnosis = "\nFunction invocation failed due to improper argument encoding."
elif encoding_matches > 1:
diagnosis = (
"\nAmbiguous argument encoding. "
"Provided arguments can be encoded to multiple functions matching this call."
)
message = (
"\nCould not identify the intended function with name `{name}`, "
"positional argument(s) of type `{arg_types}` and "
"keyword argument(s) of type `{kwarg_types}`."
"\nFound {num_candidates} function(s) with the name `{name}`: {candidates}"
"{diagnosis}"
).format(
name=fn_identifier,
arg_types=tuple(map(type, args)),
kwarg_types=valmap(type, kwargs),
num_candidates=len(matching_identifiers),
candidates=matching_function_signatures,
diagnosis=diagnosis,
)
raise ValidationError(message)
def as_samplerate(self, samplerate):
"""
Returns a copy of the current filter bank with a different samplerate.
All filters are recomputed when calling this function.
"""
# Fast path if samplerate has not changed
if samplerate == self.samplerate:
return self
# Create new bank with recomputed filters.
ret = FilterBank(samplerate)
ret.filters = valmap(
operator.methodcaller("as_samplerate", samplerate), self.filters)
return ret
def __init__(
self,
adjacency_dict: Mapping[Vertex, Set[EdgeAsTuple]],
edge_type = WeightedEdge,
):
super().__init__()
# the following line is the functional equivalent of
# {k: {edge_type(x) for x in v} for k, v in adjacency_dict.items()}
self.adjacencies: Dict[V, Set[E]] = valmap(
compose(set, partial(starmap, edge_type)),
adjacency_dict,
)
self.marks: Dict[Vertex, bool] = {}
self.in_use: Dict[Vertex, bool] = {}
def decode(r: Any) -> Any:
"""
Decodes a value from an intermediate representation `r`.
Parameters
----------
r
An intermediate representation to be decoded.
Returns
-------
Any
A Python data structure corresponding to the decoded version of ``r``.
See Also
--------
encode
Inverse function.
"""
# structural recursion over the possible shapes of r
if type(r) == dict and "__kind__" in r:
kind = r["__kind__"]
cls = cast(Any, locate(r["class"]))
assert cls is not None, f"Can not locate {r['class']}."
if kind == Kind.Type:
return cls
args = decode(r.get("args", []))
kwargs = decode(r.get("kwargs", {}))
if kind == Kind.Instance:
return cls(*args, **kwargs)
if kind == Kind.Stateful:
obj = cls.__new__(cls)
obj.__dict__.update(kwargs)
return obj
raise ValueError(f"Unknown kind {kind}.")
if type(r) == dict:
return valmap(decode, r)
if type(r) == list:
return list(map(decode, r))
return r
def getAssetCountryAllocation(date, blpData, assetTypeTuples, countryGroups, positions): """ [String] date (yyyymmdd), [Dictionary] blpData, [Iterator] assetTypeTuples, [List] countryGroups, [Iterator] positions => [Dictionary] assetypeTuple -> [Dictionary] countryGroup -> List of positions that fall into this asset type and this country group """ return \ valmap( partial(getCountryGroupAllocation, date, blpData, countryGroups) , getAssetTypeAllocation(date, blpData, assetTypeTuples, positions) )
def _get_balance(self, cr, uid, ids, name, args, context=None):
"""Computed as following:
A) Cleared Deposits, Credits, and Interest Amount: SUM of Amts of lines
Cleared Deposits, Credits, and Interest # of Items: Number of lines
B) Checks, Withdrawals, Debits, and Service Charges Amount:
Checks, Withdrawals, Debits, and Service Charges Amount # of Items:
Cleared Balance:
(Total Sum of the Deposit Amount Cleared (A) –
Total Sum of Checks Amount Cleared (B))
Difference=
(Ending Balance – Beginning Balance) - cleared balance
should be zero.
"""
res = {}
account_precision = self.pool['decimal.precision'].precision_get(
cr, uid, 'Account')
for stmt in self.browse(cr, uid, ids, context=context):
res[stmt.id] = {}
cleared = lambda l: l.cleared_bank_account and 'Cleared' or 'Uncleared'
get_amount = lambda l: [
round(v.amount, account_precision) for v in l
]
process_lines = compose(valmap(get_amount), groupby(cleared))
for line_type in ('debit', 'credit'):
r = process_lines(eval('stmt.%s_move_line_ids' % line_type))
res[stmt.id].update({
'sum_of_%ss' % line_type:
sum(r.get('Cleared', [])),
'sum_of_%ss_lines' % line_type:
len(r.get('Cleared', [])),
'sum_of_%ss_unclear' % line_type:
sum(r.get('Uncleared', [])),
'sum_of_%ss_lines_unclear' % line_type:
len(r.get('Uncleared', []))
})
res[stmt.id]['cleared_balance'] = round(
res[stmt.id]['sum_of_debits'] - res[stmt.id]['sum_of_credits'],
account_precision)
res[stmt.id]['uncleared_balance'] = round(
res[stmt.id]['sum_of_debits_unclear'] -
res[stmt.id]['sum_of_credits_unclear'], account_precision)
res[stmt.id]['difference'] = round(
(stmt.ending_balance - stmt.starting_balance) -
res[stmt.id]['cleared_balance'], account_precision)
return res
def computeRuleHitsForFileSet(self, poFiles):
"""
For each file in the given filename -> PO object dictionary,
compute the Rule -> Hits dictonary.
Stores the information in the current instance.
Does not return anything
"""
# Compute dict with sorted & prettified filenames
self.files = sorted(poFiles.keys())
# Add all futures to the executor
futures = list(itertools.chain(*(self.computeRuleHits(po, filename)
for filename, po in poFiles.items())))
# Process the results in first-received order. Also keep track of rule performance
self.fileRuleHits = collections.defaultdict(dict)
n_finished = 0
# Intermediate result storage
raw_results = collections.defaultdict(dict) # filename -> {rule: result}
for future in concurrent.futures.as_completed(futures):
# Extract result
filename, rule, result = future.result()
self.fileRuleHits[filename][rule] = result
# Track progress
n_finished += 1
if n_finished % 1000 == 0:
percent_finished = n_finished * 100. / len(futures)
print("Rule computation finished {0:.2f} %".format(percent_finished))
# Compute total stats by file
self.statsByFile = {
filename: merge(self.ruleHitsToSeverityCountMap(ruleHits), {
"translation_url": self.translationURLs[filename]})
for filename, ruleHits in self.fileRuleHits.items()
}
# Compute map filename -> {rule: numHits for rule}
self.statsByFileAndRule = {
filename: valmap(len, ruleHits)
for filename, ruleHits in self.fileRuleHits.items()
}
# Compute map rule -> numHits for rule
self.totalStatsByRule = merge_with(sum, *(self.statsByFileAndRule.values()))
from toolz.dicttoolz import valmap
from state import goal
operations = {op.__name__: op for op in (
namedtuple('partition', ['coll', 'by']),
namedtuple('len', ['coll']),
namedtuple('max', ['coll']),
namedtuple('map', ['coll', 'fn']),
namedtuple('collection', ['type']),
namedtuple('attr', ['elem', 'attr']),
namedtuple('nth', ['coll', 'n']),
)}
collections = valmap(operations['collection'], {
'goals': goal,
})
globals().update(operations)
globals().update(collections)
__all__ = operations.keys() + collections.keys()
# TODO: Should isinstance(..., walker) work?
class walker(object):
class __metaclass__(type):
def __new__(cls, name, bases, attrs):
# Make walker a regular class so it can be subclassed.
if bases == (object,):
return type.__new__(cls, name, bases, attrs)
def relatedness_each(self):
# atomistic, contiguous, integrated_projected
norm_factor = 6 * (len(self.elements) - 1)
return valmap(lambda x: x['relatedness'] / norm_factor, self.results)
def dominance_each(self):
# 0 - abscence, 2 - secondary, 4 - dominance
norm_factor = 2 * (len(self.elements) - 1)
return valmap(lambda x: x['dominance'] / norm_factor, self.results)
def test_valmap():
assert valmap(inc, {1: 1, 2: 2}) == {1: 2, 2: 3}
def valmap(self, f: Callable[[B], C]) -> 'Map[A, C]':
return Map(dicttoolz.valmap(f, dict(self)))
def test_valmap(self):
D, kw = self.D, self.kw
assert valmap(inc, D({1: 1, 2: 2}), **kw) == D({1: 2, 2: 3})
init_class_file = config.get("Batch", "init_class_file")
online_class_file = config.get("Batch", "online_class_file")
journey_cluster_file = config.get("Batch", "journey_cluster_file")
logfile = config.get("Directories", "logfile")
logging.basicConfig(filename=logfile, level=logging.ERROR)
# Load models
init_class_models = joblib.load(storedmodel_directory + init_class_file)
online_class_models = joblib.load(storedmodel_directory + online_class_file)
# Load information from historical data
historical_journey_clusters = joblib.load(storedmodel_directory + journey_cluster_file)
end_locations = dicttoolz.valmap(lambda x: [journey_cluster.averages[["EndLat", "EndLong"]].values.tolist() for
journey_cluster in x if journey_cluster.clusterID != -1], historical_journey_clusters)
mpg_insts = dicttoolz.valmap(lambda x: [journey_cluster.averages["MPG_from_MAF"] if "MPG_from_MAF" in journey_cluster.averages.keys() else float("NaN") for
journey_cluster in x if journey_cluster.clusterID != -1], historical_journey_clusters)
cluster_ids = dicttoolz.valmap(lambda x: [str(journey_cluster.clusterID) for
journey_cluster in x if journey_cluster.clusterID != -1], historical_journey_clusters)
journeys = {}
initial_predictions = {}
prob_dict = {}
time_dict = {}
def callback(body):
try:
