def resolve(msg):
request = msg.get_request()
results = msg.get_results()
slots = []
for k, v in results.items():
for m in v['adm']:
slot = {
'did': k,
'adm': m,
}
slots.append(slot)
slots.sort(key=lambda x: x['adm']['price'], reverse=True)
count = request['adunit']['param']['count']
slots = slots[:count]
floor = request['adunit']['floor']
slots = list(filter(lambda x: x['adm']['price'] >= floor, slots))
players = set(results.keys())
winners = set([i['did'] for i in slots])
losers = players - winners
result = {
'adm': [dicttoolz.keyfilter(lambda x: x not in ['price'], i['adm']) for i in slots],
'is_test': 0,
}
msg.set_decision(winners, losers)
msg.set_response(result)
def _map_event_to_dict(_include, sql_event):
"""Map event to a dictionary to be sent as an API response.
In this implementation, the goal is to restructure event data as if it
was returned by elasticsearch. This restructuration is needed because
the API in the past used elasticsearch as the backend and the client
implementation still expects data that has the same shape as
elasticsearch would return.
:param _include:
Projection used to get records from database
:type _include: list(str)
:param sql_event: Event data returned when SQL query was executed
:type sql_event: :class:`sqlalchemy.util._collections.result`
:returns: Event as would have returned by elasticsearch
:rtype: dict(str)
"""
event = {
attr: getattr(sql_event, attr)
for attr in sql_event.keys()
}
event['@timestamp'] = event['timestamp']
del event['reported_timestamp']
event['message'] = {
'text': event['message']
}
if 'node_instance_id' in event:
del event['node_instance_id']
context_fields = [
'deployment_id',
'execution_id',
'workflow_id',
'operation',
'node_id',
'node_name',
]
event['context'] = {
field: event[field]
for field in context_fields
}
for field in context_fields:
del event[field]
if event['type'] == 'cloudify_event':
event['message']['arguments'] = None
del event['logger']
del event['level']
elif event['type'] == 'cloudify_log':
del event['event_type']
# Keep only keys passed in the _include request argument
# TBD: Do the projection at the database level
if _include is not None:
event = dicttoolz.keyfilter(lambda key: key in _include, event)
return event
def is_satisfied(node: Node) -> bool:
f, args, _ = node
required = keyfilter(lambda x: x != 'return', args) #rettype = args['return']
get_ret = lambda x: x[1]['return'] # type: Callable[[Node],type]
acc_rets = map(get_ret, acc)
acc_rets = list(acc_rets)
satisfied = all([(t in acc_rets) for t in required.values()])
return satisfied
def add_to_cloud_metrics(ttl, region, group_metrics, num_tenants, config,
log=None, _print=False):
"""
Add total number of desired, actual and pending servers of a region
to Cloud metrics.
:param str region: which region's metric is collected
:param group_metrics: List of :obj:`GroupMetric`
:param int num_tenants: total number of tenants
:param dict config: Config json dict containing convergence tenants info
:param log: Optional logger
:param bool _print: Should it print activity on stdout? Useful when running
as a script
:return: `Effect` with None
"""
epoch = yield Effect(Func(time.time))
metric_part = {'collectionTime': int(epoch * 1000),
'ttlInSeconds': ttl}
tenanted_metrics, total = calc_total(group_metrics)
if log is not None:
log.msg(
'total desired: {td}, total_actual: {ta}, total pending: {tp}',
td=total.desired, ta=total.actual, tp=total.pending)
if _print:
print('total desired: {}, total actual: {}, total pending: {}'.format(
total.desired, total.actual, total.pending))
metrics = [('desired', total.desired), ('actual', total.actual),
('pending', total.pending), ('tenants', num_tenants),
('groups', len(group_metrics))]
for tenant_id, metric in sorted(tenanted_metrics.items()):
metrics.append(("{}.desired".format(tenant_id), metric.desired))
metrics.append(("{}.actual".format(tenant_id), metric.actual))
metrics.append(("{}.pending".format(tenant_id), metric.pending))
# convergence tenants desired and actual
conv_tenants = keyfilter(
partial(tenant_is_enabled,
get_config_value=lambda k: get_in([k], config)),
tenanted_metrics)
conv_desired = sum(m.desired for m in conv_tenants.itervalues())
conv_actual = sum(m.actual for m in conv_tenants.itervalues())
metrics.extend(
[("conv_desired", conv_desired), ("conv_actual", conv_actual),
("conv_divergence", conv_desired - conv_actual)])
data = [merge(metric_part,
{'metricValue': value,
'metricName': '{}.{}'.format(region, metric)})
for metric, value in metrics]
yield service_request(ServiceType.CLOUD_METRICS_INGEST,
'POST', 'ingest', data=data, log=log)
def get_todays_scaling_groups(convergence_tids, fname):
"""
Get scaling groups that from tenants that are enabled till today
"""
groups = yield Effect(GetAllGroups())
non_conv_tenants = set(groups.keys()) - set(convergence_tids)
last_tenants_len, last_date = yield get_last_info(fname)
now = yield Effect(Func(datetime.utcnow))
tenants, last_tenants_len, last_date = get_todays_tenants(
non_conv_tenants, now, last_tenants_len, last_date)
yield update_last_info(fname, last_tenants_len, last_date)
yield do_return(
keyfilter(lambda t: t in set(tenants + convergence_tids), groups))
def test_success(self):
"""
Returns todays scaling groups based on number of tenants fetched
since last time. Updates the current fetch in file
"""
seq = [
(GetAllGroups(), const(self.groups)),
(ReadFileLines("file"), const(["2", "0.0"])),
(Func(datetime.utcnow), const(datetime(1970, 1, 2))),
(WriteFileLines("file", [7, 86400.0]), noop)
]
r = perform_sequence(seq, get_todays_scaling_groups(["t1"], "file"))
self.assertEqual(
r,
keyfilter(lambda k: k in ["t{}".format(i) for i in range(1, 9)],
self.groups))
def ruleHitsToSeverityCountMap(self, rule_hit_map):
"""
In a rule -> hitlist mapping, count the total number of hits above or at a given severity
level and return a cumulative dictionary
"""
# Create a map severity -> count
severity_counts = collections.defaultdict(int)
for rule, hits in rule_hit_map.items():
severity_counts[rule.severity] += len(hits)
# Create string severity -> count map
above_severity = lambda sev: sum(keyfilter(lambda k: k >= sev, severity_counts).values())
return {"hits": above_severity(Severity.standard),
"warnings": above_severity(Severity.warning),
"errors": above_severity(Severity.dangerous),
"infos": above_severity(Severity.info),
"notices": above_severity(Severity.notice)}
def test_error_writing(self):
"""
Logs and ignores error writing to the file
"""
seq = [
(GetAllGroups(), const(self.groups)),
(ReadFileLines("file"), const(["2", "0.0"])),
(Func(datetime.utcnow), const(datetime(1970, 1, 2))),
(WriteFileLines("file", [7, 86400.0]),
lambda i: raise_(IOError("bad"))),
(LogErr(mock.ANY, "error updating number of tenants", {}), noop)
]
r = perform_sequence(seq, get_todays_scaling_groups(["t1"], "file"))
self.assertEqual(
r,
keyfilter(lambda k: k in ["t{}".format(i) for i in range(1, 9)],
self.groups))
def test_no_last_info(self):
"""
Returns first 5 non-convergence tenants if could not fetch last info
from file
"""
seq = [
(GetAllGroups(), const(self.groups)),
(ReadFileLines("file"), lambda i: raise_(IOError("e"))),
(LogErr(mock.ANY, "error reading previous number of tenants", {}),
noop),
(Func(datetime.utcnow), const(datetime(1970, 1, 2))),
(WriteFileLines("file", [5, 86400.0]), noop)
]
r = perform_sequence(seq, get_todays_scaling_groups(["t1"], "file"))
self.assertEqual(
r,
keyfilter(lambda k: k in ["t{}".format(i) for i in range(1, 7)],
self.groups))
def __call__(self, event_dict):
"""
Process event and push it to Cloud feeds
"""
if not event_dict.get("cloud_feed", False):
return
# Do further logging without cloud_feed to avoid coming back here
# in infinite recursion
log_keys = keyfilter(lambda k: k not in ("message", "cloud_feed"), event_dict)
log = self.log.bind(system="otter.cloud_feed", cf_msg=event_dict["message"][0], event_data=log_keys)
try:
eff = self.add_event(event_dict, self.tenant_id, self.region, log)
except UnsuitableMessage as me:
log.err(None, "cf-unsuitable-message", unsuitable_message=me.unsuitable_message)
else:
return perform(self.get_disp(self.reactor, self.authenticator, log, self.service_configs), eff).addErrback(
log.err, "cf-add-failure"
)
def split_execute_convergence(event, max_length=event_max_length):
"""
Try to split execute-convergence event out into multiple events if there
are too many CLB nodes, too many servers, or too many steps.
The problem is mainly the servers, since they take up the most space.
Experimentally determined that probably logs cut off at around 75k,
characters - we're going to limit it to 50k.
:param dict event: The 'execute-convergence' type event dictionary to split
:param int max_length: The maximum length of the entire JSON-formatted
dictionary.
:return: `list` of `tuple` of (`dict`, `str`). The `dict`s in the tuple
represents the spit up event dicts, and the `str` the format string
for each. If the event does not need to be split, the list will only
have one tuple.
"""
message = "Executing convergence"
if _json_len(event) <= max_length:
return [(event, message)]
events = [(event, message)]
large_things = sorted(('servers', 'lb_nodes'),
key=compose(_json_len, event.get),
reverse=True)
# simplified event which serves as a base for the split out events
base_event = keyfilter(
lambda k: k not in ('desired', 'servers', 'lb_nodes', 'steps'),
event)
for thing in large_things:
split_up_events = split(
assoc(base_event, thing), event[thing], max_length,
_json_len)
events.extend([(e, message) for e in split_up_events])
del event[thing]
if _json_len(event) <= max_length:
break
return events
def call_element_func(elval, cdepth, base_coords):
context_kwds = dtz.keyfilter(lambda k: k in XLocation._fields,
lasso._asdict())
context_kwds['base_coords'] = base_coords
context = XLocation(**context_kwds)
try:
proced, res_lasso = element_func(ranger, lasso, context, elval,
*args, **kwds)
except Exception as ex:
msg_args = (elval, context, ex)
raise ValueError("Value(%r) at %s: \n %s" % msg_args)
if proced:
if not isinstance(res_lasso, Lasso):
res_lasso = lasso._replace(values=res_lasso)
for call_spec in sub_call_specs:
res_lasso = ranger.make_call(res_lasso, *call_spec)
elval = res_lasso and res_lasso.values
return proced, elval
def _map_event_to_dict(_include, sql_event):
"""Map event to a dictionary to be sent as an API response.
In this implementation, the goal is to return a flat structure as
opposed to the nested one that was returned by Elasticsearch in the
past (see v1 implementation for more information).
:param _include:
Projection used to get records from database
:type _include: list(str)
:param sql_event: Event data returned when SQL query was executed
:type sql_event: :class:`sqlalchemy.util._collections.result`
:returns: Event as would have returned by elasticsearch
:rtype: dict(str)
"""
event = {
attr: getattr(sql_event, attr)
for attr in sql_event.keys()
}
for unused_field in Events.UNUSED_FIELDS:
if unused_field in event:
del event[unused_field]
if event['type'] == 'cloudify_event':
del event['logger']
del event['level']
elif event['type'] == 'cloudify_log':
del event['event_type']
# Keep only keys passed in the _include request argument
# TBD: Do the projection at the database level
if _include is not None:
event = dicttoolz.keyfilter(lambda key: key in _include, event)
return event
def _yield_configs_and_defaults(self, config, search_terms, merged: bool,
ciphered: bool):
verbose = self.verbose
get_classes = (self._classes_inc_parents
if verbose else self._classes_with_config_traits)
all_classes = list(get_classes(self.all_app_configurables()))
## Merging needs to visit all hierarchy.
own_traits = not (verbose or merged)
search_map = prepare_search_map(all_classes, own_traits)
if ciphered:
from . import crypto
def ciphered_filter(mapval):
_, trait = mapval
if isinstance(trait, crypto.Cipher):
return mapval
search_map = dtz.valfilter(ciphered_filter, search_map)
if search_terms:
matcher = prepare_matcher(search_terms, self.regex)
search_map = dtz.keyfilter(matcher, search_map)
items = search_map.items()
if self.sort:
items = sorted(items) # Sort by class-name (traits always sorted).
classes_configured = {}
for key, (cls, trait) in items:
if self.list:
yield key
continue
if not trait:
## Not --verbose and class not owning traits.
continue
clsname, trtname = key.split('.')
## Print own traits only, even when "merge" visits all.
#
sup = super(cls, cls)
if not verbose and getattr(sup, trtname, None) is trait:
continue
## Instanciate classes once, to merge values.
#
obj = classes_configured.get(cls)
if obj is None:
try:
## Exceptional rule for Project-zygote.
# TODO: delete when project rule is gone.
#
if cls.__name__ == 'Project':
cls.new_instance('test', None, config)
else:
obj = cls(config=config)
except Exception as ex:
self.log.warning(
"Falied initializing class '%s' due to: %r", clsname,
ex)
## Assign config-values as dummy-object's attributes.
# Note: no merging of values now!
#
class C:
pass
obj = C()
obj.__dict__ = dict(config[clsname])
classes_configured[cls] = obj
## Print 1 class-line for all its traits.
#
base_classes = ', '.join(p.__name__ for p in cls.__bases__)
yield '%s(%s)' % (clsname, base_classes)
if merged:
try:
val = getattr(obj, trtname, '??')
except trt.TraitError as ex:
self.log.warning("Cannot merge '%s' due to: %r", trtname,
ex)
val = "<invalid due to: %s>" % ex
else:
val = repr(trait.default())
yield ' +--%s = %s' % (trtname, val)
def test_keyfilter():
assert keyfilter(iseven, {1: 2, 2: 3}) == {2: 3}
def add_to_cloud_metrics(ttl,
region,
group_metrics,
num_tenants,
config,
log=None,
_print=False):
"""
Add total number of desired, actual and pending servers of a region
to Cloud metrics.
:param str region: which region's metric is collected
:param group_metrics: List of :obj:`GroupMetric`
:param int num_tenants: total number of tenants
:param dict config: Config json dict containing convergence tenants info
:param log: Optional logger
:param bool _print: Should it print activity on stdout? Useful when running
as a script
:return: `Effect` with None
"""
epoch = yield Effect(Func(time.time))
metric_part = {'collectionTime': int(epoch * 1000), 'ttlInSeconds': ttl}
tenanted_metrics, total = calc_total(group_metrics)
if log is not None:
log.msg('total desired: {td}, total_actual: {ta}, total pending: {tp}',
td=total.desired,
ta=total.actual,
tp=total.pending)
if _print:
print('total desired: {}, total actual: {}, total pending: {}'.format(
total.desired, total.actual, total.pending))
metrics = [('desired', total.desired), ('actual', total.actual),
('pending', total.pending), ('tenants', num_tenants),
('groups', len(group_metrics))]
for tenant_id, metric in sorted(tenanted_metrics.items()):
metrics.append(("{}.desired".format(tenant_id), metric.desired))
metrics.append(("{}.actual".format(tenant_id), metric.actual))
metrics.append(("{}.pending".format(tenant_id), metric.pending))
# convergence tenants desired and actual
conv_tenants = keyfilter(
partial(tenant_is_enabled,
get_config_value=lambda k: get_in([k], config)),
tenanted_metrics)
conv_desired = sum(m.desired for m in conv_tenants.itervalues())
conv_actual = sum(m.actual for m in conv_tenants.itervalues())
metrics.extend([("conv_desired", conv_desired),
("conv_actual", conv_actual),
("conv_divergence", conv_desired - conv_actual)])
data = [
merge(metric_part, {
'metricValue': value,
'metricName': '{}.{}'.format(region, metric)
}) for metric, value in metrics
]
yield service_request(ServiceType.CLOUD_METRICS_INGEST,
'POST',
'ingest',
data=data,
log=log)
def keyfilter(self, f: Callable[[A], bool]) -> 'Map[A, B]':
return Map(dicttoolz.keyfilter(f, self))
def pick(whitelist, d):
return keyfilter(lambda k: k in whitelist, d)
def test_keyfilter():
assert keyfilter(iseven, {1: 2, 2: 3}) == {2: 3}
def test_keyfilter(self):
D, kw = self.D, self.kw
assert keyfilter(iseven, D({1: 2, 2: 3}), **kw) == D({2: 3})
def keyfilter(self, f: Callable[[A], bool]) -> 'Map[A, B]':
return Map(dicttoolz.keyfilter(f, self))
def _remove_timestamp_from_plan(item):
notstamp = lambda k: k != 'timestamp'
item = dtz.keyfilter(notstamp, item)
item['flag'] = dtz.keyfilter(notstamp, item['flag'])
return item
from itertools import filterfalse
from toolz.dicttoolz import keyfilter, valfilter, itemfilter
def is_even(x):
if x % 2 == 0: return True
else: return False
def both_are_even(x):
k, v = x
if is_even(k) and is_even(v): return True
else: return False
print(list(filterfalse(is_even, range(10))))
# [1, 3, 5, 7, 9]
print(list(keyfilter(is_even, {1: 2, 2: 3, 3: 4, 4: 5, 5: 6})))
# [2, 4]
print(list(valfilter(is_even, {1: 2, 2: 3, 3: 4, 4: 5, 5: 6})))
# [1, 3, 5]
print(list(itemfilter(both_are_even, {1: 5, 2: 4, 3: 3, 4: 2, 5: 1})))
# [2, 4]
def enrich_dict_immutable(dic):
is_positive = lambda v: v >= 0
filtered = dicttoolz.keyfilter(is_positive, dic)
return dicttoolz.itemmap(_even_bar, filtered)
def test_keyfilter(self):
D, kw = self.D, self.kw
assert keyfilter(iseven, D({1: 2, 2: 3}), **kw) == D({2: 3})
def choose_kwargs(from_, which):
return keyfilter(lambda x: x in which, valfilter(lambda x: x is not None, from_))
