def __init__(self, reactor, config, log, clock=None, collect=None):
"""
Initialize the service by connecting to Cassandra and setting up
authenticator
:param reactor: Twisted reactor for connection purposes
:param dict config: All the config necessary to run the service.
Comes from config file
:param IReactorTime clock: Optional reactor for timer purpose
"""
self._client = connect_cass_servers(reactor, config['cassandra'])
self.log = log
self.reactor = reactor
self._divergent_groups = {}
self.divergent_timeout = get_in(
['metrics', 'divergent_timeout'], config, 3600)
self._service = TimerService(
get_in(['metrics', 'interval'], config, default=60),
collect or self.collect,
reactor,
config,
self.log,
client=self._client,
authenticator=generate_authenticator(reactor, config['identity']))
self._service.clock = clock or reactor
def __init__(self, reactor, config, log, clock=None, collect=None):
"""
Initialize the service by connecting to Cassandra and setting up
authenticator
:param reactor: Twisted reactor for connection purposes
:param dict config: All the config necessary to run the service.
Comes from config file
:param IReactorTime clock: Optional reactor for timer purpose
"""
self._client = connect_cass_servers(reactor, config['cassandra'])
self.log = log
self.reactor = reactor
self._divergent_groups = {}
self.divergent_timeout = get_in(
['metrics', 'divergent_timeout'], config, 3600)
self._service = TimerService(
get_in(['metrics', 'interval'], config, default=60),
collect or self.collect,
reactor,
config,
self.log,
client=self._client,
authenticator=generate_authenticator(reactor, config['identity']))
self._service.clock = clock or reactor
def prep_tweets(tweets):
"""Prepares tweets for entity analysis
This function treets retweets as tweets and
removes all duplicates. Thus, if tweet A
is retweeted 10 times in the corpus,
it will only show up once in the
tweets returned by prep_tweets.
:param tweets: iterator of tweets
:returns: generator of entities
>>> raw_tweets = [{'id': 2345, 'entities': [], ...}
>>> {'id': 9874, 'entities': [], ...}]
>>>
>>> tweets = prep_tweets(raw_tweets)
"""
pipeline = [
map(replace_retweets), deduplicate,
map(handle_truncated),
map(simplify_entities),
map(get_in(['entities']))
]
return pipe(tweets, *pipeline)
async def view(self, ctx, store=None):
""" View store """
user = await author.get(ctx.author)
store_list = ''
stores = StoreModel.find()
loaded_store = None
if store is None:
if not stores:
return await ctx.send(f'```fix\nThere are no stores setup"\n```')
if stores.count() == 1:
loaded_store = Prodict.from_dict(stores[0])
item_list = ''
loaded_store.inventory = sorted(loaded_store.inventory, key=curried.get_in(['price']), reverse=False)
for i, item in enumerate(loaded_store.inventory):
item = Prodict.from_dict(item)
if user['quote_to'] != 'USD':
rates = await coins.rate_convert()
item.price = rates[user['quote_to']] * item.price
item.payout = rates[user['quote_to']] * item.payout
formatted_price = currency.symbol(user["quote_to"]) + '{0:.2f}'.format(item.price)
formatted_payout = "{0:.2f}".format(item.payout)
item_list += f'{i + 1}. {item.name}{" " * (15 - len(item.name))}{formatted_price}{" " * (10 - len(formatted_price))}{item.about.format(payout=formatted_payout, rate=str(item.rate) + " minutes")}\n'
return await ctx.send(
f'```py\n{user.quote_to}\nStore name: {loaded_store.name}\n\nItem{" " * (18 - len("item"))}Price{" " * (10 - len("price"))}Description\n\n{item_list}\n\nQoins represent a USD value by default, the prices will convert depending upon what quote currency you have set on your account. Use the "{self.config.prefix[0]}sq <currency symbol>" command to change it```')
for i, _store in enumerate(stores):
_store = Prodict.from_dict(_store)
item_count = len(_store.inventory)
store_list += f'\n{i + 1}. {_store.name}{" " * (12 - len("Name"))}{item_count}{" " * (10 - len(str(item_count)))}{_store.about}'
store_list_head = f'\nName{" " * (15 - len("Name"))}Item Count{" " * (10 - len("item"))}Description'
await ctx.send(f'```diff\nStore list:\n{store_list_head}{store_list}\n```')
return await ctx.send_help('view')
store_list = []
for i, _store in enumerate(stores):
store_list.append(_store)
if store.isnumeric():
if int(store) - 1 in range(len(store_list)):
loaded_store = store_list[int(store) - 1]
else:
loaded_store = StoreModel.find_one({'name': store})
if not loaded_store:
return await ctx.send(f'```fix\nCould not find store "{store}"\n```')
loaded_store = Prodict.from_dict(loaded_store)
item_list = ''
for i, item in enumerate(loaded_store.inventory):
item = Prodict.from_dict(item)
if user['quote_to'] != 'USD':
rates = await coins.rate_convert()
item.price = rates[user['quote_to']] * item.price
item.payout = rates[user['quote_to']] * item.payout
formatted_price = currency.symbol(user["quote_to"]) + '{0:.2f}'.format(item.price)
formatted_payout = "{0:.2f}".format(item.payout)
item_list += f'{i + 1}. {item.name}{" " * (18 - len(item.name))}{formatted_price}{" " * (10 - len(formatted_price))}{item.about.format(payout=formatted_payout)}\n'
await ctx.send(
f'```py\n{user.quote_to}\n{loaded_store.name}\n\nItem{" " * (21 - len("item"))}Price{" " * (10 - len("price"))}Description\n\n{item_list}```')
def _county_fips_name(fips):
return pipe(
Granularity.COUNTY, geo_data, curried.get("features"),
curry(filter, lambda region: region["id"] == fips),
excepts(
StopIteration,
lambda x: pipe(x, first, curried.get_in(["properties", "NAME"])),
lambda _: None))
def effects_reactor_callback(world):
url = reactor_callback_url(world)
data = get_in(['output'], world)
data_string = json.dumps(data)
logger.info(f'Posting to {url} this data: {data_string}')
response = http.request('POST', url, body=data_string.encode('utf-8'))
logger.info(f'response {response.status_code}; text {response.text}')
assert response.status_code == 200
return response.text
def collect_metrics(reactor, config, log, client=None, authenticator=None,
_print=False):
"""
Start collecting the metrics
:param reactor: Twisted reactor
:param dict config: Configuration got from file containing all info
needed to collect metrics
:param :class:`silverberg.client.CQLClient` client:
Optional cassandra client. A new client will be created
if this is not given and disconnected before returing
:param :class:`otter.auth.IAuthenticator` authenticator:
Optional authenticator. A new authenticator will be created
if this is not given
:param bool _print: Should debug messages be printed to stdout?
:return: :class:`Deferred` fired with ``list`` of `GroupMetrics`
"""
convergence_tids = config.get('convergence-tenants', [])
_client = client or connect_cass_servers(reactor, config['cassandra'])
authenticator = authenticator or generate_authenticator(reactor,
config['identity'])
store = CassScalingGroupCollection(_client, reactor, 1000)
dispatcher = get_dispatcher(reactor, authenticator, log,
get_service_configs(config), store)
# calculate metrics
fpath = get_in(["metrics", "last_tenant_fpath"], config,
default="last_tenant.txt")
tenanted_groups = yield perform(
dispatcher,
get_todays_scaling_groups(convergence_tids, fpath))
group_metrics = yield get_all_metrics(
dispatcher, tenanted_groups, log, _print=_print)
# Add to cloud metrics
metr_conf = config.get("metrics", None)
if metr_conf is not None:
eff = add_to_cloud_metrics(
metr_conf['ttl'], config['region'], group_metrics,
len(tenanted_groups), log, _print)
eff = Effect(TenantScope(eff, metr_conf['tenant_id']))
yield perform(dispatcher, eff)
log.msg('added to cloud metrics')
if _print:
print('added to cloud metrics')
if _print:
group_metrics.sort(key=lambda g: abs(g.desired - g.actual),
reverse=True)
print('groups sorted as per divergence', *group_metrics, sep='\n')
# Disconnect only if we created the client
if not client:
yield _client.disconnect()
defer.returnValue(group_metrics)
def validate_cfn_coeffect(world):
cfn_coeffect = get_in(['coeffects', 'cloudformation'], world)
try:
return update_in(world, ['valid_cfn'],
lambda x: merge(x or {}, CFN_COEFFECT_SCHEMA(cfn_coeffect)))
except Invalid:
logger.warning(cfn_coeffect)
logger.warning('CloudFormation Coeffects are not valid; using defaults', exc_info=True)
return update_in(world, ['valid_cfn'],
lambda x: merge(x or {}, DEFAULT_CFN_COEFFECT))
def add_to_cloud_metrics(ttl, region, group_metrics, num_tenants, config,
log=None, _print=False):
"""
Add total number of desired, actual and pending servers of a region
to Cloud metrics.
:param str region: which region's metric is collected
:param group_metrics: List of :obj:`GroupMetric`
:param int num_tenants: total number of tenants
:param dict config: Config json dict containing convergence tenants info
:param log: Optional logger
:param bool _print: Should it print activity on stdout? Useful when running
as a script
:return: `Effect` with None
"""
epoch = yield Effect(Func(time.time))
metric_part = {'collectionTime': int(epoch * 1000),
'ttlInSeconds': ttl}
tenanted_metrics, total = calc_total(group_metrics)
if log is not None:
log.msg(
'total desired: {td}, total_actual: {ta}, total pending: {tp}',
td=total.desired, ta=total.actual, tp=total.pending)
if _print:
print('total desired: {}, total actual: {}, total pending: {}'.format(
total.desired, total.actual, total.pending))
metrics = [('desired', total.desired), ('actual', total.actual),
('pending', total.pending), ('tenants', num_tenants),
('groups', len(group_metrics))]
for tenant_id, metric in sorted(tenanted_metrics.items()):
metrics.append(("{}.desired".format(tenant_id), metric.desired))
metrics.append(("{}.actual".format(tenant_id), metric.actual))
metrics.append(("{}.pending".format(tenant_id), metric.pending))
# convergence tenants desired and actual
conv_tenants = keyfilter(
partial(tenant_is_enabled,
get_config_value=lambda k: get_in([k], config)),
tenanted_metrics)
conv_desired = sum(m.desired for m in conv_tenants.itervalues())
conv_actual = sum(m.actual for m in conv_tenants.itervalues())
metrics.extend(
[("conv_desired", conv_desired), ("conv_actual", conv_actual),
("conv_divergence", conv_desired - conv_actual)])
data = [merge(metric_part,
{'metricValue': value,
'metricName': '{}.{}'.format(region, metric)})
for metric, value in metrics]
yield service_request(ServiceType.CLOUD_METRICS_INGEST,
'POST', 'ingest', data=data, log=log)
def add_to_cloud_metrics(ttl, region, group_metrics, num_tenants, config,
log=None, _print=False):
"""
Add total number of desired, actual and pending servers of a region
to Cloud metrics.
:param str region: which region's metric is collected
:param group_metrics: List of :obj:`GroupMetric`
:param int num_tenants: total number of tenants
:param dict config: Config json dict containing convergence tenants info
:param log: Optional logger
:param bool _print: Should it print activity on stdout? Useful when running
as a script
:return: `Effect` with None
"""
epoch = yield Effect(Func(time.time))
metric_part = {'collectionTime': int(epoch * 1000),
'ttlInSeconds': ttl}
tenanted_metrics, total = calc_total(group_metrics)
if log is not None:
log.msg(
'total desired: {td}, total_actual: {ta}, total pending: {tp}',
td=total.desired, ta=total.actual, tp=total.pending)
if _print:
print('total desired: {}, total actual: {}, total pending: {}'.format(
total.desired, total.actual, total.pending))
metrics = [('desired', total.desired), ('actual', total.actual),
('pending', total.pending), ('tenants', num_tenants),
('groups', len(group_metrics))]
for tenant_id, metric in sorted(tenanted_metrics.items()):
metrics.append(("{}.desired".format(tenant_id), metric.desired))
metrics.append(("{}.actual".format(tenant_id), metric.actual))
metrics.append(("{}.pending".format(tenant_id), metric.pending))
# convergence tenants desired and actual
conv_tenants = keyfilter(
partial(tenant_is_enabled,
get_config_value=lambda k: get_in([k], config)),
tenanted_metrics)
conv_desired = sum(m.desired for m in conv_tenants.itervalues())
conv_actual = sum(m.actual for m in conv_tenants.itervalues())
metrics.extend(
[("conv_desired", conv_desired), ("conv_actual", conv_actual),
("conv_divergence", conv_desired - conv_actual)])
data = [merge(metric_part,
{'metricValue': value,
'metricName': '{}.{}'.format(region, metric)})
for metric, value in metrics]
yield service_request(ServiceType.CLOUD_METRICS_INGEST,
'POST', 'ingest', data=data, log=log)
def sort_coins(coins, sort_key, sort_dir, quote):
_coins = []
if sort_key == 'change':
sort_key = 'percent_change_24h'
reverse = True
if sort_dir == 'desc':
reverse = False
for coin in coins:
if type(coin) is dict:
if type(coin['quotes'][quote]['price']) and type(
coin['quotes'][quote]['percent_change_24h']) is float:
_coins.append(coin)
return sorted(_coins, key=curried.get_in(['quotes', quote, sort_key]), reverse=reverse)
def context_middleware(ctx, call):
request_id = ctx.get(_REQ_ID, None)
request_parent_id = ctx.get(_REQ_PARENT_ID, None)
uid = get_in([_SESSION, _UID], None)
ctx[_CONTEXT] = context_fn(
env=env,
sha=sha,
version=version,
request_id=request_id,
request_parent_id=request_parent_id,
scope=scope,
uid=uid,
getpid=getpid,
gettid=gettid,
)
ctx = call(ctx)
return ctx
def call_context_fnc(ctx, context_fn, namespace, scope, env, sha, version, getpid, gettid):
request_id = ctx.get(_REQ_ID, None)
request_parent_id = ctx.get(_REQ_PARENT_ID, None)
uid = get_in([_SESSION, _UID], None)
ctx[_CONTEXT] = context_fn(
env=env,
sha=sha,
namespace=namespace,
version=version,
request_id=request_id,
request_parent_id=request_parent_id,
scope=scope,
uid=uid,
getpid=getpid,
gettid=gettid,
)
return ctx
async def whalelist(self, ctx, page: int = 1):
""" View the whalelist (leaderboard) """
owners = default.get("config.json").owners
user = await author.get(ctx.author)
per_page = self.config.game.whalelist_per_page
page_count = 1
_whales = User.find()
whales = []
# add the networth to each user and remove users excluded from whale lists
for i, doc in enumerate(_whales):
if int(doc['user_id']) not in owners:
if doc['name'] != self.bot.user.name:
coin_list = await coins.get_coins(user['quote_to'])
doc_pvalue = await coins.portfolio_value(
doc, coin_list, user['quote_to'])
doc['game']['networth'] = float(
'{0:.2f}'.format(doc["game"]["money"] +
doc["game"]["in_pocket"] +
doc_pvalue))
whales.append(doc)
whales = sorted(whales,
key=curried.get_in(['game', 'networth']),
reverse=True)
if len(whales) > per_page:
whales = default.divide_chunks(whales, per_page)[page - 1]
page_count = len(whales)
whale_list = ''
if page > page_count:
return
for i, doc in enumerate(whales):
rates = await coins.rate_convert(doc['quote_to'])
doc = await coins.convert_user_currency(doc, rates,
user['quote_to'])
whale_list += f'\n{i + 1}. {doc["name"]}{" " * (25 - len(doc["name"]))}{currency.symbol(user["quote_to"])}{doc["game"]["networth"]}'
await ctx.send(
f'```py\n{user["quote_to"]}\nBiggest whales:\n{whale_list}\n\nPage {page} of {page_count}```'
)
def prepare_output(world):
valid_cfn = get_in(['valid_cfn'], world)
metadata = callback_metadata(properties(world))
message_type = 'account-link-provisioned' if request_type(world) in {'Create', 'Update'} else 'account-link-deprovisioned'
visible_cloudtrail_arns_string = null_to_none(get_in(['Discovery', 'VisibleCloudTrailArns'], valid_cfn))
visible_cloudtrail_arns = visible_cloudtrail_arns_string.split(',') if visible_cloudtrail_arns_string else None
master_payer_billing_bucket_name = (null_to_none(get_in(['Discovery', 'MasterPayerBillingBucketName'], valid_cfn)) or
null_to_none(get_in(['MasterPayerAccount', 'ReportS3Bucket'], valid_cfn)))
master_payer_billing_bucket_path = (null_to_none(get_in(['Discovery', 'MasterPayerBillingBucketPath'], valid_cfn)) or
null_to_none(get_in(['MasterPayerAccount', 'ReportS3Prefix'], valid_cfn)))
output = {
**default_metadata,
'message_type': message_type,
'data': {
'metadata': {
'cloud_region': metadata['Region'],
'external_id': metadata['ExternalId'],
'cloud_account_id': metadata['AccountId'],
'cz_account_name': metadata['AccountName'],
'reactor_id': metadata['ReactorId'],
'reactor_callback_url': metadata['ReactorCallbackUrl'],
},
'links': {
'audit': {'role_arn': null_to_none(get_in(['AuditAccount', 'RoleArn'], valid_cfn))},
'cloudtrail_owner': {
'sqs_queue_arn': null_to_none(get_in(['CloudTrailOwnerAccount', 'SQSQueueArn'], valid_cfn)),
'sqs_queue_policy_name': null_to_none(get_in(['CloudTrailOwnerAccount', 'SQSQueuePolicyName'], valid_cfn)),
},
'master_payer': {'role_arn': null_to_none(get_in(['MasterPayerAccount', 'RoleArn'], valid_cfn))},
'resource_owner': {'role_arn': null_to_none(get_in(['ResourceOwnerAccount', 'RoleArn'], valid_cfn))},
'legacy': {'role_arn': null_to_none(get_in(['LegacyAccount', 'RoleArn'], valid_cfn))},
},
'discovery': {
'audit_cloudtrail_bucket_name': null_to_none(get_in(['Discovery', 'AuditCloudTrailBucketName'], valid_cfn)),
'audit_cloudtrail_bucket_prefix': null_to_none(get_in(['Discovery', 'AuditCloudTrailBucketPrefix'], valid_cfn)),
'cloudtrail_sns_topic_arn': null_to_none(get_in(['Discovery', 'CloudTrailSNSTopicArn'], valid_cfn)),
'cloudtrail_trail_arn': null_to_none(get_in(['Discovery', 'CloudTrailTrailArn'], valid_cfn)),
'is_audit_account': string_to_bool(get_in(['Discovery', 'IsAuditAccount'], valid_cfn)),
'is_cloudtrail_owner_account': string_to_bool(get_in(['Discovery', 'IsCloudTrailOwnerAccount'], valid_cfn)),
'is_master_payer_account': string_to_bool(get_in(['Discovery', 'IsMasterPayerAccount'], valid_cfn)),
'is_organization_master_account': string_to_bool(get_in(['Discovery', 'IsOrganizationMasterAccount'], valid_cfn)),
'is_organization_trail': string_to_bool(get_in(['Discovery', 'IsOrganizationTrail'], valid_cfn)),
'is_resource_owner_account': string_to_bool(get_in(['Discovery', 'IsResourceOwnerAccount'], valid_cfn)),
'master_payer_billing_bucket_name': master_payer_billing_bucket_name,
'master_payer_billing_bucket_path': master_payer_billing_bucket_path,
'remote_cloudtrail_bucket': string_to_bool(get_in(['Discovery', 'RemoteCloudTrailBucket'], valid_cfn)),
'visible_cloudtrail_arns': visible_cloudtrail_arns,
}
}
}
return update_in(world, ['output'], lambda x: merge(x or {}, output))
'is_organization_master_account': bool,
'is_master_payer_account': bool,
'is_resource_owner_account': bool,
'master_payer_billing_bucket_name': NONEABLE_STRING,
'master_payer_billing_bucket_path': NONEABLE_STRING,
'remote_cloudtrail_bucket': bool,
}
}
}, required=True, extra=ALLOW_EXTRA)
OUTPUT_SCHEMA = Schema({
'output': ACCOUNT_LINK_PROVISIONED,
}, required=True, extra=ALLOW_EXTRA)
request_type = get_in(['event', 'RequestType'])
properties = get_in(['event', 'ResourceProperties'])
stacks = get_in(['event', 'ResourceProperties', 'Stacks'])
reactor_callback_url = get_in(['event', 'ResourceProperties', 'ReactorCallbackUrl'])
supported_metadata = {'Region', 'ExternalId', 'AccountId', 'AccountName', 'ReactorId', 'ReactorCallbackUrl'}
callback_metadata = keyfilter(lambda x: x in supported_metadata)
default_metadata = {
'version': '1',
'message_source': 'cfn',
}
#####################
#
# Coeffects, i.e. from the outside world
#
return zip(prefixed_names, arg_values)
@curry
def merge_metadata(metadata, observation):
"""Merge metadata and observation dictionaries
:rtype: dict
:param metadata: dict of query metadata
:param observation: dict of observation data
:return: data dictionary
"""
return merge(metadata, observation)
extract_observations = get_in(['history', 'observations'])
create_model = compose(
WeatherUndergroundObservation.from_dict, # create model
rename_keys(key_map) # filter and rename data
)
process_response = compose( # create a collection of models
list, map(create_model))
process_metadata = compose( # include query metadata
merge_metadata, # merge with observations
dict, # create dict from items
named_query_params(api_query_args), # query arg name, value pairs
collect # api query args as a tuple
)
'IsAuditAccount': bool,
'AuditCloudTrailBucketName': Any(None, str),
'IsResourceOwnerAccount': bool,
'IsCloudTrailOwnerAccount': bool,
'CloudTrailSNSTopicArn': Any(None, str),
'IsMasterPayerAccount': bool,
'MasterPayerBillingBucketName': Any(None, str),
},
},
required=True,
extra=ALLOW_EXTRA)
DEFAULT_PAYER_REPORTS = {'is_master_payer': False, 'report_definitions': []}
NOT_IN_ORGANIZATION_RESPONSE = {}
event_account_id = get_in(['event', 'ResourceProperties', 'AccountId'])
coeffects_traillist = get_in(['coeffects', 'cloudtrail', 'trailList'],
default=[])
coeffects_buckets = get_in(['coeffects', 's3', 'Buckets'], default=[])
coeffects_payer_reports = get_in(['coeffects', 'cur'],
default=DEFAULT_PAYER_REPORTS)
coeffects_master_account_id = get_in(
['coeffects', 'organizations', 'Organization', 'MasterAccountId'])
output_is_organization_master = get_in(
['output', 'IsOrganizationMasterAccount'])
output_is_account_outside_organization = get_in(
['output', 'IsAccountOutsideOrganization'])
#####################
#
OUTPUT_SCHEMA = Schema(
{
'output': {
'IsAuditAccount': bool,
'AuditCloudTrailBucketName': Any(None, str),
'IsResourceOwnerAccount': bool,
'IsCloudTrailOwnerAccount': bool,
'CloudTrailSNSTopicArn': Any(None, str),
'IsMasterPayerAccount': bool,
'MasterPayerBillingBucketName': Any(None, str),
},
},
required=True,
extra=ALLOW_EXTRA)
event_account_id = get_in(['event', 'ResourceProperties', 'AccountId'])
coeffects_traillist = get_in(['coeffects', 'cloudtrail', 'trailList'],
default=[])
coeffects_buckets = get_in(['coeffects', 's3', 'Buckets'], default=[])
coeffects_report_definitions = get_in(
['coeffects', 'cur', 'ReportDefinitions'], default=[])
coeffects_master_account_id = get_in(
['coeffects', 'organizations', 'Organization', 'MasterAccountId'])
#####################
#
# Coeffects, i.e. from the outside world
#
#####################
def coeffects(world):
def get_course_value(regex_dict):
if 'key' in regex_dict:
if regex_dict['key'] in course_dict:
return course_dict[regex_dict['key']]
elif 'keys' in regex_dict:
return _.get_in(regex_dict['keys'], course_dict)
