def _get_service_charging_by_diffs(self, pricing_service, **kwargs):
"""
Calculates total cost of diffs (between real and calculated costs) of
selected pricing services.
:param pricing_service: day for which calculate extra costss
:returns: dict with difference between real and calculated by specific
pricing service plugin, grouped by pricing service id (key is
another pricing service id, value is costs difference)
:rtype dict:
"""
result = {}
for ps in pricing_service.charged_by_diffs.all():
args = dict(type='total_cost',
pricing_service=ps,
for_all_service_environments=True,
**kwargs)
try:
# first run valid pricing service plugin to get "offical" costs
ps_cost = plugin_runner.run('scrooge_costs',
ps.get_plugin_name(), **args)
# then call universal plugin to get real cost
ps_real_cost = plugin_runner.run('scrooge_costs',
'pricing_service_plugin',
**args)
except (KeyError, AttributeError):
logger.warning('Invalid call for {0} total cost diff'.format(
ps.name))
else:
# calculate difference between real cost and calculated cost
diff = ps_real_cost.values()[0][0] - ps_cost.values()[0][0]
result[ps.id] = diff, {}
return result
def handle(self, today, run_only, *args, **options):
from ralph_pricing import plugins # noqa
if today:
today = datetime.datetime.strptime(today, '%Y-%m-%d').date()
else:
today = datetime.date.today()
print('Synchronizing for {0}.'.format(today.isoformat()))
if run_only:
print('Running only {0}...'.format(run_only))
success, message, context = plugin.run(
'pricing',
run_only,
today=today,
)
print('{1}: {0}'.format(message, 'Done' if success else 'Failed'))
return
done = set()
tried = set()
while True:
to_run = plugin.next('pricing', done) - tried
if not to_run:
break
name = plugin.highest_priority('pricing', to_run)
tried.add(name)
print('Running {0}...'.format(name))
success, message, context = plugin.run(
'pricing',
name,
today=today,
)
print('{1}: {0}'.format(message, 'Done' if success else 'Failed'))
if success:
done.add(name)
def handle(self, today, run_only, *args, **options):
from ralph_pricing import plugins # noqa
if today:
today = datetime.datetime.strptime(today, '%Y-%m-%d').date()
else:
today = datetime.date.today()
print('Synchronizing for {0}.'.format(today.isoformat()))
if run_only:
print('Running only {0}...'.format(run_only))
success, message, context = plugin.run(
'pricing',
run_only,
today=today,
)
print('{1}: {0}'.format(message, 'Done' if success else 'Failed'))
return
done = set()
tried = set()
while True:
to_run = plugin.next('pricing', done) - tried
if not to_run:
break
name = plugin.highest_priority('pricing', to_run)
tried.add(name)
print('Running {0}...'.format(name))
success, message, context = plugin.run(
'pricing', name, today=today,
)
print('{1}: {0}'.format(message, 'Done' if success else 'Failed'))
if success:
done.add(name)
def _get_dependent_services_cost(
self,
start,
end,
service,
forecast,
ventures
):
"""
Calculates cost of dependent services used by service.
"""
dependent_costs = 0
for dependent in service.dependency.exclude(id=service.id):
try:
dependent_schema = plugin_runner.run(
'reports',
dependent.get_plugin_name(),
service=dependent,
type='schema'
)
# find total cost column in schema
total_cost_key = None
for column_key, column_description in dependent_schema.items():
if column_description.get('total_cost'):
total_cost_key = column_key
break
if total_cost_key:
# do report of usages for service (and it's ventures)
dependent_usages = plugin_runner.run(
'reports',
dependent.get_plugin_name(),
service=dependent,
ventures=ventures,
start=start,
end=end,
forecast=forecast,
type='costs',
)
for venture, venture_data in dependent_usages.items():
dependent_costs += venture_data[total_cost_key]
else:
logger.warning(
'No total cost column for {0} dependency'.format(
dependent.name
)
)
except (KeyError, AttributeError):
logger.warning('Invalid plugin for {0} dependency'.format(
dependent.name
))
return dependent_costs
def run_deployment(deployment):
if deployment.is_running:
return
deployment.is_running = True
deployment.save()
try:
done = set(name.strip() for name in deployment.done_plugins.split(',')
if name.strip())
tried = set(done)
while True:
plugins = plugin.next('deployment', done) - tried
if not plugins:
break
name = plugin.highest_priority('deployment', plugins)
tried.add(name)
try:
success = plugin.run(
'deployment',
name,
deployment_id=deployment.id,
)
except Exception:
traceback.print_exc()
else:
if success:
done.add(name)
# The plugins might have changed the deployment object.
deployment = Deployment.objects.get(id=deployment.id)
deployment.done_plugins = ', '.join(done)
deployment.save()
finally:
deployment.is_running = False
deployment.save()
def _get_service_extra_cost(
self,
start,
end,
ventures,
):
"""
Calculates total cost of extra costs for given service.
:param datatime start: Begin of time interval
:param datatime end: End of time interval
:param list ventures: List of ventures
:returns decimal: price
:rtype decimal:
"""
try:
return plugin_runner.run(
'reports',
'extra_cost_plugin',
type='total_cost',
start=start,
end=end,
ventures=ventures,
)
except (KeyError, AttributeError):
logger.warning('Invalid call for total extra cost')
return D(0)
def _get_service_dynamic_extra_cost(
self,
date,
forecast,
service_environments,
):
"""
Calculates total cost of dynamic extra costs for given pricing_service.
:param datatime date: day for which calculate extra costs
:param list service_environments: List of service_environments
:returns decimal: price
:rtype decimal:
"""
result = {}
for dynamic_extra_cost_type in DynamicExtraCostType.objects.all():
try:
dynamic_extra_cost = plugin_runner.run(
'scrooge_costs',
'dynamic_extra_cost_plugin',
type='total_cost',
date=date,
forecast=forecast,
dynamic_extra_cost_type=dynamic_extra_cost_type,
service_environments=service_environments,
)
if dynamic_extra_cost != 0:
result.update(dynamic_extra_cost)
except (KeyError, AttributeError):
logger.warning('Invalid call for {0} total cost'.format(
dynamic_extra_cost_type.name))
return result
def _get_dependent_services_cost(
self,
date,
pricing_service,
forecast,
service_environments,
):
"""
Calculates cost of dependent services used by pricing_service.
"""
result = {}
exclude = [pricing_service]
dependent_services = pricing_service.get_dependent_services(
date,
exclude,
)
for dependent in dependent_services:
try:
dependent_cost = plugin_runner.run(
'scrooge_costs',
dependent.get_plugin_name(),
pricing_service=dependent,
type='total_cost',
date=date,
forecast=forecast,
service_environments=service_environments,
)
result.update(dependent_cost)
except (KeyError, AttributeError):
logger.warning('Invalid call for {0} total cost'.format(
dependent.name))
return result
def _get_service_teams_cost(
self,
date,
forecast,
service_environments,
):
"""
Calculates total cost of teams for given pricing service, using
real or forecast prices/costs. Total cost is calculated for period of
time (between start and end) and for specified service environments.
"""
result = {}
for team in Team.objects.all():
try:
team_cost = plugin_runner.run(
'scrooge_costs',
'team_plugin',
type='total_cost',
date=date,
team=team,
forecast=forecast,
service_environments=service_environments,
)
if team_cost != 0:
result[team.id] = team_cost, {}
except (KeyError, AttributeError):
logger.warning('Invalid call for {0} total cost'.format(
team.name))
return result
def get_header(cls, start, end, usage_types, **kwargs):
"""
Return all headers for daily usages report.
:returns list: Complete collection of headers for report
:rtype list:
"""
logger.debug("Getting headers for report")
header = [
[
(_('Service'), {
'rowspan': 2
}),
(_('Environment'), {
'rowspan': 2
}),
],
[],
]
usage_types_headers = []
for usage_type in usage_types:
usage_type_header = plugin_runner.run(
'scrooge_reports',
usage_type.get_plugin_name(),
usage_type=usage_type,
type='usages_schema',
)
usage_types_headers.append(usage_type_header)
for day in rrule.rrule(rrule.DAILY, dtstart=start, until=end):
header[0].append((day.date(), {'colspan': len(usage_types)}))
header[1].extend(usage_types_headers)
return header
def get_header(cls, start, end, usage_types, **kwargs):
"""
Return all headers for daily usages report.
:returns list: Complete collection of headers for report
:rtype list:
"""
logger.debug("Getting headers for report")
header = [
[
(_('Venture'), {'rowspan': 2})
],
[],
]
usage_types_headers = []
for usage_type in usage_types:
usage_type_header = plugin_runner.run(
'reports',
usage_type.get_plugin_name(),
usage_type=usage_type,
type='dailyusages_header',
)
usage_types_headers.append(usage_type_header)
for day in rrule.rrule(rrule.DAILY, dtstart=start, until=end):
header[0].append((day.date(), {'colspan': len(usage_types)}))
header[1].extend(usage_types_headers)
return header
def _get_report_data(cls, start, end, is_active, forecast, ventures):
"""
Use plugins to get usages data for given ventures. Plugin logic can be
so complicated but for this method, plugin must return value in
format:
data_from_plugin = {
'venture_id': {
'field1_name': value,
'field2_name': value,
...
},
...
}
:param datatime start: Start of time interval for report
:param datatime end: End of time interval for report
:param boolean forecast: Forecast prices or real
:param list ventures: List of ventures for which data must be taken
:returns dict: Complete report data for all ventures
:rtype dict:
"""
logger.debug("Getting report date")
old_queries_count = len(connection.queries)
data = {venture.id: {} for venture in ventures}
for i, plugin in enumerate(cls.get_plugins()):
try:
plugin_old_queries_count = len(connection.queries)
plugin_report = plugin_runner.run(
'reports',
plugin.plugin_name,
ventures=ventures,
start=start,
end=end,
forecast=forecast,
type='costs',
**plugin.get('plugin_kwargs', {})
)
for venture_id, venture_usage in plugin_report.iteritems():
if venture_id in data:
data[venture_id].update(venture_usage)
plugin_queries_count = (
len(connection.queries) - plugin_old_queries_count
)
if settings.DEBUG:
logger.debug('Plugin SQL queries: {0}\n'.format(
plugin_queries_count
))
except KeyError:
logger.warning(
"Usage '{0}' have no usage plugin".format(plugin.name)
)
except BaseException as e:
logger.exception("Report generate error: {0}".format(e))
queries_count = len(connection.queries) - old_queries_count
if settings.DEBUG:
logger.debug('Total SQL queries: {0}'.format(queries_count))
return data
def _get_report_data(cls, start, end, is_active, forecast, ventures):
"""
Use plugins to get usages data for given ventures. Plugin logic can be
so complicated but for this method, plugin must return value in
format:
data_from_plugin = {
'venture_id': {
'field1_name': value,
'field2_name': value,
...
},
...
}
:param datatime start: Start of time interval for report
:param datatime end: End of time interval for report
:param boolean forecast: Forecast prices or real
:param list ventures: List of ventures for which data must be taken
:returns dict: Complete report data for all ventures
:rtype dict:
"""
logger.debug("Getting report date")
old_queries_count = len(connection.queries)
data = {venture.id: {} for venture in ventures}
for i, plugin in enumerate(cls.get_plugins()):
try:
plugin_old_queries_count = len(connection.queries)
plugin_report = plugin_runner.run('reports',
plugin.plugin_name,
ventures=ventures,
start=start,
end=end,
forecast=forecast,
type='costs',
**plugin.get(
'plugin_kwargs', {}))
for venture_id, venture_usage in plugin_report.iteritems():
if venture_id in data:
data[venture_id].update(venture_usage)
plugin_queries_count = (len(connection.queries) -
plugin_old_queries_count)
if settings.DEBUG:
logger.debug('Plugin SQL queries: {0}\n'.format(
plugin_queries_count))
except KeyError:
logger.warning("Usage '{0}' have no usage plugin".format(
plugin.name))
except Exception as e:
logger.exception(
"Error while generating the report: {0}".format(e))
raise
queries_count = len(connection.queries) - old_queries_count
if settings.DEBUG:
logger.debug('Total SQL queries: {0}'.format(queries_count))
return data
def _costs(
self,
pricing_service,
date,
forecast=False,
**kwargs
):
logger.info(
(
"Calculating pricing service costs (using fixed "
"prices): {0}"
).format(
pricing_service.name,
)
)
result_dict = defaultdict(dict)
usage_types = pricing_service.usage_types.all()
for usage_type in usage_types:
try:
# results per service environment (list of costs per pricing
# object as a value)
costs = plugin_runner.run(
'scrooge_costs',
usage_type.get_plugin_name(),
type='costs',
date=date,
usage_type=usage_type,
forecast=forecast,
)
# store costs in hierarchy service environment / pricing object
# and calculate total cost of cloud per pricing object
for se, se_costs in costs.items():
for cost in se_costs:
pricing_object_id = cost.get('pricing_object_id')
if pricing_object_id not in result_dict[se]:
result_dict[se][pricing_object_id] = {
'type_id': pricing_service.id,
'pricing_object_id': pricing_object_id,
'cost': D(0),
'_children': [],
}
result_dict[se][pricing_object_id]['_children'].append(
cost
)
result_dict[se][pricing_object_id]['cost'] += (
cost['cost']
)
except (KeyError, AttributeError):
logger.warning(
'Invalid call for {0} total cost'.format(usage_type.name)
)
# return data in format accepted by collector - costs per service
# environment (without nested dict with pricing object)
return {k: v.values() for k, v in result_dict.items()}
def _run_plugin(self, name, today):
logger.info('Running {0}...'.format(name))
try:
success, message, context = plugin.run(
'pricing',
name,
today=today,
)
if not success:
raise PluginError(message)
logger.info('{0}: Done'.format(message))
return True
except Exception as e:
logger.exception("{0}: {1}".format(name, e))
def _collect_costs(
self,
date,
forecast=False,
plugins=None
):
"""
Collects costs from all plugins and stores them per service environment
"""
logger.debug("Getting report date")
old_queries_count = len(connection.queries)
data = defaultdict(list)
for i, plugin in enumerate(plugins or self.get_plugins()):
try:
plugin_old_queries_count = len(connection.queries)
plugin_report = plugin_runner.run(
'scrooge_costs',
plugin.plugin_name,
date=date,
forecast=forecast,
type='costs',
**{str(k): v for (k, v) in plugin['plugin_kwargs'].items()}
)
for service_id, service_usage in plugin_report.iteritems():
data[service_id].extend(service_usage)
plugin_queries_count = (
len(connection.queries) - plugin_old_queries_count
)
if settings.DEBUG:
logger.debug('Plugin SQL queries: {0}\n'.format(
plugin_queries_count
))
except KeyError:
logger.warning(
"Usage '{0}' has no usage plugin\n".format(plugin.name)
)
except NoPriceCostError:
logger.warning('No costs defined\n')
except MultiplePriceCostError:
logger.warning('Multiple costs defined\n')
except Exception as e:
logger.exception(
"Error while generating the report: {0}\n".format(e)
)
raise
queries_count = len(connection.queries) - old_queries_count
if settings.DEBUG:
logger.debug('Total SQL queries: {0}'.format(queries_count))
return data
def _run_plugin(self, name, today):
logger.info('Running {0}...'.format(name))
try:
success, message, context = plugin.run(
'pricing',
name,
today=today,
)
if not success:
raise PluginError(message)
logger.info('{0}: Done'.format(message))
return True
except Exception as e:
logger.exception("{0}: {1}".format(name, e))
def _get_report_data(cls, start, end, usage_types, service_environments):
"""
Use plugins to get daily usages data for given service environments.
Plugin should return data in following format:
data_from_plugin = {
'day': {
'service_environment_1_id': value,
'service_environment_2_id': value2,
},
...
}
:param datatime start: Start of time interval for report
:param datatime end: End of time interval for report
:param list usage_types: Usage types to use
:param list service_environments: List of service environments for
which data must be taken
:returns dict: Complete report data for service environments daily
usages
:rtype dict:
"""
logger.debug("Getting services environments dailyusages report data")
data = {
day.date(): {}
for day in rrule.rrule(rrule.DAILY, dtstart=start, until=end)
}
for usage_type in usage_types:
plugin_name = usage_type.get_plugin_name()
try:
plugin_report = plugin_runner.run(
'scrooge_reports',
plugin_name,
type='usages',
start=start,
end=end,
usage_type=usage_type,
service_environments=service_environments,
)
for day, day_usages in plugin_report.iteritems():
data[day][usage_type.id] = day_usages
except KeyError:
logger.warning(
"Usage '{0}' has no usage plugin".format(plugin_name))
except Exception as e:
logger.exception(
"Error while generating the report: {0}".format(e))
raise
return data
def _get_report_data(cls, start, end, usage_types, ventures):
"""
Use plugins to get daily usages data for given ventures. Plugin return
value format:
data_from_plugin = {
'day': {
'venture_id': value,
'venture_id2': value2,
},
...
}
:param datatime start: Start of time interval for report
:param datatime end: End of time interval for report
:param list usage_types: Usage types to use
:param list ventures: List of ventures for which data must be taken
:returns dict: Complete report data for ventures daily usages
:rtype dict:
"""
logger.debug("Getting ventures dailyusages report data")
data = {day.date(): {} for day in rrule.rrule(
rrule.DAILY,
dtstart=start,
until=end
)}
for usage_type in usage_types:
plugin_name = usage_type.get_plugin_name()
try:
plugin_report = plugin_runner.run(
'reports',
plugin_name,
ventures=ventures,
start=start,
end=end,
usage_type=usage_type,
type='dailyusages',
)
for day, day_usages in plugin_report.iteritems():
data[day][usage_type.symbol] = day_usages
except KeyError:
logger.warning(
"Usage '{0}' has no usage plugin".format(plugin_name)
)
except BaseException as e:
logger.error("Report generate error: {0}".format(e))
return data
def _get_report_data(cls, start, end, usage_types, ventures):
"""
Use plugins to get daily usages data for given ventures. Plugin return
value format:
data_from_plugin = {
'day': {
'venture_id': value,
'venture_id2': value2,
},
...
}
:param datatime start: Start of time interval for report
:param datatime end: End of time interval for report
:param list usage_types: Usage types to use
:param list ventures: List of ventures for which data must be taken
:returns dict: Complete report data for ventures daily usages
:rtype dict:
"""
logger.debug("Getting ventures dailyusages report data")
data = {day.date(): {} for day in rrule.rrule(
rrule.DAILY,
dtstart=start,
until=end
)}
for usage_type in usage_types:
plugin_name = usage_type.get_plugin_name()
try:
plugin_report = plugin_runner.run(
'reports',
plugin_name,
ventures=ventures,
start=start,
end=end,
usage_type=usage_type,
type='dailyusages',
)
for day, day_usages in plugin_report.iteritems():
data[day][usage_type.symbol] = day_usages
except KeyError:
logger.warning(
"Usage '{0}' has no usage plugin".format(plugin_name)
)
except BaseException as e:
logger.error("Report generate error: {0}".format(e))
return data
def _get_report_data(cls, start, end, ventures, forecast, devices):
"""
Use plugins to get usages data per device for given venture. Each
plugin has to return value in following format:
data_from_plugin = {
'device_id': {
'field1_name': value,
'field2_name': value,
...
},
...
}
:param date start: Start of date interval for report
:param date end: End of date interval for report
:param boolean forecast: Forecast prices or real
:param Venture venture: Venture to generate report for
:param list devices: List of devices to generate report for
:returns dict: Complete report data for all ventures
:rtype dict:
"""
logger.debug("Getting devices report data")
data = {device.id: {} for device in devices}
for plugin in cls.get_plugins():
try:
plugin_report = plugin_runner.run(
'reports',
plugin.plugin_name,
ventures=ventures,
start=start,
end=end,
forecast=forecast,
type='costs_per_device',
**plugin.get('plugin_kwargs', {})
)
for device_id, device_usage in plugin_report.iteritems():
if device_id in data:
data[device_id].update(device_usage)
except KeyError:
logger.warning(
"Usage {0} has no plugin connected".format(plugin.name)
)
except Exception as e:
logger.exception("Report generate error: {0}".format(e))
return data
def _run_plugin(context,
chain,
plugin_name,
requirements,
interactive,
clear_down,
done_requirements,
outputs=None):
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
stdout_verbose = output.get(interactive, verbose=True)
stderr = output.get(interactive, err=True)
message = "[{}] {}... ".format(plugin_name, context.get('ip', ''))
pad = output.WIDTH - len(message)
stdout(message, end='')
try:
new_context = {}
is_up, message, new_context = plugin.run(chain, plugin_name, **context)
if is_up:
requirements.add(plugin_name)
except plugin.Restart as e:
stdout('needs to be restarted: {}'.format(unicode(e)), end='\n')
raise
except Exception:
stdout('', end='\r')
stderr("Exception in plugin '{}' for '{}': {}".format(
plugin_name, context.get('ip', 'none'), traceback.format_exc()),
end='\n')
else:
message = message or ''
if clear_down and not is_up:
end = '\r'
else:
end = '\n'
pad -= len(message)
message = message + (" " * pad)
if is_up:
stdout(message, end=end)
else:
stdout_verbose(message, end=end)
finally:
done_requirements.add(plugin_name)
context.update(new_context)
def _run_plugin(context, chain, plugin_name, requirements, interactive,
clear_down, done_requirements, outputs=None):
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
stdout_verbose = output.get(interactive, verbose=True)
stderr = output.get(interactive, err=True)
message = "[{}] {}... ".format(plugin_name, context.get('ip', ''))
pad = output.WIDTH - len(message)
stdout(message, end='')
try:
new_context = {}
is_up, message, new_context = plugin.run(chain, plugin_name,
**context)
if is_up:
requirements.add(plugin_name)
except plugin.Restart as e:
stdout('needs to be restarted: {}'.format(unicode(e)), end='\n')
raise
except Exception:
stdout('', end='\r')
stderr("{}\nException in plugin '{}' for '{}'.".format(
traceback.format_exc()),
plugin_name,
context.get('ip', 'none'),
end='\n')
else:
message = message or ''
if clear_down and not is_up:
end = '\r'
else:
end = '\n'
pad -= len(message)
message = message + (" " * pad)
if is_up:
stdout(message, end=end)
else:
stdout_verbose(message, end=end)
finally:
done_requirements.add(plugin_name)
context.update(new_context)
context['successful_plugins'] = ', '.join(sorted(requirements))
def _get_report_data(cls, start, end, ventures, forecast, devices):
"""
Use plugins to get usages data per device for given venture. Each
plugin has to return value in following format:
data_from_plugin = {
'device_id': {
'field1_name': value,
'field2_name': value,
...
},
...
}
:param date start: Start of date interval for report
:param date end: End of date interval for report
:param boolean forecast: Forecast prices or real
:param Venture venture: Venture to generate report for
:param list devices: List of devices to generate report for
:returns dict: Complete report data for all ventures
:rtype dict:
"""
logger.debug("Getting devices report data")
data = {device.id: {} for device in devices}
for plugin in cls.get_plugins():
try:
plugin_report = plugin_runner.run('reports',
plugin.plugin_name,
ventures=ventures,
start=start,
end=end,
forecast=forecast,
type='costs_per_device',
**plugin.get(
'plugin_kwargs', {}))
for device_id, device_usage in plugin_report.iteritems():
if device_id in data:
data[device_id].update(device_usage)
except KeyError:
logger.warning("Usage {0} has no plugin connected".format(
plugin.name))
except Exception as e:
logger.exception("Report generate error: {0}".format(e))
return data
def _run_plugin(name, today):
logger.info('Running {0}...'.format(name))
success, message = False, None
sync_status = SyncStatus.objects.get_or_create(plugin=name, date=today)[0]
try:
success, message = plugin.run(
'scrooge',
name,
today=today,
)
if not success:
raise PluginError(message)
except Exception as e:
logger.exception("{0}: {1}".format(name, e))
raise PluginError(e)
finally:
sync_status.success = success
sync_status.remarks = message
sync_status.save()
logger.info('Done: {0}'.format(message))
def _run_plugin(name, today):
logger.info('Running {0}...'.format(name))
success, message = False, None
sync_status = SyncStatus.objects.get_or_create(plugin=name, date=today)[0]
try:
success, message = plugin.run(
'scrooge',
name,
today=today,
)
if not success:
raise PluginError(message)
except Exception as e:
logger.exception("{0}: {1}".format(name, e))
raise PluginError(e)
finally:
sync_status.success = success
sync_status.remarks = message
sync_status.save()
logger.info('Done: {0}'.format(message))
def _run_plugin(context,
chain,
plugin_name,
requirements,
interactive,
done_requirements,
outputs=None):
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
stderr = output.get(interactive, err=True)
message = "[{}] {}... ".format(plugin_name, _get_uid(context))
stdout(message, end='')
new_context = {}
try:
is_up, message, new_context = plugin.run(chain, plugin_name, **context)
except plugin.Restart as e:
stdout('needs to be restarted: {}'.format(unicode(e)))
raise
except Exception:
stdout('', end='\r')
stderr(
"{}\nException in plugin '{}' for '{}'.".format(
traceback.format_exc(),
plugin_name,
_get_uid(context),
),
end='\n',
)
raise
else:
if message:
stdout(message, verbose=not is_up)
if is_up:
requirements.add(plugin_name)
context['successful_plugins'] = ', '.join(sorted(requirements))
context.update(new_context)
finally:
done_requirements.add(plugin_name)
def _get_usage_type_costs(self, date, usage_type, forecast,
service_environments):
"""
Calculates total cost of usage of given type for specified
service environments in period of time (between start and end), using
real or forecast price/cost.
"""
try:
return plugin_runner.run(
'scrooge_costs',
usage_type.get_plugin_name(),
type='total_cost',
date=date,
usage_type=usage_type,
forecast=forecast,
service_environments=service_environments,
)
except (KeyError, AttributeError):
logger.warning('Invalid call for {0} total cost'.format(
usage_type.name))
return D(0)
def _get_schema(cls):
"""
Use plugins to get full schema for report
:returns dict: Complete schema for all columns in report
:rtype dict:
"""
logger.debug("Getting schema for report")
header = []
for plugin in cls.get_plugins():
try:
plugin_headers = plugin_runner.run('scrooge_reports',
plugin.plugin_name,
type=cls.schema_name,
**plugin.get(
'plugin_kwargs', {}))
header.append(plugin_headers)
except KeyError:
logger.warning("Usage '{0}' has no schema plugin".format(
plugin.name))
return header
def _get_usage_type_cost(self, start, end, usage_type, forecast, ventures):
"""
Calculates total cost of usage of given type for specified ventures in
period of time (between start and end), using real or forecast
price/cost.
"""
try:
return plugin_runner.run(
'reports',
usage_type.get_plugin_name(),
type='total_cost',
start=start,
end=end,
usage_type=usage_type,
forecast=forecast,
ventures=ventures,
)
except (KeyError, AttributeError):
logger.warning('Invalid call for {0} total cost'.format(
usage_type.name))
return D(0)
def run_deployment(deployment):
if deployment.is_running:
return
deployment.is_running = True
deployment.save()
try:
done = set(name.strip() for name in deployment.done_plugins.split(','))
tried = set(done)
while True:
plugins = plugin.next('deployment', done) - tried
if not plugins:
break
name = plugin.highest_priority('deployment', plugins)
tried.add(name)
if plugin.run('deployment', name, deployment_id=deployment.id):
done.add(name)
deployment.done_plugins = ', '.join(done)
deployment.save()
finally:
deployment.is_running = False
deployment.save()
def _get_usage_type_cost(self, start, end, usage_type, forecast, ventures):
"""
Calculates total cost of usage of given type for specified ventures in
period of time (between start and end), using real or forecast
price/cost.
"""
try:
return plugin_runner.run(
'reports',
usage_type.get_plugin_name(),
type='total_cost',
start=start,
end=end,
usage_type=usage_type,
forecast=forecast,
ventures=ventures,
)
except (KeyError, AttributeError):
logger.warning(
'Invalid call for {0} total cost'.format(usage_type.name)
)
return D(0)
def _run_plugin(context, chain, plugin_name, requirements, interactive, clear_down, done_requirements, outputs=None):
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
stdout_verbose = output.get(interactive, verbose=True)
stderr = output.get(interactive, err=True)
message = "[{}] {}... ".format(plugin_name, context["ip"])
pad = output.WIDTH - len(message)
stdout(message, end="")
try:
new_context = {}
is_up, message, new_context = plugin.run(chain, plugin_name, **context)
if is_up:
requirements.add(plugin_name)
except plugin.Restart as e:
stdout("needs to be restarted: {}".format(unicode(e)), end="\n")
raise
except Exception:
stdout("", end="\r")
stderr(
"Exception in plugin '{}' for '{}': {}".format(plugin_name, context["ip"], traceback.format_exc()), end="\n"
)
else:
message = message or ""
if clear_down and not is_up:
end = "\r"
else:
end = "\n"
pad -= len(message)
message = message + (" " * pad)
if is_up:
stdout(message, end=end)
else:
stdout_verbose(message, end=end)
finally:
done_requirements.add(plugin_name)
context.update(new_context)
def _get_schema(cls):
"""
Use plugins to get full schema for report
:returns dict: Complete schema for all columns in report
:rtype dict:
"""
logger.debug("Getting schema for report")
header = []
for plugin in cls.get_plugins():
try:
plugin_headers = plugin_runner.run(
'reports',
plugin.plugin_name,
type=cls.schema_name,
**plugin.get('plugin_kwargs', {})
)
header.append(plugin_headers)
except KeyError:
logger.warning(
"Usage '{0}' has no schema plugin".format(plugin.name)
)
return header
def run_deployment(deployment):
if deployment.is_running:
return
deployment.is_running = True
deployment.save()
try:
done = set(
name.strip() for
name in
deployment.done_plugins.split(',')
if name.strip()
)
tried = set(done)
while True:
plugins = plugin.next('deployment', done) - tried
if not plugins:
break
name = plugin.highest_priority('deployment', plugins)
tried.add(name)
try:
success = plugin.run(
'deployment',
name,
deployment_id=deployment.id,
)
except Exception:
traceback.print_exc()
else:
if success:
done.add(name)
# The plugins might have changed the deployment object.
deployment = Deployment.objects.get(id=deployment.id)
deployment.done_plugins = ', '.join(done)
deployment.save()
finally:
deployment.is_running = False
deployment.save()
def _run_plugin(context, chain, plugin_name, requirements, interactive,
done_requirements, outputs=None):
if outputs:
stdout, stdout_verbose, stderr = outputs
else:
stdout = output.get(interactive)
stderr = output.get(interactive, err=True)
message = "[{}] {}... ".format(plugin_name, _get_uid(context))
stdout(message, end='')
new_context = {}
try:
is_up, message, new_context = plugin.run(chain, plugin_name,
**context)
except plugin.Restart as e:
stdout('needs to be restarted: {}'.format(unicode(e)))
raise
except Exception:
stdout('', end='\r')
stderr(
"{}\nException in plugin '{}' for '{}'.".format(
traceback.format_exc(),
plugin_name,
_get_uid(context),
),
end='\n',
)
raise
else:
if message:
stdout(message, verbose=not is_up)
if is_up:
requirements.add(plugin_name)
context['successful_plugins'] = ', '.join(sorted(requirements))
context.update(new_context)
finally:
done_requirements.add(plugin_name)
def _get_report_data(
cls,
start,
end,
is_active,
forecast,
service_environments,
):
"""
Use plugins to get usages data for given ventures. Plugin logic can be
so complicated but for this method, plugin must return value in
format:
data_from_plugin = {
'venture_id': {
'field1_name': value,
'field2_name': value,
...
},
...
}
:param datatime start: Start of time interval for report
:param datatime end: End of time interval for report
:param boolean forecast: Forecast prices or real
:param list ventures: List of ventures for which data must be taken
:returns dict: Complete report data for all ventures
:rtype dict:
"""
logger.debug("Getting report date")
data = {se.id: {} for se in service_environments}
plugins = cls.get_plugins()
progress = 0
step = 100 / len(plugins)
for i, plugin in enumerate(plugins):
try:
logger.info('Calling plugin {} with base usage {}'.format(
plugin.plugin_name,
plugin.get('plugin_kwargs', {}).get('base_usage', '-'),
))
plugin_report = plugin_runner.run('scrooge_reports',
plugin.plugin_name,
start=start,
end=end,
forecast=forecast,
type='costs',
**plugin.get(
'plugin_kwargs', {}))
for service_id, service_usage in plugin_report.iteritems():
if service_id in data:
data[service_id].update(service_usage)
progress += step
yield progress, {}
except KeyError:
logger.warning("Usage '{0}' has no usage plugin".format(
plugin.name))
except Exception as e:
logger.exception(
"Error while generating the report: {0}".format(e))
raise
yield 100, data
def _get_report_data(
cls,
start,
end,
is_active,
forecast,
service_environments,
):
"""
Use plugins to get usages data for given ventures. Plugin logic can be
so complicated but for this method, plugin must return value in
format:
data_from_plugin = {
'venture_id': {
'field1_name': value,
'field2_name': value,
...
},
...
}
:param datatime start: Start of time interval for report
:param datatime end: End of time interval for report
:param boolean forecast: Forecast prices or real
:param list ventures: List of ventures for which data must be taken
:returns dict: Complete report data for all ventures
:rtype dict:
"""
logger.debug("Getting report date")
data = {se.id: {} for se in service_environments}
plugins = cls.get_plugins()
progress = 0
step = 100 / len(plugins)
for i, plugin in enumerate(plugins):
try:
logger.info('Calling plugin {} with base usage {}'.format(
plugin.plugin_name,
plugin.get('plugin_kwargs', {}).get('base_usage', '-'),
))
plugin_report = plugin_runner.run(
'scrooge_reports',
plugin.plugin_name,
start=start,
end=end,
forecast=forecast,
type='costs',
**plugin.get('plugin_kwargs', {})
)
for service_id, service_usage in plugin_report.iteritems():
if service_id in data:
data[service_id].update(service_usage)
progress += step
yield progress, {}
except KeyError:
logger.warning(
"Usage '{0}' has no usage plugin".format(plugin.name)
)
except Exception as e:
logger.exception(
"Error while generating the report: {0}".format(e)
)
raise
yield 100, data