def pingOnError(ctx, error):
# if not dict yet, i.e. before the cli.core.cli_core group
# Important to keep track of the didPing variable
if ctx.obj is None: return
# check if the error's ping was done
didPing = 'unhandled_error_pinged' in ctx.obj.keys()
if didPing: return
# send to sentry.io via isitfit.io (check usage of sentry_proxy in cli.core)
from sentry_sdk import capture_exception
capture_exception(error)
# proceed to ping matomo about the error (to be deprecated in full in favor of sentry)
from isitfit.utils import ping_matomo
exception_type = type(error).__name__ # https://techeplanet.com/python-catch-all-exceptions/
exception_str = ""
try:
exception_str = str(error)
except:
pass
ping_matomo("/error/unhandled/%s?message=%s"%(exception_type, exception_str))
# save a flag saying that the error sent a ping
# Note that it is not necessary to do more than that, such as storing a list of pinged errors,
# because there will be exactly one error raise at most before the program fails
ctx.obj['unhandled_error_pinged'] = True
def analyze(ctx, filter_tags, save_details):
# gather anonymous usage statistics
ping_matomo("/cost/analyze?filter_tags=%s&save_details=%s" %
(filter_tags, b2l(save_details)))
# save to click context
share_email = ctx.obj.get('share_email', [])
#logger.info("Is it fit?")
logger.info("Initializing...")
# set up pipelines for ec2, redshift, and aggregator
from isitfit.cost import ec2_cost_analyze, redshift_cost_analyze, account_cost_analyze
mm_eca = ec2_cost_analyze(ctx, filter_tags, save_details)
mm_rca = redshift_cost_analyze(share_email,
filter_region=ctx.obj['filter_region'],
ctx=ctx,
filter_tags=filter_tags)
# combine the 2 pipelines into a new pipeline
mm_all = account_cost_analyze(mm_eca, mm_rca, ctx, share_email)
# configure tqdm
from isitfit.tqdmman import TqdmL2Quiet
tqdml2 = TqdmL2Quiet(ctx)
# Run pipeline
mm_all.get_ifi(tqdml2)
def is_configured(self):
from isitfit.utils import ping_matomo
# check not None and not empty string
if os.getenv('DATADOG_API_KEY', None):
if os.getenv('DATADOG_APP_KEY', None):
if self.print_configured:
logger.info("Datadog env vars available")
ping_matomo("/cost/setting?datadog.is_configured=True")
self.print_configured = False
return True
if self.print_configured:
logger.info(
"Datadog env vars missing. Set DATADOG_API_KEY and DATADOG_APP_KEY to get memory data from Datadog."
)
ping_matomo("/cost/setting?datadog.is_configured=False")
import click
display_msg = lambda x: click.secho(x, fg='yellow')
display_msg(
"Note: without the datadog integration, memory metrics are missing, thus only CPU is used, which is not representative for memory-bound applications."
)
display_msg(
"If you gather memory metrics using another provider than datadog, please get in touch at https://www.autofitcloud.com/contact"
)
self.print_configured = False
return False
def count(self):
# method 1
# ec2_it = self.ec2_resource.instances.all()
# return len(list(ec2_it))
if self.n_entry is not None:
return self.n_entry
self.n_entry = len(list(self.iterate_core(True)))
# interim result for timer data to calculate performance (seconds per ec2 or seconds per rds)
from isitfit.utils import ping_matomo
ping_matomo(
"/cost/base_iterator/BaseIterator/count?service=%s&n_entry=%s&n_region=%s"
% (self.service_name, self.n_entry, len(self.region_include)))
# send message to logs for info
if self.n_entry == 0 and len(self.region_include) == 0:
msg_count = "Found no %s"
logger.info(msg_count % (self.service_description))
else:
msg_count = "Found a total of %i %s in %i region(s) (other regions do not hold any %s)"
logger.info(msg_count %
(self.n_entry, self.service_description,
len(self.region_include), self.service_name))
return self.n_entry
def optimize(ctx, n, filter_tags, allow_ec2_different_family):
# gather anonymous usage statistics
ping_matomo(
"/cost/optimize?n=%i&filter_tags=%s&allow_ec2_different_family=%s" %
(n, filter_tags, b2l(allow_ec2_different_family)))
# save to context
share_email = ctx.obj.get('share_email', [])
ctx.obj['allow_ec2_different_family'] = allow_ec2_different_family
#logger.info("Is it fit?")
logger.info("Initializing...")
from isitfit.cost import ec2_cost_optimize, redshift_cost_optimize, account_cost_optimize
mm_eco = ec2_cost_optimize(ctx, n, filter_tags)
mm_rco = redshift_cost_optimize(filter_region=ctx.obj['filter_region'],
ctx=ctx,
filter_tags=filter_tags)
# merge and run pipelines
mm_all = account_cost_optimize(mm_eco, mm_rco, ctx)
# configure tqdm
from isitfit.tqdmman import TqdmL2Quiet
tqdml2 = TqdmL2Quiet(ctx)
# Run pipeline
mm_all.get_ifi(tqdml2)
def version():
# gather anonymous usage statistics
from isitfit.utils import ping_matomo
ping_matomo("/version")
version_core()
return
def show(self, file=None):
# ping matomo about error
from isitfit.utils import ping_matomo
ping_matomo("/error?message=%s"%self.message)
# continue
from click._compat import get_text_stderr
if file is None:
file = get_text_stderr()
# echo wrap
color = 'red'
def wrapecho(message):
# from click.utils import echo
# echo('Error: %s' % self.format_message(), file=file, color=color)
click.secho(message, fg=color)
# main error
wrapecho('Error: %s' % self.format_message())
# if error from terminal during execution (not on program boot)
if self.ctx is not None:
if self.ctx.obj is not None:
# if isitfit installation is outdated, append a message to upgrade
if self.ctx.obj.get('is_outdated', None):
hint_1 = "Upgrade your isitfit installation with `pip3 install --upgrade isitfit` and try again."
wrapecho(hint_1)
# test that boto3 minimum command can run
# This would fail for example for: `AWS_ACCESS_KEY_ID=wrong AWS_SECRET_ACCESS_KEY=alsowrong aws iam get-user`
import boto3
iam_client = boto3.client('iam')
try:
# response = iam_client.get_user()
iam_client.get_user()
except Exception as e:
msg_e = str(e)
hint_3 = f"""Hint: The command `aws iam get-user` has also failed with the following error:
{msg_e}
This might indicate a problem with your aws user's permissions and could be related to the current error in isitfit."""
# Update 2020-01-09 Instead of raising an exception, just display a warning
#from isitfit.cli.click_descendents import IsitfitCliError
#raise IsitfitCliError(hint_3) from e
# ping matomo about warning
from isitfit.utils import ping_matomo
ping_matomo("/warning/aws-iam-get-user?message=%s"%hint_3)
# display on screen
wrapecho(hint_3)
# add link to github issues
hint_2 = "Is this my fault? 😞 Please report it at https://github.com/autofitcloud/isitfit/issues/new or reach out to me at [email protected]"
wrapecho(hint_2)
def pipeline_factory(mm_eca, mm_rca, ctx, share_email):
"""
Combines the 2 pipelines from EC2 and Redshift
mm_eca - pipeline of EC2 cost analyze
mm_rca - pipeline of Redshift cost analyze
ctx - click context
share_email - list of emails or None
"""
from isitfit.cost.mainManager import RunnerAccount
mm_all = RunnerAccount("AWS cost analyze (EC2, Redshift) in all regions",
ctx)
service_iterator = ServiceIterator(mm_eca, mm_rca)
mm_all.set_iterator(service_iterator)
# ping matomo at start of calculation
from isitfit.utils import ping_matomo
inject_timer_start = lambda context_pre: context_pre if ping_matomo(
"/cost/analyze/account/start") else context_pre
mm_all.add_listener('pre', inject_timer_start)
service_calculator_get = ServiceCalculatorGet()
mm_all.add_listener('ec2', service_calculator_get.per_service)
service_calculator_save = ServiceCalculatorSave()
mm_all.add_listener('ec2', service_calculator_save.per_service)
service_calculator_binned = ServiceCalculatorBinned()
mm_all.add_listener('ec2', service_calculator_binned.per_service)
mm_all.add_listener('all', service_calculator_binned.after_all)
# update dict and return it
# https://stackoverflow.com/a/1453013/4126114
# inject_analyzer = lambda context_all: dict({'analyzer': service_calculator_save}, **context_all)
# inject_analyzer = lambda context_all: dict({'calculator_binned': service_calculator_binned}, **context_all)
# mm_all.add_listener('all', inject_analyzer)
#
# service_reporter = ServiceReporterTotals()
# service_reporter.emailTo = share_email
# mm_all.add_listener('all', service_reporter.postprocess)
# mm_all.add_listener('all', service_reporter.display)
# mm_all.add_listener('all', service_reporter.email)
# ping matomo at end of calculation
inject_timer_end = lambda context_all: context_all if ping_matomo(
"/cost/analyze/account/end") else context_all
mm_all.add_listener('all', inject_timer_end)
# display and email
service_reporter = ServiceReporterBinned()
service_reporter.emailTo = share_email
mm_all.add_listener('all', service_reporter.display)
mm_all.add_listener('all', service_reporter.email)
# done
return mm_all
def tags(profile):
# FIXME click bug: `isitfit command subcommand --help` is calling the code in here. Workaround is to check --help and skip the whole section
import sys
if '--help' in sys.argv: return
# gather anonymous usage statistics
from isitfit.utils import ping_matomo
ping_matomo("/tags")
pass
def dump(ctx):
# gather anonymous usage statistics
from isitfit.utils import ping_matomo
ping_matomo("/tags/dump")
from ..tags.tagsDump import TagsDump
tl = TagsDump(ctx)
tl.fetch()
tl.suggest() # not really suggesting. Just dumping to csv
tl.display()
def migrate(ctx, not_dry_run):
# usage stats
from isitfit.utils import ping_matomo, b2l
ping_matomo("/migrations/migrate?not_dry_run=%s"%b2l(not_dry_run))
migman = ctx.obj['migman']
migman.not_dry_run = not_dry_run
migman.migrate_all()
if not not_dry_run:
click.echo("")
click.secho("This was a simulated execution", fg="yellow")
click.secho("Repeat using `isitfit migrations migrate --not-dry-run` for actual execution", fg='yellow')
def cost(ctx, filter_region, ndays, profile):
# FIXME click bug: `isitfit command subcommand --help` is calling the code in here. Workaround is to check --help and skip the whole section
import sys
if '--help' in sys.argv: return
# gather anonymous usage statistics
ping_matomo("/cost?filter_region=%s&ndays=%i" % (filter_region, ndays))
# save to click context
ctx.obj['ndays'] = ndays
ctx.obj['filter_region'] = filter_region
pass
def show(ctx):
# usage stats
from isitfit.utils import ping_matomo
ping_matomo("/migrations/show")
migman = ctx.obj['migman']
if migman.df_mig.shape[0]==0:
click.echo("No pending migrations")
else:
click.echo("Pending migrations")
click.echo(migman.df_mig[['migname', 'description']])
click.echo("")
click.secho("Use `isitfit migrations migrate` to execute them", fg="yellow")
def handle_pre(self, context_pre):
from isitfit.utils import ping_matomo
# set up caching if requested
self.fetch_envvars()
if self.isSetup():
self.connect()
ping_matomo("/cost/setting?redis.is_configured=True")
return context_pre
# 0th pass to count
n_ec2_total = context_pre['n_ec2_total']
ping_matomo("/cost/setting?redis.is_configured=False")
# if more than 10 servers, recommend caching with redis
cond_prompt = n_ec2_total > 10 and not self.isSetup()
if cond_prompt:
from termcolor import colored
logger.warning(
colored(
"""Since the number of EC2 instances is %i,
it is recommended to use redis for caching of downloaded CPU/memory metrics.
To do so
- install redis
[sudo] apt-get install redis-server
- export environment variables
export ISITFIT_REDIS_HOST=localhost
export ISITFIT_REDIS_PORT=6379
export ISITFIT_REDIS_DB=0
where ISITFIT_REDIS_DB is the ID of an unused database in redis.
And finally re-run isitfit as usual.
""" % n_ec2_total, "yellow"))
import click
# not using abort=True so that I can send a custom message in the abort
continue_wo_redis = click.confirm(colored(
'Would you like to continue without redis caching? ', 'cyan'),
abort=False,
default=True)
if not continue_wo_redis:
from isitfit.cli.click_descendents import IsitfitCliError
raise IsitfitCliError("Aborting to set up redis.",
context_pre['click_ctx'])
# done
return context_pre
def migrations(ctx):
# FIXME click bug: `isitfit command subcommand --help` is calling the code in here. Workaround is to check --help and skip the whole section
import sys
if '--help' in sys.argv: return
# usage stats
from isitfit.utils import ping_matomo
ping_matomo("/migrations")
from isitfit.migrations.migman import MigMan
migman = MigMan()
migman.connect()
migman.read()
ctx.obj['migman'] = migman
def push(ctx, csv_filename, not_dry_run):
# gather anonymous usage statistics
from isitfit.utils import ping_matomo, b2l
ping_matomo("/tags/push?csv_filename=%s¬_dry_run=%s" %
(csv_filename, b2l(not_dry_run)))
from ..tags.tagsPush import TagsPush
tp = TagsPush(csv_filename, ctx)
tp.read_csv()
tp.validateTagsFile()
tp.pullLatest()
tp.diffLatest()
tp.processPush(not not_dry_run)
def datadog(ctx):
# FIXME click bug: `isitfit command subcommand --help` is calling the code in here. Workaround is to check --help and skip the whole section
import sys
if '--help' in sys.argv: return
# usage stats
from isitfit.utils import ping_matomo
ping_matomo("/datadog")
# manager of redis-pandas caching
from isitfit.cost.cacheManager import RedisPandas as RedisPandasCacheManager
cache_man = RedisPandasCacheManager()
from isitfit.cost.metrics_datadog import DatadogCached
ctx.obj['ddg'] = DatadogCached(cache_man)
def suggest(ctx, advanced):
# gather anonymous usage statistics
from isitfit.utils import ping_matomo, b2l
ping_matomo("/tags/suggest?advanced=%s" % b2l(advanced))
tl = None
if not advanced:
from ..tags.tagsSuggestBasic import TagsSuggestBasic
tl = TagsSuggestBasic(ctx)
else:
from ..tags.tagsSuggestAdvanced import TagsSuggestAdvanced
tl = TagsSuggestAdvanced(ctx)
tl.prepare()
tl.fetch()
tl.suggest()
tl.display()
def pipeline_factory(mm_eco, mm_rco, ctx):
from isitfit.cost.mainManager import RunnerAccount
mm_all = RunnerAccount("AWS cost optimize (EC2, Redshift) in all regions",
ctx)
# add listener that checks the local sqlite database for a previous calculation
# and display if available, then prompt user if desires to re-calculate
# Note that if no desire to re-calculate, this raises an exception that bubbles up into get_ifi and aborts the pipeline early
sqlite_man = SqliteMan(ctx)
# Update 2019-12-27 will not display the database table ATM, in favor of using it in the interactive command
# mm_all.add_listener('pre', sqlite_man.read_sqlite)
# set up a pipeline that fetches fresh data
from .account_cost_analyze import ServiceIterator, ServiceCalculatorGet
iterator = ServiceIterator(mm_eco, mm_rco)
mm_all.set_iterator(iterator)
# ping matomo at start of calculation
from isitfit.utils import ping_matomo
inject_timer_start = lambda context_pre: context_pre if ping_matomo(
"/cost/optimize/account/start") else context_pre
mm_all.add_listener('pre', inject_timer_start)
calculator_get = ServiceCalculatorGet()
mm_all.add_listener('ec2', calculator_get.per_service)
aggregator = ServiceAggregator()
mm_all.add_listener('ec2', aggregator.per_service_save)
mm_all.add_listener('all', aggregator.concat)
mm_all.add_listener('all', sqlite_man.update_dtCreated)
# ping matomo at end of calculation
inject_timer_end = lambda context_all: context_all if ping_matomo(
"/cost/optimize/account/end") else context_all
mm_all.add_listener('all', inject_timer_end)
# whether reading from sqlite or fresh data, display
reporter = ServiceReporter()
#mm_all.add_listener('all', reporter.display)
mm_all.add_listener('all', reporter.display2)
# done
return mm_all
def cli_core(ctx, debug, verbose, optimize, version, share_email,
skip_check_upgrade, skip_prompt_email):
# FIXME click bug: `isitfit cost --help` is calling the code in here. Workaround is to check --help
import sys
if '--help' in sys.argv: return
# make sure that context is a dict
ctx.ensure_object(dict)
# set up exception aggregation in sentry.io
from isitfit import sentry_proxy
from isitfit.apiMan import BASE_URL
sp_url = f"{BASE_URL}fwd/sentry"
sentry_proxy.init(dsn=sp_url)
# test exception caught by sentry. FIXME Dont commit this! :D
# 1/0
# usage stats
# https://docs.python.org/3.5/library/string.html#format-string-syntax
from isitfit.utils import ping_matomo, b2l
ping_url = "/?debug={}&verbose={}&share_email={}&skip_check_upgrade={}"
ping_url = ping_url.format(b2l(debug), b2l(verbose),
b2l(len(share_email) > 0),
b2l(skip_check_upgrade))
ping_matomo(ping_url)
# choose log level based on debug and verbose flags
import logging
logLevel = logging.DEBUG if debug else (
logging.INFO if verbose else logging.WARNING)
ch = logging.StreamHandler()
ch.setLevel(logLevel)
logger.addHandler(ch)
logger.setLevel(logLevel)
if debug:
logger.debug("Enabled debug level")
logger.debug("-------------------")
# After adding the separate command for "cost" (i.e. `isitfit cost analyze`)
# putting a note here to notify user of new usage
# Ideally, this code would be deprecated though
if ctx.invoked_subcommand is None:
# if still used without subcommands, notify user of new usage
#from .cost import analyze as cost_analyze, optimize as cost_optimize
#if optimize:
# ctx.invoke(cost_optimize, filter_tags=filter_tags, n=n)
#else:
# ctx.invoke(cost_analyze, filter_tags=filter_tags)
from click.exceptions import UsageError
if optimize:
err_msg = "As of version 0.11, please use `isitfit cost optimize` instead of `isitfit --optimize`."
ping_matomo("/error/UsageError?message=%s" % err_msg)
raise UsageError(err_msg)
elif version:
# ctx.invoke(cli_version)
err_msg = "As of version 0.11, please use `isitfit version` instead of `isitfit --version`."
ping_matomo("/error/UsageError?message=%s" % err_msg)
raise UsageError(err_msg)
else:
err_msg = "As of version 0.11, please use `isitfit cost analyze` instead of `isitfit` to calculate the cost-weighted utilization."
ping_matomo("/error/UsageError?message=%s" % err_msg)
raise UsageError(err_msg)
# check if emailing requested
if share_email is not None:
max_n_recipients = 3
if len(share_email) > max_n_recipients:
err_msg = "Maximum allowed number of email recipients is %i. Received %i" % (
max_n_recipients, len(share_email))
ping_matomo("/error?message=%s" % err_msg)
from click.exceptions import BadParameter
raise BadParameter(err_msg, param_hint="--share-email")
ctx.obj['share_email'] = share_email
# check if current version is out-of-date
if ctx.invoked_subcommand != 'version':
if not skip_check_upgrade:
from ..utils import prompt_upgrade
is_outdated = prompt_upgrade('isitfit', isitfit_version)
ctx.obj['is_outdated'] = is_outdated
if is_outdated:
ping_matomo("/version/prompt_upgrade?is_outdated=%s" %
b2l(is_outdated))
if ctx.invoked_subcommand not in ['version', 'migrations']:
# run silent migrations
from isitfit.migrations.migman import silent_migrate
migname_l = silent_migrate()
if len(migname_l) > 0:
from isitfit.utils import l2s
migname_s = l2s(migname_l)
ping_matomo("/migrations/silent?migname=%s" % (migname_s))
# save `verbose` and `debug` for later tqdm
ctx.obj['debug'] = debug
ctx.obj['verbose'] = verbose
# save skip-prompt-email for later usage
ctx.obj['skip_prompt_email'] = skip_prompt_email
def dump(ctx, date, aws_id):
# usage stats
from isitfit.utils import ping_matomo
ping_matomo("/datadog/dump")
ddgL1 = ctx.obj['ddg']
# get as dataframe, daily
df = ddgL1.get_metrics_all(aws_id)
# drop nhours column since useless here
del df['nhours']
print("Daily usage")
print(df)
print("")
# convert aws ID to datadog hostname
dd_hostname = ddgL1.map_aws_dd[aws_id]
# higher freq
# query language, check note above in get_metrics_cpu
SECONDS_PER_POINT = 60*10 # 60*60 # *24
import datetime as dt
date_str = date.strftime("%Y-%m-%d")
dt_start="%s 00:00:00"%date_str
dt_end="%s 23:59:59"%date_str
dt_start = dt.datetime.strptime(dt_start, "%Y-%m-%d %H:%M:%S")
dt_end = dt.datetime.strptime(dt_end, "%Y-%m-%d %H:%M:%S")
import time
conv2sec = lambda x: time.mktime(x.timetuple())
ue_start = conv2sec(dt_start)
ue_end = conv2sec(dt_end)
# query datadog, result as json
# https://docs.datadoghq.com/api/?lang=python#query-timeseries-points
#datadog_api.initialize()
#m = datadog_api.api.Metric.query(start=ue_start, end=ue_end, query=query)
#print(m)
# repeat as dataframe
from isitfit.cost.metrics_datadog import DatadogApiWrap
apiwrap = DatadogApiWrap()
df_all = []
metric_all = [
('system.cpu.idle', 'cpu_idle_min', 'system.cpu.idle{host:%s}.rollup(min,%i)'),
('system.mem.free', 'mem_free_min', 'system.mem.free{host:%s}.rollup(min,%i)'),
('system.cpu.idle', 'cpu_idle_max', 'system.cpu.idle{host:%s}.rollup(max,%i)'),
('system.mem.free', 'mem_free_max', 'system.mem.free{host:%s}.rollup(max,%i)')
]
for metric_name, col_name, query_t in metric_all:
query_v = query_t%(dd_hostname, SECONDS_PER_POINT)
df_i = apiwrap.metric_query(
dd_hostname=dd_hostname,
start=ue_start,
end=ue_end,
query=query_v,
metric_name=metric_name,
dfcol_name=col_name
)
df_i.set_index('ts_dt', inplace=True)
df_all.append(df_i)
# concat all
import pandas as pd
df_all = pd.concat(df_all, axis=1)
pd.set_option("display.max_rows", None)
print("Datadog details")
print(df_all)
