def notify(t):
"""Decorates a fabric task"""
@wraps(t)
def wrapper(*args, **kwargs):
start = time.time()
try:
r = t(*args, **kwargs)
end = time.time()
duration = end - start
try:
dog_http_api.event("{0}".format(_task_details(t)),
"{0}({1}) ran for {2}.".format(_task_details(t), _format_args(args, kwargs), _human_duration(duration)),
source_type_name="fabric",
alert_type="success",
priority="normal",
aggregation_key=_task_details(t))
except Exception, e:
logger.warn("Datadog notification failed with {0} but task {1} completed".format(e, t.__name___))
return r
except Exception, e:
# If notification is on, create an error event
end = time.time()
duration = end - start
try:
dog_http_api.event("{0}".format(_task_details(t)),
"{0}({1}) failed after {2} because of {3}.".format(_task_details(t), _format_args(args, kwargs), _human_duration(duration), e),
source_type_name="fabric",
alert_type="error",
priority="normal",
aggregation_key=_task_details(t))
except Exception, e:
logger.warn("Datadog notification failed with {0} and task {1} failed".format(e, t.__name___))
def process_exception(self, request, exception):
""" Captures Django view exceptions as Datadog events """
# Get a formatted version of the traceback.
exc = traceback.format_exc()
# Make request.META json-serializable.
szble = {}
for k, v in request.META.items():
if isinstance(v, (list, basestring, bool, int, float, long)):
szble[k] = v
else:
szble[k] = str(v)
title = 'Exception from {0}'.format(request.path)
text = "Traceback:\n@@@\n{0}\n@@@\nMetadata:\n@@@\n{1}\n@@@" \
.format(exc, json.dumps(szble, indent=2))
# Submit the exception to Datadog
api.event(title, text, tags=self.event_tags, aggregation_key=request.path,
alert_type='error')
# Increment our errors metric
tags = self._get_metric_tags(request)
dog_stats_api.increment(self.error_metric, tags=tags)
def wrapper(*args, **kwargs):
start = time.time()
error = None
output = None
try:
r = t(*args, **kwargs)
if r:
if not isinstance(r, list):
r = [r]
output = '\n\n'.join(['%s\n%s\n%s' %
(res.command, res.stdout, res.stderr) for res in r]
)
except Exception as e:
error = e
end = time.time()
duration = end - start
try:
dog_http_api.event(_title(t, args, kwargs, error),
_text(t, args, kwargs, duration, output, error),
source_type_name="fabric",
alert_type="error" if error else "success",
priority="normal",
aggregation_key=_aggregation_key(t, args, kwargs, error),
tags=_tags(t, args, kwargs, error))
except Exception as e:
logger.warn("Datadog notification on task {0} failed with {1}".format(t.__name__, e))
if error:
raise error
else:
return r
def main():
parser = OptionParser()
parser.add_option('-n', '--name', action='store', type='string')
parser.add_option('-k', '--api_key', action='store', type='string')
parser.add_option('-m', '--submit_mode', action='store', type='choice', default='errors', choices=['errors', 'all'])
parser.add_option('-t', '--timeout', action='store', type='int', default=60*60*24)
parser.add_option('--sigterm_timeout', action='store', type='int', default=60*2)
parser.add_option('--sigkill_timeout', action='store', type='int', default=60)
parser.add_option('--proc_poll_interval', action='store', type='float', default=0.5)
options, args = parser.parse_args()
dog.api_key = options.api_key
cmd = []
for part in args:
cmd.extend(part.split(' '))
returncode, stdout, stderr, duration = execute(cmd, options.timeout,
options.sigterm_timeout, options.sigkill_timeout,
options.proc_poll_interval)
host = get_ec2_instance_id()
if returncode == 0:
alert_type = 'success'
event_title = u'[%s] %s succeeded in %.2fs' % (host, options.name,
duration)
elif returncode is Timeout:
alert_type = 'error'
event_title = u'[%s] %s timed out after %.2fs' % (host, options.name,
duration)
returncode = -1
else:
alert_type = 'error'
event_title = u'[%s] %s failed in %.2fs' % (host, options.name,
duration)
event_body = [u'%%%\n',
u'commmand:\n```\n', u' '.join(cmd), u'\n```\n',
u'exit code: %s\n\n' % returncode,
]
if stdout:
event_body.extend([u'stdout:\n```\n', stdout, u'\n```\n'])
if stderr:
event_body.extend([u'stderr:\n```\n', stderr, u'\n```\n'])
event_body.append(u'%%%\n')
event_body = u''.join(event_body)
event = {
'alert_type': alert_type,
'aggregation_key': options.name,
'host': host,
}
print >> sys.stderr, stderr.strip()
print >> sys.stdout, stdout.strip()
if options.submit_mode == 'all' or returncode != 0:
dog.event(event_title, event_body, **event)
sys.exit(returncode)
def import_pearson():
mode = 'import'
try:
sftp(settings.PEARSON['SFTP_IMPORT'], settings.PEARSON['LOCAL_IMPORT'], mode, deleteAfterCopy=True)
s3(settings.PEARSON['LOCAL_IMPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=False)
except Exception as e:
dog_http_api.event('Pearson Import failure', str(e))
raise e
else:
for filename in os.listdir(settings.PEARSON['LOCAL_IMPORT']):
filepath = os.path.join(settings.PEARSON['LOCAL_IMPORT'], filename)
call_command('pearson_import_conf_zip', filepath)
os.remove(filepath)
def s3(files_from, bucket, mode, deleteAfterCopy=False):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'):
try:
for filename in os.listdir(files_from):
source_file = os.path.join(files_from, filename)
# use mode as name of directory into which to write files
dest_file = os.path.join(mode, filename)
upload_file_to_s3(bucket, source_file, dest_file)
if deleteAfterCopy:
os.remove(files_from + '/' + filename)
except:
dog_http_api.event('pearson {0}'.format(mode),
's3 archiving failed')
raise
def wrapper(*args, **kwargs):
start = time.time()
try:
r = t(*args, **kwargs)
end = time.time()
duration = end - start
try:
dog_http_api.event("{0}".format(_task_details(t)),
"{0}({1}) ran for {2}.".format(_task_details(t), _format_args(args, kwargs), _human_duration(duration)),
source_type_name="fabric",
alert_type="success",
priority="normal",
aggregation_key=_task_details(t))
except Exception, e:
logger.warn("Datadog notification failed with {0} but task {1} completed".format(e, t.__name___))
return r
def wrapper(*args, **kwargs):
notify_datadog = True
if type(t) != WrappedCallableTask:
logger.warn("@notify decorator only works on a new-style Fabric Task")
notify_datadog = False
start = time.time()
try:
r = t(*args, **kwargs)
end = time.time()
duration = end - start
if notify_datadog:
try:
task_full_name = "%s.%s" % (t.__module__, t.wrapped.func_name)
dog_http_api.event("{0}".format(task_full_name),
"{0} ran for {1}.".format(task_full_name, human_duration(duration)),
source_type_name="fabric",
alert_type="success",
priority="normal",
aggregation_key=task_full_name)
except:
logger.warn("Datadog notification failed but task {0} completed".format(t.wrapped.func_name))
return r
except Exception as e:
# If notification is on, create an error event
end = time.time()
duration = end - start
if notify_datadog:
try:
task_full_name = "%s.%s" % (t.__module__, t.wrapped.func_name)
dog_http_api.event("{0}".format(task_full_name),
"{0} failed after {1} because of {2}.".format(task_full_name, human_duration(duration), e),
source_type_name="fabric",
alert_type="error",
priority="normal",
aggregation_key=task_full_name)
except:
logger.exception("Datadog notification failed")
# Reraise
raise
def sftp(files_from, files_to, mode, deleteAfterCopy=False):
with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'):
try:
t = paramiko.Transport((
settings.PEARSON['SFTP_HOSTNAME'], 22))
t.connect(username=settings.PEARSON['SFTP_USERNAME'],
password=settings.PEARSON['SFTP_PASSWORD'])
sftp = paramiko.SFTPClient.from_transport(t)
if mode == 'export':
try:
sftp.chdir(files_to)
except IOError:
raise CommandError(
'SFTP destination path does not exist: {}'.format(files_to))
for filename in os.listdir(files_from):
sftp.put(files_from + '/' + filename, filename)
if deleteAfterCopy:
os.remove(os.path.join(files_from, filename))
else:
try:
sftp.chdir(files_from)
except IOError:
raise CommandError(
'SFTP source path does not exist: {}'.format(files_from))
for filename in sftp.listdir('.'):
# skip subdirectories
if not S_ISDIR(sftp.stat(filename).st_mode):
sftp.get(filename, files_to + '/' + filename)
# delete files from sftp server once they are
# successfully pulled off:
if deleteAfterCopy:
sftp.remove(filename)
except:
dog_http_api.event('pearson {0}'.format(mode),
'sftp uploading failed',
alert_type='error')
raise
finally:
sftp.close()
t.close()
def notify(t):
"""Decorates a fabric task"""
@wraps(t)
def wrapper(*args, **kwargs):
notify_datadog = True
if type(t) != WrappedCallableTask:
logger.warn("@notify decorator only works on a new-style Fabric Task")
notify_datadog = False
start = time.time()
try:
r = t(*args, **kwargs)
end = time.time()
duration = end - start
if notify_datadog:
try:
dog_http_api.event("{0}".format(_task_details(t)),
"{0}({1}) ran for {2}.".format(_task_details(t), _format_args(args, kwargs), _human_duration(duration)),
source_type_name="fabric",
alert_type="success",
priority="normal",
aggregation_key=_task_details(t))
except Exception, e:
logger.warn("Datadog notification failed with {0} but task {1} completed".format(e, t.wrapped.func_name))
return r
except Exception, e:
# If notification is on, create an error event
end = time.time()
duration = end - start
if notify_datadog:
try:
dog_http_api.event("{0}".format(_task_details(t)),
"{0}({1}) failed after {2} because of {3}.".format(_task_details(t), _format_args(args, kwargs), _human_duration(duration), e),
source_type_name="fabric",
alert_type="error",
priority="normal",
aggregation_key=_task_details(t))
except Exception, e:
logger.warn("Datadog notification failed with {0} and task {1} failed".format(e, t.wrapped.func_name))
def notify(t):
"""Decorates a fabric task"""
@wraps(t)
def wrapper(*args, **kwargs):
start = time.time()
error = None
try:
r = t(*args, **kwargs)
except Exception, e:
error = e
end = time.time()
duration = end - start
try:
dog_http_api.event(_title(t, args, kwargs, error),
_text(t, args, kwargs, duration, error),
source_type_name="fabric",
alert_type="error" if error else "success",
priority="normal",
aggregation_key=_aggregation_key(t, args, kwargs, error),
tags=_tags(t, args, kwargs, error))
except Exception, e:
logger.warn("Datadog notification on task {0} failed with {1}".format(t.__name___, e))
def datadog_error(string, tags):
dog_http_api.event("Pearson Import", string, alert_type='error', tags=[tags])
from dogapi import dog_http_api as dog
if 'DATADOG_API_KEY' not in os.environ:
print >> sys.stderr, 'DATADOG_API_KEY environment variable not set'
sys.exit(1)
else:
dog.api_key = os.environ['DATADOG_API_KEY']
parser = OptionParser(usage="usage: %prog [options] /path/to/local_git_dir")
parser.add_option("-r", "--remote", action="store", dest="remote", help="Git remote to pull from", default="origin")
parser.add_option("-b", "--branch", action="store", dest="branch", help="Git branch to pull from", default="master")
options, args = parser.parse_args()
if len(args) < 1:
parser.print_help()
sys.exit(1)
else:
local_repo_dir = args[0]
# Execute the git pull command
git_cmd = ['git', 'pull', options.remote, options.branch]
proc = subprocess.Popen(git_cmd, cwd=local_repo_dir, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
git_message, _ = proc.communicate()
if 'Already up-to-date.' not in git_message:
# Only submit events if there were changes
print git_message
dog.event("Git Changes", git_message)
# import the simple dog client
from dogapi import dog_http_api as dog
# give dog your credentials (we're using os.environ to let you experiment via environment variables)
# in this example we're only reporting data, so we only need an API key
# see: https://github.com/DataDog/dogapi/wiki/Authentication for more on API authentication
dog.api_key = os.environ.get("DATADOG_API_KEY")
# emit points one by one. timestamp is determined at call time.
dog.metric('test.api.test_metric', 4.0, host="some_host")
time.sleep(1)
dog.metric('test.api.test_metric', 5.0, host="another_host")
# emit a list of points in one go as a list of (timestamp, value)
# here we pretend to send a point a minute for the past hour
now = dt.now()
points = []
# create the list here (a list comprehension would do too)
for i in range(60, 1, -1):
t = time.mktime((now - delta(minutes=i)).timetuple())
points.append((t, math.cos(i) + 1.0))
# and emit the data in one call
dog.metric('test.api.test_metric', points, host="some_other_host")
# send an event too
dog.event("API Testing", "Testing done, FTW")
def datadog_error(string, tags):
dog_http_api.event("Pearson Import",
string,
alert_type='error',
tags=[tags])
def main():
parser = OptionParser()
parser.add_option('-n', '--name', action='store', type='string', help="The name of the event")
parser.add_option('-k', '--api_key', action='store', type='string')
parser.add_option('-m', '--submit_mode', action='store', type='choice', default='errors', choices=['errors', 'all'])
parser.add_option('-t', '--timeout', action='store', type='int', default=60*60*24)
parser.add_option('--sigterm_timeout', action='store', type='int', default=60*2)
parser.add_option('--sigkill_timeout', action='store', type='int', default=60)
parser.add_option('--proc_poll_interval', action='store', type='float', default=0.5)
parser.add_option('--notify_success', action='store', type='string', default='')
parser.add_option('--notify_error', action='store', type='string', default='')
options, args = parser.parse_args()
dog.api_key = options.api_key
cmd = []
for part in args:
cmd.extend(part.split(' '))
returncode, stdout, stderr, duration = execute(cmd, options.timeout,
options.sigterm_timeout, options.sigkill_timeout,
options.proc_poll_interval)
host = get_ec2_instance_id()
if returncode == 0:
alert_type = SUCCESS
event_title = u'[%s] %s succeeded in %.2fs' % (host, options.name,
duration)
elif returncode is Timeout:
alert_type = ERROR
event_title = u'[%s] %s timed out after %.2fs' % (host, options.name,
duration)
returncode = -1
else:
alert_type = ERROR
event_title = u'[%s] %s failed in %.2fs' % (host, options.name,
duration)
event_body = [u'%%%\n',
u'commmand:\n```\n', u' '.join(cmd), u'\n```\n',
u'exit code: %s\n\n' % returncode,
]
if stdout:
event_body.extend([u'stdout:\n```\n', stdout, u'\n```\n'])
if stderr:
event_body.extend([u'stderr:\n```\n', stderr, u'\n```\n'])
notifications = ""
if alert_type == SUCCESS and options.notify_success:
notifications = options.notify_success
elif alert_type == ERROR and options.notify_error:
notifications = options.notify_error
if notifications:
event_body.extend([u'notifications: %s\n' % (notifications)])
event_body.append(u'%%%\n')
# ensure all strings are parsed as utf-8
event_body = [x.decode('utf-8') for x in event_body]
event_body = u''.join(event_body)
event = {
'alert_type': alert_type,
'aggregation_key': options.name,
'host': host,
}
print >> sys.stderr, stderr.strip()
print >> sys.stdout, stdout.strip()
if options.submit_mode == 'all' or returncode != 0:
dog.event(event_title, event_body, **event)
sys.exit(returncode)
