def notify(t): """Decorates a fabric task""" @wraps(t) def wrapper(*args, **kwargs): start = time.time() try: r = t(*args, **kwargs) end = time.time() duration = end - start try: dog_http_api.event("{0}".format(_task_details(t)), "{0}({1}) ran for {2}.".format(_task_details(t), _format_args(args, kwargs), _human_duration(duration)), source_type_name="fabric", alert_type="success", priority="normal", aggregation_key=_task_details(t)) except Exception, e: logger.warn("Datadog notification failed with {0} but task {1} completed".format(e, t.__name___)) return r except Exception, e: # If notification is on, create an error event end = time.time() duration = end - start try: dog_http_api.event("{0}".format(_task_details(t)), "{0}({1}) failed after {2} because of {3}.".format(_task_details(t), _format_args(args, kwargs), _human_duration(duration), e), source_type_name="fabric", alert_type="error", priority="normal", aggregation_key=_task_details(t)) except Exception, e: logger.warn("Datadog notification failed with {0} and task {1} failed".format(e, t.__name___))
def process_exception(self, request, exception): """ Captures Django view exceptions as Datadog events """ # Get a formatted version of the traceback. exc = traceback.format_exc() # Make request.META json-serializable. szble = {} for k, v in request.META.items(): if isinstance(v, (list, basestring, bool, int, float, long)): szble[k] = v else: szble[k] = str(v) title = 'Exception from {0}'.format(request.path) text = "Traceback:\n@@@\n{0}\n@@@\nMetadata:\n@@@\n{1}\n@@@" \ .format(exc, json.dumps(szble, indent=2)) # Submit the exception to Datadog api.event(title, text, tags=self.event_tags, aggregation_key=request.path, alert_type='error') # Increment our errors metric tags = self._get_metric_tags(request) dog_stats_api.increment(self.error_metric, tags=tags)
def wrapper(*args, **kwargs): start = time.time() error = None output = None try: r = t(*args, **kwargs) if r: if not isinstance(r, list): r = [r] output = '\n\n'.join(['%s\n%s\n%s' % (res.command, res.stdout, res.stderr) for res in r] ) except Exception as e: error = e end = time.time() duration = end - start try: dog_http_api.event(_title(t, args, kwargs, error), _text(t, args, kwargs, duration, output, error), source_type_name="fabric", alert_type="error" if error else "success", priority="normal", aggregation_key=_aggregation_key(t, args, kwargs, error), tags=_tags(t, args, kwargs, error)) except Exception as e: logger.warn("Datadog notification on task {0} failed with {1}".format(t.__name__, e)) if error: raise error else: return r
def main(): parser = OptionParser() parser.add_option('-n', '--name', action='store', type='string') parser.add_option('-k', '--api_key', action='store', type='string') parser.add_option('-m', '--submit_mode', action='store', type='choice', default='errors', choices=['errors', 'all']) parser.add_option('-t', '--timeout', action='store', type='int', default=60*60*24) parser.add_option('--sigterm_timeout', action='store', type='int', default=60*2) parser.add_option('--sigkill_timeout', action='store', type='int', default=60) parser.add_option('--proc_poll_interval', action='store', type='float', default=0.5) options, args = parser.parse_args() dog.api_key = options.api_key cmd = [] for part in args: cmd.extend(part.split(' ')) returncode, stdout, stderr, duration = execute(cmd, options.timeout, options.sigterm_timeout, options.sigkill_timeout, options.proc_poll_interval) host = get_ec2_instance_id() if returncode == 0: alert_type = 'success' event_title = u'[%s] %s succeeded in %.2fs' % (host, options.name, duration) elif returncode is Timeout: alert_type = 'error' event_title = u'[%s] %s timed out after %.2fs' % (host, options.name, duration) returncode = -1 else: alert_type = 'error' event_title = u'[%s] %s failed in %.2fs' % (host, options.name, duration) event_body = [u'%%%\n', u'commmand:\n```\n', u' '.join(cmd), u'\n```\n', u'exit code: %s\n\n' % returncode, ] if stdout: event_body.extend([u'stdout:\n```\n', stdout, u'\n```\n']) if stderr: event_body.extend([u'stderr:\n```\n', stderr, u'\n```\n']) event_body.append(u'%%%\n') event_body = u''.join(event_body) event = { 'alert_type': alert_type, 'aggregation_key': options.name, 'host': host, } print >> sys.stderr, stderr.strip() print >> sys.stdout, stdout.strip() if options.submit_mode == 'all' or returncode != 0: dog.event(event_title, event_body, **event) sys.exit(returncode)
def import_pearson(): mode = 'import' try: sftp(settings.PEARSON['SFTP_IMPORT'], settings.PEARSON['LOCAL_IMPORT'], mode, deleteAfterCopy=True) s3(settings.PEARSON['LOCAL_IMPORT'], settings.PEARSON['S3_BUCKET'], mode, deleteAfterCopy=False) except Exception as e: dog_http_api.event('Pearson Import failure', str(e)) raise e else: for filename in os.listdir(settings.PEARSON['LOCAL_IMPORT']): filepath = os.path.join(settings.PEARSON['LOCAL_IMPORT'], filename) call_command('pearson_import_conf_zip', filepath) os.remove(filepath)
def s3(files_from, bucket, mode, deleteAfterCopy=False): with dog_stats_api.timer('pearson.{0}'.format(mode), tags='s3'): try: for filename in os.listdir(files_from): source_file = os.path.join(files_from, filename) # use mode as name of directory into which to write files dest_file = os.path.join(mode, filename) upload_file_to_s3(bucket, source_file, dest_file) if deleteAfterCopy: os.remove(files_from + '/' + filename) except: dog_http_api.event('pearson {0}'.format(mode), 's3 archiving failed') raise
def wrapper(*args, **kwargs): start = time.time() try: r = t(*args, **kwargs) end = time.time() duration = end - start try: dog_http_api.event("{0}".format(_task_details(t)), "{0}({1}) ran for {2}.".format(_task_details(t), _format_args(args, kwargs), _human_duration(duration)), source_type_name="fabric", alert_type="success", priority="normal", aggregation_key=_task_details(t)) except Exception, e: logger.warn("Datadog notification failed with {0} but task {1} completed".format(e, t.__name___)) return r
def wrapper(*args, **kwargs): notify_datadog = True if type(t) != WrappedCallableTask: logger.warn("@notify decorator only works on a new-style Fabric Task") notify_datadog = False start = time.time() try: r = t(*args, **kwargs) end = time.time() duration = end - start if notify_datadog: try: task_full_name = "%s.%s" % (t.__module__, t.wrapped.func_name) dog_http_api.event("{0}".format(task_full_name), "{0} ran for {1}.".format(task_full_name, human_duration(duration)), source_type_name="fabric", alert_type="success", priority="normal", aggregation_key=task_full_name) except: logger.warn("Datadog notification failed but task {0} completed".format(t.wrapped.func_name)) return r except Exception as e: # If notification is on, create an error event end = time.time() duration = end - start if notify_datadog: try: task_full_name = "%s.%s" % (t.__module__, t.wrapped.func_name) dog_http_api.event("{0}".format(task_full_name), "{0} failed after {1} because of {2}.".format(task_full_name, human_duration(duration), e), source_type_name="fabric", alert_type="error", priority="normal", aggregation_key=task_full_name) except: logger.exception("Datadog notification failed") # Reraise raise
def sftp(files_from, files_to, mode, deleteAfterCopy=False): with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'): try: t = paramiko.Transport(( settings.PEARSON['SFTP_HOSTNAME'], 22)) t.connect(username=settings.PEARSON['SFTP_USERNAME'], password=settings.PEARSON['SFTP_PASSWORD']) sftp = paramiko.SFTPClient.from_transport(t) if mode == 'export': try: sftp.chdir(files_to) except IOError: raise CommandError( 'SFTP destination path does not exist: {}'.format(files_to)) for filename in os.listdir(files_from): sftp.put(files_from + '/' + filename, filename) if deleteAfterCopy: os.remove(os.path.join(files_from, filename)) else: try: sftp.chdir(files_from) except IOError: raise CommandError( 'SFTP source path does not exist: {}'.format(files_from)) for filename in sftp.listdir('.'): # skip subdirectories if not S_ISDIR(sftp.stat(filename).st_mode): sftp.get(filename, files_to + '/' + filename) # delete files from sftp server once they are # successfully pulled off: if deleteAfterCopy: sftp.remove(filename) except: dog_http_api.event('pearson {0}'.format(mode), 'sftp uploading failed', alert_type='error') raise finally: sftp.close() t.close()
def notify(t): """Decorates a fabric task""" @wraps(t) def wrapper(*args, **kwargs): notify_datadog = True if type(t) != WrappedCallableTask: logger.warn("@notify decorator only works on a new-style Fabric Task") notify_datadog = False start = time.time() try: r = t(*args, **kwargs) end = time.time() duration = end - start if notify_datadog: try: dog_http_api.event("{0}".format(_task_details(t)), "{0}({1}) ran for {2}.".format(_task_details(t), _format_args(args, kwargs), _human_duration(duration)), source_type_name="fabric", alert_type="success", priority="normal", aggregation_key=_task_details(t)) except Exception, e: logger.warn("Datadog notification failed with {0} but task {1} completed".format(e, t.wrapped.func_name)) return r except Exception, e: # If notification is on, create an error event end = time.time() duration = end - start if notify_datadog: try: dog_http_api.event("{0}".format(_task_details(t)), "{0}({1}) failed after {2} because of {3}.".format(_task_details(t), _format_args(args, kwargs), _human_duration(duration), e), source_type_name="fabric", alert_type="error", priority="normal", aggregation_key=_task_details(t)) except Exception, e: logger.warn("Datadog notification failed with {0} and task {1} failed".format(e, t.wrapped.func_name))
def notify(t): """Decorates a fabric task""" @wraps(t) def wrapper(*args, **kwargs): start = time.time() error = None try: r = t(*args, **kwargs) except Exception, e: error = e end = time.time() duration = end - start try: dog_http_api.event(_title(t, args, kwargs, error), _text(t, args, kwargs, duration, error), source_type_name="fabric", alert_type="error" if error else "success", priority="normal", aggregation_key=_aggregation_key(t, args, kwargs, error), tags=_tags(t, args, kwargs, error)) except Exception, e: logger.warn("Datadog notification on task {0} failed with {1}".format(t.__name___, e))
def datadog_error(string, tags): dog_http_api.event("Pearson Import", string, alert_type='error', tags=[tags])
from dogapi import dog_http_api as dog if 'DATADOG_API_KEY' not in os.environ: print >> sys.stderr, 'DATADOG_API_KEY environment variable not set' sys.exit(1) else: dog.api_key = os.environ['DATADOG_API_KEY'] parser = OptionParser(usage="usage: %prog [options] /path/to/local_git_dir") parser.add_option("-r", "--remote", action="store", dest="remote", help="Git remote to pull from", default="origin") parser.add_option("-b", "--branch", action="store", dest="branch", help="Git branch to pull from", default="master") options, args = parser.parse_args() if len(args) < 1: parser.print_help() sys.exit(1) else: local_repo_dir = args[0] # Execute the git pull command git_cmd = ['git', 'pull', options.remote, options.branch] proc = subprocess.Popen(git_cmd, cwd=local_repo_dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) git_message, _ = proc.communicate() if 'Already up-to-date.' not in git_message: # Only submit events if there were changes print git_message dog.event("Git Changes", git_message)
# import the simple dog client from dogapi import dog_http_api as dog # give dog your credentials (we're using os.environ to let you experiment via environment variables) # in this example we're only reporting data, so we only need an API key # see: https://github.com/DataDog/dogapi/wiki/Authentication for more on API authentication dog.api_key = os.environ.get("DATADOG_API_KEY") # emit points one by one. timestamp is determined at call time. dog.metric('test.api.test_metric', 4.0, host="some_host") time.sleep(1) dog.metric('test.api.test_metric', 5.0, host="another_host") # emit a list of points in one go as a list of (timestamp, value) # here we pretend to send a point a minute for the past hour now = dt.now() points = [] # create the list here (a list comprehension would do too) for i in range(60, 1, -1): t = time.mktime((now - delta(minutes=i)).timetuple()) points.append((t, math.cos(i) + 1.0)) # and emit the data in one call dog.metric('test.api.test_metric', points, host="some_other_host") # send an event too dog.event("API Testing", "Testing done, FTW")
def main(): parser = OptionParser() parser.add_option('-n', '--name', action='store', type='string', help="The name of the event") parser.add_option('-k', '--api_key', action='store', type='string') parser.add_option('-m', '--submit_mode', action='store', type='choice', default='errors', choices=['errors', 'all']) parser.add_option('-t', '--timeout', action='store', type='int', default=60*60*24) parser.add_option('--sigterm_timeout', action='store', type='int', default=60*2) parser.add_option('--sigkill_timeout', action='store', type='int', default=60) parser.add_option('--proc_poll_interval', action='store', type='float', default=0.5) parser.add_option('--notify_success', action='store', type='string', default='') parser.add_option('--notify_error', action='store', type='string', default='') options, args = parser.parse_args() dog.api_key = options.api_key cmd = [] for part in args: cmd.extend(part.split(' ')) returncode, stdout, stderr, duration = execute(cmd, options.timeout, options.sigterm_timeout, options.sigkill_timeout, options.proc_poll_interval) host = get_ec2_instance_id() if returncode == 0: alert_type = SUCCESS event_title = u'[%s] %s succeeded in %.2fs' % (host, options.name, duration) elif returncode is Timeout: alert_type = ERROR event_title = u'[%s] %s timed out after %.2fs' % (host, options.name, duration) returncode = -1 else: alert_type = ERROR event_title = u'[%s] %s failed in %.2fs' % (host, options.name, duration) event_body = [u'%%%\n', u'commmand:\n```\n', u' '.join(cmd), u'\n```\n', u'exit code: %s\n\n' % returncode, ] if stdout: event_body.extend([u'stdout:\n```\n', stdout, u'\n```\n']) if stderr: event_body.extend([u'stderr:\n```\n', stderr, u'\n```\n']) notifications = "" if alert_type == SUCCESS and options.notify_success: notifications = options.notify_success elif alert_type == ERROR and options.notify_error: notifications = options.notify_error if notifications: event_body.extend([u'notifications: %s\n' % (notifications)]) event_body.append(u'%%%\n') # ensure all strings are parsed as utf-8 event_body = [x.decode('utf-8') for x in event_body] event_body = u''.join(event_body) event = { 'alert_type': alert_type, 'aggregation_key': options.name, 'host': host, } print >> sys.stderr, stderr.strip() print >> sys.stdout, stdout.strip() if options.submit_mode == 'all' or returncode != 0: dog.event(event_title, event_body, **event) sys.exit(returncode)