def get_story_list(self, batch): """ get a list of stories corresponding to a list of hashes """ req_str = self.nb_endpoint + '/reader/starred_stories?' for a_hash in batch: req_str += 'h=' + a_hash + '&' stories = {} stories_req = requests.Request('GET', req_str, cookies=self.cookies) try: stories = self.request_with_backoff(stories_req) except requests.exceptions.ConnectionError as e: rollbar.report_exc_info() msg = 'Failed to get stories' logger.error(msg) logger.debug('Request string: %s', req_str) logger.error(e) statsd.event(msg, e.message, alert_type='error') logger.debug(stories.text) statsd.increment('nb.http_requests.get') story_list = [] try: story_list = json.loads(stories.text)['stories'] except ValueError as e: rollbar.report_exc_info() msg = 'Failed to parse stories response' logger.error(msg) logger.error(e) statsd.event(msg, e.message, alert_type='error') logger.debug(stories.text) return story_list
def report_suspicious_ips(disco_iter, params, job_id, host=None, port=None, database=None, user=None, password=None, bucket_name=None, db_insert_fn=insert_redshift): if db_insert_fn(disco_iter, params, job_id, host=host, port=port, database=database, user=user, password=password, bucket_name=bucket_name) > 0: msg = 'disco results %s | disco deref | ddfs xcat' % job_id statsd.event("Suspicious IPs detected. To view IPs, run: ", msg) log.debug(msg)
def get_story_list(self, batch): """ get a list of stories corresponding to a list of hashes """ req_str = self.nb_endpoint + '/reader/starred_stories?' for a_hash in batch: req_str += 'h=' + str(a_hash[0]) + '&' stories = {} stories_req = requests.Request('GET', req_str, cookies=self.cookies) try: stories = self.request_with_backoff(stories_req) except requests.exceptions.ConnectionError as e: rollbar.report_exc_info() msg = 'Failed to get stories' logger.error(msg) logger.debug('Request string: %s', req_str) logger.error(e) statsd.event(msg, e.message, alert_type='error') logger.debug(stories.text) statsd.increment('nb.http_requests.get') story_list = [] try: story_list = json.loads(stories.text)['stories'] except ValueError as e: rollbar.report_exc_info() msg = 'Failed to parse stories response' logger.error(msg) logger.error(e) statsd.event(msg, e.message, alert_type='error') logger.debug(stories.text) return story_list
def __init__(self): # Increment a counter. statsd.increment('python.breeze.reload') statsd.event('Breeze reload', '', 'info', hostname=settings.HOST_NAME)
def __init__(self, *args, **kwargs): super(HumanAI, self).__init__(*args, **kwargs) config = self.config config.from_object('config') config.from_envvar("HUMAN_AI_CONFIG_PATH", silent=True) # self.mtcnn = MTCNNDetector(model_config=config['MTCNN_DETECTOR_MODEL_CONFIG']) # self.mmod = MMODDetector(model_config=config['MMOD_DETECTOR_MODEL_CONFIG'], up=False) # self.mmod_up = MMODDetector(model_config=config['MMOD_DETECTOR_MODEL_CONFIG'], up=True) self.face_detector = TinyFaceDetector( model_configs=config['TINY_FACE_DETECTOR_MODEL_CONFIG']) # self.gender_analyzer = ResnetGenderAnalyzer(config['RESNET_GENDER_MODEL_CONFIG']) self.gender_analyzer = InceptionGenderAnalyzer( config['INCEPTION_GENDER_MODEL_CONFIG']) self.age_analyzer = InceptionAgeAnalyzer( config['INCEPTION_AGE_MODEL_CONFIG']) self.human_detector = MaskRCNNObjectDetector( config['MASK_RCNN_OBJECT_MODEL_CONFIG']) # self.human_detector = YOLOObjectDetector(config['YOLO_OBJECT_MODEL_CONFIG']) # self.human_detector = SSDObjectDetector(config['SSD_OBJECT_MODEL_CONFIG']) statsd.event(title='Vision-AI App Started', text='', alert_type='info', tags=['visionai.started'])
def event(self, metric, value, tags=[]): import socket statsd.event(title=metric, text=value, tags=tags, hostname=socket.gethostname()) if self.debug: print "{0} = {1} :: type={3} :: tags={2}".format( metric, value, tags, 'event')
def hello(): #use the global variable instead of a local one global count #increment count by 1 count += 1 statsd.increment('app.web') statsd.event('web app started', 'the web app has been started', alert_type='info') return "{}".format(count)
def add_story(self, nb_hash, added, comments_url, story_url): story = StoryModel(comments_url, nb_hash=nb_hash, added=added, url=story_url) try: story.save() except Exception as err: logger.error("Caught exception while saving Story model, wait 2 sec and retry") statsd.event('Failed to save story', err.message, alert_type='error') time.sleep(2) story.save() statsd.increment('nb.stories_added')
async def on_server_remove(self, server): channels = server.channels text_channels = sum(c.type == ChannelType.text for c in channels) voice_channels = sum(c.type == ChannelType.voice for c in channels) statsd.event(tags=self.tags, title='%s left %s :(' % (self.bot.user.name, server), text='\n'.join([ '* %i less members' % len(server.members), '* %i less text channels' % text_channels, '* %i less voice channels' % voice_channels ])) self.send_servers()
def logbook_entry_listener(event): """Listen for logbook entries and send them as events.""" name = event.data.get('name') message = event.data.get('message') statsd.event(title="Home Assistant", text="%%% \n **{}** {} \n %%%".format(name, message), tags=[ "entity:{}".format(event.data.get('entity_id')), "domain:{}".format(event.data.get('domain')) ]) _LOGGER.debug('Sent event %s', event.data.get('entity_id'))
def logbook_entry_listener(event): """Listen for logbook entries and send them as events.""" name = event.data.get("name") message = event.data.get("message") statsd.event( title="Home Assistant", text=f"%%% \n **{name}** {message} \n %%%", tags=[ f"entity:{event.data.get('entity_id')}", f"domain:{event.data.get('domain')}", ], ) _LOGGER.debug("Sent event %s", event.data.get("entity_id"))
def logbook_entry_listener(event): """Listen for logbook entries and send them as events.""" name = event.data.get('name') message = event.data.get('message') statsd.event( title="Home Assistant", text="%%% \n **{}** {} \n %%%".format(name, message), tags=[ "entity:{}".format(event.data.get('entity_id')), "domain:{}".format(event.data.get('domain')) ] ) _LOGGER.debug('Sent event %s', event.data.get('entity_id'))
def logbook_entry_listener(event): """Listen for logbook entries and send them as events.""" name = event.data.get("name") message = event.data.get("message") statsd.event( title="Home Assistant", text="%%% \n **{}** {} \n %%%".format(name, message), tags=[ "entity:{}".format(event.data.get("entity_id")), "domain:{}".format(event.data.get("domain")), ], ) _LOGGER.debug("Sent event %s", event.data.get("entity_id"))
def get_nb_hash_list(self): """ get a list of story identifiers (hashes) from NewsBlur """ hashes_req = requests.Request('GET', self.nb_endpoint + '/reader/starred_story_hashes', cookies=self.cookies) hashes = self.request_with_backoff(hashes_req) try: return hashes.json()['starred_story_hashes'] except ValueError as e: rollbar.report_exc_info() msg = 'Failed to decode JSON' logger.error(msg) logger.error(e) logger.debug(hashes) statsd.event(msg, e.message, alert_type='error') return []
def close(self, current_price): self.status = "CLOSED" self.close_time = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') if self.trade_type.upper() == "BUY": # Make oppostie trade type to close self._handle_exit_order('sell', current_price) elif self.trade_type.upper() == "SELL": # Make oppostie trade type to close self._handle_exit_order('buy', current_price) statsd.increment('close.order', tags=['name:{}'.format(BOT_NAME), 'pair:{}'.format(self.pair), 'type:{}'.format(self.trade_type), 'order:{}'.format(self.order_type), 'volume:{}'.format(self.bid_volume), 'cur_price:{}'.format(current_price), 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) statsd.event(title='Order Close', text='{}/{} - {} {} @ {} / Cost: {}'.format(self.close_order_id, self.pair, self.trade_type, self.bid_volume, self.exit_price, self.exit_cost), alert_type='success', tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) self.output.log("{c1}{trade_type} Trade closed - order_id: {order_id}{c2}".format(c1=Yellow, trade_type=self.trade_type.upper(), order_id=self.close_order_id, c2=White)) if self.profit >= 0.0: self.output.log("Profit/Loss before fees {}".format(self.profit)) statsd.increment('bottrade.profit', self.profit, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) statsd.increment('bottrade.win_rate', 1, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) else: self.output.log("Profit/Loss before fees {}".format(self.profit)) # Decrement by the absolute value if profit is negative statsd.decrement('bottrade.profit', abs(self.profit), tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) statsd.decrement('bottrade.win_rate', 1, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) self.output.log("Trade fees at closing: {}".format(self.fees)) statsd.increment('trading_fees', self.fees, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)])
def get_nb_hash_list(self): """ get a list of story identifiers (hashes) from NewsBlur """ hashes_req = requests.Request('GET', self.nb_endpoint + '/reader/starred_story_hashes', cookies=self.cookies) hashes = self.request_with_backoff(hashes_req) try: return hashes.json()['starred_story_hashes'] except AttributeError as e: logger.error("Attribute error in get_nb_has_list()") rollbar.report_exc_info() logger.error(e) except ValueError as e: if (hashes is not null): logger.error(hashes.json()) rollbar.report_exc_info() msg = 'Failed to decode JSON' logger.error(msg) logger.error(e) logger.debug(hashes) statsd.event(msg, e.message, alert_type='error') return []
def _record_pressure(self): statsd.gauge( f"{self._service_name}.pool.total_connections", self._total, tags=self._extra_tags, ) statsd.gauge( f"{self._service_name}.pool.available_connections", self.available.qsize(), tags=self._extra_tags, ) statsd.gauge( f"{self._service_name}.pool.waiting", self._waiters, tags=self._extra_tags ) statsd.gauge( f"{self._service_name}.pool.connections_used", self.in_use, tags=self._extra_tags, ) self._record_connection_acquiring() if self._total > self.max_size: if not self._is_bursting: self._is_bursting = True statsd.event( f"{self._service_name} pool using burst capacity", f"Pool max size of {self.max_size} will be exceeded temporarily, up to {self.burst_limit}", # noqa E501 alert_type="warning", tags=self._extra_tags, ) elif self._is_bursting: self._is_bursting = False self._reported_hitting_burst_limit = False statsd.event( f"{self._service_name} pool no longer bursting", f"Number of connections has dropped below {self.max_size}", alert_type="success", tags=self._extra_tags, ) if self._total == self.burst_limit: self._reported_hitting_burst_limit = True statsd.event( f"{self._service_name} pool reached burst limit", "There are not enough redis connections to satisfy all users", alert_type="error", tags=self._extra_tags, )
def main(config=Config()): try: _, _, _, sqs_region, _, sqs_queue = config.sqs_arn.split(':', 5) except Exception: raise Exception('invalid sqs arn') sqs = boto.sqs.connect_to_region(sqs_region) queue = sqs.get_queue(sqs_queue) if queue is None: raise Exception('could not connect to sqs queue: ' + config.sqs_arn) while True: message = queue.read(wait_time_seconds=20) if message is None: #out('Queue is empty') continue try: raw_body = message.get_body() out('Message received') except Exception: msg = 'Failed to get message body' out(msg) statsd.event('SQS Message Error', msg, alert_type='error') statsd.increment('tiles.processor.failed_get_body') continue try: body = json.loads( json.loads(raw_body)['Message'].replace("u'", '"').replace("'", '"')) except Exception: msg = 'Invalid message body: ' + raw_body out(msg) statsd.event('JSON parse error', msg, alert_type='error') statsd.increment('tiles.processor.invalid_body') continue try: out('Processing: ' + str(body)) process(message, body, config.ddfs_master, config.tag_prefix) statsd.increment('tiles.processor.processed') except Exception: msg = 'Failed to process: ' + str(body) out(msg) statsd.increment('tiles.processor.failed_to_process') statsd.event('Failed Processing message', msg, alert_type='error') stoppable()
async def on_guild_join(self, guild): statsd.event(tags=self.tags, title='Joined ' + guild.name, text="Server has " + str(len(guild.members)) + " users") self.send_guilds()
def process_exception(exception, e): statsd.event('Python Exception', str(e), 'warning', hostname=settings.HOST_NAME) statsd.increment('python.breeze.exception')
async def on_guild_remove(self, guild): statsd.event(tags=self.tags, title='Left ' + guild.name, text="Server had " + str(len(guild.members)) + " users") self.send_guilds()
def __init__(self, pair, current_price, trade_type=None, order_type='market', stop_loss_percent=0): self.output = BotLog() self.pair = pair self.status = "OPEN" self.entry_price = 0.0 self.exit_price = 0.0 self.entry_cost = 0.0 self.exit_cost = 0.0 self.profit = 0.0 self.fees = 0.0 self.fee_percentage = 0.0 self.exit_minus_entry_less_fees = 0.0 self.trade_pl = None self.trade_net_precent = 0.0 self.trade_type = trade_type # BUY/SELL self.order_type = order_type # Market/Limit/stop loss/settle position/etc. self.min_bid = 0.002 self.open_time = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') self.close_time = "---------- --:--:--" self.stop_loss_percent = stop_loss_percent self.open_order_id = "" self.close_order_id = "" # user reference id. 32-bit signed number. # 1-bit for sign, 3-bits for 3 decimal digit (0-999), 8-bits for for randomly generated id. self.user_ref_id = int(str(BOT_ID) + str(int(random.getrandbits(7)))) # self.usd_balance = float(Balance()) # Set volume to higher of 1% of wallet and min bid # self.bid_volume = max(self.usd_balance * 0.01 / current_price, 0.002) if current_price else 0.002 self.bid_volume = 0.002 # Instantiating a BotTrade object is equivalent to opening a trade. TODO: move open to a function if self.trade_type.upper() == 'BUY': self._handle_entry_order('buy', current_price) if self.trade_type.upper() == 'SELL': self._handle_entry_order('sell', current_price) statsd.increment('open.order', tags=['name:{}'.format(BOT_NAME), 'pair:{}'.format(self.pair), 'type:{}'.format(self.trade_type), 'order:{}'.format(self.order_type), 'volume:{}'.format(self.bid_volume), 'cur_price:{}'.format(current_price), 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) statsd.increment('total_trading_volume', self.entry_cost, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) statsd.event(title='Order Open', text='{}/{} - {} {} @ {} / Cost: {}'.format(self.open_order_id, self.pair, self.trade_type, self.bid_volume, self.entry_price, self.entry_cost), alert_type='success', tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) self.output.log("{c1}{trade_type} Trade opened - order_id: {order_id}{c2}".format(c1=Yellow, trade_type=self.trade_type.upper(), order_id=self.open_order_id, c2=White)) # TODO: Implement trailing stop loss - might be available on Kraken as a special order type # TODO: Implement a daily stop loss, 3% of wallet self.stop_loss = current_price * (1.0 - (stop_loss_percent / 100)) if stop_loss_percent else 0
import json from datadog import statsd def parse_filebeat(logger, line): attr_list = ['esb', 'Environment:prod', 'region:us-denver', 'app-name:filebeat'] if '{"monitoring": {"metrics": {' in line: myjson = line[line.find('{'):] jsonstr = json.loads(myjson) jsonstr = jsonstr['monitoring']['metrics'] statsd.gauge('filebeat.memory_total', jsonstr['beat']['memstats']['memory_total'] , tags=attr_list) statsd.gauge('filebeat.gc_next', jsonstr['beat']['memstats']['gc_next'] , tags=attr_list) statsd.gauge('filebeat.memory_alloc', jsonstr['beat']['memstats']['memory_alloc'] , tags=attr_list) statsd.gauge('filebeat.events_added', jsonstr['filebeat']['events']['added'], tags=attr_list) statsd.gauge('filebeat.events_done', jsonstr['filebeat']['events']['done'], tags=attr_list) statsd.gauge('filebeat.libbeat_pipeline.events_active', jsonstr['libbeat']['pipeline']['events']['active'], tags=attr_list) statsd.gauge('filebeat.libbeat_pipeline.events_published', jsonstr['libbeat']['pipeline']['queue']['acked'], tags=attr_list) statsd.gauge('filebeat.libbeat_pipeline.queue_acked', jsonstr['libbeat']['pipeline']['events']['published'], tags=attr_list) statsd.gauge('filebeat.libbeat_output.events_total', jsonstr['libbeat']['output']['events']['total'], tags=attr_list) statsd.gauge('filebeat.libbeat_output.events_acked', jsonstr['libbeat']['output']['events']['acked'], tags=attr_list) if 'ERROR' in line: attr_list = ['esb', 'Environment:prod', 'region:us-denver', 'app-name:filebeat', 'x-filebeat-error:error'] title = "An error has occurred" text = line statsd.event(title = title, text=text, tags=attr_list)
def event(title, text, alert_type, worker): statsd.event(title=title, text=text, alert_type=alert_type, hostname=worker)
test_images = test_images.reshape((10000, 28, 28, 1)) test_images = test_images.astype('float32') / 255 train_labels = to_categorical(train_labels) test_labels = to_categorical(test_labels) with tracer.trace('model.compile'): model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) epochs_train = 5 batch_size_train = 64 statsd.event('Convolutional Netual Network Model Training Starting', 'Training starting with Batch Size {} for {} Epochs.'.format( epochs_train, batch_size_train), alert_type='info') with tracer.trace('model.fit'): model.fit(train_images, train_labels, epochs=epochs_train, batch_size=batch_size_train, callbacks=[LogsAndMetricsCallback()], verbose=0) with tracer.trace('model.evaluate'): test_loss, test_acc = model.evaluate(test_images, test_labels, verbose=0)
def main(argv): """ Main entry point """ # Logging output = BotLog() supported_exchanges = ['kraken'] exchange = 'kraken' pair = "XXBTZUSD" # Bitcoin/USD pair on Kraken period = 5 # Time frame interval in minutes, e.g. width of candlestick. poll_time = 1 # How often an API query is made when using real time data. script_help = '\n\t-c --currency <currency pair>\n\t-x --exchange <name of the exchange {exchanges}>\n\t-t --poll <poll period length in minutes>\n\nHistorical Mode\n\t-p --period <period of frame>\n\t-s --start <start time in unix timestamp>\n\t-e --end <end time in unix timestamp>\n'.format( exchanges=supported_exchanges) start_time = False end_time = False try: opts, args = getopt.getopt( argv, "h:x:p:c:t:s:e:y:", ["exchange=", "period=", "currency=", "poll=", "start=", "end="]) except getopt.GetoptError: output.log(sys.argv[0] + script_help) sys.exit(2) for opt, arg in opts: if opt == ("-h", "--help"): output.log(sys.argv[0] + script_help) sys.exit() elif opt in ("-s", "--start"): start_time = arg elif opt in ("-e", "--end"): end_time = arg elif opt in ("-x", "--exchange"): if arg in supported_exchanges: exchange = arg else: output.log( 'Supported exchanges are {}'.format(supported_exchanges)) sys.exit(2) elif opt in ("-p", "--period"): if exchange.lower() == 'kraken': # Kraken uses minutes for getting historical data. mins = [1, 5, 15, 30, 60, 240, 1440, 10080, 21600] if (int(arg) in mins): period = int(arg) else: output.log( 'Kraken requires intervals 1, 5, 15, 30, 60, 240, 1440, 10080, 21600 minute intervals' ) sys.exit(2) else: period = int(arg) elif opt in ("-c", "--currency"): pair = arg elif opt in ("-t", "--poll"): poll_time = arg ################ Strategy in use ################ strategy = MACDStrategy(pair, period) strategy_name = strategy.get_name() ################################################# # Log bot startup event to DataDog statsd.event(title='Bot started', text='{}:{} started on {} trading {} using {}'.format( BOT_ID, BOT_NAME, exchange, pair, strategy_name), alert_type='success', tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) trade_session_details = "{bg}Trading {pair} on {exchange} with {strat}" \ " @ {period} minute period{White}".format(pair=pair, exchange=exchange.upper(), strat=strategy_name, period=period, bg=On_Cyan, White=White) if start_time: # Backtesting chart = BotChart(exchange, pair, period) for candlestick in chart.get_points(): strategy.tick(candlestick) output.log(trade_session_details) else: # Live Trading output.log(trade_session_details) chart = BotChart(exchange, pair, period, backtest=False) candlesticks = [] developing_candlestick = BotCandlestick(period) progress_counter = 0 while True: # Log trade details every so often if progress_counter == 50: output.log(trade_session_details) progress_counter = 0 progress_counter += 1 try: developing_candlestick.tick( chart.get_current_price_and_vol()[0]) except urllib2.URLError, err: # If network or site is down output.log("{}... Continuing".format(err[0])) # TODO: These calls to statsd should be Rollbar. Set up Rollbar statsd.histogram( 'main_loop.urllib2.URLError', err, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) continue except ValueError, err: # For screwy JSON output.log('{}... Continuing'.format(err[0])) statsd.histogram( 'main_loop.ValueError', err, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) continue except urllib2.ssl.SSLError, err: # For read timeouts output.log("{}... Continuing".format(err[0])) statsd.histogram( 'main_loop.urllib2.ssl.SSLError', err, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) continue
def generateEvent(): print("Generate event") statsd.event('Hello world!', 'An event from my dummy app', alert_type='info', tags=['env:' + ENV])
password=configuration['ic_options']['api_key']) consecutive_fails = 0 while True: response = requests.get( url="https://api.instaclustr.com/monitoring/v1/clusters/{0}?metrics={1}," .format(configuration['cluster_id'], configuration['metrics_list']), auth=auth_details) if not response.ok: # got an error response from the Instaclustr API - raise an alert in DataDog after 3 consecutive fails consecutive_fails += 1 print "Error retrieving metrics from Instaclustr API: {0} - {1}".format( response.status_code, response.content) if consecutive_fails > 3: statsd.event("Instaclustr monitoring API error", "Error code is: {0}".format(response.status_code)) sleep(20) continue consecutive_fails = 0 metrics = json.loads(response.content) for node in metrics: public_ip = node["publicIp"] private_ip = node["privateIp"] rack_name = node["rack"]["name"] data_centre_custom_name = node["rack"]["dataCentre"]["customDCName"] data_centre_name = node["rack"]["dataCentre"]["name"] data_centre_provider = node["rack"]["dataCentre"]["provider"] provider_account_name = node["rack"]["providerAccount"]["name"] provider_account_provider = node["rack"]["providerAccount"]["provider"]
# post-event-dogstatsd.py from datadog import initialize, statsd options = {'statsd_host': '127.0.0.1', 'statsd_port': 8125} initialize(**options) title = "An event testing DogStatsD API" text = "The DogStatsD API works fine! Just wanted to confirm it." tags = ["event-source:dogstatsd-test"] statsd.event(title, text, alert_type='info', tags=tags)
def event(self, metric, value, tags=[]): import socket statsd.event(title=metric, text=value, tags=tags, hostname=socket.gethostname()) if self.debug: print "{0} = {1} :: type={3} :: tags={2}".format(metric, value, tags, 'event')
initialize(**dd_options) auth_details = HTTPBasicAuth(username=configuration['ic_options']['user_name'], password=configuration['ic_options']['api_key']) consecutive_fails = 0 while True: response = requests.get(url="https://api.instaclustr.com/monitoring/v1/clusters/{0}?metrics={1},".format(configuration['cluster_id'], configuration['metrics_list']), auth=auth_details) if not response.ok: # got an error response from the Instaclustr API - raise an alert in DataDog after 3 consecutive fails consecutive_fails += 1 print "Error retrieving metrics from Instaclustr API: {0} - {1}".format(response.status_code, response.content) if consecutive_fails > 3: statsd.event("Instaclustr monitoring API error", "Error code is: {0}".format(response.status_code)) sleep(20) continue consecutive_fails = 0 metrics = json.loads(response.content) for node in metrics: public_ip = node["publicIp"] for metric in node["payload"]: dd_metric_name = 'instaclustr.{0}.{1}'.format(public_ip,metric["metric"]) if metric["metric"] == "nodeStatus": # node status metric maps to a data dog service check if metric["values"][0]["value"] =="WARN": statsd.service_check(dd_metric_name, 1) # WARN status else: