def process_response(self, request, response): if not hasattr(request, self.DATADOG_TIMING_ATTRIBUTE): return response request_time = time.time() - getattr(request, self.DATADOG_TIMING_ATTRIBUTE) timing_metric = '{0}.request_time'.format(self.app_name) count_metric = '{0}.no_of_requests_metric'.format(self.app_name) success_metric = '{0}.no_of_successful_requests_metric'.format( self.app_name) unsuccess_metric = '{0}.no_of_unsuccessful_requests_metric'.format( self.app_name) tags = self._get_metric_tags(request) if 200 <= response.status_code < 400: statsd.increment(success_metric, tags=tags) else: statsd.increment(unsuccess_metric, tags=tags) statsd.increment(count_metric, tags=tags) statsd.histogram(timing_metric, request_time, tags=tags) return response
async def _extract_pulls_from_stream(self, stream_name): messages = await self.redis.xrange(stream_name, count=config.STREAM_MAX_BATCH) LOG.debug("read stream", stream_name=stream_name, messages_count=len(messages)) statsd.histogram("engine.streams.size", len(messages)) # Groups stream by pull request pulls = collections.OrderedDict() for message_id, message in messages: data = msgpack.unpackb(message[b"event"], raw=False) owner = data["owner"] repo = data["repo"] source = data["source"] if data["pull_number"] is not None: key = (owner, repo, data["pull_number"]) group = pulls.setdefault(key, ([], [])) group[0].append(message_id) group[1].append(source) else: logger = daiquiri.getLogger( __name__, gh_repo=repo, gh_owner=owner, source=source ) logger.debug("unpacking event") try: converted_messages = await self._convert_event_to_messages( stream_name, owner, repo, source ) except IgnoredException: converted_messages = [] logger.debug("ignored error", exc_info=True) except StreamRetry: raise except StreamUnused: raise except Exception: # Ignore it, it will retried later logger.error("failed to process incomplete event", exc_info=True) continue logger.debug("event unpacked into %s messages", len(converted_messages)) messages.extend(converted_messages) deleted = await self.redis.xdel(stream_name, message_id) if deleted != 1: # FIXME(sileht): During shutdown, heroku may have already started # another worker that have already take the lead of this stream_name # This can create duplicate events in the streams but that should not # be a big deal as the engine will not been ran by the worker that's # shutdowning. contents = await self.redis.xrange( stream_name, start=message_id, end=message_id ) if contents: logger.error( "message `%s` have not been deleted has expected, " "(result: %s), content of current message id: %s", message_id, deleted, contents, ) return pulls
def event_handler(name, **kwargs): if name == 'request_finished': statsd.increment('mod_wsgi.request.count') application_time = kwargs.get('application_time') statsd.histogram('mod_wsgi.request.application_time', application_time) statsd.gauge('mod_wsgi.request.input_reads', kwargs.get('input_reads')) statsd.gauge('mod_wsgi.request.input_length', kwargs.get('input_length')) statsd.gauge('mod_wsgi.request.input_time', kwargs.get('input_time')) statsd.gauge('mod_wsgi.request.output_writes', kwargs.get('output_writes')) statsd.gauge('mod_wsgi.request.output_length', kwargs.get('output_length')) statsd.gauge('mod_wsgi.request.output_time', kwargs.get('output_time')) cpu_user_time = kwargs.get('cpu_user_time') cpu_system_time = kwargs.get('cpu_system_time') statsd.gauge('mod_wsgi.request.cpu_user_time', cpu_user_time) statsd.gauge('mod_wsgi.request.cpu_system_time', cpu_system_time) if cpu_user_time is not None and application_time: cpu_burn = (cpu_user_time + cpu_system_time) / application_time statsd.gauge('mod_wsgi.request.cpu_burn', cpu_burn)
def mates_metrics(sender: AlquilerMate, instance: AlquilerMate, created: bool, **kwargs): if created: statsd.increment("ceitba.mates.count") elif instance.time_returned is not None: time = instance.time_returned - instance.time_taken statsd.histogram("ceitba.mates.time", time)
async def _extract_pulls_from_stream(self, stream_name, installation): LOG.debug("read stream", stream_name=stream_name) messages = await self.redis.xrange(stream_name, count=config.STREAM_MAX_BATCH) statsd.histogram("engine.streams.size", len(messages)) # Groups stream by pull request pulls = collections.OrderedDict() for message_id, message in messages: data = msgpack.unpackb(message[b"event"], raw=False) owner = data["owner"] repo = data["repo"] source = data["source"] if data["pull_number"] is not None: key = (owner, repo, data["pull_number"]) group = pulls.setdefault(key, ([], [])) group[0].append(message_id) group[1].append(source) else: logger = logs.getLogger(__name__, gh_repo=repo, gh_owner=owner) try: messages.extend(await self._convert_event_to_messages( stream_name, installation, owner, repo, source)) except IgnoredException: logger.debug("ignored error", exc_info=True) except StreamRetry: raise except Exception: # Ignore it, it will retried later logger.error("failed to process incomplete event", exc_info=True) continue await self.redis.xdel(stream_name, message_id) return pulls
def run(self): datadog_start_time = time.time() job_cfg = self.get_appserver_job_config() logging.info('Running rollup job...') appserver_jobsubmit_url = CmvLib.get_appserver_job_submit_url( self.appserver_host_port, self.appserver_app_name, self.appserver_app_type) rslt_json = CmvLib.submit_config_to_appserver(job_cfg, appserver_jobsubmit_url) job_id = rslt_json['payload']['jobId'] appserver_jobstatus_url = CmvLib.get_appserver_job_status_url( self.appserver_host_port, self.appserver_app_name, job_id) appserver_resp = CmvLib.poll_appserver_job_status( appserver_jobstatus_url) if appserver_resp['payload']['status'] != 'Finished': logging.error( "AppServer responded with an error. AppServer Response: %s", appserver_resp['payload']['result']) raise Exception('Error in Appserver Response.') else: logging.info("Rollup job completed successfully.") self.output().touch() statsd.histogram(self.metric_name, time.time() - datadog_start_time, tags=self.tag_name)
async def _consume_pulls(self, stream_name, pulls): LOG.debug("stream contains %d pulls", len(pulls), stream_name=stream_name) for (owner, repo, pull_number), (message_ids, sources) in pulls.items(): statsd.histogram("engine.streams.batch-size", len(sources)) for source in sources: if "timestamp" in source: statsd.histogram( "engine.streams.events.latency", (datetime.datetime.utcnow() - datetime.datetime.fromisoformat( source["timestamp"])).total_seconds(), ) logger = daiquiri.getLogger(__name__, gh_repo=repo, gh_owner=owner, gh_pull=pull_number) try: logger.debug("engine start with %s sources", len(sources)) start = time.monotonic() await self._run_engine_and_translate_exception_to_retries( stream_name, owner, repo, pull_number, sources) await self.redis.execute_command("XDEL", stream_name, *message_ids) end = time.monotonic() logger.debug("engine finished in %s sec", end - start) except IgnoredException: await self.redis.execute_command("XDEL", stream_name, *message_ids) logger.debug("failed to process pull request, ignoring", exc_info=True) except MaxPullRetry as e: await self.redis.execute_command("XDEL", stream_name, *message_ids) logger.error( "failed to process pull request, abandoning", attempts=e.attempts, exc_info=True, ) except PullRetry as e: logger.info( "failed to process pull request, retrying", attempts=e.attempts, exc_info=True, ) except StreamRetry: raise except StreamUnused: raise except vcr_errors_CannotOverwriteExistingCassetteException: raise except Exception: # Ignore it, it will retried later logger.error("failed to process pull request", exc_info=True)
def GET(self): start = time.time() delay=random.uniform(0.4,0.9) sleep(delay) statsd.increment('web.get.count', tags = ["support","page:page2"]) duration = time.time() - start statsd.histogram('web.get.latency', duration, tags = ["support","page:page2"]) return "Messages Page"
def showFriends(): #metric to count the web.page_views_friends statsd.increment('web.page_views_friends', tags=["page:friends"]) #metric to count the overall number of page views statsd.increment('web.page_views_total') #start timer start_time = time() print start_time #connection to the DB connection = MySQLdb.connect(host="localhost", user="******", passwd="cacapipi", db="bucketlist") #prepare a cursor object using cursor() method cursor = connection.cursor() #execute the SQL query using execute() method cursor.execute( "select user_name, user_username, user_password from tbl_user_friends " ) #fetch all the rows from the query data = cursor.fetchall() #print the rows #THIS_DIR = os.path.dirname(os.path.abspath(__file__)) # Create the jinja2 environment # Notice the use of trim_blocks, which greatly helps control whitespace. #j2_env = Environment(loader=FileSystemLoader(THIS_DIR), trim_blocks=True) #print j2_env.get_template('community.html').render(items=data) #env = Environment(loader=PackageLoader('app', 'template')) #template = env.get_template('community.html') #print template.render(items=data) for row in data: print row[0], row[1] cursor.close() #close the connection connection.close() #return timer duration = time() - start_time print duration statsd.histogram('databaseFriends.query.time', duration, tags=["page:friends"]) statsd.gauge('test3', 300) #exit the program sys.exit()
def log_dawg(sbs): statsd.increment("adsb.message", '1') if sbs.groundSpeed is not None: statsd.gauge('adsb.airspeed', sbs.groundSpeed) if sbs.altitude is not None: statsd.gauge('adsb.altitude', sbs.altitude) if sbs.track is not None: statsd.histogram('adsb.heading', sbs.track) if sbs.verticalRate is not None: if sbs.verticalRate > 0: statsd.gauge('adsb.ascentrate', sbs.verticalRate) else: statsd.gauge('adsb.decentrate', sbs.verticalRate)
def gen_custom_metric(): print(timer) print(ran_numb) # api.Metric.send( # metric='custom.metrics.series', # points=ran_numb, # host="python.datadog_2.com", # tags=["food:pancakes"] # ) statsd.histogram('example_metric.histogram', ran_numb, tags=["environment:doghouse", "food:hotdogs"])
def showCommunity(): #metric to count the web.page_views_community statsd.increment('web.page_views_community',tags = ["page:community"]) #metric to count the overall number of page views statsd.increment('web.page_views_total') #start timer start_time = time() print start_time #connection to the DB connection = MySQLdb.connect (host = "localhost", user = "******", passwd = "cacapipi", db = "bucketlist") #prepare a cursor object using cursor() method cursor = connection.cursor() #execute the SQL query using execute() method cursor.execute("select user_name, user_username, user_password from tbl_user ") #fetch all the rows from the query data = cursor.fetchall() #print the rows #THIS_DIR = os.path.dirname(os.path.abspath(__file__)) # Create the jinja2 environment # Notice the use of trim_blocks, which greatly helps control whitespace. #j2_env = Environment(loader=FileSystemLoader(THIS_DIR), trim_blocks=True) #print j2_env.get_template('community.html').render(items=data) #env = Environment(loader=PackageLoader('app', 'template')) #template = env.get_template('community.html') #print template.render(items=data) for row in data: print row[0], row[1] cursor.close() #close the connection connection.close() #return timer duration = time() - start_time print duration statsd.histogram('database.query.time', duration, tags = ["page:community"]) statsd.gauge('test2',200) #exit the program sys.exit()
def _record_success_count(self, api_name): if settings.DEBUG: print('Record Success - %s:%s', api_name) metric = 'apiserver.success.%s' % api_name statsd.histogram(metric, 1) service_name = self._get_service_name(api_name) metric_success_all = 'apiserver.success.all' statsd.histogram( metric_success_all, 1, tags=["api_name:%s" % api_name, "service_name:%s" % service_name])
def _record_api_error(self, api_name, error_name): if settings.DEBUG: print('Record Error - %s:%s', api_name, error_name) metric = 'apiserver.errors.%s.%s' % (error_name, api_name) statsd.histogram(metric, 1) service_name = self._get_service_name(api_name) metric_all_errors = 'apiserver.errors.%s.all' % error_name statsd.histogram( metric_all_errors, 1, tags=["api_name:%s" % api_name, "service_name:%s" % service_name])
def _generate_metrics(self): nb_requests = len(self._requests) statsd.histogram( "http.client.session", nb_requests, tags=[f"hostname:{self.base_url.host}"], ) if nb_requests >= LOGGING_REQUESTS_THRESHOLD: LOG.warning( "number of GitHub requests for this session crossed the threshold (%s): %s", LOGGING_REQUESTS_THRESHOLD, nb_requests, gh_owner=self.auth.owner, requests=self._requests, ) self._requests = []
async def aclose(self): await super().aclose() nb_requests = len(self._requests) statsd.histogram( "http.client.session", nb_requests, tags=[f"hostname:{self.base_url.host}"], ) if nb_requests >= LOGGING_REQUESTS_THRESHOLD: LOG.warning( "number of GitHub requests for this session crossed the threshold (%s): %s", LOGGING_REQUESTS_THRESHOLD, nb_requests, gh_owner=self.auth.owner, gh_repo=self.auth.repo, requests=self._requests, ) self._requests = []
def api_entry(): start_time = time.time() app.logger.info('getting root endpoint') # return 'Entrypoint to the Application' name = flask_request.args.get('name', str) tracer.set_tags({'name': name}) mycursor.execute("SELECT Name, UUID, Number FROM kikeyama_table where name='%s'" % name) myresult = mycursor.fetchall() for x in myresult: result = json.dumps(x) return result duration = time.time() - start_time statsd.distribution('kikeyama.dogstatsd.distribution.latency', duration) statsd.histogram('kikeyama.dogstatsd.histogram.latency', duration)
async def _consume_pulls(self, stream_name, installation, pulls): LOG.debug("stream contains %d pulls", len(pulls), stream_name=stream_name) for (owner, repo, pull_number), (message_ids, sources) in pulls.items(): statsd.histogram("engine.streams.batch-size", len(sources)) logger = logs.getLogger(__name__, gh_repo=repo, gh_owner=owner, gh_pull=pull_number) try: logger.debug("engine start with %s sources", len(sources)) start = time.monotonic() await self._run_engine_and_translate_exception_to_retries( installation, owner, repo, pull_number, sources) await self.redis.execute_command("XDEL", stream_name, *message_ids) end = time.monotonic() logger.debug("engine finished in %s sec", end - start) except IgnoredException: await self.redis.execute_command("XDEL", stream_name, *message_ids) logger.debug("failed to process pull request, ignoring", exc_info=True) except MaxPullRetry as e: await self.redis.execute_command("XDEL", stream_name, *message_ids) logger.error( "failed to process pull request, abandoning", attempts=e.attempts, exc_info=True, ) except PullRetry as e: logger.info( "failed to process pull request, retrying", attempts=e.attempts, exc_info=True, ) except StreamRetry: raise except Exception: # Ignore it, it will retried later logger.error("failed to process pull request", exc_info=True)
def _generate_metrics(self) -> None: nb_requests = len(self._requests) statsd.histogram( "http.client.session", nb_requests, tags=[f"hostname:{self.base_url.host}"], ) if ((nb_requests / self._requests_ratio) >= LOGGING_REQUESTS_THRESHOLD or nb_requests >= LOGGING_REQUESTS_THRESHOLD_ABSOLUTE): LOG.warning( "number of GitHub requests for this session crossed the threshold (%s/%s): %s/%s", LOGGING_REQUESTS_THRESHOLD, LOGGING_REQUESTS_THRESHOLD_ABSOLUTE, nb_requests / self._requests_ratio, nb_requests, gh_owner=http.extract_organization_login(self), requests=self._requests, requests_ratio=self._requests_ratio, ) self._requests = []
def tick(self, candlestick): self.current_price = float(candlestick.typical_price) self.prices.append(self.current_price) # Highs self.currentHigh = float(candlestick.high) self.highs.append(self.currentHigh) # Lows self.currentLow = float(candlestick.low) self.lows.append(self.currentLow) # Closes self.currentClose = float(candlestick.close) self.closes.append(self.currentClose) if len(self.prices) > self.hist_period: # Price action self.output.log("\n{color}Typical Price: {price}".format( color=Cyan, price=str(candlestick.typical_price))) # MACD m, s, h = self.indicators.MACD(self.prices, self.macd_slow, self.macd_fast, self.macd_signal) self.output.log( "Last/Current indicator values:\tMACD {}\tSignal: {}\t Hist: {}" .format(str(m[-2:]), str(s[-2:]), str(h[-2:]))) statsd.histogram('macd.macd', m[-1], tags=[ 'macd.macd_line', 'bot_name:{}.bot_id:{}'.format( BOT_NAME, BOT_ID) ]) statsd.histogram('macd.signal', s[-1], tags=[ 'macd.signal_line', 'bot_name:{}.bot_id:{}'.format( BOT_NAME, BOT_ID) ]) statsd.histogram('macd.histogram', h[-1], tags=[ 'macd.histogram', 'bot_name:{}.bot_id:{}'.format( BOT_NAME, BOT_ID) ]) self.output.log(White) self.evaluate_positions() self.update_open_trades() self.show_positions()
def report_profiling_to_statsd(self, profiling, tag): """ Report status to datadog :param profiling: profile values :param tag: tags for the profile :return: """ metric = 'human-ai.request.duration' path = flask.request.path[1:].replace('/', '_') tags = ['type:' + (path or 'index'), 'tag:' + tag] # datadog prints warnings when there is no agent running, # suppress them here with this ugly check debug_or_testing = self.config.get( 'TESTING', False) or self.config.get('DEBUG', False) if debug_or_testing: return try: for k, v in profiling.items(): statsd.histogram(metric, v, tags + [tag + k]) except: pass
def tick(self, price, **kwargs): self.current = float(price) # If it's a brand new price the open price hasn't been set yet # then the current price is the open price. if self.open is None: self.open = self.current # If it's a brand new price the high price hasn't been set yet, # or if the current price is greater than the current high # then set this current price as the high. if (self.high is None) or (self.current > self.high): self.high = self.current # If it's a brand new price the low price hasn't been set yet, # or if the current price is less than the current low # then set this current price as the low. if (self.low is None) or (self.current < self.low): self.low = self.current # If the current time is at or after the start time plus the period # (i.e. this will be the last price that goes into this candlestick before # it is added to the list of past candlesticks) then set this current price # as the closing price. if time.time() >= (self.startTime + (self.period * 60)): self.close = self.current # Determine the typical price over entire period of the candlestick. self.typical_price = (self.high + self.low + self.close) / float(3) # Show OHLC data on each tick self.output.log(" Open: " + str(self.open) + " Close: " + str(self.close) + " High: " + str(self.high) + " Low: " + str(self.low) + " Current: " + str(self.current)) statsd.histogram('candlestick.price.close', self.close, tags=['candlestick.price.close', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) statsd.histogram('candlestick.price.high', self.high, tags=['candlestick.price.high', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) statsd.histogram('candlestick.price.low', self.low, tags=['candlestick.price.low', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) statsd.histogram('candlestick.price.typical_price', self.typical_price, tags=['candlestick.price.typical_price', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)])
def purchaseitem_metrics(sender: PurchaseItem, instance: PurchaseItem, created: bool, **kwargs): metric = "ceitba.purchaseitem.%s.count" % instance.product.name count = instance.quantity statsd.histogram(metric, count)
def _record_errors(service_name): print("Error {0}".format(service_name)) metric = 'availability.errors.%s' % service_name statsd.histogram(metric, 1)
async def _consume_pulls( self, installation: context.Installation, pulls: PullsToConsume, ) -> None: LOG.debug( "stream contains %d pulls", len(pulls), stream_name=installation.stream_name ) for (repo, pull_number), (message_ids, sources) in pulls.items(): statsd.histogram("engine.streams.batch-size", len(sources)) for source in sources: if "timestamp" in source: statsd.histogram( "engine.streams.events.latency", ( datetime.datetime.utcnow() - datetime.datetime.fromisoformat(source["timestamp"]) ).total_seconds(), ) logger = daiquiri.getLogger( __name__, gh_repo=repo, gh_owner=installation.owner_login, gh_pull=pull_number, ) attempts_key = f"pull~{installation.owner_login}~{repo}~{pull_number}" try: async with self._translate_exception_to_retries( installation.stream_name, attempts_key ): await run_engine(installation, repo, pull_number, sources) await self.redis_stream.hdel("attempts", attempts_key) await self.redis_stream.execute_command( "XDEL", installation.stream_name, *message_ids ) except IgnoredException: await self.redis_stream.execute_command( "XDEL", installation.stream_name, *message_ids ) logger.debug("failed to process pull request, ignoring", exc_info=True) except MaxPullRetry as e: await self.redis_stream.execute_command( "XDEL", installation.stream_name, *message_ids ) logger.error( "failed to process pull request, abandoning", attempts=e.attempts, exc_info=True, ) except PullRetry as e: logger.info( "failed to process pull request, retrying", attempts=e.attempts, exc_info=True, ) except StreamRetry: raise except StreamUnused: raise except vcr_errors_CannotOverwriteExistingCassetteException: raise except Exception: # Ignore it, it will retried later logger.error("failed to process pull request", exc_info=True)
def tick(self, candlestick): if self.cool_down_period: self.cool_down_period -= 1 # self.output.log("cool down: {}".format(self.cool_down_period)) self.current_price = float(candlestick.typical_price) self.prices.append(self.current_price) # Highs self.currentHigh = float(candlestick.high) self.highs.append(self.currentHigh) # Lows self.currentLow = float(candlestick.low) self.lows.append(self.currentLow) # Closes self.currentClose = float(candlestick.close) self.closes.append(self.currentClose) # Resistance levels - highest highs self.highest_high = float(max(self.highs)) # self.resistance.append(self.highest_high) statsd.histogram('stochastic.resistance', self.highest_high, tags=['stochastic.resistance', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) # Support levels - lowest lows self.lowest_low = float(min(self.lows)) # self.support.append(self.lowest_low) statsd.histogram('stochastic.support', self.lowest_low, tags=['stochastic.support', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) if len(self.prices) > self.hist_period: # Price action self.output.log("\n{color}Typical Price: {price}".format(color=Red, price=str(candlestick.typical_price))) # Moving average m_a = self.indicators.moving_average(self.closes, self.ma_period_fast) self.moving_avgs.append(m_a) self.output.log("Current Moving Average Value: " + str(m_a)) # self.output.log("Last {} Moving Average Values: {}".format(self.ma_period_fast, str(self.moving_avgs[-self.ma_period_fast:]))) statsd.histogram('stochastic.moving_average', m_a, tags=['stochastic.moving_average', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) # Slope of moving averages trend line slope = self.indicators.slope(self.moving_avgs, lookback=self.ma_slope_lookback) self.ma_slopes.append(slope) self.output.log("Slope over last {} periods: {}".format(str(self.ma_slope_lookback), str(self.ma_slopes[-1]))) # self.output.log("Slope values: {}".format(str(self.ma_slopes))) # Create a new list of change in value between values in an old list. e.g. [1,1,2,3,5,8] == [0, 1, 1, 2, 3] # Simple approximation for rate of change. self.slope_difference = [j-i for i, j in zip(self.ma_slopes[:-1], self.ma_slopes[1:])] # self.output.log("Slope differences: {}".format(self.slope_difference)) statsd.histogram('stochastic.slope', slope, tags=['stochastic.slope', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) # Trend direction trend_dir = self.price_to_ma_trend(candlestick.typical_price, m_a) self.output.log("Price to MA trend direction: {} (price to moving average) ".format(str(trend_dir))) statsd.histogram('stochastic.trend', trend_dir, tags=['stochastic.trend', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) # Stochastic k, d = self.indicators.STOCH(self.highs, self.lows, self.closes, fastk_period=self.stoch_period) self.output.log("\n{color}%K: {k}\t%D: {d}".format(color=Yellow, k=str(k[-1:]), d=str(d[-1:]))) # Stochastic Ready to Buy - True if (k[-1] < self.mid_line and d[-1] < self.mid_line) and k[-1] > d[-1]: self.stoch_ready_to_buy = True # Stochastic Ready to Buy - False if (k[-1] > self.mid_line and d[-1] > self.mid_line) and k[-1] < d[-1]: self.stoch_ready_to_buy = False self.output.log("Stochastic ready to buy: {}".format(self.stoch_ready_to_buy)) statsd.histogram('stochastic.percent_k', k[-1], tags=['stochastic.percent_k', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) statsd.histogram('stochastic.percent_d', d[-1], tags=['stochastic.percent_d', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) # Support and Resistance self.sup, self.res = self.indicators.support_resistance(self.closes, n=self.sr_n) self.output.log("\n{color}n={n} - {prices} available prices".format(color=Cyan, n=self.sr_n, prices=len(self.closes))) self.output.log("Support: {sup}\nResistance: {res}{color}".format(sup=self.sup, res=self.res, color=White)) statsd.histogram('sr.support_major', min(self.sup), tags=['sr.support_major', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) statsd.histogram('sr.resistance_major', max(self.res), tags=['sr.resistance_major', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) statsd.histogram('sr.support_minor', max(self.sup), tags=['sr.support_minor', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) statsd.histogram('sr.resistance_minor', min(self.res), tags=['sr.resistance_minor', 'bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) self.evaluate_positions() self.update_open_trades() self.show_positions()
async def _extract_pulls_from_stream( self, installation: context.Installation ) -> PullsToConsume: messages: typing.List[ typing.Tuple[T_MessageID, T_MessagePayload] ] = await self.redis_stream.xrange( installation.stream_name, count=config.STREAM_MAX_BATCH ) LOG.debug( "read stream", stream_name=installation.stream_name, messages_count=len(messages), ) statsd.histogram("engine.streams.size", len(messages)) statsd.gauge("engine.streams.max_size", config.STREAM_MAX_BATCH) # TODO(sileht): Put this cache in Repository context opened_pulls_by_repo: typing.Dict[ github_types.GitHubRepositoryName, typing.List[github_types.GitHubPullRequest], ] = {} # Groups stream by pull request pulls: PullsToConsume = PullsToConsume(collections.OrderedDict()) for message_id, message in messages: data = msgpack.unpackb(message[b"event"], raw=False) repo = github_types.GitHubRepositoryName(data["repo"]) source = typing.cast(context.T_PayloadEventSource, data["source"]) if data["pull_number"] is not None: key = (repo, github_types.GitHubPullRequestNumber(data["pull_number"])) group = pulls.setdefault(key, ([], [])) group[0].append(message_id) group[1].append(source) else: logger = daiquiri.getLogger( __name__, gh_repo=repo, gh_owner=installation.owner_login, source=source, ) if repo not in opened_pulls_by_repo: try: opened_pulls_by_repo[repo] = [ p async for p in installation.client.items( f"/repos/{installation.owner_login}/{repo}/pulls" ) ] except Exception as e: if exceptions.should_be_ignored(e): opened_pulls_by_repo[repo] = [] else: raise converted_messages = await self._convert_event_to_messages( installation, repo, source, opened_pulls_by_repo[repo], ) logger.debug("event unpacked into %s messages", len(converted_messages)) messages.extend(converted_messages) deleted = await self.redis_stream.xdel( installation.stream_name, message_id ) if deleted != 1: # FIXME(sileht): During shutdown, heroku may have already started # another worker that have already take the lead of this stream_name # This can create duplicate events in the streams but that should not # be a big deal as the engine will not been ran by the worker that's # shutdowning. contents = await self.redis_stream.xrange( installation.stream_name, start=message_id, end=message_id ) if contents: logger.error( "message `%s` have not been deleted has expected, " "(result: %s), content of current message id: %s", message_id, deleted, contents, ) return pulls
def event_handler(name, **kwargs): if name == 'request_finished': statsd.increment('mod_wsgi.request.count') application_time = kwargs.get('application_time') statsd.histogram('mod_wsgi.request.application_time', application_time)
def dogstats(pagetag): start = time.time() statsd.increment('web.get.count', tags = ["support",pagetag]) duration = time.time() - start statsd.histogram('web.get.latency', duration, tags = ["support",pagetag])
err, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) continue except urllib2.ssl.SSLError, err: # For read timeouts output.log("{}... Continuing".format(err[0])) statsd.histogram( 'main_loop.urllib2.ssl.SSLError', err, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) continue except Exception as err: # Something else happened but we still want ot log it, send it to DD, and keep going. output.log("Unexpected error: {}".format(sys.exc_info()[0])) statsd.histogram( 'main_loop.unknown_exception', err, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) time.sleep(30) continue # When close price is present if developing_candlestick.isClosed(): # Add the closed candlestick to the list candlesticks.append(developing_candlestick) # Enact the strategy strategy.tick(developing_candlestick) # Create a new candlestick developing_candlestick = BotCandlestick(period) time.sleep(float(poll_time))
async def _extract_pulls_from_stream(self, stream_name: str) -> PullsToConsume: messages = await self.redis.xrange(stream_name, count=config.STREAM_MAX_BATCH) LOG.debug("read stream", stream_name=stream_name, messages_count=len(messages)) statsd.histogram("engine.streams.size", len(messages)) statsd.gauge("engine.streams.max_size", config.STREAM_MAX_BATCH) opened_pulls_by_repo: typing.Dict[typing.Tuple[ str, str], typing.List[github_types.GitHubPullRequest]] = {} # Groups stream by pull request pulls: PullsToConsume = PullsToConsume(collections.OrderedDict()) for message_id, message in messages: data = msgpack.unpackb(message[b"event"], raw=False) owner = data["owner"] repo = data["repo"] source = data["source"] if data["pull_number"] is not None: key = (owner, repo, data["pull_number"]) group = pulls.setdefault(key, ([], [])) group[0].append(message_id) group[1].append(source) else: logger = daiquiri.getLogger(__name__, gh_repo=repo, gh_owner=owner, source=source) if (owner, repo) not in opened_pulls_by_repo: try: opened_pulls_by_repo[( owner, repo)] = await self._get_pulls_for( stream_name, owner, repo) except IgnoredException: opened_pulls_by_repo[(owner, repo)] = [] converted_messages = await self._convert_event_to_messages( owner, repo, source, opened_pulls_by_repo[(owner, repo)], ) logger.debug("event unpacked into %s messages", len(converted_messages)) messages.extend(converted_messages) deleted = await self.redis.xdel(stream_name, message_id) if deleted != 1: # FIXME(sileht): During shutdown, heroku may have already started # another worker that have already take the lead of this stream_name # This can create duplicate events in the streams but that should not # be a big deal as the engine will not been ran by the worker that's # shutdowning. contents = await self.redis.xrange(stream_name, start=message_id, end=message_id) if contents: logger.error( "message `%s` have not been deleted has expected, " "(result: %s), content of current message id: %s", message_id, deleted, contents, ) return pulls
from datadog import initialize, statsd import time import random options = { 'statsd_host': '127.0.0.1', 'statsd_port': 8125 } initialize(**options) statsd.increment('example_metric.increment', tags=["environment:dev"]) statsd.decrement('example_metric.decrement', tags=["environment:dev"]) statsd.gauge('example_metric.gauge', 40, tags=["environment:dev"]) statsd.set('example_metric.set', 40, tags=["environment:dev"]) statsd.histogram('example_metric.histogram', random.randint(0, 20), tags=["environment:dev"]) with statsd.timed('example_metric.timer', tags=["environment:dev"]): # do something to be measured time.sleep(random.randint(0, 10)) statsd.distribution('example_metric.distribution', random.randint(0, 20), tags=["environment:dev"])
def main(argv): """ Main entry point """ # Logging output = BotLog() supported_exchanges = ['kraken'] exchange = 'kraken' pair = "XXBTZUSD" # Bitcoin/USD pair on Kraken period = 5 # Time frame interval in minutes, e.g. width of candlestick. poll_time = 1 # How often an API query is made when using real time data. script_help = '\n\t-c --currency <currency pair>\n\t-x --exchange <name of the exchange {exchanges}>\n\t-t --poll <poll period length in minutes>\n\nHistorical Mode\n\t-p --period <period of frame>\n\t-s --start <start time in unix timestamp>\n\t-e --end <end time in unix timestamp>\n'.format( exchanges=supported_exchanges) start_time = False end_time = False try: opts, args = getopt.getopt( argv, "h:x:p:c:t:s:e:y:", ["exchange=", "period=", "currency=", "poll=", "start=", "end="]) except getopt.GetoptError: output.log(sys.argv[0] + script_help) sys.exit(2) for opt, arg in opts: if opt == ("-h", "--help"): output.log(sys.argv[0] + script_help) sys.exit() elif opt in ("-s", "--start"): start_time = arg elif opt in ("-e", "--end"): end_time = arg elif opt in ("-x", "--exchange"): if arg in supported_exchanges: exchange = arg else: output.log( 'Supported exchanges are {}'.format(supported_exchanges)) sys.exit(2) elif opt in ("-p", "--period"): if exchange.lower() == 'kraken': # Kraken uses minutes for getting historical data. mins = [1, 5, 15, 30, 60, 240, 1440, 10080, 21600] if (int(arg) in mins): period = int(arg) else: output.log( 'Kraken requires intervals 1, 5, 15, 30, 60, 240, 1440, 10080, 21600 minute intervals' ) sys.exit(2) else: period = int(arg) elif opt in ("-c", "--currency"): pair = arg elif opt in ("-t", "--poll"): poll_time = arg ################ Strategy in use ################ strategy = MACDStrategy(pair, period) strategy_name = strategy.get_name() ################################################# # Log bot startup event to DataDog statsd.event(title='Bot started', text='{}:{} started on {} trading {} using {}'.format( BOT_ID, BOT_NAME, exchange, pair, strategy_name), alert_type='success', tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) trade_session_details = "{bg}Trading {pair} on {exchange} with {strat}" \ " @ {period} minute period{White}".format(pair=pair, exchange=exchange.upper(), strat=strategy_name, period=period, bg=On_Cyan, White=White) if start_time: # Backtesting chart = BotChart(exchange, pair, period) for candlestick in chart.get_points(): strategy.tick(candlestick) output.log(trade_session_details) else: # Live Trading output.log(trade_session_details) chart = BotChart(exchange, pair, period, backtest=False) candlesticks = [] developing_candlestick = BotCandlestick(period) progress_counter = 0 while True: # Log trade details every so often if progress_counter == 50: output.log(trade_session_details) progress_counter = 0 progress_counter += 1 try: developing_candlestick.tick( chart.get_current_price_and_vol()[0]) except urllib2.URLError, err: # If network or site is down output.log("{}... Continuing".format(err[0])) # TODO: These calls to statsd should be Rollbar. Set up Rollbar statsd.histogram( 'main_loop.urllib2.URLError', err, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) continue except ValueError, err: # For screwy JSON output.log('{}... Continuing'.format(err[0])) statsd.histogram( 'main_loop.ValueError', err, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) continue except urllib2.ssl.SSLError, err: # For read timeouts output.log("{}... Continuing".format(err[0])) statsd.histogram( 'main_loop.urllib2.ssl.SSLError', err, tags=['bot_name:{}.bot_id:{}'.format(BOT_NAME, BOT_ID)]) continue
def histogram(self, metric, value, tags=[]): statsd.histogram(metric=metric, value=value, tags=tags) if self.debug: print "{0} = {1} :: type={3} :: tags={2}".format(metric, value, tags, 'histogram')
def generateHistogram(): print("Generate histogram") statsd.histogram('demo.python_statsd.histogram', random.randint(0, 20), tags=["env:" + ENV])