def create_with_random_short_path(cls, long_url, prefix): """Generate an unused, valid random short path for prefix""" nb_tries = 0 while True: # Generate a seed from the long url and the current date (with milliseconds) seed = "%s%s%s" % (long_url, datetime.utcnow(), nb_tries) hashed = int(sha1(seed).hexdigest(), 16) mod = 1 while hashed > mod: mod *= 10 nb_tries += 1 short_path = int_to_alnum.encode(hashed % mod) if not cls.is_valid_random_short_path(short_path): continue link, created = cls.__get_or_create(prefix, short_path, long_url) if created: # Short path didn't exist, store number of tries and we're done statsd.histogram('workforus.nb_tries_to_generate', nb_tries, tags=['prefix:' + prefix]) return link
def create_with_random_short_path(cls, long_url, prefix): """Generate an unused, valid random short path for prefix""" nb_tries = 0 while True: # Generate a seed from the long url and the current date (with milliseconds) seed = "%s%s%s" % (long_url, datetime.utcnow(), nb_tries) hashed = int(sha1(seed).hexdigest(), 16) mod = 1 while hashed > mod: mod *= 10 nb_tries += 1 short_path = int_to_alnum_encode(hashed % mod, LOWERALPHANUM) if not cls._is_valid_random_short_path(short_path): continue link = cls.__get_or_create(prefix, short_path, long_url) if link.long_url == long_url: # Short path didn't exist, store number of tries and we're done # (it's also possible that we got lucky and guessed a short_url that matched our long_url. Could # happen.) statsd.histogram('workforus.nb_tries_to_generate', nb_tries, tags=['prefix:' + prefix]) return link
def _deliver_submission(self, submission): payload = {'xqueue_body': submission.xqueue_body, 'xqueue_files': submission.s3_urls} submission.grader_id = self.worker_url submission.push_time = timezone.now() start = time.time() (grading_success, grader_reply) = _http_post(self.worker_url, json.dumps(payload), settings.GRADING_TIMEOUT) statsd.histogram('xqueue.consumer.consumer_callback.grading_time', time.time() - start, tags=['queue:{0}'.format(self.queue_name)]) job_count = get_queue_length(self.queue_name) statsd.gauge('xqueue.consumer.consumer_callback.queue_length', job_count, tags=['queue:{0}'.format(self.queue_name)]) submission.return_time = timezone.now() # TODO: For the time being, a submission in a push interface gets one chance at grading, # with no requeuing logic if grading_success: submission.grader_reply = grader_reply submission.lms_ack = post_grade_to_lms(submission.xqueue_header, grader_reply) else: log.error("Submission {} to grader {} failure: Reply: {}, ".format(submission.id, self.worker_url, grader_reply)) submission.num_failures += 1 submission.lms_ack = post_failure_to_lms(submission.xqueue_header) # NOTE: retiring pushed submissions after one shot regardless of grading_success submission.retired = True submission.save()
def onChannelHangupComplete(self, ev): try: statsd.histogram('freeswitch.rtp.skipped_packet.in', ev.variable_rtp_audio_in_skip_packet_count) statsd.histogram('freeswitch.rtp.skipped_packet.out', ev.variable_rtp_audio_out_skip_packet_count) except: log.msg("Unable to read variable_rtp_audio_in_skip_packet_count and / or variable_rtp_audio_out_skip_packet_count ") statsd.increment('freeswitch.caller.context.'+ev.Caller_Context) statsd.increment('freeswitch.caller.source.'+ev.Caller_Source) self.g729_metrics()
def main_page(): statsd.increment('web.page_count', tags=['page:main']) #time diff for histograms start_time = time.time() #putting randomly to sleep to generate delays. time.sleep(random.randint(1, 10)) duration = time.time() - start_time #paging data for histogram statsd.histogram('page.load.hist_timer', duration, tags=['type:support', 'page:main']) return "App Main Page"
def process_item(self, content, queue=None): try: statsd.increment('xqueuewatcher.process-item') body = content['xqueue_body'] files = content['xqueue_files'] # Delivery from the lms print("____ DEBUG ____") print(body) body = json.loads(body) student_response = body['student_response'] payload = body['grader_payload'] print(student_response) print(payload) try: grader_config = json.loads(payload) except ValueError as err: # If parsing json fails, erroring is fine--something is wrong in the content. # However, for debugging, still want to see what the problem is statsd.increment('xqueuewatcher.grader_payload_error') self.log.debug("error parsing: '{0}' -- {1}".format(payload, err)) raise self.log.debug("Processing submission, grader payload: {0}".format(payload)) #relative_grader_path = 'lesson1_hw2/grader.py' # TODO actually have a grader in the config relative_grader_path = grader_config['grader'] grader_path = (self.grader_root / relative_grader_path).abspath() print("___ DEBUG ___") print("Grader path", grader_path) print("Relative path", relative_grader_path) start = time.time() results = self.grade(grader_path, grader_config, student_response) statsd.histogram('xqueuewatcher.grading-time', time.time() - start) # Make valid JSON message reply = {'correct': results['correct'], 'score': results['score'], 'msg': self.render_results(results)} statsd.increment('xqueuewatcher.replies (non-exception)') except Exception as e: self.log.exception("process_item") if queue: queue.put(e) else: raise else: if queue: queue.put(reply) return reply
def onChannelHangupComplete(self, ev): try: statsd.histogram('freeswitch.rtp.skipped_packet.in', ev.variable_rtp_audio_in_skip_packet_count) statsd.histogram('freeswitch.rtp.skipped_packet.out', ev.variable_rtp_audio_out_skip_packet_count) except: log.msg( "Unable to read variable_rtp_audio_in_skip_packet_count and / or variable_rtp_audio_out_skip_packet_count " ) statsd.increment('freeswitch.caller.context.' + ev.Caller_Context) statsd.increment('freeswitch.caller.source.' + ev.Caller_Source) self.g729_metrics()
def do_POST(data): statsd.increment('xserver.post-requests') # This server expects jobs to be pushed to it from the queue xpackage = json.loads(data) if xpackage.get('myreq', None): body = xpackage['xqueue_body'] files = xpackage['xqueue_files'] student_response = body['student_response'] payload = body['grader_payload'] grader_config = payload else: body = xpackage['xqueue_body'] files = xpackage['xqueue_files'] # Delivery from the lms body = json.loads(body) student_response = body['student_response'] payload = body['grader_payload'] try: grader_config = json.loads(payload) except ValueError as err: # If parsing json fails, erroring is fine--something is wrong in the content. # However, for debugging, still want to see what the problem is statsd.increment('xserver.grader_payload_error') log.debug("error parsing: '{0}' -- {1}".format(payload, err)) raise log.debug("Processing submission, grader payload: {0}".format(payload)) relative_grader_path = grader_config['grader'] grader_path = os.path.join(settings.GRADER_ROOT, relative_grader_path) start = time() results = grade.grade(grader_path, grader_config, student_response, sandbox) statsd.histogram('xserver.grading-time', time() - start) # Make valid JSON message reply = { 'correct': results['correct'], 'score': results['score'], 'msg': render_results(results) } statsd.increment('xserver.post-replies (non-exception)') return json.dumps(reply)
def _deliver_submission(self, submission): payload = { 'xqueue_body': submission.xqueue_body, 'xqueue_files': submission.urls } submission.grader_id = self.worker_url submission.push_time = timezone.now() start = time.time() (grading_success, grader_reply) = _http_post(self.worker_url, json.dumps(payload), settings.GRADING_TIMEOUT) grading_time = time.time() - start statsd.histogram('xqueue.consumer.consumer_callback.grading_time', grading_time, tags=['queue:{0}'.format(self.queue_name)]) if grading_time > settings.GRADING_TIMEOUT: log.error( "Grading time above {} for submission. grading_time: {}s body: {} files: {}" .format(settings.GRADING_TIMEOUT, grading_time, submission.xqueue_body, submission.urls)) job_count = get_queue_length(self.queue_name) statsd.gauge('xqueue.consumer.consumer_callback.queue_length', job_count, tags=['queue:{0}'.format(self.queue_name)]) submission.return_time = timezone.now() # TODO: For the time being, a submission in a push interface gets one chance at grading, # with no requeuing logic if grading_success: submission.grader_reply = grader_reply submission.lms_ack = post_grade_to_lms(submission.xqueue_header, grader_reply) else: log.error("Submission {} to grader {} failure: Reply: {}, ".format( submission.id, self.worker_url, grader_reply)) submission.num_failures += 1 submission.lms_ack = post_failure_to_lms(submission.xqueue_header) # NOTE: retiring pushed submissions after one shot regardless of grading_success submission.retired = True submission.save()
def process_item(self, content, queue=None): try: statsd.increment('xqueuewatcher.process-item') body = content['xqueue_body'] files = content['xqueue_files'] # Delivery from the lms body = json.loads(body) student_response = body['student_response'] payload = body['grader_payload'] files = json.loads(files) student_info = json.loads(body['student_info']) try: grader_config = json.loads(payload) except ValueError as err: # If parsing json fails, erroring is fine--something is wrong in the content. # However, for debugging, still want to see what the problem is statsd.increment('xqueuewatcher.grader_payload_error') self.log.debug("error parsing: '{0}' -- {1}".format(payload, err)) raise self.log.debug("Processing submission, grader payload: {0}".format(payload)) relative_grader_path = grader_config['grader'] grader_path = (self.grader_root / relative_grader_path).abspath() start = time.time() results = self.grade(grader_path, grader_config, student_response, files, student_info) statsd.histogram('xqueuewatcher.grading-time', time.time() - start) # Make valid JSON message reply = {'correct': results['correct'], 'score': results['score'], 'msg': self.render_results(results)} statsd.increment('xqueuewatcher.replies (non-exception)') except Exception as e: self.log.exception("process_item") if queue: queue.put(e) else: raise else: if queue: queue.put(reply) return reply
def do_POST(data): statsd.increment('xserver.post-requests') # This server expects jobs to be pushed to it from the queue xpackage = json.loads(data) body = xpackage['xqueue_body'] files = xpackage['xqueue_files'] # Delivery from the lms files = files and json.loads(files) or {} body = json.loads(body) student_response = body['student_response'] payload = body['grader_payload'] try: grader_config = json.loads(payload) except ValueError as err: # If parsing json fails, erroring is fine--something is wrong in the content. # However, for debugging, still want to see what the problem is statsd.increment('xserver.grader_payload_error') log.debug("error parsing: '{0}' -- {1}".format(payload, err)) raise log.debug("Processing submission, grader payload: {0}".format(payload)) relative_grader_path = grader_config['grader'] grader_path = os.path.join(settings.GRADER_ROOT, relative_grader_path) start = time() results = grade.grade(grader_path, grader_config, student_response, sandbox, files) statsd.histogram('xserver.grading-time', time() - start) # Make valid JSON message reply = { 'correct': results['correct'], 'score': results['score'], 'msg': render_results(results) } statsd.increment('xserver.post-replies (non-exception)') return json.dumps(reply)
def emit(self, name, value, tags): statsd.histogram(name, value, tags=tags)