def create_with_random_short_path(cls, long_url, prefix):
"""Generate an unused, valid random short path for prefix"""
nb_tries = 0
while True:
# Generate a seed from the long url and the current date (with milliseconds)
seed = "%s%s%s" % (long_url, datetime.utcnow(), nb_tries)
hashed = int(sha1(seed).hexdigest(), 16)
mod = 1
while hashed > mod:
mod *= 10
nb_tries += 1
short_path = int_to_alnum.encode(hashed % mod)
if not cls.is_valid_random_short_path(short_path):
continue
link, created = cls.__get_or_create(prefix, short_path,
long_url)
if created:
# Short path didn't exist, store number of tries and we're done
statsd.histogram('workforus.nb_tries_to_generate',
nb_tries,
tags=['prefix:' + prefix])
return link
def create_with_random_short_path(cls, long_url, prefix):
"""Generate an unused, valid random short path for prefix"""
nb_tries = 0
while True:
# Generate a seed from the long url and the current date (with milliseconds)
seed = "%s%s%s" % (long_url, datetime.utcnow(), nb_tries)
hashed = int(sha1(seed).hexdigest(), 16)
mod = 1
while hashed > mod:
mod *= 10
nb_tries += 1
short_path = int_to_alnum_encode(hashed % mod, LOWERALPHANUM)
if not cls._is_valid_random_short_path(short_path):
continue
link = cls.__get_or_create(prefix, short_path, long_url)
if link.long_url == long_url:
# Short path didn't exist, store number of tries and we're done
# (it's also possible that we got lucky and guessed a short_url that matched our long_url. Could
# happen.)
statsd.histogram('workforus.nb_tries_to_generate', nb_tries, tags=['prefix:' + prefix])
return link
def _deliver_submission(self, submission):
payload = {'xqueue_body': submission.xqueue_body,
'xqueue_files': submission.s3_urls}
submission.grader_id = self.worker_url
submission.push_time = timezone.now()
start = time.time()
(grading_success, grader_reply) = _http_post(self.worker_url, json.dumps(payload), settings.GRADING_TIMEOUT)
statsd.histogram('xqueue.consumer.consumer_callback.grading_time', time.time() - start,
tags=['queue:{0}'.format(self.queue_name)])
job_count = get_queue_length(self.queue_name)
statsd.gauge('xqueue.consumer.consumer_callback.queue_length', job_count,
tags=['queue:{0}'.format(self.queue_name)])
submission.return_time = timezone.now()
# TODO: For the time being, a submission in a push interface gets one chance at grading,
# with no requeuing logic
if grading_success:
submission.grader_reply = grader_reply
submission.lms_ack = post_grade_to_lms(submission.xqueue_header, grader_reply)
else:
log.error("Submission {} to grader {} failure: Reply: {}, ".format(submission.id, self.worker_url, grader_reply))
submission.num_failures += 1
submission.lms_ack = post_failure_to_lms(submission.xqueue_header)
# NOTE: retiring pushed submissions after one shot regardless of grading_success
submission.retired = True
submission.save()
def onChannelHangupComplete(self, ev):
try:
statsd.histogram('freeswitch.rtp.skipped_packet.in', ev.variable_rtp_audio_in_skip_packet_count)
statsd.histogram('freeswitch.rtp.skipped_packet.out', ev.variable_rtp_audio_out_skip_packet_count)
except:
log.msg("Unable to read variable_rtp_audio_in_skip_packet_count and / or variable_rtp_audio_out_skip_packet_count ")
statsd.increment('freeswitch.caller.context.'+ev.Caller_Context)
statsd.increment('freeswitch.caller.source.'+ev.Caller_Source)
self.g729_metrics()
def main_page():
statsd.increment('web.page_count', tags=['page:main'])
#time diff for histograms
start_time = time.time()
#putting randomly to sleep to generate delays.
time.sleep(random.randint(1, 10))
duration = time.time() - start_time
#paging data for histogram
statsd.histogram('page.load.hist_timer', duration, tags=['type:support', 'page:main'])
return "App Main Page"
def process_item(self, content, queue=None):
try:
statsd.increment('xqueuewatcher.process-item')
body = content['xqueue_body']
files = content['xqueue_files']
# Delivery from the lms
print("____ DEBUG ____")
print(body)
body = json.loads(body)
student_response = body['student_response']
payload = body['grader_payload']
print(student_response)
print(payload)
try:
grader_config = json.loads(payload)
except ValueError as err:
# If parsing json fails, erroring is fine--something is wrong in the content.
# However, for debugging, still want to see what the problem is
statsd.increment('xqueuewatcher.grader_payload_error')
self.log.debug("error parsing: '{0}' -- {1}".format(payload, err))
raise
self.log.debug("Processing submission, grader payload: {0}".format(payload))
#relative_grader_path = 'lesson1_hw2/grader.py' # TODO actually have a grader in the config
relative_grader_path = grader_config['grader']
grader_path = (self.grader_root / relative_grader_path).abspath()
print("___ DEBUG ___")
print("Grader path", grader_path)
print("Relative path", relative_grader_path)
start = time.time()
results = self.grade(grader_path, grader_config, student_response)
statsd.histogram('xqueuewatcher.grading-time', time.time() - start)
# Make valid JSON message
reply = {'correct': results['correct'],
'score': results['score'],
'msg': self.render_results(results)}
statsd.increment('xqueuewatcher.replies (non-exception)')
except Exception as e:
self.log.exception("process_item")
if queue:
queue.put(e)
else:
raise
else:
if queue:
queue.put(reply)
return reply
def onChannelHangupComplete(self, ev):
try:
statsd.histogram('freeswitch.rtp.skipped_packet.in',
ev.variable_rtp_audio_in_skip_packet_count)
statsd.histogram('freeswitch.rtp.skipped_packet.out',
ev.variable_rtp_audio_out_skip_packet_count)
except:
log.msg(
"Unable to read variable_rtp_audio_in_skip_packet_count and / or variable_rtp_audio_out_skip_packet_count "
)
statsd.increment('freeswitch.caller.context.' + ev.Caller_Context)
statsd.increment('freeswitch.caller.source.' + ev.Caller_Source)
self.g729_metrics()
def do_POST(data):
statsd.increment('xserver.post-requests')
# This server expects jobs to be pushed to it from the queue
xpackage = json.loads(data)
if xpackage.get('myreq', None):
body = xpackage['xqueue_body']
files = xpackage['xqueue_files']
student_response = body['student_response']
payload = body['grader_payload']
grader_config = payload
else:
body = xpackage['xqueue_body']
files = xpackage['xqueue_files']
# Delivery from the lms
body = json.loads(body)
student_response = body['student_response']
payload = body['grader_payload']
try:
grader_config = json.loads(payload)
except ValueError as err:
# If parsing json fails, erroring is fine--something is wrong in the content.
# However, for debugging, still want to see what the problem is
statsd.increment('xserver.grader_payload_error')
log.debug("error parsing: '{0}' -- {1}".format(payload, err))
raise
log.debug("Processing submission, grader payload: {0}".format(payload))
relative_grader_path = grader_config['grader']
grader_path = os.path.join(settings.GRADER_ROOT, relative_grader_path)
start = time()
results = grade.grade(grader_path, grader_config, student_response, sandbox)
statsd.histogram('xserver.grading-time', time() - start)
# Make valid JSON message
reply = { 'correct': results['correct'],
'score': results['score'],
'msg': render_results(results) }
statsd.increment('xserver.post-replies (non-exception)')
return json.dumps(reply)
def _deliver_submission(self, submission):
payload = {
'xqueue_body': submission.xqueue_body,
'xqueue_files': submission.urls
}
submission.grader_id = self.worker_url
submission.push_time = timezone.now()
start = time.time()
(grading_success, grader_reply) = _http_post(self.worker_url,
json.dumps(payload),
settings.GRADING_TIMEOUT)
grading_time = time.time() - start
statsd.histogram('xqueue.consumer.consumer_callback.grading_time',
grading_time,
tags=['queue:{0}'.format(self.queue_name)])
if grading_time > settings.GRADING_TIMEOUT:
log.error(
"Grading time above {} for submission. grading_time: {}s body: {} files: {}"
.format(settings.GRADING_TIMEOUT, grading_time,
submission.xqueue_body, submission.urls))
job_count = get_queue_length(self.queue_name)
statsd.gauge('xqueue.consumer.consumer_callback.queue_length',
job_count,
tags=['queue:{0}'.format(self.queue_name)])
submission.return_time = timezone.now()
# TODO: For the time being, a submission in a push interface gets one chance at grading,
# with no requeuing logic
if grading_success:
submission.grader_reply = grader_reply
submission.lms_ack = post_grade_to_lms(submission.xqueue_header,
grader_reply)
else:
log.error("Submission {} to grader {} failure: Reply: {}, ".format(
submission.id, self.worker_url, grader_reply))
submission.num_failures += 1
submission.lms_ack = post_failure_to_lms(submission.xqueue_header)
# NOTE: retiring pushed submissions after one shot regardless of grading_success
submission.retired = True
submission.save()
def process_item(self, content, queue=None):
try:
statsd.increment('xqueuewatcher.process-item')
body = content['xqueue_body']
files = content['xqueue_files']
# Delivery from the lms
body = json.loads(body)
student_response = body['student_response']
payload = body['grader_payload']
files = json.loads(files)
student_info = json.loads(body['student_info'])
try:
grader_config = json.loads(payload)
except ValueError as err:
# If parsing json fails, erroring is fine--something is wrong in the content.
# However, for debugging, still want to see what the problem is
statsd.increment('xqueuewatcher.grader_payload_error')
self.log.debug("error parsing: '{0}' -- {1}".format(payload, err))
raise
self.log.debug("Processing submission, grader payload: {0}".format(payload))
relative_grader_path = grader_config['grader']
grader_path = (self.grader_root / relative_grader_path).abspath()
start = time.time()
results = self.grade(grader_path, grader_config, student_response, files, student_info)
statsd.histogram('xqueuewatcher.grading-time', time.time() - start)
# Make valid JSON message
reply = {'correct': results['correct'],
'score': results['score'],
'msg': self.render_results(results)}
statsd.increment('xqueuewatcher.replies (non-exception)')
except Exception as e:
self.log.exception("process_item")
if queue:
queue.put(e)
else:
raise
else:
if queue:
queue.put(reply)
return reply
def do_POST(data):
statsd.increment('xserver.post-requests')
# This server expects jobs to be pushed to it from the queue
xpackage = json.loads(data)
body = xpackage['xqueue_body']
files = xpackage['xqueue_files']
# Delivery from the lms
files = files and json.loads(files) or {}
body = json.loads(body)
student_response = body['student_response']
payload = body['grader_payload']
try:
grader_config = json.loads(payload)
except ValueError as err:
# If parsing json fails, erroring is fine--something is wrong in the content.
# However, for debugging, still want to see what the problem is
statsd.increment('xserver.grader_payload_error')
log.debug("error parsing: '{0}' -- {1}".format(payload, err))
raise
log.debug("Processing submission, grader payload: {0}".format(payload))
relative_grader_path = grader_config['grader']
grader_path = os.path.join(settings.GRADER_ROOT, relative_grader_path)
start = time()
results = grade.grade(grader_path, grader_config, student_response,
sandbox, files)
statsd.histogram('xserver.grading-time', time() - start)
# Make valid JSON message
reply = {
'correct': results['correct'],
'score': results['score'],
'msg': render_results(results)
}
statsd.increment('xserver.post-replies (non-exception)')
return json.dumps(reply)
def create_with_random_short_path(cls, long_url, prefix):
"""Generate an unused, valid random short path for prefix"""
nb_tries = 0
while True:
# Generate a seed from the long url and the current date (with milliseconds)
seed = "%s%s%s" % (long_url, datetime.utcnow(), nb_tries)
hashed = int(sha1(seed).hexdigest(), 16)
mod = 1
while hashed > mod:
mod *= 10
nb_tries += 1
short_path = int_to_alnum.encode(hashed % mod)
if not cls.is_valid_random_short_path(short_path):
continue
link, created = cls.__get_or_create(prefix, short_path, long_url)
if created:
# Short path didn't exist, store number of tries and we're done
statsd.histogram('workforus.nb_tries_to_generate', nb_tries, tags=['prefix:' + prefix])
return link
def emit(self, name, value, tags):
statsd.histogram(name, value, tags=tags)