class UserMetaResource(WigoResource):
model = User
@user_token_required
@api.response(200,
'Success',
model=api.model(
'UserMeta', {
'last_message_received': fields.DateTime(),
'last_friend_request': fields.DateTime(),
'last_notification': fields.DateTime(),
'is_tapped': fields.Boolean(),
'is_friend': fields.Boolean(),
'is_blocked': fields.Boolean(),
'attending_event_id': fields.Integer(),
'friend_request': fields.String(),
'num_friends_in_common': fields.Integer()
}))
def get(self, user_id):
user_id = self.get_id(user_id)
meta = {}
if user_id == g.user.id:
user_meta = wigo_db.get_redis().hgetall(
skey('user', user_id, 'meta'))
if user_meta:
def format_date(field):
return datetime.utcfromtimestamp(float(
user_meta[field])).isoformat()
if 'last_message_received' in user_meta:
meta['last_message_received'] = format_date(
'last_message_received')
if 'last_friend_request' in user_meta:
meta['last_friend_request'] = format_date(
'last_friend_request')
if 'last_notification' in user_meta:
meta['last_notification'] = format_date(
'last_notification')
meta['attending_event_id'] = g.user.get_attending_id()
else:
meta['is_tapped'] = g.user.is_tapped(user_id)
meta['is_friend'] = g.user.is_friend(user_id)
meta['is_blocked'] = g.user.is_blocked(user_id)
if g.user.is_friend_request_sent(user_id):
meta['friend_request'] = 'sent'
elif g.user.is_friend_request_received(user_id):
meta['friend_request'] = 'received'
if request.args.get('num_friends_in_common') == 'true':
meta['num_friends_in_common'] = len(
g.user.get_friend_ids_in_common(user_id))
meta['num_friends'] = wigo_db.get_sorted_set_size(
skey('user', user_id, 'friends'))
return meta
def create_event(self):
requirement = self.api.model(
'Requirement', {
'name':
fields.String(required=True, description='Event name'),
'quantity':
fields.Integer(required=True, description='Name of event')
})
event = self.api.model(
'Event',
{
'tag':
fields.String(required=False, description='Tag of event'),
'name':
fields.String(required=True, description='Name of event'),
'date':
fields.String(required=True, description='Date of event'
), # Cambiar el type por lo que corresponde.
'time':
fields.String(required=False, description='Time of event'),
'capacity':
fields.String(required=False, description='Capacity of event'),
'venue':
fields.Nested(self.venueDocument.venue,
required=False,
description='Venue of event'),
'image':
fields.String(required=False, description='Image of event'),
'description':
fields.String(required=False,
description='Description of event'),
'visibility':
fields.String(required=True,
description='Visibility of event'),
'hasAssistance':
fields.Boolean(required=False, description=''),
'isOwner':
fields.Boolean(required=False, description=''),
'soldOut':
fields.Boolean(required=False, description=''),
'gests':
fields.List(fields.String(),
required=False,
description='Description of event'),
'requirementMissing':
fields.List(fields.Nested(requirement),
required=False,
description='Requirements missing'),
'requirement':
fields.List(fields.Nested(requirement),
required=False,
description='Requirements')
})
return event
def test_boolean_field_with_readonly(self):
field = fields.Boolean(readonly=True)
self.assertFalse(field.required)
self.assertEqual(field.__schema__, {
'type': 'boolean',
'readOnly': True
})
class GetQueueNames(Resource):
"""Get list of job queues and return as JSON."""
resp_model = api.model('Queue Listing Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'result': fields.Raw(required=True, description="queue response")
})
parser = api.parser()
parser.add_argument('id', required=False, type=str, help="Job Type Specification ID")
@api.expect(parser)
@api.marshal_with(resp_model)
def get(self):
'''
Gets a listing of non-celery queues handling jobs.
'''
try:
ident = request.form.get('id',request.args.get('id', None))
queues = mozart.lib.queue_utils.get_queue_names(ident)
app.logger.warn("Queues:"+str(queues))
except Exception as e:
message = "Failed to list job queues. {0}:{1}".format(type(e),str(e))
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return {'success': False, 'message': message}, 500
return { 'success': True,
'message': "",
'result': queues }
def test_boolean_field_with_title(self):
field = fields.Boolean(title='A title')
self.assertFalse(field.required)
self.assertEqual(field.__schema__, {
'type': 'boolean',
'title': 'A title'
})
def test_boolean_field_with_description(self):
field = fields.Boolean(description='A description')
self.assertFalse(field.required)
self.assertEqual(field.__schema__, {
'type': 'boolean',
'description': 'A description'
})
class GetHySDSIOTypes(Resource):
"""Get list of registered hysds-io and return as JSON."""
resp_model_job_types = api.model('HySDS IO List Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'result': fields.List(fields.String, required=True,
description="list of hysds-io types")
})
@api.marshal_with(resp_model_job_types)
def get(self):
'''
List HySDS IO specifications
'''
try:
ids = hysds_commons.hysds_io_utils.get_hysds_io_types(app.config["ES_URL"], logger=app.logger)
except Exception as e:
message = "Failed to query ES for HySDS IO types. {0}:{1}".format(type(e),str(e))
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return {'success': False, 'message': message}, 500
return { 'success': True,
'message': "",
'result': ids }
class AddHySDSIOType(Resource):
"""Add job spec"""
resp_model = api.model('HySDS IO Addition Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'result': fields.String(required=True, description="HySDS IO ID")
})
parser = api.parser()
parser.add_argument('spec', required=True, type=str, help="HySDS IO JSON Object")
@api.expect(parser)
@api.marshal_with(resp_model)
def post(self):
'''
Add a HySDS IO specification
'''
try:
spec = request.form.get('spec',request.args.get('spec', None))
if spec is None:
raise Exception("'spec' must be supplied")
obj = json.loads(spec)
ident = hysds_commons.hysds_io_utils.add_hysds_io(app.config["ES_URL"], obj, logger=app.logger)
except Exception as e:
message = "Failed to add ES for HySDS IO. {0}:{1}".format(type(e),str(e))
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return {'success': False, 'message': message}, 500
return { 'success': True,
'message': "",
'result': ident }
class RemoveHySDSIOType(Resource):
"""Remove job spec"""
resp_model = api.model('HySDS IO Removal Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
})
parser = api.parser()
parser.add_argument('id', required=True, type=str, help="HySDS IO ID")
@api.expect(parser)
@api.marshal_with(resp_model)
def get(self):
'''
Remove HySDS IO for the given ID
'''
try:
ident = request.form.get('id',request.args.get('id', None))
hysds_commons.hysds_io_utils.remove_hysds_io(app.config["ES_URL"], ident, logger=app.logger)
except Exception as e:
message = "Failed to add ES for HySDS IO. {0}:{1}".format(type(e),str(e))
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return {'success': False, 'message': message}, 500
return { 'success': True,
'message': ""}
class GetContainerInfo(Resource):
"""Info a container"""
resp_model = api.model('Container Info Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'result': fields.Raw(required=True,description="Container Info")
})
parser = api.parser()
parser.add_argument('id', required=True, type=str, help="Container ID")
@api.expect(parser)
@api.marshal_with(resp_model)
def get(self):
'''
Get information on container by ID
'''
try:
# get job id
ident = request.form.get('id', request.args.get('id', None))
info = hysds_commons.container_utils.get_container(app.config['ES_URL'], ident, logger=app.logger)
except Exception as e:
message = "Failed to get info for container {2}. {0}:{1}".format(type(e),str(e),ident)
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return {'success': False, 'message': message}, 500
return { 'success': True,
'message': "",
'result': info }
def test_with_default(self):
field = fields.Boolean(default=True)
self.assertFalse(field.required)
self.assertEqual(field.__schema__, {
'type': 'boolean',
'default': True
})
class QueryJson(Resource):
"""Get PROV-ES document by ID and return as JSON."""
resp_model = api.model('JsonResponse', {
'success': fields.Boolean(required=True, description="if 'false', encountered exception; otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure"),
'result': fields.Raw(required=True, description="PROV-ES JSON document")
})
@api.doc(params={ 'id': 'ID of PROV-ES document'})
@api.marshal_with(resp_model)
def get(self):
id = request.args.get('id', None)
if id is None:
return { 'success': False,
'message': "Missing id parameter.",
'result': {} }, 400
# query
try: pej = get_prov_es_json(id)
except Exception, e:
message = "Failed to get PROV-ES document for id %s: %s" % (id, str(e))
current_app.logger.debug(message)
return { 'success': False,
'message': message,
'result': {} }, 500
# set message
if len(pej) == 0: message = "No document found with id %s." % id
else: message = ""
# return result
return { 'success': True,
'message': message,
'result': pej.get('_source', {}) }
class GetJobs(Resource):
"""Get list of job IDs."""
resp_model = api.model('Jobs Listing Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'result': fields.List(fields.String, required=True,
description="list of job IDs")
})
parser = api.parser()
parser.add_argument('page_size', required=False, type=str, help="Job Listing Pagination Size")
parser = api.parser()
parser.add_argument('offset', required=False, type=str, help="Job Listing Pagination Offset")
@api.marshal_with(resp_model)
def get(self):
'''
Paginated list submitted jobs
'''
try:
page_size = request.form.get('page_size', request.args.get('page_size', 100))
offset = request.form.get('offset', request.args.get('id', 0))
jobs = mozart.lib.job_utils.get_job_list(page_size,offset)
except Exception as e:
message = "Failed to get job listing(page: {2}, offset: {3}). {0}:{1}".format(type(e),str(e),page_size,offset)
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return {'success': False, 'message': message}, 500
return { 'success': True,
'message': "",
'result': jobs }
class GetJobSpecType(Resource):
"""Get list of job queues and return as JSON."""
resp_model = api.model('Job Type Specification Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'result': fields.Raw(required=True, description="Job Type Specification")
})
parser = api.parser()
parser.add_argument('id', required=True, type=str, help="Job Type ID")
@api.expect(parser)
@api.marshal_with(resp_model)
def get(self):
'''
Gets a Job Type specification object for the given ID.
'''
try:
ident = request.form.get('id',request.args.get('id', None))
spec = hysds_commons.job_spec_utils.get_job_spec(app.config['ES_URL'], ident, logger=app.logger)
except Exception as e:
message = "Failed to query ES for Job spec. {0}:{1}".format(type(e),str(e))
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return {'success': False, 'message': message}, 500
return { 'success': True,
'message': "",
'result': spec }
class GetJobInfo(Resource):
"""Get info of job IDs."""
resp_model = api.model('Job Info Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'result': fields.Raw(required=True,description="Job Info Object")
})
parser = api.parser()
parser.add_argument('id', required=True, type=str, help="Job ID")
@api.expect(parser)
@api.marshal_with(resp_model)
def get(self):
'''
Get complete infor on submitted job based on id
'''
try:
# get job id
ident = request.form.get('id', request.args.get('id', None))
info = mozart.lib.job_utils.get_job_info(ident)
except Exception as e:
message = "Failed to get job info for {2}. {0}:{1}".format(type(e),str(e),ident)
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return {'success': False, 'message': message}, 500
return { 'success': True,
'message': "",
'result': info }
class IndexDataset(Resource):
"""Dataset indexing API."""
resp_model = api.model(
'JsonResponse', {
'success':
fields.Boolean(
required=True,
description="if 'false', " +
"encountered exception; otherwise no errors " + "occurred"),
'message':
fields.String(
required=True,
description="message describing " + "success or failure"),
'objectid':
fields.String(required=True, description="ID of indexed dataset"),
'index':
fields.String(required=True, description="dataset index name"),
})
parser = api.parser()
parser.add_argument('dataset_info',
required=True,
type=str,
location='form',
help="HySDS dataset info JSON")
@api.marshal_with(resp_model)
@api.expect(parser, validate=True)
def post(self):
# get info
info = request.form.get('dataset_info',
request.args.get('dataset_info', None))
if info is not None:
try:
info = json.loads(info)
except Exception, e:
message = "Failed to parse dataset info JSON."
app.logger.debug(message)
return {
'success': False,
'message': message,
'job_id': None
}, 500
# update
try:
return updateDataset(info)
except Exception, e:
message = "Failed index dataset. {0}:{1}\n{2}".format(
type(e), e, traceback.format_exc())
app.logger.debug(message)
return {'success': False, 'message': message}, 500
class GetContainerAdd(Resource):
"""Add a container"""
resp_model = api.model('Container Add Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'result': fields.String(required=True,description="Container ID")
})
parser = api.parser()
parser.add_argument('name', required=True, type=str, help="Container Name")
parser.add_argument('url', required=True, type=str, help="Container URL")
parser.add_argument('version', required=True, type=str, help="Container Version")
parser.add_argument('digest', required=True, type=str, help="Container Digest")
@api.expect(parser)
@api.marshal_with(resp_model)
def post(self):
'''
Add a container specification to Mozart
'''
try:
# get job id
print request.form
print request.args
name = request.form.get('name', request.args.get('name', None))
url = request.form.get('url', request.args.get('url', None))
version = request.form.get('version', request.args.get('version', None))
digest = request.form.get('digest', request.args.get('digest', None))
if name is None:
raise Exception("'name' must be supplied")
if url is None:
raise Exception("'url' must be supplied")
if version is None:
raise Exception("'version' must be supplied")
if digest is None:
raise Exception("'digest' must be supplied")
ident = hysds_commons.container_utils.add_container(app.config['ES_URL'],
name, url, version,
digest, logger=app.logger)
except Exception as e:
message = "Failed to add container {2}. {0}:{1}".format(type(e),str(e),name)
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return {'success': False, 'message': message}, 500
return { 'success': True,
'message': "",
'result': ident }
class GetJobStatus(Resource):
"""Get status of job ID."""
resp_model = api.model(
'Job Status Response(JSON)', {
'success':
fields.Boolean(
required=True,
description="if 'false', " +
"encountered exception; otherwise no errors " + "occurred"),
'message':
fields.String(
required=True,
description="message describing " + "success or failure"),
'status':
fields.String(required=True,
enum=[
'job-queued', 'job-started', 'job-failed',
'job-completed', 'job-offline', 'job-revoked'
],
description='job status')
})
parser = api.parser()
parser.add_argument('id', required=True, type=str, help="Job ID")
@api.expect(parser)
@api.marshal_with(resp_model)
def get(self):
'''
Gets the status of a submitted job based on job id
'''
try:
# get job id
ident = request.form.get('id', request.args.get('id', None))
status = mozart.lib.job_utils.get_job_status(ident)
except Exception as e:
message = "Failed to get job status for {2}. {0}:{1}".format(
type(e), str(e), ident)
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return {'success': False, 'message': message}, 500
# return result
return {'success': True, 'message': "", 'status': status}
class QueryTurtle(Resource):
"""Get PROV-ES document by ID and return as Turtle."""
resp_model = api.model('TurtleResponse', {
'success': fields.Boolean(required=True, description="if 'false', encountered exception; otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure"),
'result': fields.String(required=True, description="PROV-ES Turtle document")
})
@api.doc(params={ 'id': 'ID of PROV-ES document'})
@api.marshal_with(resp_model)
def get(self):
id = request.args.get('id', None)
if id is None:
return { 'success': False,
'message': "Missing id parameter.",
'result': "" }, 400
# query
try: pej = get_prov_es_json(id)
except Exception, e:
message = "Failed to get PROV-ES document for id %s: %s" % (id, str(e))
current_app.logger.debug(message)
return { 'success': False,
'message': message,
'result': "" }, 500
# if no document found, return
if len(pej) == 0:
message = "No PROV-ES document found with id %s." % id
current_app.logger.debug(message)
return { 'success': False,
'message': message,
'result': "" }, 500
# convert PROV-ES JSON to turtle
try: ttl = get_ttl(pej['_source']['prov_es_json'])
except Exception, e:
message = "Failed to transform PROV-ES JSON document to Turtle for id %s: %s" % (id, str(e))
current_app.logger.debug(message)
return { 'success': False,
'message': message,
'result': "" }, 500
class ImportProvEs(Resource):
"""Import PROV-ES document."""
resp_model = api.model('ImportResponse', {
'success': fields.Boolean(required=True, description="if 'false', encountered exception; otherwise no errors occurred"),
'message': fields.String(required=True, description="message describing success or failure")
})
@api.doc(params={ 'prov_es': 'PROV-ES JSON document string'})
@api.marshal_with(resp_model)
def post(self):
# get PROV-ES json
prov_es = request.form.get('prov_es', request.args.get('prov_es', None))
if prov_es is None:
return { 'success': False,
'message': "Missing prov_es parameter.",
'result': {} }, 400
# load JSON
try: pej = json.loads(prov_es)
except Exception, e:
message = "Failed to parse PROV-ES json. Check that your PROV-ES JSON conforms to PROV-JSON."
current_app.logger.debug(message)
return { 'success': False,
'message': message,
'result': {} }, 500
# import prov
es_url = current_app.config['ES_URL']
dt = datetime.utcnow()
es_index = "%s-%04d.%02d.%02d" % (current_app.config['PROVES_ES_PREFIX'],
dt.year, dt.month, dt.day)
alias = current_app.config['PROVES_ES_ALIAS']
conn = get_es_conn(es_url, es_index, alias)
try: import_prov(conn, es_index, alias, pej)
except Exception, e:
current_app.logger.debug("Got error: %s" % e)
current_app.logger.debug("Traceback: %s" % traceback.format_exc())
message = "Failed to import PROV-ES json. Check that your PROV-ES JSON conforms to PROV-JSON."
current_app.logger.debug(message)
return { 'success': False,
'message': message,
'result': {} }, 500
class AddLogEvent(Resource):
"""Add log event."""
resp_model = api.model('HySDS Event Log Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'result': fields.String(required=True, description="HySDS custom event log ID")
})
parser = api.parser()
parser.add_argument('type', required=True, type=str,
help="Event type, e.g. aws_autoscaling, verdi_anomalies")
parser.add_argument('status', required=True, type=str,
help="Event status, e.g. spot_termination, docker_daemon_failed")
parser.add_argument('event', required=True, type=str,
help="""Arbitrary JSON event payload, e.g. {} or {
"ec2_instance_id": "i-07b8989f41ce23880",
"private_ip": "100.64.134.145",
"az": "us-west-2a",
"reservation": "r-02fd006170749a0a8",
"termination_date": "2015-01-02T15:49:05.571384"
}""")
parser.add_argument('tags', required=False, type=str,
help='JSON list of tags, e.g. ["dumby", "test_job"]')
parser.add_argument('hostname', required=False, type=str,
help='Event-related hostname, e.g. "job.hysds.net", "192.168.0.1"')
@api.expect(parser)
@api.marshal_with(resp_model)
def post(self):
"""Log HySDS custom event."""
try:
#app.logger.info("data: %s %d" % (request.data, len(request.data)))
#app.logger.info("form: %s" % request.form)
#app.logger.info("args: %s" % request.args)
if len(request.data) > 0:
try: form = json.loads(request.data)
except Exception as e:
raise Exception("Failed to parse request data. '{0}' is malformed JSON".format(request.data))
else: form = request.form
event_type = form.get('type', request.args.get('type', None))
event_status = form.get('status', request.args.get('status', None))
event = form.get('event', request.args.get('event', '{}'))
try:
if event is not None and not isinstance(event, dict):
event = json.loads(event)
except Exception as e:
raise Exception("Failed to parse input event. '{0}' is malformed JSON".format(event))
tags = form.get('tags', request.args.get('tags', None))
try:
if tags is not None and not isinstance(tags, list):
tags = json.loads(tags)
except Exception as e:
raise Exception("Failed to parse input tags. '{0}' is malformed JSON".format(tags))
hostname = form.get('hostname', request.args.get('hostname', None))
app.logger.info("type: %s" % event_type)
app.logger.info("status: %s" % event_status)
app.logger.info("event: %s" % event)
app.logger.info("tags: %s" % tags)
app.logger.info("hostname: %s" % hostname)
uuid = log_custom_event(event_type, event_status, event, tags, hostname)
except Exception as e:
message = "Failed to log custom event. {0}:{1}".format(type(e),str(e))
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return { 'success': False, 'message': message }, 500
return { 'success': True,
'message': '',
'result': uuid }
class VertexUpdate(Resource):
vertexUpdate = api.model(
'VertexUpdate', {
'vertex':
fields.String(required=True, description='Vertex Name'),
'vtype':
fields.String(required=True,
description='Vertex Type Name (None = 0)'),
'set_weight':
fields.String(required=False, description='Edge type'),
'incr_weight':
fields.Integer(required=False, description='Timestamp')
})
vertexUpdateSpec = api.model(
'Update', {
'vertexUpdates':
fields.List(fields.Nested(vertexUpdate),
description='List of Vertex Updates',
required=True),
'immediate':
fields.Boolean(
description=
'Instructs the API to send this batch to STINGER immediately upon receipt',
required=False,
default=False)
})
@api.expect(vertexUpdateSpec)
@api.doc(
responses={
201: 'Vertices Updated',
400: 'Bad Request',
503: 'Unable to reach STINGER'
})
def post(self):
exec_time = time.time()
if not request.json:
r = json.dumps({"error": "Invalid input"})
return Response(response=r,
status=400,
mimetype="application/json")
# grab the lock
counter_lock.acquire()
try:
data = request.json
send_immediate = False
if 'immediate' in data:
if data['immediate'] == True:
send_immediate = True
if isinstance(data["vertexUpdates"], list):
vertexUpdates = data["vertexUpdates"]
print "Received batch of size", len(
vertexUpdates), 'at', strftime("%Y%m%d%H%M%S",
gmtime()), ""
for x in vertexUpdates:
try:
vtx = str(x["vertex"])
vtype = str(x["vtype"])
set_weight = x["set_weight"] if 'set_weight' in x else 0
incr_weight = int(
x["incr_weight"]) if 'incr_weight' in x else 0
s.add_vertex_update(vtx, vtype, set_weight,
incr_weight)
except Exception as e:
print(traceback.format_exc())
pass
# send immediately if the batch is now large
current_batch_size = s.insertions_count + s.deletions_count + s.vertex_updates_count
if current_batch_size > BATCH_THRESHOLD and BATCH_THRESHOLD > 0 or send_immediate:
print "Sending batch of size", current_batch_size, 'at', strftime(
"%Y%m%d%H%M%S", gmtime()), ""
s.send_batch()
except:
print(traceback.format_exc())
r = json.dumps({"error": "Unable to parse object"})
return Response(response=r,
status=400,
mimetype="application/json")
# end critical section
finally:
counter_lock.release()
exec_time = time.time() - exec_time
r = json.dumps({
"status": "success",
"time": exec_time,
"current_batch_size": current_batch_size
})
return Response(response=r, status=201, mimetype="application/json")
class Insert(Resource):
edge = api.model(
'Edge', {
'src': fields.String(required=True, description='Source vertex'),
'dest': fields.String(required=True,
description='Destination vertex'),
'type': fields.String(required=False, description='Edge type'),
'weight': fields.Integer(required=False,
description='Edge weight'),
'time': fields.Integer(required=False, description='Timestamp')
})
edgesSpec = api.model(
'Insert', {
'edges':
fields.List(fields.Nested(edge),
description='List of edges',
required=True),
'immediate':
fields.Boolean(
description=
'Instructs the API to send this batch to STINGER immediately upon receipt',
required=False,
default=False),
'strings':
fields.Boolean(
description=
'Instructs the API to interpret integer vertex names as strings rather than integer vertex IDs',
required=False,
default=True)
})
@api.expect(edgesSpec)
@api.doc(responses={
201: 'Edge Inserted',
400: 'Bad Request',
503: 'Unable to reach STINGER'
})
def post(self):
setupSTINGERConnection()
exec_time = time.time()
if not request.json:
r = json.dumps({"error": "Invalid input"})
return Response(response=r,
status=400,
mimetype="application/json")
# grab the lock
counter_lock.acquire()
try:
data = request.json
send_immediate = False if 'immediate' not in data else data[
'immediate']
only_strings = True if 'strings' not in data else data['strings']
if isinstance(data["edges"], list):
edges = data["edges"]
print "Received batch of size", len(edges), 'at', strftime(
"%Y%m%d%H%M%S", gmtime()), ""
for x in edges:
try:
if only_strings:
source = str(x["src"])
destination = str(x["dest"])
else:
source = x["src"]
destination = x["dest"]
edge_type = x["type"] if 'type' in x else 0
edge_weight = int(x["weight"]) if 'weight' in x else 0
timestamp = int(x["time"]) if 'time' in x else 0
s.add_insert(source,
destination,
edge_type,
weight=edge_weight,
ts=timestamp,
insert_strings=only_strings)
# print "added edge", source, destination, edge_type, timestamp
except Exception as e:
print(traceback.format_exc())
pass
# send immediately if the batch is now large
current_batch_size = s.insertions_count + s.deletions_count + s.vertex_updates_count
if current_batch_size > BATCH_THRESHOLD and BATCH_THRESHOLD > 0 or send_immediate:
s.send_batch()
print "Sending batch of size", current_batch_size, 'at', strftime(
"%Y%m%d%H%M%S", gmtime()), ""
except:
print(traceback.format_exc())
r = json.dumps({"error": "Unable to parse object"})
return Response(response=r,
status=400,
mimetype="application/json")
# end critical section
finally:
counter_lock.release()
exec_time = time.time() - exec_time
r = json.dumps({
"status": "success",
"time": exec_time,
"current_batch_size": current_batch_size
})
return Response(response=r, status=201, mimetype="application/json")
def test_simple_boolean_field(self):
field = fields.Boolean()
self.assertFalse(field.required)
self.assertEqual(field.__schema__, {'type': 'boolean'})
def test_boolean_field_with_required(self):
field = fields.Boolean(required=True)
self.assertTrue(field.required)
self.assertEqual(field.__schema__, {'type': 'boolean'})
def test_defaults(self):
field = fields.Boolean()
self.assertFalse(field.required)
self.assertEqual(field.__schema__, {'type': 'boolean'})
class SubmitJob(Resource):
"""Submit job for execution in HySDS."""
resp_model = api.model('SubmitJobResponse', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'result': fields.String(required=True, description="HySDS job ID")
})
parser = api.parser()
parser.add_argument('type', required=True, type=str,
help="a job type from jobspec/list")
parser.add_argument('queue', required=True, type=str, help="Job queue from /queue/list e.g. grfn-job_worker-small")
parser.add_argument('priority', required=False, type=int, help='Job priority in the range of 0 to 9')
parser.add_argument('tags', required=False, type=str, help='JSON list of tags, e.g. ["dumby", "test_job"]')
parser.add_argument('payload_hash', required=False, type=str, help='user-generated payload hash')
parser.add_argument('enable_dedup', required=False, type=bool, help='flag to enable/disable job dedup')
parser.add_argument('params', required=False, type=str,
help="""JSON job context, e.g. {
"entity_id": "LC80101172015002LGN00",
"min_lat": -79.09923,
"max_lon": -125.09297,
"id": "dumby-product-20161114180506209624",
"acq_time": "2015-01-02T15:49:05.571384",
"min_sleep": 1,
"max_lat": -77.7544,
"min_lon": -139.66082,
"max_sleep": 10
}""")
@api.marshal_with(resp_model)
@api.expect(parser, validate=True)
def post(self):
'''
Submits a job to run inside HySDS
'''
try:
app.logger.warning(request.form)
job_type = request.form.get('type', request.args.get('type', None))
job_queue = request.form.get('queue', request.args.get('queue', None))
priority = int(request.form.get('priority', request.args.get('priority', 0)))
tags = request.form.get('tags', request.args.get('tags', None))
payload_hash = request.form.get('payload_hash', request.args.get('payload_hash', None))
enable_dedup = str(request.form.get('enable_dedup', request.args.get('enable_dedup', "true")))
if enable_dedup.strip().lower() == "true": enable_dedup = True
elif enable_dedup.strip().lower() == "false": enable_dedup = False
else: raise Exception("Invalid value for param 'enable_dedup': {0}".format(enable_dedup))
try:
if not tags is None:
tags = json.loads(tags)
except Exception as e:
raise Exception("Failed to parse input tags. '{0}' is malformed".format(tags))
params = request.form.get('params', request.args.get('params', "{}"))
app.logger.warning(params)
try:
if not params is None:
params = json.loads(params)
except Exception as e:
raise Exception("Failed to parse input params. '{0}' is malformed".format(params))
app.logger.warning(job_type)
app.logger.warning(job_queue)
job_json = hysds_commons.job_utils.resolve_hysds_job(job_type, job_queue, priority,
tags, params,
payload_hash=payload_hash,
enable_dedup=enable_dedup)
ident = hysds_commons.job_utils.submit_hysds_job(job_json)
except Exception as e:
message = "Failed to submit job. {0}:{1}".format(type(e),str(e))
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return {'success': False, 'message': message}, 500
return { 'success': True,
'message': '',
'result': ident }
required=True),
'paid_hourly_rate':
fields.Float(
description='hourly cost for paid parking here (if applicable)')
})
slots_field = api.model(
'SlotsField', {
'way_name':
fields.String,
'button_locations':
fields.List(fields.Nested(button_locations), required=True),
'restrict_types':
fields.List(fields.String),
'compact':
fields.Boolean(True)
})
slots_field_full = api.model(
'SlotsFieldFull', {
'way_name':
fields.String,
'rules':
fields.List(fields.Nested(rules_field)),
'button_locations':
fields.List(fields.Nested(button_locations), required=True),
'compact':
fields.Boolean(False)
})
slots_fields = api.model(
# Fields that are expected and need to be exposed from a Job
fields = api.model(
'Job', {
'id':
fields.String(),
'name':
fields.String(),
'task_class':
fields.String(attribute=lambda x: x.func_ref.replace(':', '.').replace(
'.execute', '')),
'next_run_time':
fields.DateTime(dt_format='iso8601'),
'misfire_grace_time':
fields.String(),
'coalesce':
fields.Boolean(),
'trigger':
fields.String(),
'args':
fields.List(fields.String),
'start_date':
fields.DateTime(attribute=lambda x: x.trigger.start_date,
dt_format='iso8601'),
'end_date':
fields.DateTime(attribute=lambda x: x.trigger.end_date,
dt_format='iso8601'),
'timezone':
fields.String(),
'year':
fields.String(attribute=lambda x: x.trigger.fields[0]
if not x.trigger.fields[0].is_default else None),