class UserMetaResource(WigoResource): model = User @user_token_required @api.response(200, 'Success', model=api.model( 'UserMeta', { 'last_message_received': fields.DateTime(), 'last_friend_request': fields.DateTime(), 'last_notification': fields.DateTime(), 'is_tapped': fields.Boolean(), 'is_friend': fields.Boolean(), 'is_blocked': fields.Boolean(), 'attending_event_id': fields.Integer(), 'friend_request': fields.String(), 'num_friends_in_common': fields.Integer() })) def get(self, user_id): user_id = self.get_id(user_id) meta = {} if user_id == g.user.id: user_meta = wigo_db.get_redis().hgetall( skey('user', user_id, 'meta')) if user_meta: def format_date(field): return datetime.utcfromtimestamp(float( user_meta[field])).isoformat() if 'last_message_received' in user_meta: meta['last_message_received'] = format_date( 'last_message_received') if 'last_friend_request' in user_meta: meta['last_friend_request'] = format_date( 'last_friend_request') if 'last_notification' in user_meta: meta['last_notification'] = format_date( 'last_notification') meta['attending_event_id'] = g.user.get_attending_id() else: meta['is_tapped'] = g.user.is_tapped(user_id) meta['is_friend'] = g.user.is_friend(user_id) meta['is_blocked'] = g.user.is_blocked(user_id) if g.user.is_friend_request_sent(user_id): meta['friend_request'] = 'sent' elif g.user.is_friend_request_received(user_id): meta['friend_request'] = 'received' if request.args.get('num_friends_in_common') == 'true': meta['num_friends_in_common'] = len( g.user.get_friend_ids_in_common(user_id)) meta['num_friends'] = wigo_db.get_sorted_set_size( skey('user', user_id, 'friends')) return meta
def create_event(self): requirement = self.api.model( 'Requirement', { 'name': fields.String(required=True, description='Event name'), 'quantity': fields.Integer(required=True, description='Name of event') }) event = self.api.model( 'Event', { 'tag': fields.String(required=False, description='Tag of event'), 'name': fields.String(required=True, description='Name of event'), 'date': fields.String(required=True, description='Date of event' ), # Cambiar el type por lo que corresponde. 'time': fields.String(required=False, description='Time of event'), 'capacity': fields.String(required=False, description='Capacity of event'), 'venue': fields.Nested(self.venueDocument.venue, required=False, description='Venue of event'), 'image': fields.String(required=False, description='Image of event'), 'description': fields.String(required=False, description='Description of event'), 'visibility': fields.String(required=True, description='Visibility of event'), 'hasAssistance': fields.Boolean(required=False, description=''), 'isOwner': fields.Boolean(required=False, description=''), 'soldOut': fields.Boolean(required=False, description=''), 'gests': fields.List(fields.String(), required=False, description='Description of event'), 'requirementMissing': fields.List(fields.Nested(requirement), required=False, description='Requirements missing'), 'requirement': fields.List(fields.Nested(requirement), required=False, description='Requirements') }) return event
def test_boolean_field_with_readonly(self): field = fields.Boolean(readonly=True) self.assertFalse(field.required) self.assertEqual(field.__schema__, { 'type': 'boolean', 'readOnly': True })
class GetQueueNames(Resource): """Get list of job queues and return as JSON.""" resp_model = api.model('Queue Listing Response(JSON)', { 'success': fields.Boolean(required=True, description="if 'false', " + "encountered exception; otherwise no errors " + "occurred"), 'message': fields.String(required=True, description="message describing " + "success or failure"), 'result': fields.Raw(required=True, description="queue response") }) parser = api.parser() parser.add_argument('id', required=False, type=str, help="Job Type Specification ID") @api.expect(parser) @api.marshal_with(resp_model) def get(self): ''' Gets a listing of non-celery queues handling jobs. ''' try: ident = request.form.get('id',request.args.get('id', None)) queues = mozart.lib.queue_utils.get_queue_names(ident) app.logger.warn("Queues:"+str(queues)) except Exception as e: message = "Failed to list job queues. {0}:{1}".format(type(e),str(e)) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 return { 'success': True, 'message': "", 'result': queues }
def test_boolean_field_with_title(self): field = fields.Boolean(title='A title') self.assertFalse(field.required) self.assertEqual(field.__schema__, { 'type': 'boolean', 'title': 'A title' })
def test_boolean_field_with_description(self): field = fields.Boolean(description='A description') self.assertFalse(field.required) self.assertEqual(field.__schema__, { 'type': 'boolean', 'description': 'A description' })
class GetHySDSIOTypes(Resource): """Get list of registered hysds-io and return as JSON.""" resp_model_job_types = api.model('HySDS IO List Response(JSON)', { 'success': fields.Boolean(required=True, description="if 'false', " + "encountered exception; otherwise no errors " + "occurred"), 'message': fields.String(required=True, description="message describing " + "success or failure"), 'result': fields.List(fields.String, required=True, description="list of hysds-io types") }) @api.marshal_with(resp_model_job_types) def get(self): ''' List HySDS IO specifications ''' try: ids = hysds_commons.hysds_io_utils.get_hysds_io_types(app.config["ES_URL"], logger=app.logger) except Exception as e: message = "Failed to query ES for HySDS IO types. {0}:{1}".format(type(e),str(e)) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 return { 'success': True, 'message': "", 'result': ids }
class AddHySDSIOType(Resource): """Add job spec""" resp_model = api.model('HySDS IO Addition Response(JSON)', { 'success': fields.Boolean(required=True, description="if 'false', " + "encountered exception; otherwise no errors " + "occurred"), 'message': fields.String(required=True, description="message describing " + "success or failure"), 'result': fields.String(required=True, description="HySDS IO ID") }) parser = api.parser() parser.add_argument('spec', required=True, type=str, help="HySDS IO JSON Object") @api.expect(parser) @api.marshal_with(resp_model) def post(self): ''' Add a HySDS IO specification ''' try: spec = request.form.get('spec',request.args.get('spec', None)) if spec is None: raise Exception("'spec' must be supplied") obj = json.loads(spec) ident = hysds_commons.hysds_io_utils.add_hysds_io(app.config["ES_URL"], obj, logger=app.logger) except Exception as e: message = "Failed to add ES for HySDS IO. {0}:{1}".format(type(e),str(e)) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 return { 'success': True, 'message': "", 'result': ident }
class RemoveHySDSIOType(Resource): """Remove job spec""" resp_model = api.model('HySDS IO Removal Response(JSON)', { 'success': fields.Boolean(required=True, description="if 'false', " + "encountered exception; otherwise no errors " + "occurred"), 'message': fields.String(required=True, description="message describing " + "success or failure"), }) parser = api.parser() parser.add_argument('id', required=True, type=str, help="HySDS IO ID") @api.expect(parser) @api.marshal_with(resp_model) def get(self): ''' Remove HySDS IO for the given ID ''' try: ident = request.form.get('id',request.args.get('id', None)) hysds_commons.hysds_io_utils.remove_hysds_io(app.config["ES_URL"], ident, logger=app.logger) except Exception as e: message = "Failed to add ES for HySDS IO. {0}:{1}".format(type(e),str(e)) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 return { 'success': True, 'message': ""}
class GetContainerInfo(Resource): """Info a container""" resp_model = api.model('Container Info Response(JSON)', { 'success': fields.Boolean(required=True, description="if 'false', " + "encountered exception; otherwise no errors " + "occurred"), 'message': fields.String(required=True, description="message describing " + "success or failure"), 'result': fields.Raw(required=True,description="Container Info") }) parser = api.parser() parser.add_argument('id', required=True, type=str, help="Container ID") @api.expect(parser) @api.marshal_with(resp_model) def get(self): ''' Get information on container by ID ''' try: # get job id ident = request.form.get('id', request.args.get('id', None)) info = hysds_commons.container_utils.get_container(app.config['ES_URL'], ident, logger=app.logger) except Exception as e: message = "Failed to get info for container {2}. {0}:{1}".format(type(e),str(e),ident) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 return { 'success': True, 'message': "", 'result': info }
def test_with_default(self): field = fields.Boolean(default=True) self.assertFalse(field.required) self.assertEqual(field.__schema__, { 'type': 'boolean', 'default': True })
class QueryJson(Resource): """Get PROV-ES document by ID and return as JSON.""" resp_model = api.model('JsonResponse', { 'success': fields.Boolean(required=True, description="if 'false', encountered exception; otherwise no errors occurred"), 'message': fields.String(required=True, description="message describing success or failure"), 'result': fields.Raw(required=True, description="PROV-ES JSON document") }) @api.doc(params={ 'id': 'ID of PROV-ES document'}) @api.marshal_with(resp_model) def get(self): id = request.args.get('id', None) if id is None: return { 'success': False, 'message': "Missing id parameter.", 'result': {} }, 400 # query try: pej = get_prov_es_json(id) except Exception, e: message = "Failed to get PROV-ES document for id %s: %s" % (id, str(e)) current_app.logger.debug(message) return { 'success': False, 'message': message, 'result': {} }, 500 # set message if len(pej) == 0: message = "No document found with id %s." % id else: message = "" # return result return { 'success': True, 'message': message, 'result': pej.get('_source', {}) }
class GetJobs(Resource): """Get list of job IDs.""" resp_model = api.model('Jobs Listing Response(JSON)', { 'success': fields.Boolean(required=True, description="if 'false', " + "encountered exception; otherwise no errors " + "occurred"), 'message': fields.String(required=True, description="message describing " + "success or failure"), 'result': fields.List(fields.String, required=True, description="list of job IDs") }) parser = api.parser() parser.add_argument('page_size', required=False, type=str, help="Job Listing Pagination Size") parser = api.parser() parser.add_argument('offset', required=False, type=str, help="Job Listing Pagination Offset") @api.marshal_with(resp_model) def get(self): ''' Paginated list submitted jobs ''' try: page_size = request.form.get('page_size', request.args.get('page_size', 100)) offset = request.form.get('offset', request.args.get('id', 0)) jobs = mozart.lib.job_utils.get_job_list(page_size,offset) except Exception as e: message = "Failed to get job listing(page: {2}, offset: {3}). {0}:{1}".format(type(e),str(e),page_size,offset) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 return { 'success': True, 'message': "", 'result': jobs }
class GetJobSpecType(Resource): """Get list of job queues and return as JSON.""" resp_model = api.model('Job Type Specification Response(JSON)', { 'success': fields.Boolean(required=True, description="if 'false', " + "encountered exception; otherwise no errors " + "occurred"), 'message': fields.String(required=True, description="message describing " + "success or failure"), 'result': fields.Raw(required=True, description="Job Type Specification") }) parser = api.parser() parser.add_argument('id', required=True, type=str, help="Job Type ID") @api.expect(parser) @api.marshal_with(resp_model) def get(self): ''' Gets a Job Type specification object for the given ID. ''' try: ident = request.form.get('id',request.args.get('id', None)) spec = hysds_commons.job_spec_utils.get_job_spec(app.config['ES_URL'], ident, logger=app.logger) except Exception as e: message = "Failed to query ES for Job spec. {0}:{1}".format(type(e),str(e)) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 return { 'success': True, 'message': "", 'result': spec }
class GetJobInfo(Resource): """Get info of job IDs.""" resp_model = api.model('Job Info Response(JSON)', { 'success': fields.Boolean(required=True, description="if 'false', " + "encountered exception; otherwise no errors " + "occurred"), 'message': fields.String(required=True, description="message describing " + "success or failure"), 'result': fields.Raw(required=True,description="Job Info Object") }) parser = api.parser() parser.add_argument('id', required=True, type=str, help="Job ID") @api.expect(parser) @api.marshal_with(resp_model) def get(self): ''' Get complete infor on submitted job based on id ''' try: # get job id ident = request.form.get('id', request.args.get('id', None)) info = mozart.lib.job_utils.get_job_info(ident) except Exception as e: message = "Failed to get job info for {2}. {0}:{1}".format(type(e),str(e),ident) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 return { 'success': True, 'message': "", 'result': info }
class IndexDataset(Resource): """Dataset indexing API.""" resp_model = api.model( 'JsonResponse', { 'success': fields.Boolean( required=True, description="if 'false', " + "encountered exception; otherwise no errors " + "occurred"), 'message': fields.String( required=True, description="message describing " + "success or failure"), 'objectid': fields.String(required=True, description="ID of indexed dataset"), 'index': fields.String(required=True, description="dataset index name"), }) parser = api.parser() parser.add_argument('dataset_info', required=True, type=str, location='form', help="HySDS dataset info JSON") @api.marshal_with(resp_model) @api.expect(parser, validate=True) def post(self): # get info info = request.form.get('dataset_info', request.args.get('dataset_info', None)) if info is not None: try: info = json.loads(info) except Exception, e: message = "Failed to parse dataset info JSON." app.logger.debug(message) return { 'success': False, 'message': message, 'job_id': None }, 500 # update try: return updateDataset(info) except Exception, e: message = "Failed index dataset. {0}:{1}\n{2}".format( type(e), e, traceback.format_exc()) app.logger.debug(message) return {'success': False, 'message': message}, 500
class GetContainerAdd(Resource): """Add a container""" resp_model = api.model('Container Add Response(JSON)', { 'success': fields.Boolean(required=True, description="if 'false', " + "encountered exception; otherwise no errors " + "occurred"), 'message': fields.String(required=True, description="message describing " + "success or failure"), 'result': fields.String(required=True,description="Container ID") }) parser = api.parser() parser.add_argument('name', required=True, type=str, help="Container Name") parser.add_argument('url', required=True, type=str, help="Container URL") parser.add_argument('version', required=True, type=str, help="Container Version") parser.add_argument('digest', required=True, type=str, help="Container Digest") @api.expect(parser) @api.marshal_with(resp_model) def post(self): ''' Add a container specification to Mozart ''' try: # get job id print request.form print request.args name = request.form.get('name', request.args.get('name', None)) url = request.form.get('url', request.args.get('url', None)) version = request.form.get('version', request.args.get('version', None)) digest = request.form.get('digest', request.args.get('digest', None)) if name is None: raise Exception("'name' must be supplied") if url is None: raise Exception("'url' must be supplied") if version is None: raise Exception("'version' must be supplied") if digest is None: raise Exception("'digest' must be supplied") ident = hysds_commons.container_utils.add_container(app.config['ES_URL'], name, url, version, digest, logger=app.logger) except Exception as e: message = "Failed to add container {2}. {0}:{1}".format(type(e),str(e),name) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 return { 'success': True, 'message': "", 'result': ident }
class GetJobStatus(Resource): """Get status of job ID.""" resp_model = api.model( 'Job Status Response(JSON)', { 'success': fields.Boolean( required=True, description="if 'false', " + "encountered exception; otherwise no errors " + "occurred"), 'message': fields.String( required=True, description="message describing " + "success or failure"), 'status': fields.String(required=True, enum=[ 'job-queued', 'job-started', 'job-failed', 'job-completed', 'job-offline', 'job-revoked' ], description='job status') }) parser = api.parser() parser.add_argument('id', required=True, type=str, help="Job ID") @api.expect(parser) @api.marshal_with(resp_model) def get(self): ''' Gets the status of a submitted job based on job id ''' try: # get job id ident = request.form.get('id', request.args.get('id', None)) status = mozart.lib.job_utils.get_job_status(ident) except Exception as e: message = "Failed to get job status for {2}. {0}:{1}".format( type(e), str(e), ident) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 # return result return {'success': True, 'message': "", 'status': status}
class QueryTurtle(Resource): """Get PROV-ES document by ID and return as Turtle.""" resp_model = api.model('TurtleResponse', { 'success': fields.Boolean(required=True, description="if 'false', encountered exception; otherwise no errors occurred"), 'message': fields.String(required=True, description="message describing success or failure"), 'result': fields.String(required=True, description="PROV-ES Turtle document") }) @api.doc(params={ 'id': 'ID of PROV-ES document'}) @api.marshal_with(resp_model) def get(self): id = request.args.get('id', None) if id is None: return { 'success': False, 'message': "Missing id parameter.", 'result': "" }, 400 # query try: pej = get_prov_es_json(id) except Exception, e: message = "Failed to get PROV-ES document for id %s: %s" % (id, str(e)) current_app.logger.debug(message) return { 'success': False, 'message': message, 'result': "" }, 500 # if no document found, return if len(pej) == 0: message = "No PROV-ES document found with id %s." % id current_app.logger.debug(message) return { 'success': False, 'message': message, 'result': "" }, 500 # convert PROV-ES JSON to turtle try: ttl = get_ttl(pej['_source']['prov_es_json']) except Exception, e: message = "Failed to transform PROV-ES JSON document to Turtle for id %s: %s" % (id, str(e)) current_app.logger.debug(message) return { 'success': False, 'message': message, 'result': "" }, 500
class ImportProvEs(Resource): """Import PROV-ES document.""" resp_model = api.model('ImportResponse', { 'success': fields.Boolean(required=True, description="if 'false', encountered exception; otherwise no errors occurred"), 'message': fields.String(required=True, description="message describing success or failure") }) @api.doc(params={ 'prov_es': 'PROV-ES JSON document string'}) @api.marshal_with(resp_model) def post(self): # get PROV-ES json prov_es = request.form.get('prov_es', request.args.get('prov_es', None)) if prov_es is None: return { 'success': False, 'message': "Missing prov_es parameter.", 'result': {} }, 400 # load JSON try: pej = json.loads(prov_es) except Exception, e: message = "Failed to parse PROV-ES json. Check that your PROV-ES JSON conforms to PROV-JSON." current_app.logger.debug(message) return { 'success': False, 'message': message, 'result': {} }, 500 # import prov es_url = current_app.config['ES_URL'] dt = datetime.utcnow() es_index = "%s-%04d.%02d.%02d" % (current_app.config['PROVES_ES_PREFIX'], dt.year, dt.month, dt.day) alias = current_app.config['PROVES_ES_ALIAS'] conn = get_es_conn(es_url, es_index, alias) try: import_prov(conn, es_index, alias, pej) except Exception, e: current_app.logger.debug("Got error: %s" % e) current_app.logger.debug("Traceback: %s" % traceback.format_exc()) message = "Failed to import PROV-ES json. Check that your PROV-ES JSON conforms to PROV-JSON." current_app.logger.debug(message) return { 'success': False, 'message': message, 'result': {} }, 500
class AddLogEvent(Resource): """Add log event.""" resp_model = api.model('HySDS Event Log Response(JSON)', { 'success': fields.Boolean(required=True, description="if 'false', " + "encountered exception; otherwise no errors " + "occurred"), 'message': fields.String(required=True, description="message describing " + "success or failure"), 'result': fields.String(required=True, description="HySDS custom event log ID") }) parser = api.parser() parser.add_argument('type', required=True, type=str, help="Event type, e.g. aws_autoscaling, verdi_anomalies") parser.add_argument('status', required=True, type=str, help="Event status, e.g. spot_termination, docker_daemon_failed") parser.add_argument('event', required=True, type=str, help="""Arbitrary JSON event payload, e.g. {} or { "ec2_instance_id": "i-07b8989f41ce23880", "private_ip": "100.64.134.145", "az": "us-west-2a", "reservation": "r-02fd006170749a0a8", "termination_date": "2015-01-02T15:49:05.571384" }""") parser.add_argument('tags', required=False, type=str, help='JSON list of tags, e.g. ["dumby", "test_job"]') parser.add_argument('hostname', required=False, type=str, help='Event-related hostname, e.g. "job.hysds.net", "192.168.0.1"') @api.expect(parser) @api.marshal_with(resp_model) def post(self): """Log HySDS custom event.""" try: #app.logger.info("data: %s %d" % (request.data, len(request.data))) #app.logger.info("form: %s" % request.form) #app.logger.info("args: %s" % request.args) if len(request.data) > 0: try: form = json.loads(request.data) except Exception as e: raise Exception("Failed to parse request data. '{0}' is malformed JSON".format(request.data)) else: form = request.form event_type = form.get('type', request.args.get('type', None)) event_status = form.get('status', request.args.get('status', None)) event = form.get('event', request.args.get('event', '{}')) try: if event is not None and not isinstance(event, dict): event = json.loads(event) except Exception as e: raise Exception("Failed to parse input event. '{0}' is malformed JSON".format(event)) tags = form.get('tags', request.args.get('tags', None)) try: if tags is not None and not isinstance(tags, list): tags = json.loads(tags) except Exception as e: raise Exception("Failed to parse input tags. '{0}' is malformed JSON".format(tags)) hostname = form.get('hostname', request.args.get('hostname', None)) app.logger.info("type: %s" % event_type) app.logger.info("status: %s" % event_status) app.logger.info("event: %s" % event) app.logger.info("tags: %s" % tags) app.logger.info("hostname: %s" % hostname) uuid = log_custom_event(event_type, event_status, event, tags, hostname) except Exception as e: message = "Failed to log custom event. {0}:{1}".format(type(e),str(e)) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return { 'success': False, 'message': message }, 500 return { 'success': True, 'message': '', 'result': uuid }
class VertexUpdate(Resource): vertexUpdate = api.model( 'VertexUpdate', { 'vertex': fields.String(required=True, description='Vertex Name'), 'vtype': fields.String(required=True, description='Vertex Type Name (None = 0)'), 'set_weight': fields.String(required=False, description='Edge type'), 'incr_weight': fields.Integer(required=False, description='Timestamp') }) vertexUpdateSpec = api.model( 'Update', { 'vertexUpdates': fields.List(fields.Nested(vertexUpdate), description='List of Vertex Updates', required=True), 'immediate': fields.Boolean( description= 'Instructs the API to send this batch to STINGER immediately upon receipt', required=False, default=False) }) @api.expect(vertexUpdateSpec) @api.doc( responses={ 201: 'Vertices Updated', 400: 'Bad Request', 503: 'Unable to reach STINGER' }) def post(self): exec_time = time.time() if not request.json: r = json.dumps({"error": "Invalid input"}) return Response(response=r, status=400, mimetype="application/json") # grab the lock counter_lock.acquire() try: data = request.json send_immediate = False if 'immediate' in data: if data['immediate'] == True: send_immediate = True if isinstance(data["vertexUpdates"], list): vertexUpdates = data["vertexUpdates"] print "Received batch of size", len( vertexUpdates), 'at', strftime("%Y%m%d%H%M%S", gmtime()), "" for x in vertexUpdates: try: vtx = str(x["vertex"]) vtype = str(x["vtype"]) set_weight = x["set_weight"] if 'set_weight' in x else 0 incr_weight = int( x["incr_weight"]) if 'incr_weight' in x else 0 s.add_vertex_update(vtx, vtype, set_weight, incr_weight) except Exception as e: print(traceback.format_exc()) pass # send immediately if the batch is now large current_batch_size = s.insertions_count + s.deletions_count + s.vertex_updates_count if current_batch_size > BATCH_THRESHOLD and BATCH_THRESHOLD > 0 or send_immediate: print "Sending batch of size", current_batch_size, 'at', strftime( "%Y%m%d%H%M%S", gmtime()), "" s.send_batch() except: print(traceback.format_exc()) r = json.dumps({"error": "Unable to parse object"}) return Response(response=r, status=400, mimetype="application/json") # end critical section finally: counter_lock.release() exec_time = time.time() - exec_time r = json.dumps({ "status": "success", "time": exec_time, "current_batch_size": current_batch_size }) return Response(response=r, status=201, mimetype="application/json")
class Insert(Resource): edge = api.model( 'Edge', { 'src': fields.String(required=True, description='Source vertex'), 'dest': fields.String(required=True, description='Destination vertex'), 'type': fields.String(required=False, description='Edge type'), 'weight': fields.Integer(required=False, description='Edge weight'), 'time': fields.Integer(required=False, description='Timestamp') }) edgesSpec = api.model( 'Insert', { 'edges': fields.List(fields.Nested(edge), description='List of edges', required=True), 'immediate': fields.Boolean( description= 'Instructs the API to send this batch to STINGER immediately upon receipt', required=False, default=False), 'strings': fields.Boolean( description= 'Instructs the API to interpret integer vertex names as strings rather than integer vertex IDs', required=False, default=True) }) @api.expect(edgesSpec) @api.doc(responses={ 201: 'Edge Inserted', 400: 'Bad Request', 503: 'Unable to reach STINGER' }) def post(self): setupSTINGERConnection() exec_time = time.time() if not request.json: r = json.dumps({"error": "Invalid input"}) return Response(response=r, status=400, mimetype="application/json") # grab the lock counter_lock.acquire() try: data = request.json send_immediate = False if 'immediate' not in data else data[ 'immediate'] only_strings = True if 'strings' not in data else data['strings'] if isinstance(data["edges"], list): edges = data["edges"] print "Received batch of size", len(edges), 'at', strftime( "%Y%m%d%H%M%S", gmtime()), "" for x in edges: try: if only_strings: source = str(x["src"]) destination = str(x["dest"]) else: source = x["src"] destination = x["dest"] edge_type = x["type"] if 'type' in x else 0 edge_weight = int(x["weight"]) if 'weight' in x else 0 timestamp = int(x["time"]) if 'time' in x else 0 s.add_insert(source, destination, edge_type, weight=edge_weight, ts=timestamp, insert_strings=only_strings) # print "added edge", source, destination, edge_type, timestamp except Exception as e: print(traceback.format_exc()) pass # send immediately if the batch is now large current_batch_size = s.insertions_count + s.deletions_count + s.vertex_updates_count if current_batch_size > BATCH_THRESHOLD and BATCH_THRESHOLD > 0 or send_immediate: s.send_batch() print "Sending batch of size", current_batch_size, 'at', strftime( "%Y%m%d%H%M%S", gmtime()), "" except: print(traceback.format_exc()) r = json.dumps({"error": "Unable to parse object"}) return Response(response=r, status=400, mimetype="application/json") # end critical section finally: counter_lock.release() exec_time = time.time() - exec_time r = json.dumps({ "status": "success", "time": exec_time, "current_batch_size": current_batch_size }) return Response(response=r, status=201, mimetype="application/json")
def test_simple_boolean_field(self): field = fields.Boolean() self.assertFalse(field.required) self.assertEqual(field.__schema__, {'type': 'boolean'})
def test_boolean_field_with_required(self): field = fields.Boolean(required=True) self.assertTrue(field.required) self.assertEqual(field.__schema__, {'type': 'boolean'})
def test_defaults(self): field = fields.Boolean() self.assertFalse(field.required) self.assertEqual(field.__schema__, {'type': 'boolean'})
class SubmitJob(Resource): """Submit job for execution in HySDS.""" resp_model = api.model('SubmitJobResponse', { 'success': fields.Boolean(required=True, description="if 'false', " + "encountered exception; otherwise no errors " + "occurred"), 'message': fields.String(required=True, description="message describing " + "success or failure"), 'result': fields.String(required=True, description="HySDS job ID") }) parser = api.parser() parser.add_argument('type', required=True, type=str, help="a job type from jobspec/list") parser.add_argument('queue', required=True, type=str, help="Job queue from /queue/list e.g. grfn-job_worker-small") parser.add_argument('priority', required=False, type=int, help='Job priority in the range of 0 to 9') parser.add_argument('tags', required=False, type=str, help='JSON list of tags, e.g. ["dumby", "test_job"]') parser.add_argument('payload_hash', required=False, type=str, help='user-generated payload hash') parser.add_argument('enable_dedup', required=False, type=bool, help='flag to enable/disable job dedup') parser.add_argument('params', required=False, type=str, help="""JSON job context, e.g. { "entity_id": "LC80101172015002LGN00", "min_lat": -79.09923, "max_lon": -125.09297, "id": "dumby-product-20161114180506209624", "acq_time": "2015-01-02T15:49:05.571384", "min_sleep": 1, "max_lat": -77.7544, "min_lon": -139.66082, "max_sleep": 10 }""") @api.marshal_with(resp_model) @api.expect(parser, validate=True) def post(self): ''' Submits a job to run inside HySDS ''' try: app.logger.warning(request.form) job_type = request.form.get('type', request.args.get('type', None)) job_queue = request.form.get('queue', request.args.get('queue', None)) priority = int(request.form.get('priority', request.args.get('priority', 0))) tags = request.form.get('tags', request.args.get('tags', None)) payload_hash = request.form.get('payload_hash', request.args.get('payload_hash', None)) enable_dedup = str(request.form.get('enable_dedup', request.args.get('enable_dedup', "true"))) if enable_dedup.strip().lower() == "true": enable_dedup = True elif enable_dedup.strip().lower() == "false": enable_dedup = False else: raise Exception("Invalid value for param 'enable_dedup': {0}".format(enable_dedup)) try: if not tags is None: tags = json.loads(tags) except Exception as e: raise Exception("Failed to parse input tags. '{0}' is malformed".format(tags)) params = request.form.get('params', request.args.get('params', "{}")) app.logger.warning(params) try: if not params is None: params = json.loads(params) except Exception as e: raise Exception("Failed to parse input params. '{0}' is malformed".format(params)) app.logger.warning(job_type) app.logger.warning(job_queue) job_json = hysds_commons.job_utils.resolve_hysds_job(job_type, job_queue, priority, tags, params, payload_hash=payload_hash, enable_dedup=enable_dedup) ident = hysds_commons.job_utils.submit_hysds_job(job_json) except Exception as e: message = "Failed to submit job. {0}:{1}".format(type(e),str(e)) app.logger.warning(message) app.logger.warning(traceback.format_exc(e)) return {'success': False, 'message': message}, 500 return { 'success': True, 'message': '', 'result': ident }
required=True), 'paid_hourly_rate': fields.Float( description='hourly cost for paid parking here (if applicable)') }) slots_field = api.model( 'SlotsField', { 'way_name': fields.String, 'button_locations': fields.List(fields.Nested(button_locations), required=True), 'restrict_types': fields.List(fields.String), 'compact': fields.Boolean(True) }) slots_field_full = api.model( 'SlotsFieldFull', { 'way_name': fields.String, 'rules': fields.List(fields.Nested(rules_field)), 'button_locations': fields.List(fields.Nested(button_locations), required=True), 'compact': fields.Boolean(False) }) slots_fields = api.model(
# Fields that are expected and need to be exposed from a Job fields = api.model( 'Job', { 'id': fields.String(), 'name': fields.String(), 'task_class': fields.String(attribute=lambda x: x.func_ref.replace(':', '.').replace( '.execute', '')), 'next_run_time': fields.DateTime(dt_format='iso8601'), 'misfire_grace_time': fields.String(), 'coalesce': fields.Boolean(), 'trigger': fields.String(), 'args': fields.List(fields.String), 'start_date': fields.DateTime(attribute=lambda x: x.trigger.start_date, dt_format='iso8601'), 'end_date': fields.DateTime(attribute=lambda x: x.trigger.end_date, dt_format='iso8601'), 'timezone': fields.String(), 'year': fields.String(attribute=lambda x: x.trigger.fields[0] if not x.trigger.fields[0].is_default else None),