def create_event(self):
requirement = self.api.model(
'Requirement', {
'name':
fields.String(required=True, description='Event name'),
'quantity':
fields.Integer(required=True, description='Name of event')
})
event = self.api.model(
'Event',
{
'tag':
fields.String(required=False, description='Tag of event'),
'name':
fields.String(required=True, description='Name of event'),
'date':
fields.String(required=True, description='Date of event'
), # Cambiar el type por lo que corresponde.
'time':
fields.String(required=False, description='Time of event'),
'capacity':
fields.String(required=False, description='Capacity of event'),
'venue':
fields.Nested(self.venueDocument.venue,
required=False,
description='Venue of event'),
'image':
fields.String(required=False, description='Image of event'),
'description':
fields.String(required=False,
description='Description of event'),
'visibility':
fields.String(required=True,
description='Visibility of event'),
'hasAssistance':
fields.Boolean(required=False, description=''),
'isOwner':
fields.Boolean(required=False, description=''),
'soldOut':
fields.Boolean(required=False, description=''),
'gests':
fields.List(fields.String(),
required=False,
description='Description of event'),
'requirementMissing':
fields.List(fields.Nested(requirement),
required=False,
description='Requirements missing'),
'requirement':
fields.List(fields.Nested(requirement),
required=False,
description='Requirements')
})
return event
def test_marshal_list_of_lists(self):
model = OrderedDict([
('foo', fields.Raw),
('fee', fields.List(fields.List(
fields.String)))
])
marshal_fields = OrderedDict([('foo', 'bar'),
('bat', 'baz'),
('fee', [['fye'], ['fum']])])
output = marshal(marshal_fields, model)
expected = OrderedDict([('foo', 'bar'), ('fee', [['fye'], ['fum']])])
self.assertEquals(output, expected)
class GetJobs(Resource):
"""Get list of job IDs."""
resp_model = api.model('Jobs Listing Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'result': fields.List(fields.String, required=True,
description="list of job IDs")
})
parser = api.parser()
parser.add_argument('page_size', required=False, type=str, help="Job Listing Pagination Size")
parser = api.parser()
parser.add_argument('offset', required=False, type=str, help="Job Listing Pagination Offset")
@api.marshal_with(resp_model)
def get(self):
'''
Paginated list submitted jobs
'''
try:
page_size = request.form.get('page_size', request.args.get('page_size', 100))
offset = request.form.get('offset', request.args.get('id', 0))
jobs = mozart.lib.job_utils.get_job_list(page_size,offset)
except Exception as e:
message = "Failed to get job listing(page: {2}, offset: {3}). {0}:{1}".format(type(e),str(e),page_size,offset)
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return {'success': False, 'message': message}, 500
return { 'success': True,
'message': "",
'result': jobs }
def test_model_as_dict_with_list(self):
model = self.api.model(
'Person', {
'name': fields.String,
'age': fields.Integer,
'tags': fields.List(fields.String),
})
self.assertEqual(
model.__schema__, {
'properties': {
'name': {
'type': 'string'
},
'age': {
'type': 'integer'
},
'tags': {
'type': 'array',
'items': {
'type': 'string'
}
}
}
})
class GetHySDSIOTypes(Resource):
"""Get list of registered hysds-io and return as JSON."""
resp_model_job_types = api.model('HySDS IO List Response(JSON)', {
'success': fields.Boolean(required=True, description="if 'false', " +
"encountered exception; otherwise no errors " +
"occurred"),
'message': fields.String(required=True, description="message describing " +
"success or failure"),
'result': fields.List(fields.String, required=True,
description="list of hysds-io types")
})
@api.marshal_with(resp_model_job_types)
def get(self):
'''
List HySDS IO specifications
'''
try:
ids = hysds_commons.hysds_io_utils.get_hysds_io_types(app.config["ES_URL"], logger=app.logger)
except Exception as e:
message = "Failed to query ES for HySDS IO types. {0}:{1}".format(type(e),str(e))
app.logger.warning(message)
app.logger.warning(traceback.format_exc(e))
return {'success': False, 'message': message}, 500
return { 'success': True,
'message': "",
'result': ids }
def paginate_events(self):
return self.api.model(
'EventsAsistances', {
'page': fields.Integer(required=False),
'totalPages': fields.Integer(required=False),
'events': fields.List(fields.Nested(self.event,
required=False))
})
def test_defaults(self):
field = fields.List(fields.String)
self.assertFalse(field.required)
self.assertEqual(field.__schema__, {
'type': 'array',
'items': {
'type': 'string'
}
})
def test_with_nested_field(self):
nested_fields = self.api.model('NestedModel', {'name': fields.String})
field = fields.List(fields.Nested(nested_fields))
self.assertEqual(field.__schema__, {
'type': 'array',
'items': {
'$ref': '#/definitions/NestedModel'
}
})
def test_list_field_with_readonly(self):
field = fields.List(fields.String, readonly=True)
self.assertEqual(field.__schema__, {
'type': 'array',
'items': {
'type': 'string'
},
'readOnly': True
})
def test_list_field_with_required(self):
field = fields.List(fields.String, required=True)
self.assertTrue(field.required)
self.assertEqual(field.__schema__, {
'type': 'array',
'items': {
'type': 'string'
}
})
def test_list_field_with_title(self):
field = fields.List(fields.String, title='A title')
self.assertEqual(field.__schema__, {
'type': 'array',
'items': {
'type': 'string'
},
'title': 'A title'
})
def test_list_field_with_description(self):
field = fields.List(fields.String, description='A description')
self.assertEqual(
field.__schema__, {
'type': 'array',
'items': {
'type': 'string'
},
'description': 'A description'
})
def paginate_assistances(self):
return self.api.model(
'PaginateAsistances', {
'page':
fields.Integer(required=False),
'totalPages':
fields.Integer(required=False),
'assistances':
fields.List(
fields.Nested(self.assistance,
required=False,
description='Requerimientos'))
})
def test_marshal_list_of_nesteds(self):
model = OrderedDict([
('foo', fields.Raw),
('fee', fields.List(fields.Nested({
'fye': fields.String
})))
])
marshal_fields = OrderedDict([('foo', 'bar'),
('bat', 'baz'),
('fee', {'fye': 'fum'})])
output = marshal(marshal_fields, model)
expected = OrderedDict([('foo', 'bar'),
('fee', [OrderedDict([('fye', 'fum')])])])
self.assertEquals(output, expected)
def test_inherit_inline(self):
parent = self.api.model('Person', {
'name': fields.String,
'age': fields.Integer,
})
child = self.api.inherit('Child', parent, {
'extra': fields.String,
})
self.api.model(
'Output', {
'child': fields.Nested(child),
'children': fields.List(fields.Nested(child))
})
self.assertIn('Person', self.api.models)
self.assertIn('Child', self.api.models)
def test_model_as_nested_dict_with_list(self):
address = self.api.model('Address', {
'road': fields.String,
})
person = self.api.model(
'Person', {
'name': fields.String,
'age': fields.Integer,
'birthdate': fields.DateTime,
'addresses': fields.List(fields.Nested(address))
})
self.assertEqual(
person.__schema__,
{
# 'required': ['address'],
'properties': {
'name': {
'type': 'string'
},
'age': {
'type': 'integer'
},
'birthdate': {
'type': 'string',
'format': 'date-time'
},
'addresses': {
'type': 'array',
'items': {
'$ref': '#/definitions/Address',
}
}
}
})
self.assertEqual(address.__schema__,
{'properties': {
'road': {
'type': 'string'
},
}})
def test_unique(self):
field = fields.List(fields.String, unique=True)
self.assertIn('uniqueItems', field.__schema__)
self.assertEqual(field.__schema__['uniqueItems'], True)
def test_max_items(self):
field = fields.List(fields.String, max_items=42)
self.assertIn('maxItems', field.__schema__)
self.assertEqual(field.__schema__['maxItems'], 42)
def test_min_items(self):
field = fields.List(fields.String, min_items=5)
self.assertIn('minItems', field.__schema__)
self.assertEqual(field.__schema__['minItems'], 5)
# define header parser for the API key
api_key_parser = api.parser()
api_key_parser.add_argument('X-API-KEY',
type=str,
location='headers',
help='Prkng API Key',
required=True)
# define response models
geometry_point = api.model(
'GeometryPoint', {
'type':
fields.String(required=True, enum=['Point']),
'coordinates':
fields.List(fields.Float,
description='The geometry as coordinates lists',
required=True),
})
geometry_linestring = api.model(
'GeometryLinestring', {
'type':
fields.String(required=True, enum=['LineString']),
'coordinates':
fields.List(fields.List(fields.Float),
description='The geometry as coordinates lists',
required=True),
})
agenda_view = api.model('AgendaView', {
'%s' % day: fields.List(fields.List(fields.Float))
class Insert(Resource):
edge = api.model(
'Edge', {
'src': fields.String(required=True, description='Source vertex'),
'dest': fields.String(required=True,
description='Destination vertex'),
'type': fields.String(required=False, description='Edge type'),
'weight': fields.Integer(required=False,
description='Edge weight'),
'time': fields.Integer(required=False, description='Timestamp')
})
edgesSpec = api.model(
'Insert', {
'edges':
fields.List(fields.Nested(edge),
description='List of edges',
required=True),
'immediate':
fields.Boolean(
description=
'Instructs the API to send this batch to STINGER immediately upon receipt',
required=False,
default=False),
'strings':
fields.Boolean(
description=
'Instructs the API to interpret integer vertex names as strings rather than integer vertex IDs',
required=False,
default=True)
})
@api.expect(edgesSpec)
@api.doc(responses={
201: 'Edge Inserted',
400: 'Bad Request',
503: 'Unable to reach STINGER'
})
def post(self):
setupSTINGERConnection()
exec_time = time.time()
if not request.json:
r = json.dumps({"error": "Invalid input"})
return Response(response=r,
status=400,
mimetype="application/json")
# grab the lock
counter_lock.acquire()
try:
data = request.json
send_immediate = False if 'immediate' not in data else data[
'immediate']
only_strings = True if 'strings' not in data else data['strings']
if isinstance(data["edges"], list):
edges = data["edges"]
print "Received batch of size", len(edges), 'at', strftime(
"%Y%m%d%H%M%S", gmtime()), ""
for x in edges:
try:
if only_strings:
source = str(x["src"])
destination = str(x["dest"])
else:
source = x["src"]
destination = x["dest"]
edge_type = x["type"] if 'type' in x else 0
edge_weight = int(x["weight"]) if 'weight' in x else 0
timestamp = int(x["time"]) if 'time' in x else 0
s.add_insert(source,
destination,
edge_type,
weight=edge_weight,
ts=timestamp,
insert_strings=only_strings)
# print "added edge", source, destination, edge_type, timestamp
except Exception as e:
print(traceback.format_exc())
pass
# send immediately if the batch is now large
current_batch_size = s.insertions_count + s.deletions_count + s.vertex_updates_count
if current_batch_size > BATCH_THRESHOLD and BATCH_THRESHOLD > 0 or send_immediate:
s.send_batch()
print "Sending batch of size", current_batch_size, 'at', strftime(
"%Y%m%d%H%M%S", gmtime()), ""
except:
print(traceback.format_exc())
r = json.dumps({"error": "Unable to parse object"})
return Response(response=r,
status=400,
mimetype="application/json")
# end critical section
finally:
counter_lock.release()
exec_time = time.time() - exec_time
r = json.dumps({
"status": "success",
"time": exec_time,
"current_batch_size": current_batch_size
})
return Response(response=r, status=201, mimetype="application/json")
class VertexUpdate(Resource):
vertexUpdate = api.model(
'VertexUpdate', {
'vertex':
fields.String(required=True, description='Vertex Name'),
'vtype':
fields.String(required=True,
description='Vertex Type Name (None = 0)'),
'set_weight':
fields.String(required=False, description='Edge type'),
'incr_weight':
fields.Integer(required=False, description='Timestamp')
})
vertexUpdateSpec = api.model(
'Update', {
'vertexUpdates':
fields.List(fields.Nested(vertexUpdate),
description='List of Vertex Updates',
required=True),
'immediate':
fields.Boolean(
description=
'Instructs the API to send this batch to STINGER immediately upon receipt',
required=False,
default=False)
})
@api.expect(vertexUpdateSpec)
@api.doc(
responses={
201: 'Vertices Updated',
400: 'Bad Request',
503: 'Unable to reach STINGER'
})
def post(self):
exec_time = time.time()
if not request.json:
r = json.dumps({"error": "Invalid input"})
return Response(response=r,
status=400,
mimetype="application/json")
# grab the lock
counter_lock.acquire()
try:
data = request.json
send_immediate = False
if 'immediate' in data:
if data['immediate'] == True:
send_immediate = True
if isinstance(data["vertexUpdates"], list):
vertexUpdates = data["vertexUpdates"]
print "Received batch of size", len(
vertexUpdates), 'at', strftime("%Y%m%d%H%M%S",
gmtime()), ""
for x in vertexUpdates:
try:
vtx = str(x["vertex"])
vtype = str(x["vtype"])
set_weight = x["set_weight"] if 'set_weight' in x else 0
incr_weight = int(
x["incr_weight"]) if 'incr_weight' in x else 0
s.add_vertex_update(vtx, vtype, set_weight,
incr_weight)
except Exception as e:
print(traceback.format_exc())
pass
# send immediately if the batch is now large
current_batch_size = s.insertions_count + s.deletions_count + s.vertex_updates_count
if current_batch_size > BATCH_THRESHOLD and BATCH_THRESHOLD > 0 or send_immediate:
print "Sending batch of size", current_batch_size, 'at', strftime(
"%Y%m%d%H%M%S", gmtime()), ""
s.send_batch()
except:
print(traceback.format_exc())
r = json.dumps({"error": "Unable to parse object"})
return Response(response=r,
status=400,
mimetype="application/json")
# end critical section
finally:
counter_lock.release()
exec_time = time.time() - exec_time
r = json.dumps({
"status": "success",
"time": exec_time,
"current_batch_size": current_batch_size
})
return Response(response=r, status=201, mimetype="application/json")
collectdConf = '/etc/collectd/collectd.conf'
lsfConf = '/etc/logstash-forwarder.conf'
lsfList = os.path.join(tmpDir, 'logstashforwarder.list')
lsfGPG = os.path.join(tmpDir, 'GPG-KEY-elasticsearch')
certLoc = '/opt/certs/logstash-forwarder.crt'
stormLogDir = '/home/ubuntu/apache-storm-0.9.5/logs'
# supported aux components
# auxList = ['collectd', 'lsf', 'jmx']
nodeRoles = api.model(
'query details Model', {
'roles':
fields.List(
fields.String(required=False,
default='hdfs',
description='Roles assigned to this node!'))
})
collectdConfModel = api.model(
'configuration details Model for collectd', {
'LogstashIP':
fields.String(required=True,
default='127.0.0.1',
description='IP of the Logstash Server'),
'UDPPort':
fields.String(required=True,
default='25680',
description='Port of UDP plugin from Logstash Server'),
'Interval':
fields.String(required=False,
fields.String(),
'name':
fields.String(),
'task_class':
fields.String(attribute=lambda x: x.func_ref.replace(':', '.').replace(
'.execute', '')),
'next_run_time':
fields.DateTime(dt_format='iso8601'),
'misfire_grace_time':
fields.String(),
'coalesce':
fields.Boolean(),
'trigger':
fields.String(),
'args':
fields.List(fields.String),
'start_date':
fields.DateTime(attribute=lambda x: x.trigger.start_date,
dt_format='iso8601'),
'end_date':
fields.DateTime(attribute=lambda x: x.trigger.end_date,
dt_format='iso8601'),
'timezone':
fields.String(),
'year':
fields.String(attribute=lambda x: x.trigger.fields[0]
if not x.trigger.fields[0].is_default else None),
'month':
fields.String(attribute=lambda x: x.trigger.fields[1]
if not x.trigger.fields[1].is_default else None),
'day':
import db_cellresults
import db_cellmarks
from utilities import getRequestValues
mark_fields = api.model('cellmarks', {
'x': fields.Integer,
'y': fields.Integer,
'typeofmarking': fields.Float
})
cell_fields = api.model('cells', {
'image_name' : fields.String(),
'browser': fields.String(required=False),
'deleted': fields.String(required=False),
'source_ip': fields.String(required=False),
'cellmarks': fields.List(fields.Nested(mark_fields)),
'created': fields.DateTime
})
class TaskCellsAPI(Resource):
def __init__(self):
pass
@api.doc(responses={404: 'Cell not found'})
@api.marshal_list_with(cell_fields)
def get(self):
"""
Show all results
"""
results = db_cellresults.getall()
from flask.ext.restplus import fields
## TODO: create enum type for possible statuses.
## TODO: document this
build_process_list_response = {
'numberOfProcesses': fields.Integer(default=0),
'listOfProcesses': fields.List(fields.String)
}
## TODO: document this
build_basic_status_response = {
'token': fields.String,
'status': fields.String,
'context': fields.String,
'description': fields.String,
'imageName': fields.String,
'tag': fields.String
}
## TODO: document this
buildbase_basic_status_response = {
'name': fields.String,
'status': fields.String,
'description': fields.String
}
## TODO: document this
build_detailed_status_response = build_basic_status_response.copy()
build_detailed_status_response['log'] = fields.String
import db_points
from utilities import getRequestValues
point_fields = api.model('points', {
'x': fields.Float,
'y': fields.Float,
'width': fields.Float
})
image_fields = api.model(
'Image', {
'description': fields.String,
'browser': fields.String(required=False),
'deleted': fields.String(required=False),
'source_ip': fields.String(required=False),
'points': fields.List(fields.Nested(point_fields)),
'created': fields.DateTime
})
class TaskImagesAPI(Resource):
@api.doc(responses={404: 'Image not found'})
@api.marshal_list_with(image_fields)
def get(self):
"""
Show all images
"""
images = db_images.getall()
if images is None:
api.abort(404)
def parse(self, value):
return mask.parse('{' + value + '}')
class DObject(object):
'''A dead simple object built from a dictionnary (no recursion)'''
def __init__(self, data):
self.__dict__.update(data)
person_fields = {'name': fields.String, 'age': fields.Integer}
family_fields = {
'father': fields.Nested(person_fields),
'mother': fields.Nested(person_fields),
'children': fields.List(fields.Nested(person_fields))
}
class ApplyMaskTest(TestCase):
def test_empty(self):
data = {
'integer': 42,
'string': 'a string',
'boolean': True,
}
result = mask.apply(data, '{}')
self.assertEqual(result, {})
def test_single_field(self):
data = {
