def get_db_id():
"""Get the db_id from the request path."""
# logger.debug("top of get_db_id. request.path: {}".format(request.path))
path_split = request.path.split("/")
if len(path_split) < 3:
logger.error(
"Unrecognized request -- could not find the actor id. path_split: {}"
.format(path_split))
raise PermissionsException("Not authorized.")
# logger.debug("path_split: {}".format(path_split))
actor_identifier = path_split[2]
# logger.debug("actor_identifier: {}; tenant: {}".format(actor_identifier, g.tenant))
try:
actor_id = Actor.get_actor_id(g.tenant, actor_identifier)
except KeyError:
logger.info(
"Unrecoginzed actor_identifier: {}. Actor not found".format(
actor_identifier))
raise ResourceError(
"Actor with identifier '{}' not found".format(actor_identifier),
404)
except Exception as e:
msg = "Unrecognized exception trying to resolve actor identifier: {}; " \
"exception: {}".format(actor_identifier, e)
logger.error(msg)
raise ResourceError(msg)
logger.debug("actor_id: {}".format(actor_id))
return Actor.get_dbid(g.tenant, actor_id)
def get(self, actor_id):
logger.debug("top of GET /actors/{}/workers for tenant {}.".format(
actor_id, g.tenant))
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
Actor.from_db(actors_store[dbid])
except KeyError:
logger.debug("did not find actor: {}.".format(actor_id))
raise ResourceError("No actor found with id: {}.".format(actor_id),
404)
try:
workers = Worker.get_workers(dbid)
except WorkerException as e:
logger.debug(
"did not find workers for actor: {}.".format(actor_id))
raise ResourceError(e.msg, 404)
result = []
for id, worker in workers.items():
worker.update({'id': id})
try:
w = Worker(**worker)
result.append(w.display())
except Exception as e:
logger.error(
"Unable to instantiate worker in workers endpoint from description: {}. "
.format(worker))
return ok(result=result, msg="Workers retrieved successfully.")
def get(self, actor_id, execution_id):
def get_hypermedia(actor, exc):
return {'_links': {'self': '{}/actors/v2/{}/executions/{}/logs'.format(actor.api_server, actor.id, exc.id),
'owner': '{}/profiles/v2/{}'.format(actor.api_server, actor.owner),
'execution': '{}/actors/v2/{}/executions/{}'.format(actor.api_server, actor.id, exc.id)},
}
logger.debug("top of GET /actors/{}/executions/{}/logs.".format(actor_id, execution_id))
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[dbid])
except KeyError:
logger.debug("did not find actor: {}.".format(actor_id))
raise ResourceError(
"No actor found with id: {}.".format(actor_id), 404)
try:
excs = executions_store[dbid]
except KeyError:
logger.debug("did not find executions. actor: {}.".format(actor_id))
raise ResourceError("No executions found for actor {}.".format(actor_id))
try:
exc = Execution.from_db(excs[execution_id])
except KeyError:
logger.debug("did not find execution: {}. actor: {}.".format(execution_id, actor_id))
raise ResourceError("Execution {} not found.".format(execution_id))
try:
logs = logs_store[execution_id]
except KeyError:
logger.debug("did not find logs. execution: {}. actor: {}.".format(execution_id, actor_id))
logs = ""
result={'logs': logs}
result.update(get_hypermedia(actor, exc))
return ok(result, msg="Logs retrieved successfully.")
def validate_post(self):
logger.debug("validating message payload.")
parser = RequestParser()
parser.add_argument('message',
type=str,
required=False,
help="The message to send to the actor.")
args = parser.parse_args()
# if a special 'message' object isn't passed, use entire POST payload as message
if not args.get('message'):
logger.debug("POST body did not have a message field.")
# first check for binary data:
if request.headers.get(
'Content-Type') == 'application/octet-stream':
# ensure not sending too much data
length = request.headers.get('Content-Length')
if not length:
raise ResourceError(
"Content Length required for application/octet-stream."
)
try:
int(length)
except Exception:
raise ResourceError("Content Length must be an integer.")
if int(length) > int(Config.get('web', 'max_content_length')):
raise ResourceError(
"Message exceeds max content length of: {}".format(
Config.get('web', 'max_content_length')))
logger.debug(
"using get_data, setting content type to application/octet-stream."
)
args['message'] = request.get_data()
args['_abaco_Content_Type'] = 'application/octet-stream'
return args
json_data = request.get_json()
if json_data:
logger.debug("message was JSON data.")
args['message'] = json_data
args['_abaco_Content_Type'] = 'application/json'
else:
logger.debug("message was NOT JSON data.")
# try to get data for mime types not recognized by flask. flask creates a python string for these
try:
args['message'] = json.loads(request.data)
except TypeError:
logger.debug(
"message POST body could not be serialized. args: {}".
format(args))
raise DAOError(
'message POST body could not be serialized. Pass JSON data or use the message attribute.'
)
args['_abaco_Content_Type'] = 'str'
else:
# the special message object is a string
logger.debug(
"POST body has a message field. Setting _abaco_Content_type to 'str'."
)
args['_abaco_Content_Type'] = 'str'
return args
def post(self, actor_id):
"""Ensure a certain number of workers are running for an actor"""
logger.debug("top of POST /actors/{}/workers.".format(actor_id))
id = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[id])
except KeyError:
logger.debug("did not find actor: {}.".format(actor_id))
raise ResourceError("No actor found with id: {}.".format(actor_id),
404)
args = self.validate_post()
logger.debug(
"workers POST params validated. actor: {}.".format(actor_id))
num = args.get('num')
if not num or num == 0:
logger.debug("did not get a num: {}.".format(actor_id))
num = 1
logger.debug("ensuring at least {} workers. actor: {}.".format(
num, actor_id))
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
workers = Worker.get_workers(dbid)
except WorkerException as e:
logger.debug(
"did not find workers for actor: {}.".format(actor_id))
raise ResourceError(e.msg, 404)
current_number_workers = len(workers.items())
if current_number_workers < num:
logger.debug(
"There were only {} workers for actor: {} so we're adding more."
.format(current_number_workers, actor_id))
worker_ids = []
num_to_add = int(num) - len(workers.items())
logger.info("adding {} more workers for actor {}".format(
num_to_add, actor_id))
for idx in range(num_to_add):
worker_ids.append(
Worker.request_worker(tenant=g.tenant, actor_id=actor_id))
logger.info("New worker ids: {}".format(worker_ids))
ch = CommandChannel()
ch.put_cmd(actor_id=actor.db_id,
worker_ids=worker_ids,
image=actor.image,
tenant=g.tenant,
num=num_to_add,
stop_existing=False)
ch.close()
logger.info(
"Message put on command channel for new worker ids: {}".format(
worker_ids))
return ok(
result=None,
msg="Scheduled {} new worker(s) to start. There were only".
format(num_to_add))
else:
return ok(result=None,
msg="Actor {} already had {} worker(s).".format(
actor_id, num))
def get(self, actor_id):
id = Actor.get_dbid(g.tenant, actor_id)
try:
Actor.from_db(actors_store[id])
except KeyError:
raise ResourceError("actor not found: {}'".format(actor_id), 404)
try:
permissions = get_permissions(id)
except PermissionsException as e:
raise ResourceError(e.msg, 404)
return ok(result=permissions,
msg="Permissions retrieved successfully.")
def post(self, actor_id):
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[dbid])
except KeyError:
raise ResourceError("actor not found: {}'".format(actor_id), 404)
if actor.stateless:
raise ResourceError("actor is stateless.", 404)
args = self.validate_post()
state = args['state']
actors_store.update(dbid, 'state', state)
actor = Actor.from_db(actors_store[dbid])
return ok(result=actor.display(), msg="State updated successfully.")
def get(self, actor_id):
logger.debug("top of GET /actors/{}/permissions.".format(actor_id))
id = Actor.get_dbid(g.tenant, actor_id)
try:
Actor.from_db(actors_store[id])
except KeyError:
logger.debug("Did not find actor: {}.".format(actor_id))
raise ResourceError("No actor found with id: {}.".format(actor_id), 404)
try:
permissions = get_permissions(id)
except PermissionsException as e:
logger.debug("Did not find permissions. actor: {}.".format(actor_id))
raise ResourceError(e.msg, 404)
return ok(result=permissions, msg="Permissions retrieved successfully.")
def get(self, actor_id, worker_id):
logger.debug("top of GET /actors/{}/workers/{}.".format(actor_id, worker_id))
id = Actor.get_dbid(g.tenant, actor_id)
try:
Actor.from_db(actors_store[id])
except KeyError:
logger.debug("Did not find actor: {}.".format(actor_id))
raise ResourceError("No actor found with id: {}.".format(actor_id), 404)
try:
worker = Worker.get_worker(id, worker_id)
except WorkerException as e:
logger.debug("Did not find worker: {}. actor: {}.".format(worker_id, actor_id))
raise ResourceError(e.msg, 404)
return ok(result=worker, msg="Worker retrieved successfully.")
def get(self, actor_id):
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
Actor.from_db(actors_store[dbid])
except KeyError:
raise ResourceError("actor not found: {}'".format(actor_id), 400)
try:
workers = Worker.get_workers(dbid)
except WorkerException as e:
raise ResourceError(e.msg, 404)
result = []
for id, worker in workers.items():
worker.update({'id': id})
result.append(worker)
return ok(result=result, msg="Workers retrieved successfully.")
def post(self, actor_id):
"""Ensure a certain number of workers are running for an actor"""
id = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[id])
except KeyError:
raise ResourceError("actor not found: {}'".format(actor_id), 404)
args = self.validate_post()
num = args.get('num')
if not num or num == 0:
num = 1
dbid = Actor.get_dbid(g.tenant, actor_id)
workers = Worker.get_workers(dbid)
if len(workers.items()) < num:
worker_ids = []
num_to_add = int(num) - len(workers.items())
for idx in range(num_to_add):
worker_ids.append(Worker.request_worker(actor_id))
ch = CommandChannel()
ch.put_cmd(actor_id=actor.db_id,
worker_ids=worker_ids,
image=actor.image,
tenant=g.tenant,
num=num_to_add,
stop_existing=False)
return ok(
result=None,
msg="Scheduled {} new worker(s) to start. There were only".
format(num_to_add))
else:
return ok(result=None,
msg="Actor {} already had {} worker(s).".format(
actor_id, num))
def validate_post(self):
parser = RequestParser()
parser.add_argument('runtime',
type=str,
required=True,
help="Runtime, in milliseconds, of the execution.")
parser.add_argument(
'cpu',
type=str,
required=True,
help="CPU usage, in user jiffies, of the execution.")
parser.add_argument(
'io',
type=str,
required=True,
help=
"Block I/O usage, in number of 512-byte sectors read from and written to, by the execution."
)
# Accounting for memory is quite hard -- probably easier to cap all containers at a fixed amount or perhaps have
# a graduated list of cap sized (e.g. small, medium and large).
# parser.add_argument('mem', type=str, required=True, help="Memory usage, , of the execution.")
args = parser.parse_args()
for k, v in args.items():
try:
int(v)
except ValueError:
raise ResourceError(message="Argument " + k +
" must be an integer.")
return args
def post(self, actor_id):
"""Create a new nonce for an actor."""
logger.debug("top of POST /actors/{}/nonces".format(actor_id))
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
Actor.from_db(actors_store[dbid])
except KeyError:
logger.debug("did not find actor: {}.".format(actor_id))
raise ResourceError(
"No actor found with id: {}.".format(actor_id), 404)
args = self.validate_post()
logger.debug("nonce post args validated: {}.".format(actor_id))
# supply "provided" fields:
args['tenant'] = g.tenant
args['api_server'] = g.api_server
args['db_id'] = dbid
args['owner'] = g.user
args['roles'] = g.roles
# create and store the nonce:
nonce = Nonce(**args)
Nonce.add_nonce(dbid, nonce)
logger.info("nonce added for actor: {}.".format(actor_id))
return ok(result=nonce.display(), msg="Actor nonce created successfully.")
def put(self, actor_id):
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[dbid])
except KeyError:
raise ResourceError("actor not found: {}'".format(actor_id), 404)
previous_image = actor.image
args = self.validate_put(actor)
args['tenant'] = g.tenant
update_image = False
if args['image'] == previous_image:
args['status'] = actor.status
else:
update_image = True
args['status'] = SUBMITTED
args['api_server'] = g.api_server
args['owner'] = g.user
actor = Actor(**args)
actors_store[actor.db_id] = actor.to_db()
worker_ids = Worker.request_worker(actor.db_id)
if update_image:
ch = CommandChannel()
ch.put_cmd(actor_id=actor.db_id,
worker_ids=worker_ids,
image=actor.image,
tenant=args['tenant'])
# return ok(result={'update_image': str(update_image)},
# msg="Actor updated successfully.")
return ok(result=actor.display(), msg="Actor updated successfully.")
def put(self, actor_id):
logger.debug("top of PUT /actors/{}".format(actor_id))
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[dbid])
except KeyError:
logger.debug("did not find actor {} in store.".format(dbid))
raise ResourceError(
"No actor found with id: {}.".format(actor_id), 404)
previous_image = actor.image
args = self.validate_put(actor)
logger.debug("PUT args validated successfully.")
args['tenant'] = g.tenant
update_image = False
if args['image'] == previous_image:
logger.debug("new image is the same. not updating actor.")
args['status'] = actor.status
else:
update_image = True
args['status'] = SUBMITTED
logger.debug("new image is different. updating actor.")
args['api_server'] = g.api_server
args['owner'] = g.user
actor = Actor(**args)
actors_store[actor.db_id] = actor.to_db()
logger.info("updated actor {} stored in db.".format(actor_id))
worker_ids = Worker.request_worker(actor.db_id)
if update_image:
ch = CommandChannel()
ch.put_cmd(actor_id=actor.db_id, worker_ids=worker_ids, image=actor.image, tenant=args['tenant'])
logger.debug("put new command on command channel to update actor.")
return ok(result=actor.display(),
msg="Actor updated successfully.")
def get(self, actor_id):
def get_hypermedia(actor):
return {
'_links': {
'self':
'{}/actors/v2/{}/messages'.format(actor.api_server,
actor.id),
'owner':
'{}/profiles/v2/{}'.format(actor.api_server, actor.owner),
},
}
logger.debug("top of GET /actors/{}/messages".format(actor_id))
# check that actor exists
id = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[id])
except KeyError:
logger.debug("did not find actor: {}.".format(actor_id))
raise ResourceError("No actor found with id: {}.".format(actor_id),
404)
ch = ActorMsgChannel(actor_id=id)
result = {'messages': len(ch._queue._queue)}
ch.close()
logger.debug("messages found for actor: {}.".format(actor_id))
result.update(get_hypermedia(actor))
return ok(result)
def get(self, actor_id):
logger.debug("top of GET /actors/{}/executions".format(actor_id))
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[dbid])
except KeyError:
logger.debug("did not find actor: {}.".format(actor_id))
raise ResourceError(
"No actor found with id: {}.".format(actor_id), 404)
try:
summary = ExecutionsSummary(db_id=dbid)
except DAOError as e:
logger.debug("did not find executions summary: {}".format(actor_id))
raise ResourceError("Could not retrieve executions summary for actor: {}. "
"Details: {}".format(actor_id, e), 404)
return ok(result=summary.display(), msg="Actor executions retrieved successfully.")
def get(self, actor_id):
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[dbid])
except KeyError:
raise ResourceError(
"actor not found: {}. db_id:{}'".format(actor_id, dbid), 404)
return ok(result=actor.display(), msg="Actor retrieved successfully.")
def get(self, actor_id, execution_id):
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
actors_store[dbid]
except KeyError:
raise ResourceError("actor not found: {}'".format(actor_id), 404)
try:
excs = executions_store[dbid]
except KeyError:
raise ResourceError(
"No executions found for actor {}.".format(actor_id))
try:
exc = Execution.from_db(excs[execution_id])
except KeyError:
raise ResourceError("Execution not found {}.".format(execution_id))
return ok(result=exc.display(),
msg="Actor execution retrieved successfully.")
def delete(self, actor_id, worker_id):
id = Actor.get_dbid(g.tenant, actor_id)
try:
worker = Worker.get_worker(id, worker_id)
except WorkerException as e:
raise ResourceError(e.msg, 404)
shutdown_worker(worker['ch_name'])
return ok(result=None, msg="Worker scheduled to be stopped.")
def get_db_id():
"""Get the db_id and actor_identifier from the request path."""
# the location of the actor identifier is different for aliases vs actor_id's.
# for actors, it is in index 2:
# /actors/<actor_id>
# for aliases, it is in index 3:
# /actors/aliases/<alias_id>
idx = 2
if 'aliases' in request.path:
idx = 3
path_split = request.path.split("/")
if len(path_split) < 3:
logger.error(
"Unrecognized request -- could not find the actor id. path_split: {}"
.format(path_split))
raise PermissionsException("Not authorized.")
logger.debug("path_split: {}".format(path_split))
try:
actor_identifier = path_split[idx]
except IndexError:
raise ResourceError(
"Unable to parse actor identifier: is it missing from the URL?",
404)
logger.debug("actor_identifier: {}; tenant: {}".format(
actor_identifier, g.tenant))
if actor_identifier == 'search':
raise ResourceError(
"'x-nonce' query parameter on the '/actors/search/{database}' endpoint does not resolve.",
404)
try:
actor_id = Actor.get_actor_id(g.tenant, actor_identifier)
except KeyError:
logger.info(
"Unrecognized actor_identifier: {}. Actor not found".format(
actor_identifier))
raise ResourceError(
"Actor with identifier '{}' not found".format(actor_identifier),
404)
except Exception as e:
msg = "Unrecognized exception trying to resolve actor identifier: {}; " \
"exception: {}".format(actor_identifier, e)
logger.error(msg)
raise ResourceError(msg)
logger.debug("actor_id: {}".format(actor_id))
return Actor.get_dbid(g.tenant, actor_id), actor_identifier
def get(self, actor_id, worker_id):
logger.debug("top of GET /actors/{}/workers/{}.".format(actor_id, worker_id))
id = Actor.get_dbid(g.tenant, actor_id)
try:
Actor.from_db(actors_store[id])
except KeyError:
logger.debug("Did not find actor: {}.".format(actor_id))
raise ResourceError("No actor found with id: {}.".format(actor_id), 404)
try:
worker = Worker.get_worker(id, worker_id)
except WorkerException as e:
logger.debug("Did not find worker: {}. actor: {}.".format(worker_id, actor_id))
raise ResourceError(e.msg, 404)
# worker is an honest python dictionary with a single key, the id of the worker. need to
# convert it to a Worker object
worker.update({'id': worker_id})
w = Worker(**worker)
return ok(result=w.display(), msg="Worker retrieved successfully.")
def post(self, actor_id):
logger.debug("top of POST /actors/{}/state".format(actor_id))
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[dbid])
except KeyError:
logger.debug("did not find actor with id: {}.".format(actor_id))
raise ResourceError(
"No actor found with id: {}.".format(actor_id), 404)
if actor.stateless:
logger.debug("cannot update state for stateless actor: {}".format(actor_id))
raise ResourceError("actor is stateless.", 404)
state = self.validate_post()
logger.debug("state post params validated: {}".format(actor_id))
actors_store.update(dbid, 'state', state)
logger.info("state updated: {}".format(actor_id))
actor = Actor.from_db(actors_store[dbid])
return ok(result=actor.display(), msg="State updated successfully.")
def get(self, actor_id):
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
summary = ExecutionsSummary(db_id=dbid)
except DAOError as e:
raise ResourceError(
"actor not found: {}. DAOError: {}'".format(actor_id, e), 404)
return ok(result=summary.display(),
msg="Actor executions retrieved successfully.")
def get(self, actor_id):
logger.debug("top of GET /actors/{}/state".format(actor_id))
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[dbid])
except KeyError:
raise ResourceError(
"No actor found with id: {}.".format(actor_id), 404)
return ok(result={'state': actor.get('state') }, msg="Actor state retrieved successfully.")
def validate_post(self):
parser = RequestParser()
parser.add_argument('user', type=str, required=True, help="User owning the permission.")
parser.add_argument('level', type=str, required=True,
help="Level of the permission: {}".format(PERMISSION_LEVELS))
args = parser.parse_args()
if not args['level'] in PERMISSION_LEVELS:
raise ResourceError("Invalid permission level: {}. \
The valid values are {}".format(args['level'], PERMISSION_LEVELS))
return args
def post(self, actor_id):
id = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[id])
except KeyError:
raise ResourceError("actor not found: {}'".format(actor_id), 404)
args = self.validate_post()
Execution.add_execution(id, args)
return ok(result=actor.display(),
msg="Actor execution added successfully.")
def get(self, actor_id):
logger.debug("top of GET /actors/{}/nonces".format(actor_id))
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
Actor.from_db(actors_store[dbid])
except KeyError:
logger.debug("did not find actor: {}.".format(actor_id))
raise ResourceError(
"No actor found with id: {}.".format(actor_id), 404)
nonces = Nonce.get_nonces(actor_id=dbid)
return ok(result=[n.display() for n in nonces], msg="Actor nonces retrieved successfully.")
def post(self, actor_id):
"""Add new permissions for an actor"""
id = Actor.get_dbid(g.tenant, actor_id)
try:
Actor.from_db(actors_store[id])
except KeyError:
raise ResourceError("actor not found: {}'".format(actor_id), 404)
args = self.validate_post()
add_permission(args['user'], id, args['level'])
permissions = get_permissions(id)
return ok(result=permissions, msg="Permission added successfully.")
def get(self, actor_id):
logger.debug("top of GET /actors/{}".format(actor_id))
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[dbid])
except KeyError:
logger.debug("did not find actor with id: {}".format(actor_id))
raise ResourceError(
"No actor found with id: {}.".format(actor_id), 404)
logger.debug("found actor {}".format(actor_id))
return ok(result=actor.display(), msg="Actor retrieved successfully.")
