def scale_up(actor_id):
tenant, aid = actor_id.decode('utf8').split('_')
logger.debug(
'METRICS Attempting to create a new worker for {}'.format(actor_id))
try:
# create a worker & add to this actor
actor = Actor.from_db(actors_store[actor_id])
worker_id = Worker.request_worker(tenant=tenant, actor_id=actor_id)
logger.info("New worker id: {}".format(worker_id))
if actor.queue:
channel_name = actor.queue
else:
channel_name = 'default'
ch = CommandChannel(name=channel_name)
ch.put_cmd(actor_id=actor.db_id,
worker_id=worker_id,
image=actor.image,
tenant=tenant,
stop_existing=False)
ch.close()
logger.debug(
'METRICS Added worker successfully for {}'.format(actor_id))
return channel_name
except Exception as e:
logger.debug("METRICS - SOMETHING BROKE: {} - {} - {}".format(
type(e), e, e.args))
return None
def post(self, actor_id):
"""Ensure a certain number of workers are running for an actor"""
logger.debug("top of POST /actors/{}/workers.".format(actor_id))
id = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[id])
except KeyError:
logger.debug("did not find actor: {}.".format(actor_id))
raise ResourceError("No actor found with id: {}.".format(actor_id),
404)
args = self.validate_post()
logger.debug(
"workers POST params validated. actor: {}.".format(actor_id))
num = args.get('num')
if not num or num == 0:
logger.debug("did not get a num: {}.".format(actor_id))
num = 1
logger.debug("ensuring at least {} workers. actor: {}.".format(
num, actor_id))
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
workers = Worker.get_workers(dbid)
except WorkerException as e:
logger.debug(
"did not find workers for actor: {}.".format(actor_id))
raise ResourceError(e.msg, 404)
current_number_workers = len(workers.items())
if current_number_workers < num:
logger.debug(
"There were only {} workers for actor: {} so we're adding more."
.format(current_number_workers, actor_id))
worker_ids = []
num_to_add = int(num) - len(workers.items())
logger.info("adding {} more workers for actor {}".format(
num_to_add, actor_id))
for idx in range(num_to_add):
worker_ids.append(
Worker.request_worker(tenant=g.tenant, actor_id=actor_id))
logger.info("New worker ids: {}".format(worker_ids))
ch = CommandChannel()
ch.put_cmd(actor_id=actor.db_id,
worker_ids=worker_ids,
image=actor.image,
tenant=g.tenant,
num=num_to_add,
stop_existing=False)
ch.close()
logger.info(
"Message put on command channel for new worker ids: {}".format(
worker_ids))
return ok(
result=None,
msg="Scheduled {} new worker(s) to start. There were only".
format(num_to_add))
else:
return ok(result=None,
msg="Actor {} already had {} worker(s).".format(
actor_id, num))
def ensure_one_worker(self):
"""This method will check the workers store for the actor and request a new worker if none exist."""
logger.debug("top of Actor.ensure_one_worker().")
worker_id = Worker.ensure_one_worker(self.db_id, self.tenant)
logger.debug("Worker.ensure_one_worker returned worker_id: {}".format(worker_id))
if worker_id:
worker_ids = [worker_id]
logger.info("Actor.ensure_one_worker() putting message on command channel for worker_id: {}".format(
worker_id))
ch = CommandChannel()
ch.put_cmd(actor_id=self.db_id,
worker_ids=worker_ids,
image=self.image,
tenant=self.tenant,
num=1,
stop_existing=False)
ch.close()
return worker_ids
else:
logger.debug("Actor.ensure_one_worker() returning None.")
return None
def check_metrics(self, actor_ids):
for actor_id in actor_ids:
logger.debug("TOP OF CHECK METRICS")
query = {
'query': 'message_count_for_actor_{}'.format(actor_id.decode("utf-8").replace('-', '_')),
'time': datetime.datetime.utcnow().isoformat() + "Z"
}
r = requests.get(PROMETHEUS_URL + '/api/v1/query', params=query)
data = json.loads(r.text)['data']['result']
change_rate = 0
try:
previous_data = last_metric[actor_id]
try:
change_rate = int(data[0]['value'][1]) - int(previous_data[0]['value'][1])
except:
logger.debug("Could not calculate change rate.")
except:
logger.info("No previous data yet for new actor {}".format(actor_id))
last_metric.update({actor_id: data})
# Add a worker if message count reaches a given number
try:
logger.debug("METRICS current message count: {}".format(data[0]['value'][1]))
if int(data[0]['value'][1]) >= 1:
tenant, aid = actor_id.decode('utf8').split('_')
logger.debug('METRICS Attempting to create a new worker for {}'.format(actor_id))
try:
# create a worker & add to this actor
actor = Actor.from_db(actors_store[actor_id])
worker_ids = [Worker.request_worker(tenant=tenant, actor_id=aid)]
logger.info("New worker id: {}".format(worker_ids[0]))
ch = CommandChannel()
ch.put_cmd(actor_id=actor.db_id,
worker_ids=worker_ids,
image=actor.image,
tenant=tenant,
num=1,
stop_existing=False)
ch.close()
logger.debug('METRICS Added worker successfully for {}'.format(actor_id))
except Exception as e:
logger.debug("METRICS - SOMETHING BROKE: {} - {} - {}".format(type(e), e, e.args))
elif int(data[0]['value'][1]) <= 1:
logger.debug("METRICS made it to scale down block")
# Check the number of workers for this actor before deciding to scale down
workers = Worker.get_workers(actor_id)
logger.debug('METRICS NUMBER OF WORKERS: {}'.format(len(workers)))
try:
if len(workers) == 1:
logger.debug("METRICS only one worker, won't scale down")
else:
while len(workers) > 0:
logger.debug('METRICS made it STATUS check')
worker = workers.popitem()[1]
logger.debug('METRICS SCALE DOWN current worker: {}'.format(worker['status']))
# check status of the worker is ready
if worker['status'] == 'READY':
logger.debug("METRICS I MADE IT")
# scale down
try:
shutdown_worker(worker['id'])
continue
except Exception as e:
logger.debug('METRICS ERROR shutting down worker: {} - {} - {}'.format(type(e), e, e.args))
logger.debug('METRICS shut down worker {}'.format(worker['id']))
except IndexError:
logger.debug('METRICS only one worker found for actor {}. '
'Will not scale down'.format(actor_id))
except Exception as e:
logger.debug("METRICS SCALE UP FAILED: {}".format(e))
except Exception as e:
logger.debug("METRICS - ANOTHER ERROR: {} - {} - {}".format(type(e), e, e.args))
def put(self, actor_id):
logger.debug("top of PUT /actors/{}".format(actor_id))
dbid = Actor.get_dbid(g.tenant, actor_id)
try:
actor = Actor.from_db(actors_store[dbid])
except KeyError:
logger.debug("did not find actor {} in store.".format(dbid))
raise ResourceError(
"No actor found with id: {}.".format(actor_id), 404)
previous_image = actor.image
previous_status = actor.status
previous_owner = actor.owner
args = self.validate_put(actor)
logger.debug("PUT args validated successfully.")
args['tenant'] = g.tenant
# user can force an update by setting the force param:
update_image = args.get('force')
if not update_image and args['image'] == previous_image:
logger.debug("new image is the same and force was false. not updating actor.")
logger.debug("Setting status to the actor's previous status which is: {}".format(previous_status))
args['status'] = previous_status
else:
update_image = True
args['status'] = SUBMITTED
logger.debug("new image is different. updating actor.")
args['api_server'] = g.api_server
# we do not allow a PUT to override the owner in case the PUT is issued by another user
args['owner'] = previous_owner
use_container_uid = args.get('use_container_uid')
if Config.get('web', 'case') == 'camel':
use_container_uid = args.get('useContainerUid')
try:
use_tas = Config.get('workers', 'use_tas_uid')
except configparser.NoOptionError:
logger.debug("no use_tas_uid config.")
use_tas = False
if hasattr(use_tas, 'lower'):
use_tas = use_tas.lower() == 'true'
else:
logger.error("use_tas_uid configured but not as a string. use_tas_uid: {}".format(use_tas))
logger.debug("use_tas={}. user_container_uid={}".format(use_tas, use_container_uid))
if use_tas and not use_container_uid:
uid, gid, tasdir = get_tas_data(g.user, g.tenant)
if uid and gid:
args['uid'] = uid
args['gid'] = gid
if tasdir:
args['tasdir'] = tasdir
args['mounts'] = get_all_mounts(args)
args['last_update_time'] = get_current_utc_time()
logger.debug("update args: {}".format(args))
actor = Actor(**args)
actors_store[actor.db_id] = actor.to_db()
logger.info("updated actor {} stored in db.".format(actor_id))
if update_image:
worker_ids = [Worker.request_worker(tenant=g.tenant, actor_id=actor.db_id)]
ch = CommandChannel()
ch.put_cmd(actor_id=actor.db_id, worker_ids=worker_ids, image=actor.image, tenant=args['tenant'])
ch.close()
logger.debug("put new command on command channel to update actor.")
# put could have been issued by a user with
if not previous_owner == g.user:
set_permission(g.user, actor.db_id, UPDATE)
return ok(result=actor.display(),
msg="Actor updated successfully.")
