def _assign_tickets(self):
with session_scope() as session:
qticket = QTickets(session)
tickets = [
x.id
for x in qticket.waiting().order_by(models.Ticket.id).all()
]
for ticket_id in tickets:
notify_ticket = False
with session_scope() as session:
ticket = session.query(models.Ticket).get(ticket_id)
qres = QResources(session)
resources = qres.ready().all()
ticket_tags = ticket.tag_set
for resource in resources:
res_tags = resource.tag_set
if ticket_tags.issubset(res_tags):
# We have found appropriate resource!
ticket.resource_id = resource.id
if ticket.tid:
notify_ticket = ticket.tid
session.add_all([ticket])
break
if notify_ticket:
self._notify_waiting(notify_ticket)
def _assign_tickets(self):
with session_scope() as session:
qticket = QTickets(session)
tickets = [x.id for x in qticket.waiting().order_by(models.Ticket.id).all()]
for ticket_id in tickets:
notify_ticket = False
with session_scope() as session:
ticket = session.query(models.Ticket).get(ticket_id)
qres = QResources(session)
resources = qres.ready().all()
ticket_tags = ticket.tag_set
for resource in resources:
res_tags = resource.tag_set
if resource.sandbox and resource.sandbox != ticket.sandbox:
continue
if not ticket_tags.issubset(res_tags):
continue
# We have found appropriate resource!
log.debug("Assigning %s to %s", resource.name, ticket.id)
assign_ticket(resource, ticket)
if ticket.tid:
notify_ticket = ticket.tid
break
if notify_ticket:
self._notify_waiting(notify_ticket)
def loop(self):
log.debug("Watcher loop")
pools = reload_config()
to_check = {}
with session_scope() as session:
# Even though we never terminate resources that have assigned
# ticket, we still check them. This raises the check limit before
# user releases the ticket and the resource can be terminated as
# soon as possible.
up = QResources(session).up().all()
for item in up:
if not item.pool in pools:
continue
to_check[item.id] = {
'name': item.name,
'pool': item.pool,
'last': item.check_last_time,
'fail': item.check_failed_count,
'id_in_pool': item.id_in_pool,
'data': item.data,
}
for res_id, data in to_check.items():
pool = pools[data['pool']]
if not pool.cmd_livecheck:
continue
if data['last'] + pool.livecheck_period > time.time():
# Not yet needed check.
continue
rc = run_command(
pool.id,
res_id,
data['name'],
data['id_in_pool'],
pool.cmd_livecheck,
'watch',
data=data["data"],
)
with session_scope() as session:
res = session.query(models.Resource).get(res_id)
res.check_last_time = time.time()
if rc['status']:
res.check_failed_count = res.check_failed_count + 1
log.debug("failed check #{0} for {1}"\
.format(res.check_failed_count, res_id))
else:
res.check_failed_count = 0
session.add(res)
session.flush()
def job(self):
self.log.debug(
"Allocating new resource id={id} in pool '{pool}' by {cmd}"\
.format(
id=self.resource_id,
pool=self.pool.name,
cmd=self.pool.cmd_new
)
)
id_in_pool = None
with session_scope() as session:
resource = session.query(models.Resource).get(self.resource_id)
id_in_pool = resource.id_in_pool
session.expunge(resource)
# Run the allocation script.
output = run_command(
self.pool.id,
resource.id,
resource.name,
id_in_pool,
self.pool.cmd_new,
catch_stdout_bytes=512,
)
with session_scope() as session:
resource.state = RState.ENDED if output['status'] else RState.UP
resource.data = output['stdout']
tags = []
if type(self.pool.tags) != type([]):
msg = "Pool {pool} has set 'tags' set, but that's not an array"\
.format(pool=self.name)
warnings.warn(msg)
else:
for tag in self.pool.tags:
tag_obj = models.ResourceTag()
tag_obj.id = tag
tag_obj.resource_id = resource.id
tags.append(tag_obj)
self.log.debug("Allocator ends with state={0}".format(resource.state))
session.add_all(tags + [resource])
if resource.state == RState.ENDED:
session.delete(resource.id_in_pool_object)
# Notify manager that it is worth doing re-spin.
self.event.set()
def closeTicket(self, ticket_id):
with session_scope() as session:
ticket = session.query(models.Ticket).get(ticket_id)
if not ticket:
raise ServerAPIException("no such ticket {0}".format(ticket_id))
ticket.state = TState.CLOSED
self.sync.ticket.set()
def job(self):
id_in_pool = None
with session_scope() as session:
resource = session.query(models.Resource).get(self.resource_id)
if resource.ticket:
if resource.ticket.state == helpers.TState.OPEN:
log.warning("can't delete {0}, ticket opened"\
.format(resource.name))
return
resource.state = RState.DELETING
session.add(resource)
session.flush()
id_in_pool = resource.id_in_pool
session.expunge(resource)
self.log.debug("TerminateWorker(pool={0}): name={1} by: \"{2}\""\
.format(self.pool.name, resource.name, self.pool.cmd_delete))
if not self.pool.cmd_delete:
self.close()
return
run_command(
self.pool.id,
resource.id,
resource.name,
id_in_pool,
self.pool.cmd_delete,
'terminate',
)
self.close()
def allocate(self, event):
resource_id = None
with session_scope() as session:
dbinfo = session.query(models.Pool).get(self.name)
dbinfo.last_start = time.time()
resource = models.Resource()
resource.pool = self.name
session.add_all([resource, dbinfo])
session.flush()
pool_id = self._allocate_pool_id(session, resource)
session.add(pool_id)
session.flush()
log.debug("id in pool: {0}".format(pool_id.id))
resource_id = resource.id
fill_dict = dict(
id=str(resource_id).zfill(8),
pool_name=self.name)
resource.name = helpers.careful_string_format(
self.name_pattern, fill_dict)
session.add(resource)
if resource_id:
self.last_start = time.time()
AllocWorker(event, self, int(resource_id)).start()
def close(self):
with session_scope() as session:
resource = session.query(models.Resource).get(self.resource_id)
resource.state = RState.ENDED
session.add(resource)
if resource.id_in_pool_object:
session.delete(resource.id_in_pool_object)
self.event.set()
def loop(self):
log.debug("Watcher loop")
pools = reload_config()
to_check = {}
with session_scope() as session:
up = QResources(session).up().all()
for item in up:
if not item.pool in pools:
continue
to_check[item.id] = {
'name': item.name,
'pool': item.pool,
'last': item.check_last_time,
'fail': item.check_failed_count,
'id_in_pool': item.id_in_pool,
}
for res_id, data in to_check.items():
pool = pools[data['pool']]
if not pool.cmd_livecheck:
continue
if data['last'] + pool.livecheck_period > time.time():
# Not yet needed check.
continue
failed_count = 0
rc = run_command(pool.id, res_id, data['name'], data['id_in_pool'],
pool.cmd_livecheck, 'watch')
with session_scope() as session:
res = session.query(models.Resource).get(res_id)
res.check_last_time = time.time()
if rc['status']:
res.check_failed_count = res.check_failed_count + 1
log.debug("failed check #{0} for {1}"\
.format(res.check_failed_count, res_id))
else:
res.check_failed_count = 0
session.add(res)
session.flush()
failed_count = res.check_failed_count
if failed_count >= 3:
log.debug("Watcher plans to kill {0}".format(res_id))
TerminateWorker(self.event, pool, res_id).start()
def resource_delete(self, resources=None):
if not resources or type(resources) != list:
log.error("no resources specified")
return
for res_id in resources:
with session_scope() as session:
resources = QResources(session=session)
resources.kill(res_id)
def job(self):
with session_scope() as session:
resource = session.query(models.Resource).get(self.resource_id)
id_in_pool = resource.id_in_pool
session.expunge(resource)
out = run_command(self.pool.id, resource.id, resource.name, id_in_pool,
self.pool.cmd_release, "release", data=resource.data)
status = out["status"]
with session_scope() as session:
resource = session.query(models.Resource).get(self.resource_id)
if status:
# mark it for removal
resource.releases_counter = self.pool.reuse_max_count + 1
resource.state = RState.UP
if not status:
self.event.set()
def waitTicket(self, ticket_id):
""" ... blocking! ... """
output = ""
with session_scope() as session:
ticket = session.query(models.Ticket).get(ticket_id)
ticket.tid = self.my_id()
session.add(ticket)
while True:
with session_scope() as session:
ticket = session.query(models.Ticket).get(ticket_id)
if ticket.resource:
return ticket.resource.data
with self.sync.resource_ready:
while self.sync.resource_ready.wait(timeout=10):
if self.sync.tid==self.my_id():
break
def collectTicket(self, ticket_id):
output = {
'ready': False,
'output': None,
}
with session_scope() as session:
resource = self._checkTicket(ticket_id, session)
if resource:
output['output'] = resource.data
output['ready'] = True
return output
def collectTicket(self, ticket_id):
output = {
'ready': False,
'output': None,
'closed': None
}
with session_scope() as session:
ticket = self._checkTicket(ticket_id, session)
if ticket.resource:
output['output'] = ticket.resource.data
output['ready'] = True
output['closed'] = ticket.state == TState.CLOSED
return output
def takeTicket(self, tags=None):
with session_scope() as session:
ticket = models.Ticket()
tag_objects = []
for tag in (tags or []):
to = models.TicketTag()
to.ticket = ticket
to.id = tag
tag_objects.append(to)
session.add_all([ticket] + tag_objects)
session.flush()
ticket_id = ticket.id
self.sync.ticket.set()
return ticket_id
def _too_soon(self):
last_start = 0.0
with session_scope() as session:
dbinfo = session.query(models.Pool).get(self.name)
if not dbinfo:
dbinfo = models.Pool()
dbinfo.name = self.name
dbinfo.last_start = 0.0
session.add(dbinfo)
else:
last_start = dbinfo.last_start
is_too_soon = last_start + self.start_delay > time.time()
if is_too_soon:
log.debug("too soon for Pool('{0}')".format(self.name))
return is_too_soon
def _detect_closed_tickets(self, event):
with session_scope() as session:
qres = QResources(session, pool=self.name)
for resource in qres.taken():
ticket = resource.ticket
assert ticket
if ticket.state == helpers.TState.CLOSED:
release_resource(ticket)
if self.cmd_release:
# UP → RELEASING → UP, TODO: we might want to optimize
# this a bit, and stop calling the releasing script when
# the resource is not releasable anymore (max_reuses
# reached, etc.).
resource.state = helpers.RState.RELEASING
ReleaseWorker(event, self, int(resource.id)).start()
def ticket_list(self):
with session_scope() as session:
tq = QTickets(session)
for ticket in tq.not_closed().all():
output = ''
ticket_line = '{id} - state={state} tags={tags}'
tags = ','.join(list(ticket.tag_set))
output = ticket_line.format(
id=ticket.id,
state=ticket.state,
tags=tags,
)
if ticket.resource:
output += ' resource=' + ticket.resource.name
print(output)
def _allocate_more_resources(self, event):
while True:
with session_scope() as session:
qres = QResources(session, pool=self.name)
stats = qres.stats()
msg = "=> POOL('{0}'):".format(self.name)
for key, val in stats.items():
msg = msg + ' {0}={1}'.format(key, val)
log.debug(msg)
if stats['on'] >= self.max \
or stats['ready'] + stats['start'] >= self.max_prealloc \
or stats['start'] >= self.max_starting \
or self._too_soon():
# Quota reached, don't allocate more.
break
self.allocate(event)
def resource_list(self, up=None):
with session_scope() as session:
resources = QResources(session)
if up:
resources = resources.up()
else:
resources = resources.on()
for resource in resources.all():
msg = "{id} - {name} pool={pool} tags={tags} status={status}"
tags = ','.join(list(resource.tag_set))
print(
msg.format(
id=resource.id,
name=resource.name,
pool=resource.pool,
tags=tags,
status=resource.state,
))
def _request_resource_removal(self):
with session_scope() as session:
now = time.time()
qres = QResources(session, pool=self.name)
for res in qres.check_failure_candidates():
if res.check_failed_count >= 3:
log.debug("Removing %s, continuous failures", res.name)
res.state = RState.DELETE_REQUEST
continue
for res in qres.clean_candidates():
if not self.reuse_opportunity_time:
# reuse turned off by default, remove no matter what
log.debug("Removing %s, not reusable", res.name)
res.state = RState.DELETE_REQUEST
continue
if res.released_at < (now - self.reuse_opportunity_time):
log.debug("Removing %s, not taken quickly enough", res.name)
res.state = RState.DELETE_REQUEST
continue
if self.reuse_max_time:
last_allowed = now - self.reuse_max_time
if res.sandboxed_since < last_allowed:
log.debug("Removing %s, too long in one sandbox, "
"since %s, last_allowed %s, now %s",
res.name, res.sandboxed_since, last_allowed,
now)
res.state = RState.DELETE_REQUEST
continue
if self.reuse_max_count and \
res.releases_counter > self.reuse_max_count:
log.debug("Removing %s, max reuses reached", res.name)
res.state = RState.DELETE_REQUEST
continue
def resource_list(self, up=None):
with session_scope() as session:
resources = QResources(session)
if up:
resources = resources.up()
else:
resources = resources.on()
for resource in resources.all():
msg = ("{id} - {name} pool={pool} tags={tags} status={status} "
"releases={releases} ticket={ticket}")
tags = ','.join(list(resource.tag_set))
print(
msg.format(
id=resource.id,
name=resource.name,
pool=resource.pool,
tags=tags,
status=resource.state,
releases=resource.releases_counter,
ticket=resource.ticket.id
if resource.ticket else 'NULL',
))
def _garbage_collector(self, event):
to_terminate = []
with session_scope() as session:
qres = QResources(session, pool=self.name)
for res in qres.clean().all():
TerminateWorker(event, self, int(res.id)).start()
def _detect_closed_tickets(self):
with session_scope() as session:
qres = QResources(session, pool=self.name)
for res in qres.clean_candidates().all():
res.state = RState.DELETE_REQUEST
session.add(res)
