def _assign_tickets(self):
with session_scope() as session:
qticket = QTickets(session)
tickets = [
x.id
for x in qticket.waiting().order_by(models.Ticket.id).all()
]
for ticket_id in tickets:
notify_ticket = False
with session_scope() as session:
ticket = session.query(models.Ticket).get(ticket_id)
qres = QResources(session)
resources = qres.ready().all()
ticket_tags = ticket.tag_set
for resource in resources:
res_tags = resource.tag_set
if ticket_tags.issubset(res_tags):
# We have found appropriate resource!
ticket.resource_id = resource.id
if ticket.tid:
notify_ticket = ticket.tid
session.add_all([ticket])
break
if notify_ticket:
self._notify_waiting(notify_ticket)
def _assign_tickets(self):
with session_scope() as session:
qticket = QTickets(session)
tickets = [x.id for x in qticket.waiting().order_by(models.Ticket.id).all()]
for ticket_id in tickets:
notify_ticket = False
with session_scope() as session:
ticket = session.query(models.Ticket).get(ticket_id)
qres = QResources(session)
resources = qres.ready().all()
ticket_tags = ticket.tag_set
for resource in resources:
res_tags = resource.tag_set
if resource.sandbox and resource.sandbox != ticket.sandbox:
continue
if not ticket_tags.issubset(res_tags):
continue
# We have found appropriate resource!
log.debug("Assigning %s to %s", resource.name, ticket.id)
assign_ticket(resource, ticket)
if ticket.tid:
notify_ticket = ticket.tid
break
if notify_ticket:
self._notify_waiting(notify_ticket)
def resource_delete(self, resources=None):
if not resources or type(resources) != list:
log.error("no resources specified")
return
for res_id in resources:
with session_scope() as session:
resources = QResources(session=session)
resources.kill(res_id)
def _assign_tickets(self):
with session_scope() as session:
qticket = QTickets(session)
tickets = [x.id for x in qticket.waiting().order_by(models.Ticket.id).all()]
for ticket_id in tickets:
notify_ticket = False
with session_scope() as session:
ticket = session.query(models.Ticket).get(ticket_id)
qres = QResources(session)
resources = qres.ready().all()
if not resources:
app.log.debug("No available resource, skipping %s", ticket)
continue
queue = PriorityQueue()
ticket_tags = ticket.tag_set
for resource in resources:
res_tags = resource.tag_set
if resource.sandbox and resource.sandbox != ticket.sandbox:
continue
if not ticket_tags.issubset(res_tags):
continue
priority = 0
for tag in resource.tags:
if tag.priority is not None and tag.id in ticket_tags:
priority += tag.priority
if resource.sandbox:
# Re-used resources should be preferred to avoid
# allocating new and new resources for the same
# sandboxes. TODO, make this configurable once needed.
priority += REUSED_RESOURCE_PRIORITY
queue.add_task(resource, priority)
try:
resource = queue.pop_task()
except KeyError:
app.log.debug("%d resources UP but unusable for %s",
len(resources), ticket)
continue
# we found an appropriate resource
app.log.debug("Assigning %s to %s", resource.name, ticket.id)
assign_ticket(resource, ticket)
if ticket.tid:
notify_ticket = ticket.tid
# notify ticket when the session is closed (to have short sessions)
if notify_ticket:
self._notify_waiting(notify_ticket)
def _detect_closed_tickets(self, event):
with session_scope() as session:
qres = QResources(session, pool=self.name)
for resource in qres.taken():
ticket = resource.ticket
assert ticket
if ticket.state == helpers.TState.CLOSED:
release_resource(ticket)
if self.cmd_release:
# UP → RELEASING → UP, TODO: we might want to optimize
# this a bit, and stop calling the releasing script when
# the resource is not releasable anymore (max_reuses
# reached, etc.).
resource.state = helpers.RState.RELEASING
ReleaseWorker(event, self, int(resource.id)).start()
def _allocate_more_resources(self, event):
while True:
with session_scope() as session:
qres = QResources(session, pool=self.name)
stats = qres.stats()
msg = "=> POOL('{0}'):".format(self.name)
for key, val in stats.items():
msg = msg + ' {0}={1}'.format(key, val)
log.debug(msg)
if stats['on'] >= self.max \
or stats['ready'] + stats['start'] >= self.max_prealloc \
or stats['start'] >= self.max_starting \
or self._too_soon():
# Quota reached, don't allocate more.
break
self.allocate(event)
def resource_list(self, up=None):
with session_scope() as session:
resources = QResources(session)
if up:
resources = resources.up()
else:
resources = resources.on()
for resource in resources.all():
msg = "{id} - {name} pool={pool} tags={tags} status={status}"
tags = ','.join(list(resource.tag_set))
print(
msg.format(
id=resource.id,
name=resource.name,
pool=resource.pool,
tags=tags,
status=resource.state,
))
def foreach_resource(self, args):
""" Execute shell command for each resource """
command = args.command
with session_scope() as session:
resources = QResources(session)
for resource in resources.on().all():
try:
utf_data = ""
if resource.data:
utf_data = resource.data.decode("utf8")
command = args.command.format(
name=resource.name,
state=resource.state,
data_utf8=utf_data,
)
except KeyError as err:
sys.stderr.write(str(err))
subprocess.call(command, shell=True)
def _request_resource_removal(self):
with session_scope() as session:
now = time.time()
qres = QResources(session, pool=self.name)
for res in qres.check_failure_candidates():
if res.check_failed_count >= 3:
app.log.debug("Removing %s, continuous failures", res.name)
res.state = RState.DELETE_REQUEST
continue
for res in qres.clean_candidates():
if not self.reuse_opportunity_time:
# reuse turned off by default, remove no matter what
app.log.debug("Removing %s, not reusable", res.name)
res.state = RState.DELETE_REQUEST
continue
if res.released_at < (now - self.reuse_opportunity_time):
app.log.debug("Removing %s, not taken quickly enough", res.name)
res.state = RState.DELETE_REQUEST
continue
if self.reuse_max_time:
last_allowed = now - self.reuse_max_time
if res.sandboxed_since < last_allowed:
app.log.debug(
"Removing %s, too long in one sandbox, "
"since %s, last_allowed %s, now %s",
res.name, res.sandboxed_since, last_allowed,
now)
res.state = RState.DELETE_REQUEST
continue
if self.reuse_max_count and \
res.releases_counter > self.reuse_max_count:
app.log.debug("Removing %s, max reuses reached", res.name)
res.state = RState.DELETE_REQUEST
continue
def loop(self):
log.debug("Watcher loop")
pools = reload_config()
to_check = {}
with session_scope() as session:
# Even though we never terminate resources that have assigned
# ticket, we still check them. This raises the check limit before
# user releases the ticket and the resource can be terminated as
# soon as possible.
up = QResources(session).up().all()
for item in up:
if not item.pool in pools:
continue
to_check[item.id] = {
'name': item.name,
'pool': item.pool,
'last': item.check_last_time,
'fail': item.check_failed_count,
'id_in_pool': item.id_in_pool,
'data': item.data,
}
for res_id, data in to_check.items():
pool = pools[data['pool']]
if not pool.cmd_livecheck:
continue
if data['last'] + pool.livecheck_period > time.time():
# Not yet needed check.
continue
rc = run_command(
pool.id,
res_id,
data['name'],
data['id_in_pool'],
pool.cmd_livecheck,
'watch',
data=data["data"],
)
with session_scope() as session:
res = session.query(models.Resource).get(res_id)
res.check_last_time = time.time()
if rc['status']:
res.check_failed_count = res.check_failed_count + 1
log.debug("failed check #{0} for {1}"\
.format(res.check_failed_count, res_id))
else:
res.check_failed_count = 0
session.add(res)
session.flush()
def resource_list(self, up=None):
with session_scope() as session:
resources = QResources(session)
if up:
resources = resources.up()
else:
resources = resources.on()
for resource in resources.all():
msg = ("{id} - {name} pool={pool} tags={tags} status={status} "
"releases={releases} ticket={ticket}")
tags = ','.join(list(resource.tag_set))
print(
msg.format(
id=resource.id,
name=resource.name,
pool=resource.pool,
tags=tags,
status=resource.state,
releases=resource.releases_counter,
ticket=resource.ticket.id
if resource.ticket else 'NULL',
))
def resource_delete(self, args):
resources = args.resource
with session_scope() as session:
qresources = QResources(session=session)
if args.all:
resources = [res.id for res in qresources.up()]
elif args.unused:
resources = [res.id for res in qresources.ready()]
if not resources or type(resources) != list:
log.error("no resources specified")
return
for res_id in resources:
with session_scope() as session:
qresources = QResources(session=session)
qresources.kill(res_id)
def loop(self):
log.debug("Watcher loop")
pools = reload_config()
to_check = {}
with session_scope() as session:
up = QResources(session).up().all()
for item in up:
if not item.pool in pools:
continue
to_check[item.id] = {
'name': item.name,
'pool': item.pool,
'last': item.check_last_time,
'fail': item.check_failed_count,
'id_in_pool': item.id_in_pool,
}
for res_id, data in to_check.items():
pool = pools[data['pool']]
if not pool.cmd_livecheck:
continue
if data['last'] + pool.livecheck_period > time.time():
# Not yet needed check.
continue
failed_count = 0
rc = run_command(pool.id, res_id, data['name'], data['id_in_pool'],
pool.cmd_livecheck, 'watch')
with session_scope() as session:
res = session.query(models.Resource).get(res_id)
res.check_last_time = time.time()
if rc['status']:
res.check_failed_count = res.check_failed_count + 1
log.debug("failed check #{0} for {1}"\
.format(res.check_failed_count, res_id))
else:
res.check_failed_count = 0
session.add(res)
session.flush()
failed_count = res.check_failed_count
if failed_count >= 3:
log.debug("Watcher plans to kill {0}".format(res_id))
TerminateWorker(self.event, pool, res_id).start()
def main():
""" module entrypoint """
# Create the database, if not exist yet.
init_by_alembic()
# Synchronization tool.
sync = Synchronizer()
# Delete leftovers from previous session, we need to run everything
# asynchronously, see https://github.com/praiskup/resalloc/issues/41
with session_scope() as session:
QResources(session=session).fix_broken_after_restart(app.log)
# Start server on background.
server = Server()
server.sync = sync
server.start()
try:
Manager(sync).run()
except KeyboardInterrupt:
pass
finally:
server.shutdown()
def _garbage_collector(self, event):
to_terminate = []
with session_scope() as session:
qres = QResources(session, pool=self.name)
for res in qres.clean().all():
TerminateWorker(event, self, int(res.id)).start()
def _detect_closed_tickets(self):
with session_scope() as session:
qres = QResources(session, pool=self.name)
for res in qres.clean_candidates().all():
res.state = RState.DELETE_REQUEST
session.add(res)