def _assign_tickets(self):
with session_scope() as session:
qticket = QTickets(session)
tickets = [x.id for x in qticket.waiting().order_by(models.Ticket.id).all()]
for ticket_id in tickets:
notify_ticket = False
with session_scope() as session:
ticket = session.query(models.Ticket).get(ticket_id)
qres = QResources(session)
resources = qres.ready().all()
if not resources:
app.log.debug("No available resource, skipping %s", ticket)
continue
queue = PriorityQueue()
ticket_tags = ticket.tag_set
for resource in resources:
res_tags = resource.tag_set
if resource.sandbox and resource.sandbox != ticket.sandbox:
continue
if not ticket_tags.issubset(res_tags):
continue
priority = 0
for tag in resource.tags:
if tag.priority is not None and tag.id in ticket_tags:
priority += tag.priority
if resource.sandbox:
# Re-used resources should be preferred to avoid
# allocating new and new resources for the same
# sandboxes. TODO, make this configurable once needed.
priority += REUSED_RESOURCE_PRIORITY
queue.add_task(resource, priority)
try:
resource = queue.pop_task()
except KeyError:
app.log.debug("%d resources UP but unusable for %s",
len(resources), ticket)
continue
# we found an appropriate resource
app.log.debug("Assigning %s to %s", resource.name, ticket.id)
assign_ticket(resource, ticket)
if ticket.tid:
notify_ticket = ticket.tid
# notify ticket when the session is closed (to have short sessions)
if notify_ticket:
self._notify_waiting(notify_ticket)
def test_resource_tag_priority(self, passengers, expected_car):
self.prepare_database({
"resources": {
"car1": {
"data": b"red",
"tags": {
"mother": {"priority": 1},
"father": {"priority": 2},
"son": {"priority": 3},
},
},
"car2": {
"data": b"green",
"tags": {
"mother": {"priority": 1},
"father": {"priority": 3},
"daughter": {"priority": 2},
},
},
"car3": {
"data": b"blue",
"tags": {
"mother": {"priority": 3},
"father": {"priority": 2},
"daughter": {},
},
},
},
})
with session_scope() as session:
ticket = models.Ticket(
state=TState.OPEN,
sandbox=str(random.random()),
)
session.add(ticket)
for tag in passengers:
print(tag)
tag = models.TicketTag(
ticket=ticket,
id=tag,
)
session.add(tag)
sync = Synchronizer()
manager = Manager(sync)
manager._assign_tickets()
with session_scope() as session:
ticket = session.query(models.Ticket).get(1)
assert ticket.resource.name == expected_car
def loop(self):
app.log.debug("Watcher loop")
pools = reload_config()
to_check = {}
with session_scope() as session:
# Even though we never terminate resources that have assigned
# ticket, we still check them. This raises the check limit before
# user releases the ticket and the resource can be terminated as
# soon as possible.
up = QResources(session).up().all()
for item in up:
if not item.pool in pools:
continue
to_check[item.id] = {
'name': item.name,
'pool': item.pool,
'last': item.check_last_time,
'fail': item.check_failed_count,
'id_in_pool': item.id_in_pool,
'data': item.data,
}
for res_id, data in to_check.items():
pool = pools[data['pool']]
if not pool.cmd_livecheck:
continue
if data['last'] + pool.livecheck_period > time.time():
# Not yet needed check.
continue
rc = run_command(
pool.id,
res_id,
data['name'],
data['id_in_pool'],
pool.cmd_livecheck,
'watch',
data=data["data"],
)
with session_scope() as session:
res = session.query(models.Resource).get(res_id)
res.check_last_time = time.time()
if rc['status']:
res.check_failed_count = res.check_failed_count + 1
app.log.debug("failed check #{0} for {1}"\
.format(res.check_failed_count, res_id))
else:
res.check_failed_count = 0
session.add(res)
session.flush()
def allocate(self, event):
resource_id = None
with session_scope() as session:
dbinfo = session.query(models.Pool).get(self.name)
dbinfo.last_start = time.time()
resource = models.Resource()
resource.pool = self.name
session.add_all([resource, dbinfo])
session.flush()
pool_id = self._allocate_pool_id(session, resource)
session.add(pool_id)
session.flush()
app.log.debug("id in pool: {0}".format(pool_id.id))
resource_id = resource.id
fill_dict = dict(
id=str(resource_id).zfill(8),
pool_name=self.name)
resource.name = helpers.careful_string_format(
self.name_pattern, fill_dict)
session.add(resource)
if resource_id:
self.last_start = time.time()
AllocWorker(event, self, int(resource_id)).start()
def job(self):
id_in_pool = None
with session_scope() as session:
resource = session.query(models.Resource).get(self.resource_id)
if resource.ticket:
if resource.ticket.state == helpers.TState.OPEN:
self.log.warning("can't delete {0}, ticket opened"\
.format(resource.name))
return
resource.state = RState.DELETING
session.add(resource)
session.flush()
id_in_pool = resource.id_in_pool
session.expunge(resource)
self.log.debug("TerminateWorker(pool={0}): name={1} by: \"{2}\""\
.format(self.pool.name, resource.name, self.pool.cmd_delete))
if not self.pool.cmd_delete:
self.close()
return
run_command(
self.pool.id,
resource.id,
resource.name,
id_in_pool,
self.pool.cmd_delete,
'terminate',
data=resource.data,
)
self.close()
def close(self):
with session_scope() as session:
resource = session.query(models.Resource).get(self.resource_id)
resource.state = RState.ENDED
session.add(resource)
if resource.id_in_pool_object:
session.delete(resource.id_in_pool_object)
self.event.set()
def closeTicket(self, ticket_id):
with session_scope() as session:
ticket = session.query(models.Ticket).get(ticket_id)
if not ticket:
raise ServerAPIException(
"no such ticket {0}".format(ticket_id))
ticket.state = TState.CLOSED
self.sync.ticket.set()
def collectTicket(self, ticket_id):
output = {'ready': False, 'output': None, 'closed': None}
with session_scope() as session:
ticket = self._checkTicket(ticket_id, session)
if ticket.resource:
output['output'] = ticket.resource.data
output['ready'] = True
output['closed'] = ticket.state == TState.CLOSED
return output
def resource_delete(self, args):
resources = args.resource
with session_scope() as session:
qresources = QResources(session=session)
if args.all:
resources = [res.id for res in qresources.up()]
elif args.unused:
resources = [res.id for res in qresources.ready()]
if not resources or type(resources) != list:
log.error("no resources specified")
return
for res_id in resources:
with session_scope() as session:
qresources = QResources(session=session)
qresources.kill(res_id)
def resource_info(self, resource):
with session_scope() as session:
query = Query(Resource)
query = query.with_session(session)
if resource.isnumeric():
query = query.filter(Resource.id == resource)
else:
query = query.filter(Resource.name == resource)
resource = query.one()
print(json.dumps(resource.to_dict(), indent=4))
def job(self):
with session_scope() as session:
resource = session.query(models.Resource).get(self.resource_id)
id_in_pool = resource.id_in_pool
session.expunge(resource)
out = run_command(self.pool.id, resource.id, resource.name, id_in_pool,
self.pool.cmd_release, "release", data=resource.data)
status = out["status"]
with session_scope() as session:
resource = session.query(models.Resource).get(self.resource_id)
if status:
# mark it for removal
resource.releases_counter = self.pool.reuse_max_count + 1
resource.state = RState.UP
if not status:
self.event.set()
def waitTicket(self, ticket_id):
""" ... blocking! ... """
output = ""
with session_scope() as session:
ticket = session.query(models.Ticket).get(ticket_id)
ticket.tid = self.my_id()
session.add(ticket)
while True:
with session_scope() as session:
ticket = session.query(models.Ticket).get(ticket_id)
if ticket.resource:
return ticket.resource.data
with self.sync.resource_ready:
while self.sync.resource_ready.wait(timeout=10):
if self.sync.tid == self.my_id():
break
def _too_soon(self):
last_start = 0.0
with session_scope() as session:
dbinfo = session.query(models.Pool).get(self.name)
if not dbinfo:
dbinfo = models.Pool()
dbinfo.name = self.name
dbinfo.last_start = 0.0
session.add(dbinfo)
else:
last_start = dbinfo.last_start
is_too_soon = last_start + self.start_delay > time.time()
if is_too_soon:
app.log.debug("too soon for Pool('{0}')".format(self.name))
return is_too_soon
def _detect_closed_tickets(self, event):
with session_scope() as session:
qres = QResources(session, pool=self.name)
for resource in qres.taken():
ticket = resource.ticket
assert ticket
if ticket.state == helpers.TState.CLOSED:
release_resource(ticket)
if self.cmd_release:
# UP → RELEASING → UP, TODO: we might want to optimize
# this a bit, and stop calling the releasing script when
# the resource is not releasable anymore (max_reuses
# reached, etc.).
resource.state = helpers.RState.RELEASING
ReleaseWorker(event, self, int(resource.id)).start()
def takeTicket(self, tags=None, sandbox=None):
with session_scope() as session:
ticket = models.Ticket()
ticket.sandbox = sandbox
tag_objects = []
for tag in (tags or []):
to = models.TicketTag()
to.ticket = ticket
to.id = tag
tag_objects.append(to)
session.add_all([ticket] + tag_objects)
session.flush()
ticket_id = ticket.id
self.sync.ticket.set()
return ticket_id
def ticket_list(self):
with session_scope() as session:
tq = QTickets(session)
for ticket in tq.not_closed().all():
output = ''
ticket_line = '{id} - state={state} tags={tags}'
tags = ','.join(list(ticket.tag_set))
output = ticket_line.format(
id=ticket.id,
state=ticket.state,
tags=tags,
)
if ticket.resource:
output += ' resource=' + ticket.resource.name
print(output)
def foreach_resource(self, args):
""" Execute shell command for each resource """
command = args.command
with session_scope() as session:
resources = QResources(session)
for resource in resources.on().all():
try:
utf_data = ""
if resource.data:
utf_data = resource.data.decode("utf8")
command = args.command.format(
name=resource.name,
state=resource.state,
data_utf8=utf_data,
)
except KeyError as err:
sys.stderr.write(str(err))
subprocess.call(command, shell=True)
def _allocate_more_resources(self, event):
while True:
with session_scope() as session:
qres = QResources(session, pool=self.name)
stats = qres.stats()
msg = "=> POOL('{0}'):".format(self.name)
for key, val in stats.items():
msg = msg + ' {0}={1}'.format(key,val)
app.log.debug(msg)
if stats['on'] >= self.max \
or stats['free'] + stats['start'] >= self.max_prealloc \
or stats['start'] >= self.max_starting \
or self._too_soon():
# Quota reached, don't allocate more.
break
self.allocate(event)
def resource_logs(self, resources=None):
hooks_dir = os.path.join(app.config["logdir"], "hooks")
paths = []
for resource in resources:
# If the resource was specified by its name instead of its ID
if not resource.isnumeric():
with session_scope() as session:
query = Query(Resource)
query = query.with_session(session)
query = query.filter(Resource.name == resource)
resource = query.one().id
# We can't wildcard everything because then `tail' wouldn't
# discover newly created log files
suffixes = ["_alloc", "_watch", "_terminate"]
path = os.path.join(hooks_dir, str(resource).zfill(6))
paths.extend([path + suffix for suffix in suffixes])
cmd = ["tail", "-F", "-n+0"] + paths
subprocess.call(cmd)
def _request_resource_removal(self):
with session_scope() as session:
now = time.time()
qres = QResources(session, pool=self.name)
for res in qres.check_failure_candidates():
if res.check_failed_count >= 3:
app.log.debug("Removing %s, continuous failures", res.name)
res.state = RState.DELETE_REQUEST
continue
for res in qres.clean_candidates():
if not self.reuse_opportunity_time:
# reuse turned off by default, remove no matter what
app.log.debug("Removing %s, not reusable", res.name)
res.state = RState.DELETE_REQUEST
continue
if res.released_at < (now - self.reuse_opportunity_time):
app.log.debug("Removing %s, not taken quickly enough", res.name)
res.state = RState.DELETE_REQUEST
continue
if self.reuse_max_time:
last_allowed = now - self.reuse_max_time
if res.sandboxed_since < last_allowed:
app.log.debug(
"Removing %s, too long in one sandbox, "
"since %s, last_allowed %s, now %s",
res.name, res.sandboxed_since, last_allowed,
now)
res.state = RState.DELETE_REQUEST
continue
if self.reuse_max_count and \
res.releases_counter > self.reuse_max_count:
app.log.debug("Removing %s, max reuses reached", res.name)
res.state = RState.DELETE_REQUEST
continue
def resource_list(self, up=None):
with session_scope() as session:
resources = QResources(session)
if up:
resources = resources.up()
else:
resources = resources.on()
for resource in resources.all():
msg = ("{id} - {name} pool={pool} tags={tags} status={status} "
"releases={releases} ticket={ticket}")
tags = ','.join(list(resource.tag_set))
print(
msg.format(
id=resource.id,
name=resource.name,
pool=resource.pool,
tags=tags,
status=resource.state,
releases=resource.releases_counter,
ticket=resource.ticket.id
if resource.ticket else 'NULL',
))
def main():
""" module entrypoint """
# Create the database, if not exist yet.
init_by_alembic()
# Synchronization tool.
sync = Synchronizer()
# Delete leftovers from previous session, we need to run everything
# asynchronously, see https://github.com/praiskup/resalloc/issues/41
with session_scope() as session:
QResources(session=session).fix_broken_after_restart(app.log)
# Start server on background.
server = Server()
server.sync = sync
server.start()
try:
Manager(sync).run()
except KeyboardInterrupt:
pass
finally:
server.shutdown()
def _garbage_collector(self, event):
to_terminate = []
with session_scope() as session:
qres = QResources(session, pool=self.name)
for res in qres.clean().all():
TerminateWorker(event, self, int(res.id)).start()
def job(self):
self.log.debug(
"Allocating new resource id={id} in pool '{pool}' by {cmd}"\
.format(
id=self.resource_id,
pool=self.pool.name,
cmd=self.pool.cmd_new
)
)
id_in_pool = None
with session_scope() as session:
resource = session.query(models.Resource).get(self.resource_id)
id_in_pool = resource.id_in_pool
session.expunge(resource)
# Run the allocation script.
output = run_command(
self.pool.id,
resource.id,
resource.name,
id_in_pool,
self.pool.cmd_new,
catch_stdout_bytes=512,
)
with session_scope() as session:
resource.state = RState.ENDED if output['status'] else RState.UP
resource.data = output['stdout']
tags = []
if not isinstance(self.pool.tags, list):
msg = "Pool {pool} has set 'tags' set, but that's not an array"\
.format(pool=self.name)
warnings.warn(msg)
else:
for tag in self.pool.tags:
tag_name = None
tag_priority = 0
if isinstance(tag, str):
# older format
tag_name = tag
elif isinstance(tag, dict):
tag_name = tag['name']
tag_priority = tag.get('priority', 0)
else:
assert False
tag_obj = models.ResourceTag()
tag_obj.id = tag_name
tag_obj.resource_id = resource.id
tag_obj.priority = tag_priority
tags.append(tag_obj)
self.log.debug("Allocator ends with state={0}".format(resource.state))
session.add_all(tags + [resource])
if resource.state == RState.ENDED:
session.delete(resource.id_in_pool_object)
# Notify manager that it is worth doing re-spin.
self.event.set()
def prepare_database(self, data):
"""
Using the DATA dictionary pre-populate the database.
"""
with session_scope() as session:
self._prepare_database(data, session)
def init_by_models():
with session_scope() as session:
models.Base.metadata.create_all(session.get_bind())
