def _assign_tickets(self): with session_scope() as session: qticket = QTickets(session) tickets = [ x.id for x in qticket.waiting().order_by(models.Ticket.id).all() ] for ticket_id in tickets: notify_ticket = False with session_scope() as session: ticket = session.query(models.Ticket).get(ticket_id) qres = QResources(session) resources = qres.ready().all() ticket_tags = ticket.tag_set for resource in resources: res_tags = resource.tag_set if ticket_tags.issubset(res_tags): # We have found appropriate resource! ticket.resource_id = resource.id if ticket.tid: notify_ticket = ticket.tid session.add_all([ticket]) break if notify_ticket: self._notify_waiting(notify_ticket)
def _assign_tickets(self): with session_scope() as session: qticket = QTickets(session) tickets = [x.id for x in qticket.waiting().order_by(models.Ticket.id).all()] for ticket_id in tickets: notify_ticket = False with session_scope() as session: ticket = session.query(models.Ticket).get(ticket_id) qres = QResources(session) resources = qres.ready().all() ticket_tags = ticket.tag_set for resource in resources: res_tags = resource.tag_set if resource.sandbox and resource.sandbox != ticket.sandbox: continue if not ticket_tags.issubset(res_tags): continue # We have found appropriate resource! log.debug("Assigning %s to %s", resource.name, ticket.id) assign_ticket(resource, ticket) if ticket.tid: notify_ticket = ticket.tid break if notify_ticket: self._notify_waiting(notify_ticket)
def loop(self): log.debug("Watcher loop") pools = reload_config() to_check = {} with session_scope() as session: # Even though we never terminate resources that have assigned # ticket, we still check them. This raises the check limit before # user releases the ticket and the resource can be terminated as # soon as possible. up = QResources(session).up().all() for item in up: if not item.pool in pools: continue to_check[item.id] = { 'name': item.name, 'pool': item.pool, 'last': item.check_last_time, 'fail': item.check_failed_count, 'id_in_pool': item.id_in_pool, 'data': item.data, } for res_id, data in to_check.items(): pool = pools[data['pool']] if not pool.cmd_livecheck: continue if data['last'] + pool.livecheck_period > time.time(): # Not yet needed check. continue rc = run_command( pool.id, res_id, data['name'], data['id_in_pool'], pool.cmd_livecheck, 'watch', data=data["data"], ) with session_scope() as session: res = session.query(models.Resource).get(res_id) res.check_last_time = time.time() if rc['status']: res.check_failed_count = res.check_failed_count + 1 log.debug("failed check #{0} for {1}"\ .format(res.check_failed_count, res_id)) else: res.check_failed_count = 0 session.add(res) session.flush()
def job(self): self.log.debug( "Allocating new resource id={id} in pool '{pool}' by {cmd}"\ .format( id=self.resource_id, pool=self.pool.name, cmd=self.pool.cmd_new ) ) id_in_pool = None with session_scope() as session: resource = session.query(models.Resource).get(self.resource_id) id_in_pool = resource.id_in_pool session.expunge(resource) # Run the allocation script. output = run_command( self.pool.id, resource.id, resource.name, id_in_pool, self.pool.cmd_new, catch_stdout_bytes=512, ) with session_scope() as session: resource.state = RState.ENDED if output['status'] else RState.UP resource.data = output['stdout'] tags = [] if type(self.pool.tags) != type([]): msg = "Pool {pool} has set 'tags' set, but that's not an array"\ .format(pool=self.name) warnings.warn(msg) else: for tag in self.pool.tags: tag_obj = models.ResourceTag() tag_obj.id = tag tag_obj.resource_id = resource.id tags.append(tag_obj) self.log.debug("Allocator ends with state={0}".format(resource.state)) session.add_all(tags + [resource]) if resource.state == RState.ENDED: session.delete(resource.id_in_pool_object) # Notify manager that it is worth doing re-spin. self.event.set()
def closeTicket(self, ticket_id): with session_scope() as session: ticket = session.query(models.Ticket).get(ticket_id) if not ticket: raise ServerAPIException("no such ticket {0}".format(ticket_id)) ticket.state = TState.CLOSED self.sync.ticket.set()
def job(self): id_in_pool = None with session_scope() as session: resource = session.query(models.Resource).get(self.resource_id) if resource.ticket: if resource.ticket.state == helpers.TState.OPEN: log.warning("can't delete {0}, ticket opened"\ .format(resource.name)) return resource.state = RState.DELETING session.add(resource) session.flush() id_in_pool = resource.id_in_pool session.expunge(resource) self.log.debug("TerminateWorker(pool={0}): name={1} by: \"{2}\""\ .format(self.pool.name, resource.name, self.pool.cmd_delete)) if not self.pool.cmd_delete: self.close() return run_command( self.pool.id, resource.id, resource.name, id_in_pool, self.pool.cmd_delete, 'terminate', ) self.close()
def allocate(self, event): resource_id = None with session_scope() as session: dbinfo = session.query(models.Pool).get(self.name) dbinfo.last_start = time.time() resource = models.Resource() resource.pool = self.name session.add_all([resource, dbinfo]) session.flush() pool_id = self._allocate_pool_id(session, resource) session.add(pool_id) session.flush() log.debug("id in pool: {0}".format(pool_id.id)) resource_id = resource.id fill_dict = dict( id=str(resource_id).zfill(8), pool_name=self.name) resource.name = helpers.careful_string_format( self.name_pattern, fill_dict) session.add(resource) if resource_id: self.last_start = time.time() AllocWorker(event, self, int(resource_id)).start()
def close(self): with session_scope() as session: resource = session.query(models.Resource).get(self.resource_id) resource.state = RState.ENDED session.add(resource) if resource.id_in_pool_object: session.delete(resource.id_in_pool_object) self.event.set()
def loop(self): log.debug("Watcher loop") pools = reload_config() to_check = {} with session_scope() as session: up = QResources(session).up().all() for item in up: if not item.pool in pools: continue to_check[item.id] = { 'name': item.name, 'pool': item.pool, 'last': item.check_last_time, 'fail': item.check_failed_count, 'id_in_pool': item.id_in_pool, } for res_id, data in to_check.items(): pool = pools[data['pool']] if not pool.cmd_livecheck: continue if data['last'] + pool.livecheck_period > time.time(): # Not yet needed check. continue failed_count = 0 rc = run_command(pool.id, res_id, data['name'], data['id_in_pool'], pool.cmd_livecheck, 'watch') with session_scope() as session: res = session.query(models.Resource).get(res_id) res.check_last_time = time.time() if rc['status']: res.check_failed_count = res.check_failed_count + 1 log.debug("failed check #{0} for {1}"\ .format(res.check_failed_count, res_id)) else: res.check_failed_count = 0 session.add(res) session.flush() failed_count = res.check_failed_count if failed_count >= 3: log.debug("Watcher plans to kill {0}".format(res_id)) TerminateWorker(self.event, pool, res_id).start()
def resource_delete(self, resources=None): if not resources or type(resources) != list: log.error("no resources specified") return for res_id in resources: with session_scope() as session: resources = QResources(session=session) resources.kill(res_id)
def job(self): with session_scope() as session: resource = session.query(models.Resource).get(self.resource_id) id_in_pool = resource.id_in_pool session.expunge(resource) out = run_command(self.pool.id, resource.id, resource.name, id_in_pool, self.pool.cmd_release, "release", data=resource.data) status = out["status"] with session_scope() as session: resource = session.query(models.Resource).get(self.resource_id) if status: # mark it for removal resource.releases_counter = self.pool.reuse_max_count + 1 resource.state = RState.UP if not status: self.event.set()
def waitTicket(self, ticket_id): """ ... blocking! ... """ output = "" with session_scope() as session: ticket = session.query(models.Ticket).get(ticket_id) ticket.tid = self.my_id() session.add(ticket) while True: with session_scope() as session: ticket = session.query(models.Ticket).get(ticket_id) if ticket.resource: return ticket.resource.data with self.sync.resource_ready: while self.sync.resource_ready.wait(timeout=10): if self.sync.tid==self.my_id(): break
def collectTicket(self, ticket_id): output = { 'ready': False, 'output': None, } with session_scope() as session: resource = self._checkTicket(ticket_id, session) if resource: output['output'] = resource.data output['ready'] = True return output
def collectTicket(self, ticket_id): output = { 'ready': False, 'output': None, 'closed': None } with session_scope() as session: ticket = self._checkTicket(ticket_id, session) if ticket.resource: output['output'] = ticket.resource.data output['ready'] = True output['closed'] = ticket.state == TState.CLOSED return output
def takeTicket(self, tags=None): with session_scope() as session: ticket = models.Ticket() tag_objects = [] for tag in (tags or []): to = models.TicketTag() to.ticket = ticket to.id = tag tag_objects.append(to) session.add_all([ticket] + tag_objects) session.flush() ticket_id = ticket.id self.sync.ticket.set() return ticket_id
def _too_soon(self): last_start = 0.0 with session_scope() as session: dbinfo = session.query(models.Pool).get(self.name) if not dbinfo: dbinfo = models.Pool() dbinfo.name = self.name dbinfo.last_start = 0.0 session.add(dbinfo) else: last_start = dbinfo.last_start is_too_soon = last_start + self.start_delay > time.time() if is_too_soon: log.debug("too soon for Pool('{0}')".format(self.name)) return is_too_soon
def _detect_closed_tickets(self, event): with session_scope() as session: qres = QResources(session, pool=self.name) for resource in qres.taken(): ticket = resource.ticket assert ticket if ticket.state == helpers.TState.CLOSED: release_resource(ticket) if self.cmd_release: # UP → RELEASING → UP, TODO: we might want to optimize # this a bit, and stop calling the releasing script when # the resource is not releasable anymore (max_reuses # reached, etc.). resource.state = helpers.RState.RELEASING ReleaseWorker(event, self, int(resource.id)).start()
def ticket_list(self): with session_scope() as session: tq = QTickets(session) for ticket in tq.not_closed().all(): output = '' ticket_line = '{id} - state={state} tags={tags}' tags = ','.join(list(ticket.tag_set)) output = ticket_line.format( id=ticket.id, state=ticket.state, tags=tags, ) if ticket.resource: output += ' resource=' + ticket.resource.name print(output)
def _allocate_more_resources(self, event): while True: with session_scope() as session: qres = QResources(session, pool=self.name) stats = qres.stats() msg = "=> POOL('{0}'):".format(self.name) for key, val in stats.items(): msg = msg + ' {0}={1}'.format(key, val) log.debug(msg) if stats['on'] >= self.max \ or stats['ready'] + stats['start'] >= self.max_prealloc \ or stats['start'] >= self.max_starting \ or self._too_soon(): # Quota reached, don't allocate more. break self.allocate(event)
def resource_list(self, up=None): with session_scope() as session: resources = QResources(session) if up: resources = resources.up() else: resources = resources.on() for resource in resources.all(): msg = "{id} - {name} pool={pool} tags={tags} status={status}" tags = ','.join(list(resource.tag_set)) print( msg.format( id=resource.id, name=resource.name, pool=resource.pool, tags=tags, status=resource.state, ))
def _request_resource_removal(self): with session_scope() as session: now = time.time() qres = QResources(session, pool=self.name) for res in qres.check_failure_candidates(): if res.check_failed_count >= 3: log.debug("Removing %s, continuous failures", res.name) res.state = RState.DELETE_REQUEST continue for res in qres.clean_candidates(): if not self.reuse_opportunity_time: # reuse turned off by default, remove no matter what log.debug("Removing %s, not reusable", res.name) res.state = RState.DELETE_REQUEST continue if res.released_at < (now - self.reuse_opportunity_time): log.debug("Removing %s, not taken quickly enough", res.name) res.state = RState.DELETE_REQUEST continue if self.reuse_max_time: last_allowed = now - self.reuse_max_time if res.sandboxed_since < last_allowed: log.debug("Removing %s, too long in one sandbox, " "since %s, last_allowed %s, now %s", res.name, res.sandboxed_since, last_allowed, now) res.state = RState.DELETE_REQUEST continue if self.reuse_max_count and \ res.releases_counter > self.reuse_max_count: log.debug("Removing %s, max reuses reached", res.name) res.state = RState.DELETE_REQUEST continue
def resource_list(self, up=None): with session_scope() as session: resources = QResources(session) if up: resources = resources.up() else: resources = resources.on() for resource in resources.all(): msg = ("{id} - {name} pool={pool} tags={tags} status={status} " "releases={releases} ticket={ticket}") tags = ','.join(list(resource.tag_set)) print( msg.format( id=resource.id, name=resource.name, pool=resource.pool, tags=tags, status=resource.state, releases=resource.releases_counter, ticket=resource.ticket.id if resource.ticket else 'NULL', ))
def _garbage_collector(self, event): to_terminate = [] with session_scope() as session: qres = QResources(session, pool=self.name) for res in qres.clean().all(): TerminateWorker(event, self, int(res.id)).start()
def _detect_closed_tickets(self): with session_scope() as session: qres = QResources(session, pool=self.name) for res in qres.clean_candidates().all(): res.state = RState.DELETE_REQUEST session.add(res)