def test_double_lock(self): node_name = 'goldfish' user = '******' node = Node(name=node_name) node.update(dict(locked=True, locked_by=user)) with pytest.raises(ForbiddenRequestError): node.update(dict(locked=True, locked_by=user))
def test_double_lock(self): node_name = 'goldfish' user = '******' node = Node(name=node_name) node.update(dict(locked=True, locked_by=user)) with pytest.raises(ForbiddenRequestError): node.update(dict(locked=True, locked_by=user))
def test_locked_since_unlocked(self): node_name = 'cats' user = '******' old_locked_since = datetime(2000, 1, 1, 0, 0) node = Node(name=node_name) node.update(dict(locked=True, locked_by=user)) node.locked_since = old_locked_since node.update(dict(locked=False, locked_by=user)) assert node.locked_since is None
def test_locked_since_locked(self): node_name = 'cats' user = '******' node = Node(name=node_name) node.update(dict(locked=True, locked_by=user)) # This used to take <100us; since we started flushing on node updates, # it takes around 2-3ms. assert (datetime.utcnow() - node.locked_since) < timedelta(milliseconds=5)
def test_locked_since_unlocked(self): node_name = 'cats' user = '******' old_locked_since = datetime(2000, 1, 1, 0, 0) node = Node(name=node_name) node.update(dict(locked=True, locked_by=user)) node.locked_since = old_locked_since node.update(dict(locked=False, locked_by=user)) assert node.locked_since is None
class NodesController(object): @expose(generic=True, template='json') def index(self, locked=None, machine_type='', os_type=None, os_version=None, locked_by=None, up=None, count=None): query = Node.query if locked is not None: query = query.filter(Node.locked == locked) if machine_type: if '|' in machine_type: machine_types = machine_type.split('|') query = query.filter(Node.machine_type.in_(machine_types)) else: query = query.filter(Node.machine_type == machine_type) if os_type: query = query.filter(Node.os_type == os_type) if os_version: query = query.filter(Node.os_version == os_version) if locked_by: query = query.filter(Node.locked_by == locked_by) if up is not None: query = query.filter(Node.up == up) if count is not None: if not count.isdigit() or isinstance(count, int): error('/errors/invalid/', 'count must be an integer') query = query.limit(count) return [node.__json__() for node in query.all()] @index.when(method='POST', template='json') def index_post(self): """ Create a new node """ try: data = request.json name = data.get('name') except ValueError: rollback() error('/errors/invalid/', 'could not decode JSON body') # we allow empty data to be pushed if not name: error('/errors/invalid/', "could not find required key: 'name'") if Node.filter_by(name=name).first(): error('/errors/invalid/', "Node with name %s already exists" % name) else: self.node = Node(name=name) try: self.node.update(data) except PaddlesError as exc: error(exc.url, str(exc)) log.info("Created {node}: {data}".format( node=self.node, data=data, )) return dict() @expose(generic=True, template='json') def lock_many(self): error('/errors/invalid/', "this URI only supports POST requests") @isolation_level('SERIALIZABLE') @lock_many.when(method='POST', template='json') def lock_many_post(self): req = request.json fields = set(('count', 'locked_by', 'machine_type', 'description')) if not fields.issubset(set(req.keys())): error('/errors/invalid/', "must pass these fields: %s" % ', '.join(fields)) req['locked'] = True count = req.pop('count', 0) if count < 1: error('/errors/invalid/', "cannot lock less than 1 node") machine_type = req.pop('machine_type', None) if not machine_type: error('/errors/invalid/', "must specify machine_type") locked_by = req.get('locked_by') description = req.get('description') os_type = req.get('os_type') os_version = req.get('os_version') arch = req.get('arch') if os_version is not None: os_version = str(os_version) attempts = 2 log.debug("Locking {count} {mtype} nodes for {locked_by}".format( count=count, mtype=machine_type, locked_by=locked_by)) while attempts > 0: try: result = Node.lock_many(count=count, locked_by=locked_by, machine_type=machine_type, description=description, os_type=os_type, os_version=os_version, arch=arch) if description: desc_str = " with description %s" % description else: desc_str = "" log.info("Locked {names} for {locked_by}{desc_str}".format( names=" ".join([str(node) for node in result]), locked_by=locked_by, desc_str=desc_str, )) return result except RaceConditionError as exc: log.warn("lock_many() detected race condition") attempts -= 1 if attempts > 0: log.info("retrying after race avoidance (%s tries left)", attempts) else: error(exc.url, str(exc)) except PaddlesError as exc: error(exc.url, str(exc)) @expose(generic=True, template='json') def unlock_many(self): error('/errors/invalid/', "this URI only supports POST requests") @unlock_many.when(method='POST', template='json') def unlock_many_post(self): req = request.json fields = ['names', 'locked_by'] if sorted(req.keys()) != sorted(fields): error('/errors/invalid/', "must pass these fields: %s" % ', '.join(fields)) locked_by = req.get('locked_by') names = req.get('names') if not isinstance(names, list): error('/errors/invalid/', "'names' must be a list; got: %s" % str(type(names))) base_query = Node.query query = base_query.filter(Node.name.in_(names)) if query.count() != len(names): error('/errors/invalid/', "Could not find all nodes!") log.info("Unlocking {count} nodes for {locked_by}".format( count=len(names), locked_by=locked_by)) result = [] for node in query.all(): result.append( NodeController._lock(node, dict(locked=False, locked_by=locked_by), 'unlock') ) return result @expose('json') def job_stats(self, machine_type='', since_days=14): since_days = int(since_days) if since_days < 1: error('/errors/invalid/', "since_days must be a positive integer") now = datetime.utcnow() past = now - timedelta(days=since_days) recent_jobs = Job.query.filter(Job.posted.between(past, now)).subquery() RecentJob = aliased(Job, recent_jobs) query = Session.query(Node.name, RecentJob.status, func.count('*')) if machine_type: # Note: filtering by Job.machine_type (as below) greatly improves # performance but could lead slightly incorrect values if many jobs # are being scheduled using mixed machine types. We work around # this by including the 'multi' machine type (which is the name of # the queue Inktank uses for such jobs. query = query.filter(RecentJob.machine_type.in_((machine_type, 'multi'))) query = query.filter(Node.machine_type == machine_type) query = query.join(RecentJob.target_nodes).group_by(Node)\ .group_by(RecentJob.status) all_stats = {} results = query.all() for (name, status, count) in results: node_stats = all_stats.get(name, {}) node_stats[status] = count all_stats[name] = node_stats stats_sorter = lambda t: sum(t[1].values()) ordered_stats = OrderedDict(sorted(all_stats.items(), key=stats_sorter)) return ordered_stats @expose('json') def machine_types(self): query = Node.query.values(Node.machine_type) return sorted(list(set([item[0] for item in query if item[0]]))) @expose('json') def _lookup(self, name, *remainder): return NodeController(name), remainder
class NodesController(object): @expose(generic=True, template='json') def index(self, locked=None, machine_type='', os_type=None, os_version=None, locked_by=None, up=None, count=None): query = Node.query if locked is not None: query = query.filter(Node.locked == locked) if machine_type: if '|' in machine_type: machine_types = machine_type.split('|') query = query.filter(Node.machine_type.in_(machine_types)) else: query = query.filter(Node.machine_type == machine_type) if os_type: query = query.filter(Node.os_type == os_type) if os_version: query = query.filter(Node.os_version == os_version) if locked_by: query = query.filter(Node.locked_by == locked_by) if up is not None: query = query.filter(Node.up == up) if count is not None: if not count.isdigit() or isinstance(count, int): error('/errors/invalid/', 'count must be an integer') query = query.limit(count) return [node.__json__() for node in query.all()] @index.when(method='POST', template='json') def index_post(self): """ Create a new node """ try: data = request.json name = data.get('name') except ValueError: rollback() error('/errors/invalid/', 'could not decode JSON body') # we allow empty data to be pushed if not name: error('/errors/invalid/', "could not find required key: 'name'") if Node.filter_by(name=name).first(): error('/errors/invalid/', "Node with name %s already exists" % name) else: self.node = Node(name=name) try: self.node.update(data) except PaddlesError as exc: error(exc.url, str(exc)) log.info("Created {node}: {data}".format( node=self.node, data=data, )) return dict() @expose(generic=True, template='json') def lock_many(self): error('/errors/invalid/', "this URI only supports POST requests") @isolation_level('SERIALIZABLE') @lock_many.when(method='POST', template='json') def lock_many_post(self): req = request.json fields = set(('count', 'locked_by', 'machine_type', 'description')) if not fields.issubset(set(req.keys())): error('/errors/invalid/', "must pass these fields: %s" % ', '.join(fields)) req['locked'] = True count = req.pop('count', 0) if count < 1: error('/errors/invalid/', "cannot lock less than 1 node") machine_type = req.pop('machine_type', None) if not machine_type: error('/errors/invalid/', "must specify machine_type") locked_by = req.get('locked_by') description = req.get('description') os_type = req.get('os_type') os_version = req.get('os_version') arch = req.get('arch') if os_version is not None: os_version = str(os_version) attempts = 2 log.debug("Locking {count} {mtype} nodes for {locked_by}".format( count=count, mtype=machine_type, locked_by=locked_by)) while attempts > 0: try: result = Node.lock_many(count=count, locked_by=locked_by, machine_type=machine_type, description=description, os_type=os_type, os_version=os_version, arch=arch) if description: desc_str = " with description %s" % description else: desc_str = "" log.info("Locked {names} for {locked_by}{desc_str}".format( names=" ".join([str(node) for node in result]), locked_by=locked_by, desc_str=desc_str, )) return result except RaceConditionError as exc: log.warn("lock_many() detected race condition") attempts -= 1 if attempts > 0: log.info("retrying after race avoidance (%s tries left)", attempts) else: error(exc.url, str(exc)) except PaddlesError as exc: error(exc.url, str(exc)) @expose(generic=True, template='json') def unlock_many(self): error('/errors/invalid/', "this URI only supports POST requests") @unlock_many.when(method='POST', template='json') def unlock_many_post(self): req = request.json fields = ['names', 'locked_by'] if sorted(req.keys()) != sorted(fields): error('/errors/invalid/', "must pass these fields: %s" % ', '.join(fields)) locked_by = req.get('locked_by') names = req.get('names') if not isinstance(names, list): error('/errors/invalid/', "'names' must be a list; got: %s" % str(type(names))) base_query = Node.query query = base_query.filter(Node.name.in_(names)) if query.count() != len(names): error('/errors/invalid/', "Could not find all nodes!") log.info("Unlocking {count} nodes for {locked_by}".format( count=len(names), locked_by=locked_by)) result = [] for node in query.all(): result.append( NodeController._lock(node, dict(locked=False, locked_by=locked_by), 'unlock')) return result @expose('json') def job_stats(self, machine_type='', since_days=14): since_days = int(since_days) if since_days < 1: error('/errors/invalid/', "since_days must be a positive integer") now = datetime.utcnow() past = now - timedelta(days=since_days) recent_jobs = Job.query.filter(Job.posted.between(past, now)).subquery() RecentJob = aliased(Job, recent_jobs) query = Session.query(Node.name, RecentJob.status, func.count('*')) if machine_type: # Note: filtering by Job.machine_type (as below) greatly improves # performance but could lead slightly incorrect values if many jobs # are being scheduled using mixed machine types. We work around # this by including the 'multi' machine type (which is the name of # the queue Inktank uses for such jobs. query = query.filter( RecentJob.machine_type.in_((machine_type, 'multi'))) query = query.filter(Node.machine_type == machine_type) query = query.join(RecentJob.target_nodes).group_by(Node)\ .group_by(RecentJob.status) all_stats = {} results = query.all() for (name, status, count) in results: node_stats = all_stats.get(name, {}) node_stats[status] = count all_stats[name] = node_stats stats_sorter = lambda t: sum(t[1].values()) ordered_stats = OrderedDict(sorted(all_stats.items(), key=stats_sorter)) return ordered_stats @expose('json') def machine_types(self): query = Node.query.values(Node.machine_type) return sorted(list(set([item[0] for item in query if item[0]]))) @expose('json') def _lookup(self, name, *remainder): return NodeController(name), remainder
def test_locked_since_locked(self): node_name = 'cats' user = '******' node = Node(name=node_name) node.update(dict(locked=True, locked_by=user)) assert (datetime.utcnow() - node.locked_since) < timedelta(0, 0, 100)