def test_double_lock(self):
node_name = 'goldfish'
user = '******'
node = Node(name=node_name)
node.update(dict(locked=True, locked_by=user))
with pytest.raises(ForbiddenRequestError):
node.update(dict(locked=True, locked_by=user))
def test_double_lock(self):
node_name = 'goldfish'
user = '******'
node = Node(name=node_name)
node.update(dict(locked=True, locked_by=user))
with pytest.raises(ForbiddenRequestError):
node.update(dict(locked=True, locked_by=user))
def test_locked_since_unlocked(self):
node_name = 'cats'
user = '******'
old_locked_since = datetime(2000, 1, 1, 0, 0)
node = Node(name=node_name)
node.update(dict(locked=True, locked_by=user))
node.locked_since = old_locked_since
node.update(dict(locked=False, locked_by=user))
assert node.locked_since is None
def test_locked_since_locked(self):
node_name = 'cats'
user = '******'
node = Node(name=node_name)
node.update(dict(locked=True, locked_by=user))
# This used to take <100us; since we started flushing on node updates,
# it takes around 2-3ms.
assert (datetime.utcnow() -
node.locked_since) < timedelta(milliseconds=5)
def test_locked_since_unlocked(self):
node_name = 'cats'
user = '******'
old_locked_since = datetime(2000, 1, 1, 0, 0)
node = Node(name=node_name)
node.update(dict(locked=True, locked_by=user))
node.locked_since = old_locked_since
node.update(dict(locked=False, locked_by=user))
assert node.locked_since is None
class NodesController(object):
@expose(generic=True, template='json')
def index(self, locked=None, machine_type='', os_type=None,
os_version=None, locked_by=None, up=None, count=None):
query = Node.query
if locked is not None:
query = query.filter(Node.locked == locked)
if machine_type:
if '|' in machine_type:
machine_types = machine_type.split('|')
query = query.filter(Node.machine_type.in_(machine_types))
else:
query = query.filter(Node.machine_type == machine_type)
if os_type:
query = query.filter(Node.os_type == os_type)
if os_version:
query = query.filter(Node.os_version == os_version)
if locked_by:
query = query.filter(Node.locked_by == locked_by)
if up is not None:
query = query.filter(Node.up == up)
if count is not None:
if not count.isdigit() or isinstance(count, int):
error('/errors/invalid/', 'count must be an integer')
query = query.limit(count)
return [node.__json__() for node in query.all()]
@index.when(method='POST', template='json')
def index_post(self):
"""
Create a new node
"""
try:
data = request.json
name = data.get('name')
except ValueError:
rollback()
error('/errors/invalid/', 'could not decode JSON body')
# we allow empty data to be pushed
if not name:
error('/errors/invalid/', "could not find required key: 'name'")
if Node.filter_by(name=name).first():
error('/errors/invalid/',
"Node with name %s already exists" % name)
else:
self.node = Node(name=name)
try:
self.node.update(data)
except PaddlesError as exc:
error(exc.url, str(exc))
log.info("Created {node}: {data}".format(
node=self.node,
data=data,
))
return dict()
@expose(generic=True, template='json')
def lock_many(self):
error('/errors/invalid/',
"this URI only supports POST requests")
@isolation_level('SERIALIZABLE')
@lock_many.when(method='POST', template='json')
def lock_many_post(self):
req = request.json
fields = set(('count', 'locked_by', 'machine_type', 'description'))
if not fields.issubset(set(req.keys())):
error('/errors/invalid/',
"must pass these fields: %s" % ', '.join(fields))
req['locked'] = True
count = req.pop('count', 0)
if count < 1:
error('/errors/invalid/',
"cannot lock less than 1 node")
machine_type = req.pop('machine_type', None)
if not machine_type:
error('/errors/invalid/',
"must specify machine_type")
locked_by = req.get('locked_by')
description = req.get('description')
os_type = req.get('os_type')
os_version = req.get('os_version')
arch = req.get('arch')
if os_version is not None:
os_version = str(os_version)
attempts = 2
log.debug("Locking {count} {mtype} nodes for {locked_by}".format(
count=count, mtype=machine_type, locked_by=locked_by))
while attempts > 0:
try:
result = Node.lock_many(count=count, locked_by=locked_by,
machine_type=machine_type,
description=description,
os_type=os_type, os_version=os_version,
arch=arch)
if description:
desc_str = " with description %s" % description
else:
desc_str = ""
log.info("Locked {names} for {locked_by}{desc_str}".format(
names=" ".join([str(node) for node in result]),
locked_by=locked_by,
desc_str=desc_str,
))
return result
except RaceConditionError as exc:
log.warn("lock_many() detected race condition")
attempts -= 1
if attempts > 0:
log.info("retrying after race avoidance (%s tries left)",
attempts)
else:
error(exc.url, str(exc))
except PaddlesError as exc:
error(exc.url, str(exc))
@expose(generic=True, template='json')
def unlock_many(self):
error('/errors/invalid/',
"this URI only supports POST requests")
@unlock_many.when(method='POST', template='json')
def unlock_many_post(self):
req = request.json
fields = ['names', 'locked_by']
if sorted(req.keys()) != sorted(fields):
error('/errors/invalid/',
"must pass these fields: %s" % ', '.join(fields))
locked_by = req.get('locked_by')
names = req.get('names')
if not isinstance(names, list):
error('/errors/invalid/',
"'names' must be a list; got: %s" % str(type(names)))
base_query = Node.query
query = base_query.filter(Node.name.in_(names))
if query.count() != len(names):
error('/errors/invalid/',
"Could not find all nodes!")
log.info("Unlocking {count} nodes for {locked_by}".format(
count=len(names), locked_by=locked_by))
result = []
for node in query.all():
result.append(
NodeController._lock(node,
dict(locked=False, locked_by=locked_by),
'unlock')
)
return result
@expose('json')
def job_stats(self, machine_type='', since_days=14):
since_days = int(since_days)
if since_days < 1:
error('/errors/invalid/', "since_days must be a positive integer")
now = datetime.utcnow()
past = now - timedelta(days=since_days)
recent_jobs = Job.query.filter(Job.posted.between(past,
now)).subquery()
RecentJob = aliased(Job, recent_jobs)
query = Session.query(Node.name,
RecentJob.status,
func.count('*'))
if machine_type:
# Note: filtering by Job.machine_type (as below) greatly improves
# performance but could lead slightly incorrect values if many jobs
# are being scheduled using mixed machine types. We work around
# this by including the 'multi' machine type (which is the name of
# the queue Inktank uses for such jobs.
query = query.filter(RecentJob.machine_type.in_((machine_type,
'multi')))
query = query.filter(Node.machine_type == machine_type)
query = query.join(RecentJob.target_nodes).group_by(Node)\
.group_by(RecentJob.status)
all_stats = {}
results = query.all()
for (name, status, count) in results:
node_stats = all_stats.get(name, {})
node_stats[status] = count
all_stats[name] = node_stats
stats_sorter = lambda t: sum(t[1].values())
ordered_stats = OrderedDict(sorted(all_stats.items(),
key=stats_sorter))
return ordered_stats
@expose('json')
def machine_types(self):
query = Node.query.values(Node.machine_type)
return sorted(list(set([item[0] for item in query if item[0]])))
@expose('json')
def _lookup(self, name, *remainder):
return NodeController(name), remainder
class NodesController(object):
@expose(generic=True, template='json')
def index(self,
locked=None,
machine_type='',
os_type=None,
os_version=None,
locked_by=None,
up=None,
count=None):
query = Node.query
if locked is not None:
query = query.filter(Node.locked == locked)
if machine_type:
if '|' in machine_type:
machine_types = machine_type.split('|')
query = query.filter(Node.machine_type.in_(machine_types))
else:
query = query.filter(Node.machine_type == machine_type)
if os_type:
query = query.filter(Node.os_type == os_type)
if os_version:
query = query.filter(Node.os_version == os_version)
if locked_by:
query = query.filter(Node.locked_by == locked_by)
if up is not None:
query = query.filter(Node.up == up)
if count is not None:
if not count.isdigit() or isinstance(count, int):
error('/errors/invalid/', 'count must be an integer')
query = query.limit(count)
return [node.__json__() for node in query.all()]
@index.when(method='POST', template='json')
def index_post(self):
"""
Create a new node
"""
try:
data = request.json
name = data.get('name')
except ValueError:
rollback()
error('/errors/invalid/', 'could not decode JSON body')
# we allow empty data to be pushed
if not name:
error('/errors/invalid/', "could not find required key: 'name'")
if Node.filter_by(name=name).first():
error('/errors/invalid/',
"Node with name %s already exists" % name)
else:
self.node = Node(name=name)
try:
self.node.update(data)
except PaddlesError as exc:
error(exc.url, str(exc))
log.info("Created {node}: {data}".format(
node=self.node,
data=data,
))
return dict()
@expose(generic=True, template='json')
def lock_many(self):
error('/errors/invalid/', "this URI only supports POST requests")
@isolation_level('SERIALIZABLE')
@lock_many.when(method='POST', template='json')
def lock_many_post(self):
req = request.json
fields = set(('count', 'locked_by', 'machine_type', 'description'))
if not fields.issubset(set(req.keys())):
error('/errors/invalid/',
"must pass these fields: %s" % ', '.join(fields))
req['locked'] = True
count = req.pop('count', 0)
if count < 1:
error('/errors/invalid/', "cannot lock less than 1 node")
machine_type = req.pop('machine_type', None)
if not machine_type:
error('/errors/invalid/', "must specify machine_type")
locked_by = req.get('locked_by')
description = req.get('description')
os_type = req.get('os_type')
os_version = req.get('os_version')
arch = req.get('arch')
if os_version is not None:
os_version = str(os_version)
attempts = 2
log.debug("Locking {count} {mtype} nodes for {locked_by}".format(
count=count, mtype=machine_type, locked_by=locked_by))
while attempts > 0:
try:
result = Node.lock_many(count=count,
locked_by=locked_by,
machine_type=machine_type,
description=description,
os_type=os_type,
os_version=os_version,
arch=arch)
if description:
desc_str = " with description %s" % description
else:
desc_str = ""
log.info("Locked {names} for {locked_by}{desc_str}".format(
names=" ".join([str(node) for node in result]),
locked_by=locked_by,
desc_str=desc_str,
))
return result
except RaceConditionError as exc:
log.warn("lock_many() detected race condition")
attempts -= 1
if attempts > 0:
log.info("retrying after race avoidance (%s tries left)",
attempts)
else:
error(exc.url, str(exc))
except PaddlesError as exc:
error(exc.url, str(exc))
@expose(generic=True, template='json')
def unlock_many(self):
error('/errors/invalid/', "this URI only supports POST requests")
@unlock_many.when(method='POST', template='json')
def unlock_many_post(self):
req = request.json
fields = ['names', 'locked_by']
if sorted(req.keys()) != sorted(fields):
error('/errors/invalid/',
"must pass these fields: %s" % ', '.join(fields))
locked_by = req.get('locked_by')
names = req.get('names')
if not isinstance(names, list):
error('/errors/invalid/',
"'names' must be a list; got: %s" % str(type(names)))
base_query = Node.query
query = base_query.filter(Node.name.in_(names))
if query.count() != len(names):
error('/errors/invalid/', "Could not find all nodes!")
log.info("Unlocking {count} nodes for {locked_by}".format(
count=len(names), locked_by=locked_by))
result = []
for node in query.all():
result.append(
NodeController._lock(node,
dict(locked=False, locked_by=locked_by),
'unlock'))
return result
@expose('json')
def job_stats(self, machine_type='', since_days=14):
since_days = int(since_days)
if since_days < 1:
error('/errors/invalid/', "since_days must be a positive integer")
now = datetime.utcnow()
past = now - timedelta(days=since_days)
recent_jobs = Job.query.filter(Job.posted.between(past,
now)).subquery()
RecentJob = aliased(Job, recent_jobs)
query = Session.query(Node.name, RecentJob.status, func.count('*'))
if machine_type:
# Note: filtering by Job.machine_type (as below) greatly improves
# performance but could lead slightly incorrect values if many jobs
# are being scheduled using mixed machine types. We work around
# this by including the 'multi' machine type (which is the name of
# the queue Inktank uses for such jobs.
query = query.filter(
RecentJob.machine_type.in_((machine_type, 'multi')))
query = query.filter(Node.machine_type == machine_type)
query = query.join(RecentJob.target_nodes).group_by(Node)\
.group_by(RecentJob.status)
all_stats = {}
results = query.all()
for (name, status, count) in results:
node_stats = all_stats.get(name, {})
node_stats[status] = count
all_stats[name] = node_stats
stats_sorter = lambda t: sum(t[1].values())
ordered_stats = OrderedDict(sorted(all_stats.items(),
key=stats_sorter))
return ordered_stats
@expose('json')
def machine_types(self):
query = Node.query.values(Node.machine_type)
return sorted(list(set([item[0] for item in query if item[0]])))
@expose('json')
def _lookup(self, name, *remainder):
return NodeController(name), remainder
def test_locked_since_locked(self):
node_name = 'cats'
user = '******'
node = Node(name=node_name)
node.update(dict(locked=True, locked_by=user))
assert (datetime.utcnow() - node.locked_since) < timedelta(0, 0, 100)
