def test_idle_is_set(self, session): session.app.post_json('/api/nodes/', params={ 'name': 'wheezy-slave', 'provider': 'openstack', 'keyname': 'ci-key', 'image_name': 'beefy-wheezy', 'size': '3xlarge', 'script': '#!/bin/bash echo hello world! %s', 'labels': ['wheezy', 'amd64'], }) node = Node.get(1) session.app.post('/api/nodes/%s/idle' % node.identifier) node = Node.get(1) assert node.idle_since is not None
def test_make_node_active(self, session): session.app.post_json('/api/nodes/', params={ 'name': 'wheezy-slave', 'provider': 'openstack', 'keyname': 'ci-key', 'image_name': 'beefy-wheezy', 'size': '3xlarge', 'script': '#!/bin/bash echo hello world! %s', 'labels': ['wheezy', 'amd64'], }) node = Node.get(1) uuid = node.identifier # delete it session.app.post_json('/api/nodes/%s/delete/' % uuid, params={}) assert Node.get(1) is None
def test_idle_is_set(self, session): session.app.post_json( '/api/nodes/', params={ 'name': 'wheezy-slave', 'provider': 'openstack', 'keyname': 'ci-key', 'image_name': 'beefy-wheezy', 'size': '3xlarge', 'script': '#!/bin/bash echo hello world! %s', 'labels': ['wheezy', 'amd64'], } ) node = Node.get(1) session.app.post('/api/nodes/%s/idle' % node.identifier) node = Node.get(1) assert node.idle_since is not None
def test_make_node_active(self, session): session.app.post_json( '/api/nodes/', params={ 'name': 'wheezy-slave', 'provider': 'openstack', 'keyname': 'ci-key', 'image_name': 'beefy-wheezy', 'size': '3xlarge', 'script': '#!/bin/bash echo hello world! %s', 'labels': ['wheezy', 'amd64'], } ) node = Node.get(1) uuid = node.identifier # delete it session.app.post_json('/api/nodes/%s/delete/' % uuid, params={}) assert Node.get(1) is None
def test_node_not_in_jenkins_gets_skipped_(self, session, monkeypatch): fake_conn = fake_jenkins() fake_conn.node_exists = lambda x: True monkeypatch.setattr(nodes, "jenkins_connection", lambda *a: fake_conn) session.app.post_json('/api/nodes/', params={ 'name': 'wheezy-slave', 'provider': 'openstack', 'keyname': 'ci-key', 'image_name': 'beefy-wheezy', 'size': '3xlarge', 'script': '#!/bin/bash echo hello world! %s', 'labels': ['wheezy', 'amd64'], }) node = Node.get(1) # make it idle for more than a day node.idle_since = datetime.utcnow() - timedelta(seconds=2000) session.commit() session.app.post('/api/nodes/%s/idle/' % node.identifier) assert Node.get(1) is not None
def index(self): provider = providers.get(request.json['provider']) # request.json is read-only, since we are going to add extra metadata # to get the classes created, make a clean copy _json = deepcopy(request.json) # Before creating a node, check if it has already been created by us: name = _json['name'] keyname = _json['keyname'] image_name = _json['image_name'] size = _json['size'] labels = _json['labels'] script = _json['script'] existing_nodes = Node.filter_by( name=name, keyname=keyname, image_name=image_name, size=size, ).all() matching_nodes = [n for n in existing_nodes if n.labels_match(labels)] if not matching_nodes: # we don't have anything that matches this that has been ever created logger.info('requested node does not exist, will create one') # slap the UUID into the new node details _id = str(uuid.uuid4()) logger.info('changing name: %s' % _json['name']) _json['name'] = "%s__%s" % (name, _id) logger.info('changed name into something else: %s' % _json['name']) # try to slap it into the script, it is not OK if we are not allowed to, assume we should try: _json['script'] = script % _id except TypeError: logger.error('attempted to add a UUID to the script but failed') logger.error('ensure that a formatting entry exists, like: %%s') return # do not add anything if we haven't been able to format logger.warning('creating node with details: %s' % str(_json)) provider.create_node(**_json) _json.pop('name') Node( name=request.json['name'], identifier=_id, **_json )
def index(self): provider = providers.get(request.json['provider']) # request.json is read-only, since we are going to add extra metadata # to get the classes created, make a clean copy _json = deepcopy(request.json) # Before creating a node, check if it has already been created by us: name = _json['name'] keyname = _json['keyname'] image_name = _json['image_name'] size = _json['size'] labels = _json['labels'] script = _json['script'] count = _json.get('count', 1) # a buffered count is 3/4 what is needed rounded up buffered_count = int(round(count * 0.75)) existing_nodes = Node.filter_by( name=name, keyname=keyname, image_name=image_name, size=size, ).all() # try to slap it into the script, it is not OK if we are not allowed to, assume we should # this is just a validation step, should be taken care of by proper schema validation. try: script % '0000-aaaaa' except TypeError: logger.error('attempted to add a UUID to the script but failed') logger.error( 'ensure that a formatting entry for %s["script"] exists, like: %%s' % name ) return # do not add anything if we haven't been able to format logger.info('checking if an existing node matches required labels: %s', str(labels)) matching_nodes = [n for n in existing_nodes if n.labels_match(labels)] if not matching_nodes: # we don't have anything that matches this that has been ever created logger.info('job needs %s nodes to get unstuck', count) logger.info( 'no matching nodes were found, will create new ones. count: %s', buffered_count ) for i in range(buffered_count): # slap the UUID into the new node details node_kwargs = deepcopy(request.json) _id = str(uuid.uuid4()) node_kwargs['name'] = "%s__%s" % (name, _id) node_kwargs['script'] = script % _id provider.create_node(**node_kwargs) node_kwargs.pop('name') Node( name=name, identifier=_id, **node_kwargs ) models.commit() else: logger.info('found existing nodes that match labels: %s', len(matching_nodes)) now = datetime.utcnow() # we have something that matches, go over all of them and check: # if *all of them* are over 6 (by default) minutes since creation. # that means that they are probably busy, so create a new one already_created_nodes = 0 for n in matching_nodes: difference = now - n.created if difference.seconds < 360: # 6 minutes already_created_nodes += 1 if already_created_nodes > count: logger.info('job needs %s nodes to get unstuck', count) logger.info( 'but there are %s node(s) already created 6 minutes ago', already_created_nodes ) logger.info('will not create one') return logger.info('job needs %s nodes to get unstuck', count) logger.info( 'no nodes created recently enough, will create new ones. count: %s', buffered_count ) for i in range(buffered_count): # slap the UUID into the new node details node_kwargs = deepcopy(request.json) _id = str(uuid.uuid4()) node_kwargs['name'] = "%s__%s" % (name, _id) node_kwargs['script'] = script % _id provider.create_node(**node_kwargs) node_kwargs.pop('name') Node( name=name, identifier=_id, **node_kwargs ) models.commit()
def index(self): provider = providers.get(request.json['provider']) # request.json is read-only, since we are going to add extra metadata # to get the classes created, make a clean copy _json = deepcopy(request.json) # Before creating a node, check if it has already been created by us: name = _json['name'] keyname = _json['keyname'] image_name = _json['image_name'] size = _json['size'] labels = _json['labels'] script = _json['script'] count = _json.get('count', 1) # a buffered count is 3/4 what is needed rounded up buffered_count = int(round(count * 0.75)) existing_nodes = Node.filter_by( name=name, keyname=keyname, image_name=image_name, size=size, ).all() # try to slap it into the script, it is not OK if we are not allowed to, assume we should # this is just a validation step, should be taken care of by proper schema validation. try: script % '0000-aaaaa' except TypeError: logger.error('attempted to add a UUID to the script but failed') logger.error( 'ensure that a formatting entry for %s["script"] exists, like: %%s' % name ) return # do not add anything if we haven't been able to format logger.info('checking if an existing node matches required labels: %s', str(labels)) matching_nodes = [n for n in existing_nodes if n.labels_match(labels)] if not matching_nodes: # we don't have anything that matches this that has been ever created logger.info('job needs %s nodes to get unstuck', count) logger.info( 'no matching nodes were found, will create new ones. count: %s', buffered_count ) for i in range(buffered_count): # slap the UUID into the new node details node_kwargs = deepcopy(request.json) _id = str(uuid.uuid4()) node_kwargs['name'] = "%s__%s" % (name, _id) node_kwargs['script'] = script % _id provider.create_node(**node_kwargs) node_kwargs.pop('name') Node( name=name, identifier=_id, **node_kwargs ) models.commit() else: logger.info('found existing nodes that match labels: %s', len(matching_nodes)) now = datetime.utcnow() # we have something that matches, go over all of them and check: # if *all of them* are over 6 (by default) minutes since creation. # that means that they are probably busy, so create a new one already_created_nodes = 0 for n in matching_nodes: difference = now - n.created if difference.seconds < 360: # 6 minutes already_created_nodes += 1 if already_created_nodes >= count: logger.info('job needs %s nodes to get unstuck', count) logger.info( 'but there are %s node(s) already created 6 minutes ago', already_created_nodes ) logger.info('will not create one') return logger.info('job needs %s nodes to get unstuck', count) logger.info( 'no nodes created recently enough, will create new ones. count: %s', buffered_count ) for i in range(buffered_count): # slap the UUID into the new node details node_kwargs = deepcopy(request.json) _id = str(uuid.uuid4()) node_kwargs['name'] = "%s__%s" % (name, _id) node_kwargs['script'] = script % _id provider.create_node(**node_kwargs) node_kwargs.pop('name') Node( name=name, identifier=_id, **node_kwargs ) models.commit()
def index(self): provider = providers.get(request.json['provider']) # request.json is read-only, since we are going to add extra metadata # to get the classes created, make a clean copy _json = deepcopy(request.json) # Before creating a node, check if it has already been created by us: name = _json['name'] keyname = _json['keyname'] image_name = _json['image_name'] size = _json['size'] labels = _json['labels'] script = _json['script'] existing_nodes = Node.filter_by( name=name, keyname=keyname, image_name=image_name, size=size, ).all() # slap the UUID into the new node details _id = str(uuid.uuid4()) _json['name'] = "%s__%s" % (name, _id) # try to slap it into the script, it is not OK if we are not allowed to, assume we should try: _json['script'] = script % _id except TypeError: logger.error('attempted to add a UUID to the script but failed') logger.error( 'ensure that a formatting entry for %s["script"] exists, like: %%s' % name ) return # do not add anything if we haven't been able to format logger.info('checking if an existing node matches required labels: %s', str(labels)) matching_nodes = [n for n in existing_nodes if n.labels_match(labels)] if not matching_nodes: # we don't have anything that matches this that has been ever created logger.info('no matching nodes were found, will create one') logger.warning('creating node with details: %s' % str(_json)) provider.create_node(**_json) _json.pop('name') Node( name=request.json['name'], identifier=_id, **_json ) else: logger.info('found existing nodes that match labels: %s', len(matching_nodes)) now = datetime.utcnow() # we have something that matches, go over all of them and check: # if *all of them* are over 8 (by default) minutes since creation. # that means that they are probably busy, so create a new one for n in matching_nodes: difference = now - n.created if difference.seconds < 480: # 8 minutes logger.info( 'a matching node was already created in the past 8 minutes: %s', n.name ) logger.info('will not create one') return # FIXME: need to check with cloud provider and see if this # node is running, otherwise it means this node is dead and # should be removed from the DB logger.info('no nodes created recently, will create a new one') provider.create_node(**_json) _json.pop('name') Node( name=request.json['name'], identifier=_id, **_json )