def index(self):
provider = providers.get(request.json['provider'])
# request.json is read-only, since we are going to add extra metadata
# to get the classes created, make a clean copy
_json = deepcopy(request.json)
# Before creating a node, check if it has already been created by us:
name = _json['name']
keyname = _json['keyname']
image_name = _json['image_name']
size = _json['size']
labels = _json['labels']
script = _json['script']
count = _json.get('count', 1)
# a buffered count is 3/4 what is needed rounded up
buffered_count = int(round(count * 0.75))
existing_nodes = Node.filter_by(
name=name,
keyname=keyname,
image_name=image_name,
size=size,
).all()
# try to slap it into the script, it is not OK if we are not allowed to, assume we should
# this is just a validation step, should be taken care of by proper schema validation.
try:
script % '0000-aaaaa'
except TypeError:
logger.error('attempted to add a UUID to the script but failed')
logger.error(
'ensure that a formatting entry for %s["script"] exists, like: %%s' % name
)
return # do not add anything if we haven't been able to format
logger.info('checking if an existing node matches required labels: %s', str(labels))
matching_nodes = [n for n in existing_nodes if n.labels_match(labels)]
if not matching_nodes: # we don't have anything that matches this that has been ever created
logger.info('job needs %s nodes to get unstuck', count)
logger.info(
'no matching nodes were found, will create new ones. count: %s',
buffered_count
)
for i in range(buffered_count):
# slap the UUID into the new node details
node_kwargs = deepcopy(request.json)
_id = str(uuid.uuid4())
node_kwargs['name'] = "%s__%s" % (name, _id)
node_kwargs['script'] = script % _id
provider.create_node(**node_kwargs)
node_kwargs.pop('name')
Node(
name=name,
identifier=_id,
**node_kwargs
)
models.commit()
else:
logger.info('found existing nodes that match labels: %s', len(matching_nodes))
now = datetime.utcnow()
# we have something that matches, go over all of them and check:
# if *all of them* are over 6 (by default) minutes since creation.
# that means that they are probably busy, so create a new one
already_created_nodes = 0
for n in matching_nodes:
difference = now - n.created
if difference.seconds < 360: # 6 minutes
already_created_nodes += 1
if already_created_nodes >= count:
logger.info('job needs %s nodes to get unstuck', count)
logger.info(
'but there are %s node(s) already created 6 minutes ago',
already_created_nodes
)
logger.info('will not create one')
return
logger.info('job needs %s nodes to get unstuck', count)
logger.info(
'no nodes created recently enough, will create new ones. count: %s',
buffered_count
)
for i in range(buffered_count):
# slap the UUID into the new node details
node_kwargs = deepcopy(request.json)
_id = str(uuid.uuid4())
node_kwargs['name'] = "%s__%s" % (name, _id)
node_kwargs['script'] = script % _id
provider.create_node(**node_kwargs)
node_kwargs.pop('name')
Node(
name=name,
identifier=_id,
**node_kwargs
)
models.commit()
def index(self):
provider = providers.get(request.json['provider'])
# request.json is read-only, since we are going to add extra metadata
# to get the classes created, make a clean copy
_json = deepcopy(request.json)
# Before creating a node, check if it has already been created by us:
name = _json['name']
keyname = _json['keyname']
image_name = _json['image_name']
size = _json['size']
labels = _json['labels']
script = _json['script']
existing_nodes = Node.filter_by(
name=name,
keyname=keyname,
image_name=image_name,
size=size,
).all()
# slap the UUID into the new node details
_id = str(uuid.uuid4())
_json['name'] = "%s__%s" % (name, _id)
# try to slap it into the script, it is not OK if we are not allowed to, assume we should
try:
_json['script'] = script % _id
except TypeError:
logger.error('attempted to add a UUID to the script but failed')
logger.error(
'ensure that a formatting entry for %s["script"] exists, like: %%s' % name
)
return # do not add anything if we haven't been able to format
logger.info('checking if an existing node matches required labels: %s', str(labels))
matching_nodes = [n for n in existing_nodes if n.labels_match(labels)]
if not matching_nodes: # we don't have anything that matches this that has been ever created
logger.info('no matching nodes were found, will create one')
logger.warning('creating node with details: %s' % str(_json))
provider.create_node(**_json)
_json.pop('name')
Node(
name=request.json['name'],
identifier=_id,
**_json
)
else:
logger.info('found existing nodes that match labels: %s', len(matching_nodes))
now = datetime.utcnow()
# we have something that matches, go over all of them and check:
# if *all of them* are over 8 (by default) minutes since creation.
# that means that they are probably busy, so create a new one
for n in matching_nodes:
difference = now - n.created
if difference.seconds < 480: # 8 minutes
logger.info(
'a matching node was already created in the past 8 minutes: %s', n.name
)
logger.info('will not create one')
return
# FIXME: need to check with cloud provider and see if this
# node is running, otherwise it means this node is dead and
# should be removed from the DB
logger.info('no nodes created recently, will create a new one')
provider.create_node(**_json)
_json.pop('name')
Node(
name=request.json['name'],
identifier=_id,
**_json
)
