def put(self, request, provider_id, identity_id, instance_id):
"""
TODO:
Options for put
- Instance status change (suspend,resume,etc.)
- DB changes (Name, tags)
"""
user = request.user
data = request.DATA
#Ensure item exists on the server first
esh_driver = prepare_driver(request, provider_id, identity_id)
if not esh_driver:
return invalid_creds(provider_id, identity_id)
esh_instance = esh_driver.get_instance(instance_id)
if not esh_instance:
return instance_not_found(instance_id)
#Gather the DB related item and update
core_instance = convert_esh_instance(esh_driver, esh_instance,
provider_id, identity_id, user)
serializer = InstanceSerializer(core_instance, data=data)
if serializer.is_valid():
logger.info('metadata = %s' % data)
update_instance_metadata(esh_driver, esh_instance, data)
serializer.save()
response = Response(serializer.data)
logger.info('data = %s' % serializer.data)
response['Cache-Control'] = 'no-cache'
return response
else:
return Response(serializer.errors, status=status.HTTP_400)
def deploy_failed(task_uuid, driverCls, provider, identity, instance_id,
**celery_task_args):
from core.models.instance import Instance
from core.email import send_deploy_failed_email
try:
logger.debug("deploy_failed task started at %s." % datetime.now())
logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "DEPLOYERROR::%s" % (result.traceback, )
logger.error(err_str)
driver = get_driver(driverCls, provider, identity)
instance = driver.get_instance(instance_id)
update_instance_metadata(driver,
instance,
data={'tmp_status': 'deploy_error'},
replace=False)
#Send deploy email
core_instance = Instance.objects.get(provider_alias=instance_id)
send_deploy_failed_email(core_instance, err_str)
logger.debug("deploy_failed task finished at %s." % datetime.now())
except Exception as exc:
logger.warn(exc)
deploy_failed.retry(exc=exc)
def patch(self, request, provider_id, identity_id, instance_id):
"""
"""
user = request.user
data = request.DATA
esh_driver = prepare_driver(request, provider_id, identity_id)
if not esh_driver:
return invalid_creds(provider_id, identity_id)
esh_instance = esh_driver.get_instance(instance_id)
if not esh_instance:
return instance_not_found(instance_id)
#Gather the DB related item and update
core_instance = convert_esh_instance(esh_driver, esh_instance,
provider_id, identity_id, user)
serializer = InstanceSerializer(core_instance, data=data, partial=True)
if serializer.is_valid():
logger.info('metadata = %s' % data)
update_instance_metadata(esh_driver, esh_instance, data)
serializer.save()
response = Response(serializer.data)
logger.info('data = %s' % serializer.data)
response['Cache-Control'] = 'no-cache'
return response
else:
return Response(
serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
def patch(self, request, provider_uuid, identity_uuid, instance_id):
"""Authentication Required, update metadata about the instance"""
user = request.user
data = request.DATA
esh_driver = prepare_driver(request, provider_uuid, identity_uuid)
if not esh_driver:
return invalid_creds(provider_uuid, identity_uuid)
esh_instance = esh_driver.get_instance(instance_id)
if not esh_instance:
return instance_not_found(instance_id)
#Gather the DB related item and update
core_instance = convert_esh_instance(esh_driver, esh_instance,
provider_uuid, identity_uuid, user)
serializer = InstanceSerializer(core_instance, data=data,
context={"request":request}, partial=True)
if serializer.is_valid():
logger.info('metadata = %s' % data)
update_instance_metadata(esh_driver, esh_instance, data,
replace=False)
serializer.save()
boot_scripts = data.pop('boot_scripts', [])
if boot_scripts:
_save_scripts_to_instance(serializer.object, boot_scripts)
invalidate_cached_instances(identity=Identity.objects.get(uuid=identity_uuid))
response = Response(serializer.data)
logger.info('data = %s' % serializer.data)
response['Cache-Control'] = 'no-cache'
return response
else:
return Response(
serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
def put(self, request, provider_id, identity_id, instance_id):
"""Authentication Required, update metadata about the instance"""
user = request.user
data = request.DATA
#Ensure item exists on the server first
esh_driver = prepare_driver(request, provider_id, identity_id)
if not esh_driver:
return invalid_creds(provider_id, identity_id)
esh_instance = esh_driver.get_instance(instance_id)
if not esh_instance:
return instance_not_found(instance_id)
#Gather the DB related item and update
core_instance = convert_esh_instance(esh_driver, esh_instance,
provider_id, identity_id, user)
serializer = InstanceSerializer(core_instance, data=data,
context={"request":request})
if serializer.is_valid():
logger.info('metadata = %s' % data)
update_instance_metadata(esh_driver, esh_instance, data)
serializer.save()
response = Response(serializer.data)
logger.info('data = %s' % serializer.data)
response['Cache-Control'] = 'no-cache'
return response
else:
return Response(serializer.errors, status=status.HTTP_400)
def put(self, request, provider_id, identity_id, instance_id):
"""
TODO:
Options for put
- Instance status change (suspend,resume,etc.)
- DB changes (Name, tags)
"""
user = request.user
data = request.DATA
#Ensure item exists on the server first
esh_driver = prepare_driver(request, identity_id)
try:
esh_instance = esh_driver.get_instance(instance_id)
except InvalidCredsError:
return invalid_creds(provider_id, identity_id)
if not esh_instance:
return instance_not_found(instance_id)
#Gather the DB related item and update
core_instance = convert_esh_instance(esh_driver, esh_instance,
provider_id, identity_id, user)
serializer = InstanceSerializer(core_instance, data=data)
if serializer.is_valid():
logger.info('metadata = %s' % data)
update_instance_metadata(esh_driver, esh_instance, data)
serializer.save()
response = Response(serializer.data)
logger.info('data = %s' % serializer.data)
response['Cache-Control'] = 'no-cache'
return response
else:
return Response(serializer.errors, status=status.HTTP_400)
def patch(self, request, provider_id, identity_id, instance_id):
"""
"""
user = request.user
data = request.DATA
#Ensure item exists on the server first
esh_driver = prepare_driver(request, identity_id)
try:
esh_instance = esh_driver.get_instance(instance_id)
except InvalidCredsError:
return invalid_creds(provider_id, identity_id)
if not esh_instance:
return instance_not_found(instance_id)
#Gather the DB related item and update
core_instance = convert_esh_instance(esh_driver, esh_instance,
provider_id, identity_id, user)
serializer = InstanceSerializer(core_instance, data=data, partial=True)
if serializer.is_valid():
logger.info('metadata = %s' % data)
update_instance_metadata(esh_driver, esh_instance, data)
serializer.save()
response = Response(serializer.data)
logger.info('data = %s' % serializer.data)
response['Cache-Control'] = 'no-cache'
return response
else:
return Response(
serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
def put(self, request, provider_id, identity_id, instance_id):
"""Authentication Required, update metadata about the instance"""
user = request.user
data = request.DATA
#Ensure item exists on the server first
esh_driver = prepare_driver(request, provider_id, identity_id)
if not esh_driver:
return invalid_creds(provider_id, identity_id)
esh_instance = esh_driver.get_instance(instance_id)
if not esh_instance:
return instance_not_found(instance_id)
#Gather the DB related item and update
core_instance = convert_esh_instance(esh_driver, esh_instance,
provider_id, identity_id, user)
serializer = InstanceSerializer(core_instance,
data=data,
context={"request": request})
if serializer.is_valid():
logger.info('metadata = %s' % data)
update_instance_metadata(esh_driver, esh_instance, data)
serializer.save()
response = Response(serializer.data)
logger.info('data = %s' % serializer.data)
response['Cache-Control'] = 'no-cache'
return response
else:
return Response(serializer.errors, status=status.HTTP_400)
def add_floating_ip(driverCls, provider, identity,
instance_alias, delete_status=True,
*args, **kwargs):
#For testing ONLY.. Test cases ignore countdown..
if app.conf.CELERY_ALWAYS_EAGER:
logger.debug("Eager task waiting 15 seconds")
time.sleep(15)
try:
logger.debug("add_floating_ip task started at %s." % datetime.now())
#Remove unused floating IPs first, so they can be re-used
driver = get_driver(driverCls, provider, identity)
driver._clean_floating_ip()
#assign if instance doesn't already have an IP addr
instance = driver.get_instance(instance_alias)
if not instance:
logger.debug("Instance has been teminated: %s." % instance_alias)
return None
floating_ips = driver._connection.neutron_list_ips(instance)
if floating_ips:
floating_ip = floating_ips[0]["floating_ip_address"]
else:
floating_ip = driver._connection.neutron_associate_ip(
instance, *args, **kwargs)["floating_ip_address"]
_update_status_log(instance, "Networking Complete")
#TODO: Implement this as its own task, with the result from
#'floating_ip' passed in. Add it to the deploy_chain before deploy_to
hostname = ""
if floating_ip.startswith('128.196'):
regex = re.compile(
"(?P<one>[0-9]+)\.(?P<two>[0-9]+)\."
"(?P<three>[0-9]+)\.(?P<four>[0-9]+)")
r = regex.search(floating_ip)
(one, two, three, four) = r.groups()
hostname = "vm%s-%s.iplantcollaborative.org" % (three, four)
else:
# Find a way to convert new floating IPs to hostnames..
hostname = floating_ip
update_instance_metadata(driver, instance, data={
'public-hostname': hostname,
'public-ip':floating_ip}, replace=False)
logger.info("Assigned IP:%s - Hostname:%s" % (floating_ip, hostname))
#End
logger.debug("add_floating_ip task finished at %s." % datetime.now())
return {"floating_ip":floating_ip, "hostname":hostname}
except Exception as exc:
logger.exception("Error occurred while assigning a floating IP")
#Networking can take a LONG time when an instance first launches,
#it can also be one of those things you 'just miss' by a few seconds..
#So we will retry 30 times using limited exp.backoff
#Max Time: 53min
countdown = min(2**current.request.retries, 128)
add_floating_ip.retry(exc=exc,
countdown=countdown)
def add_floating_ip(driverCls, provider, identity,
instance_alias, delete_status=True,
*args, **kwargs):
#For testing ONLY.. Test cases ignore countdown..
if app.conf.CELERY_ALWAYS_EAGER:
logger.debug("Eager task waiting 15 seconds")
time.sleep(15)
try:
logger.debug("add_floating_ip task started at %s." % datetime.now())
#Remove unused floating IPs first, so they can be re-used
driver = get_driver(driverCls, provider, identity)
driver._clean_floating_ip()
#assign if instance doesn't already have an IP addr
instance = driver.get_instance(instance_alias)
if not instance:
logger.debug("Instance has been teminated: %s." % instance_alias)
return None
floating_ips = driver._connection.neutron_list_ips(instance)
if floating_ips:
floating_ip = floating_ips[0]["floating_ip_address"]
else:
floating_ip = driver._connection.neutron_associate_ip(
instance, *args, **kwargs)["floating_ip_address"]
_update_status_log(instance, "Networking Complete")
#TODO: Implement this as its own task, with the result from
#'floating_ip' passed in. Add it to the deploy_chain before deploy_to
hostname = ""
if floating_ip.startswith('128.196'):
regex = re.compile(
"(?P<one>[0-9]+)\.(?P<two>[0-9]+)\."
"(?P<three>[0-9]+)\.(?P<four>[0-9]+)")
r = regex.search(floating_ip)
(one, two, three, four) = r.groups()
hostname = "vm%s-%s.iplantcollaborative.org" % (three, four)
else:
# Find a way to convert new floating IPs to hostnames..
hostname = floating_ip
update_instance_metadata(driver, instance, data={
'public-hostname': hostname,
'public-ip': floating_ip}, replace=False)
logger.info("Assigned IP:%s - Hostname:%s" % (floating_ip, hostname))
#End
logger.debug("add_floating_ip task finished at %s." % datetime.now())
return {"floating_ip": floating_ip, "hostname": hostname}
except Exception as exc:
logger.exception("Error occurred while assigning a floating IP")
#Networking can take a LONG time when an instance first launches,
#it can also be one of those things you 'just miss' by a few seconds..
#So we will retry 30 times using limited exp.backoff
#Max Time: 53min
countdown = min(2**current.request.retries, 128)
add_floating_ip.retry(exc=exc,
countdown=countdown)
def deploy_failed(driverCls, provider, identity, instance_id, task_uuid):
try:
logger.debug("deploy_failed task started at %s." % datetime.now())
driver = get_driver(driverCls, provider, identity)
instance = driver.get_instance(instance_id)
update_instance_metadata(driver, instance,
data={'tmp_status': 'deploy_error'},
replace=False)
logger.debug("deploy_failed task finished at %s." % datetime.now())
except Exception as exc:
logger.warn(exc)
deploy_failed.retry(exc=exc)
def update_metadata(self, request, pk=None):
"""
Until a better method comes about, we will handle Updating metadata here.
"""
data = request.data
metadata = data.pop('metadata')
instance_id = pk
instance = find_instance(instance_id)
try:
update_instance_metadata(instance, metadata)
return Response(status=status.HTTP_204_NO_CONTENT)
except Exception as exc:
logger.exception("Error occurred updating v2 instance metadata")
return Response(exc.message, status=status.HTTP_409_CONFLICT)
def put(self, request, provider_uuid, identity_uuid, instance_id):
"""Authentication Required, update metadata about the instance"""
user = request.user
data = request.DATA
# Ensure item exists on the server first
esh_driver = prepare_driver(request, provider_uuid, identity_uuid)
if not esh_driver:
return invalid_creds(provider_uuid, identity_uuid)
try:
esh_instance = esh_driver.get_instance(instance_id)
except (socket_error, ConnectionFailure):
return connection_failure(provider_uuid, identity_uuid)
except InvalidCredsError:
return invalid_creds(provider_uuid, identity_uuid)
except Exception as exc:
logger.exception("Encountered a generic exception. "
"Returning 409-CONFLICT")
return failure_response(status.HTTP_409_CONFLICT,
str(exc.message))
if not esh_instance:
return instance_not_found(instance_id)
# Gather the DB related item and update
core_instance = convert_esh_instance(esh_driver, esh_instance,
provider_uuid, identity_uuid,
user)
serializer = InstanceSerializer(core_instance, data=data,
context={"request": request})
if serializer.is_valid():
logger.info('metadata = %s' % data)
#NOTE: We shouldn't allow 'full replacement' of metadata..
# We should also validate against potentional updating of 'atmo-used metadata'
update_instance_metadata(esh_driver, esh_instance, data, replace=False)
new_instance = serializer.save()
boot_scripts = data.pop('boot_scripts', [])
if boot_scripts:
new_instance = _save_scripts_to_instance(new_instance,
boot_scripts)
serializer = InstanceSerializer(
new_instance,
context={"request": request})
invalidate_cached_instances(
identity=Identity.objects.get(
uuid=identity_uuid))
response = Response(serializer.data)
logger.info('data = %s' % serializer.data)
response['Cache-Control'] = 'no-cache'
return response
else:
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
def update_metadata(self, request, pk=None):
"""
Until a better method comes about, we will handle Updating metadata here.
"""
data = request.data
metadata = data.pop('metadata')
instance_id = pk
instance = find_instance(instance_id)
try:
update_instance_metadata(instance, metadata)
return Response(status=status.HTTP_204_NO_CONTENT)
except Exception as exc:
logger.exception("Error occurred updating v2 instance metadata")
return Response(exc.message, status=status.HTTP_409_CONFLICT)
def put(self, request, provider_uuid, identity_uuid, instance_id):
"""Authentication Required, update metadata about the instance"""
user = request.user
data = request.DATA
# Ensure item exists on the server first
esh_driver = prepare_driver(request, provider_uuid, identity_uuid)
if not esh_driver:
return invalid_creds(provider_uuid, identity_uuid)
try:
esh_instance = esh_driver.get_instance(instance_id)
except (socket_error, ConnectionFailure):
return connection_failure(provider_uuid, identity_uuid)
except InvalidCredsError:
return invalid_creds(provider_uuid, identity_uuid)
except Exception as exc:
logger.exception("Encountered a generic exception. "
"Returning 409-CONFLICT")
return failure_response(status.HTTP_409_CONFLICT,
str(exc.message))
if not esh_instance:
return instance_not_found(instance_id)
# Gather the DB related item and update
core_instance = convert_esh_instance(esh_driver, esh_instance,
provider_uuid, identity_uuid,
user)
serializer = InstanceSerializer(core_instance, data=data,
context={"request": request})
if serializer.is_valid():
logger.info('metadata = %s' % data)
update_instance_metadata(esh_driver, esh_instance, data)
serializer.save()
new_instance = serializer.object
boot_scripts = data.pop('boot_scripts', [])
if boot_scripts:
new_instance = _save_scripts_to_instance(new_instance,
boot_scripts)
serializer = InstanceSerializer(
new_instance,
context={"request": request})
invalidate_cached_instances(
identity=Identity.objects.get(
uuid=identity_uuid))
response = Response(serializer.data)
logger.info('data = %s' % serializer.data)
response['Cache-Control'] = 'no-cache'
return response
else:
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
def update_metadata(driverCls,
provider,
identity,
instance_alias,
metadata,
replace_metadata=False):
"""
#NOTE: While this looks like a large number (250 ?!) of retries
# we expect this task to fail often when the image is building
# and large, uncached images can have a build time.
"""
try:
logger.debug("update_metadata task started at %s." % datetime.now())
driver = get_driver(driverCls, provider, identity)
instance = driver.get_instance(instance_alias)
if not instance:
return
#NOTE: This task will only be executed in TEST mode
if app.conf.CELERY_ALWAYS_EAGER:
eager_update_metadata(driver, instance, metadata)
return update_instance_metadata(driver,
instance,
data=metadata,
replace=replace_metadata)
logger.debug("update_metadata task finished at %s." % datetime.now())
except Exception as exc:
logger.exception(exc)
update_metadata.retry(exc=exc)
def patch(self, request, provider_uuid, identity_uuid, instance_id):
"""Authentication Required, update metadata about the instance"""
user = request.user
data = request.DATA
esh_driver = prepare_driver(request, provider_uuid, identity_uuid)
if not esh_driver:
return invalid_creds(provider_uuid, identity_uuid)
try:
esh_instance = esh_driver.get_instance(instance_id)
except ConnectionFailure:
return connection_failure(provider_uuid, identity_uuid)
except InvalidCredsError:
return invalid_creds(provider_uuid, identity_uuid)
except Exception as exc:
logger.exception("Encountered a generic exception. "
"Returning 409-CONFLICT")
return failure_response(status.HTTP_409_CONFLICT,
str(exc.message))
if not esh_instance:
return instance_not_found(instance_id)
# Gather the DB related item and update
core_instance = convert_esh_instance(esh_driver, esh_instance,
provider_uuid, identity_uuid,
user)
serializer = InstanceSerializer(
core_instance, data=data,
context={"request": request}, partial=True)
if serializer.is_valid():
logger.info('metadata = %s' % data)
update_instance_metadata(esh_driver, esh_instance, data,
replace=False)
serializer.save()
boot_scripts = data.pop('boot_scripts', [])
if boot_scripts:
_save_scripts_to_instance(serializer.object, boot_scripts)
invalidate_cached_instances(
identity=Identity.objects.get(uuid=identity_uuid))
response = Response(serializer.data)
logger.info('data = %s' % serializer.data)
response['Cache-Control'] = 'no-cache'
return response
else:
return Response(
serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
def deploy_failed(task_uuid, driverCls, provider, identity, instance_id, **celery_task_args):
from core.models.instance import Instance
from core.email import send_deploy_failed_email
try:
logger.debug("deploy_failed task started at %s." % datetime.now())
logger.info("task_uuid=%s" % task_uuid)
result = app.AsyncResult(task_uuid)
with allow_join_result():
exc = result.get(propagate=False)
err_str = "DEPLOYERROR::%s" % (result.traceback,)
logger.error(err_str)
driver = get_driver(driverCls, provider, identity)
instance = driver.get_instance(instance_id)
update_instance_metadata(driver, instance, data={"tmp_status": "deploy_error"}, replace=False)
logger.debug("deploy_failed task finished at %s." % datetime.now())
except Exception as exc:
logger.warn(exc)
deploy_failed.retry(exc=exc)
def update_metadata(driverCls, provider, identity, instance_alias, metadata):
"""
#NOTE: While this looks like a large number (250 ?!) of retries
# we expect this task to fail often when the image is building
# and large, uncached images can have a build time
"""
try:
logger.debug("update_metadata task started at %s." % datetime.now())
driver = get_driver(driverCls, provider, identity)
instance = driver.get_instance(instance_alias)
if not instance:
return
return update_instance_metadata(
driver, instance, data=metadata, replace=False)
logger.debug("update_metadata task finished at %s." % datetime.now())
except Exception as exc:
logger.exception(exc)
update_metadata.retry(exc=exc)
def eager_update_metadata(driver, instance, metadata):
"""
Used for TESTING ONLY. NEVER called in normal celery operation.
"""
while 1:
#Check if instance is terminated or no longer building.
if not instance or instance.extra['status'] != 'build':
break
#Wait 1min try again
wait_time = 1*60
logger.info("Always Eager Detected and instance is not active"
". Will wait 1 minute and check again to avoid"
" stack overflow from immediately retrying.."
)
time.sleep(wait_time*60)
# Update reference for the instance to see if its 'done'
instance = driver.get_instance(instance_id)
return update_instance_metadata(
driver, instance, data=metadata, replace=False)
def eager_update_metadata(driver, instance, metadata):
"""
Used for TESTING ONLY. NEVER called in normal celery operation.
"""
while 1:
#Check if instance is terminated or no longer building.
if not instance or instance.extra['status'] != 'build':
break
#Wait 1min try again
wait_time = 1*60
logger.info("Always Eager Detected and instance is not active"
". Will wait 1 minute and check again to avoid"
" stack overflow from immediately retrying.."
)
time.sleep(wait_time*60)
# Update reference for the instance to see if its 'done'
instance = driver.get_instance(instance_id)
return update_instance_metadata(
driver, instance, data=metadata, replace=False)
def update_metadata(driverCls, provider, identity, instance_alias, metadata):
"""
#NOTE: While this looks like a large number (250 ?!) of retries
# we expect this task to fail often when the image is building
# and large, uncached images can have a build time
"""
try:
logger.debug("update_metadata task started at %s." % datetime.now())
driver = get_driver(driverCls, provider, identity)
instance = driver.get_instance(instance_alias)
#NOTE: This task will only be executed in TEST mode
if app.conf.CELERY_ALWAYS_EAGER:
eager_update_metadata(driver, instance, metadata)
return update_instance_metadata(
driver, instance, data=metadata, replace=False)
logger.debug("update_metadata task finished at %s." % datetime.now())
except Exception as exc:
logger.warn(exc)
update_metadata.retry(exc=exc)
