def setup_cloud_service(self, sctx):
# get context from super context
ctx = sctx.job_ctxs[sctx.current_job_index]
adapter = self.__get_adapter_from_sctx(sctx, CloudServiceAdapter)
try:
if not adapter.cloud_service_exists(ctx.cloud_service_name):
if not adapter.create_cloud_service(
name=ctx.cloud_service_name,
label=ctx.cloud_service_label,
location=ctx.cloud_service_host):
self.log.error(
"azure virtual environment %d create remote cloud service failed via creation"
% sctx.current_job_index)
self._on_virtual_environment_failed(sctx)
return
# create the cloud service remote successfully, record
sctx.remote_created.append(
Context(type=REMOTE_CREATED_RECORD.TYPE_CLOUD_SERVICE,
name=ctx.cloud_service_name))
self.log.debug(
"azure virtual environment %d cloud service setup done" %
sctx.current_job_index)
# next step: setup storage
self.scheduler.add_once("azure_vm",
"setup_storage",
id="setup_storage_" +
str(sctx.experiment_id),
context=sctx,
seconds=0)
except Exception as e:
self.log.error(
"azure virtual environment %d create remote cloud service failed: %r"
% (sctx.current_job_index, str(e)))
self._on_virtual_environment_failed(sctx)
def upload_files(self):
"""Handle uploaded files from http request"""
self.__validate_upload_files()
images = []
storage = RequiredFeature("storage")
for file_name in request.files:
file_storage = request.files[file_name]
self.log.debug("upload image file : " + file_name)
context = Context(hackathon_name=g.hackathon.name,
file_name=file_storage.filename,
file_type=FILE_TYPE.HACK_IMAGE,
content=file_storage)
context = storage.save(context)
image = {
"name": file_storage.filename,
"url": context.url,
"thumbnailUrl": context.url,
"deleteUrl": '/api/admin/file?key=' + context.file_name
}
# context.file_name is a random name created by server, file.filename is the original name
images.append(image)
return {"files": images}
def __stop_virtual_machine_done(self, sctx):
self.log.debug("azure virtual environment %d stop vm done" %
sctx.current_job_index)
try:
# update the status of virtual environment
expr = Experiment.objects(id=sctx.experiment_id).first()
ve = expr.virtual_environments[sctx.current_job_index]
self._on_virtual_environment_stopped(
Context(experiment_id=expr.id,
virtual_environment_name=ve.name))
self.log.debug(
"azure virtual environment %d vm success callback done, step to next"
% sctx.current_job_index)
# step to config next unit
sctx.current_job_index += 1
self.__schedule_setup(sctx)
except Exception as e:
self.log.error(
"azure virtual environment %d error while stopping vm: %r" %
(sctx.current_job_index, e.message))
self.__on_stop_virtual_machine_failed(sctx)
def create_certificate(self, subscription_id, management_host, hackathon):
"""Create certificate for specific subscription and hackathon
1. check certificate dir
2. generate pem file
3. generate cert file
4. add azure key to db
5. add hackathon azure key to db
:param subscription_id:
:param management_host:
:param hackathon:
:return:
"""
base_url = '%s/%s' % (self.CERT_BASE, subscription_id)
pem_url = base_url + '.pem'
# avoid duplicate pem generation
if not os.path.isfile(pem_url):
pem_command = 'openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout %s -out %s -batch' % \
(pem_url, pem_url)
commands.getstatusoutput(pem_command)
else:
self.log.debug('%s exists' % pem_url)
cert_url = base_url + '.cer'
# avoid duplicate cert generation
if not os.path.isfile(cert_url):
cert_command = 'openssl x509 -inform pem -in %s -outform der -out %s' % (
pem_url, cert_url)
commands.getstatusoutput(cert_command)
else:
self.log.debug('%s exists' % cert_url)
azure_key = self.db.find_first_object_by(
AzureKey,
cert_url=cert_url,
pem_url=pem_url,
subscription_id=subscription_id,
management_host=management_host)
# avoid duplicate azure key
if azure_key is None:
azure_key = self.db.add_object_kwargs(
AzureKey,
cert_url=cert_url,
pem_url=pem_url,
subscription_id=subscription_id,
management_host=management_host)
self.db.commit()
else:
self.log.debug('azure key exists')
hackathon_azure_key = self.db.find_first_object_by(
HackathonAzureKey,
hackathon_id=hackathon.id,
azure_key_id=azure_key.id)
# avoid duplicate hackathon azure key
if hackathon_azure_key is None:
self.db.add_object_kwargs(HackathonAzureKey,
hackathon_id=hackathon.id,
azure_key_id=azure_key.id)
self.db.commit()
else:
self.log.debug('hackathon azure key exists')
file_name = subscription_id + '.cer'
context = Context(hackathon_name=hackathon.name,
file_name=file_name,
file_type=FILE_TYPE.AZURE_CERT,
content=file(cert_url))
self.log.debug("saving cerf file [%s] to azure" % file_name)
context = self.storage.save(context)
azure_key.cert_url = context.url
self.db.commit()
return azure_key.cert_url
def test_start_expr(self):
#TODO: mock record in db
ctx = Context(template="test_template",
user="******",
hackathon="test_hackathon")
self.assertTrue(self.service.start_expr(ctx))
def create_certificate(self, subscription_id, management_host, hackathon):
"""Create certificate for specific subscription and hackathon
1. check certificate dir
2. generate pem file
3. generate cert file
4. add azure key to db
5. add hackathon azure key to db
:param subscription_id:
:param management_host:
:param hackathon:
:return:
"""
base_url = '%s/%s' % (self.CERT_BASE, subscription_id)
pem_url = base_url + '.pem'
cert_url = base_url + '.cer'
# avoid duplicate pem generation
if not os.path.isfile(pem_url):
pem_command = 'openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout %s -out %s -batch' % \
(pem_url, pem_url)
commands.getstatusoutput(pem_command)
else:
self.log.debug('%s exists' % pem_url)
# avoid duplicate cert generation
if not os.path.isfile(cert_url):
cert_command = 'openssl x509 -inform pem -in %s -outform der -out %s' % (pem_url, cert_url)
commands.getstatusoutput(cert_command)
else:
self.log.debug('%s exists' % cert_url)
azure_key = AzureKey.objects(subscription_id=subscription_id, management_host=management_host).first()
if azure_key is None:
azure_key = AzureKey(
cert_url=base_url + '.cer',
pem_url=base_url + '.pem',
subscription_id=subscription_id,
management_host=management_host,
verified=False
)
azure_key.save()
hackathon.azure_keys.append(azure_key)
hackathon.save()
else:
self.log.debug('azure key exists')
if not (azure_key in hackathon.azure_keys):
hackathon.azure_keys.append(azure_key)
hackathon.save()
else:
self.log.debug('hackathon azure key exists')
# store cer file
cer_context = Context(
hackathon_name=hackathon.name,
file_name=subscription_id + '.cer',
file_type=FILE_TYPE.AZURE_CERT,
content=file(cert_url)
)
self.log.debug("saving cerf file [%s] to azure" % cer_context.file_name)
cer_context = self.storage.save(cer_context)
azure_key.cert_url = cer_context.url
# store pem file
# encrypt certification file before upload to storage
encrypted_pem_url = self.__encrypt_content(pem_url)
pem_contex = Context(
hackathon_name=hackathon.name,
file_name=subscription_id + '.pem',
file_type=FILE_TYPE.AZURE_CERT,
content=file(encrypted_pem_url)
)
self.log.debug("saving pem file [%s] to azure" % pem_contex.file_name)
pem_contex = self.storage.save(pem_contex)
os.remove(encrypted_pem_url)
azure_key.pem_url = pem_contex.url
azure_key.save()
return azure_key.dic()
def start_new_docker_host_vm(self, hackathon):
"""
create docker host VM for hackathon whose id is hackathon_id
:param hackathon: hackathon
:type hackathon: Hackathon
:return: True if send an creating VM request via API successfully after validating storage, container and
service
Otherwise, False
:rtype: bool
"""
# todo debug this logic and make sure DON'T start two or more VM at the same time
return False
hackathon_id = hackathon.id
sms = self.__get_sms_object(hackathon_id)
if sms is None:
self.log.error('No account found for Hackathon:%d' % hackathon_id)
return False
# get storage and container
res, storage_account_name, container_name = self.__get_available_storage_account_and_container(
hackathon_id)
if not res:
return False
# get service
res, service_name = self.__get_available_cloud_service(hackathon_id)
if not res:
return False
# set host_name to ensure its uniqueness
host_name = str(uuid1())[0:9] + strftime("%Y%m%d%H%M%S")
# set vm os image and hard disk storage
image_name_default = 'b549f4301d0b4295b8e76ceb65df47d4__Ubuntu-14_04-LTS-amd64-server-20140606.1-en-us-30GB'
image_name = self.util.safe_get_config(
'dockerhostserver.vm.image_name', image_name_default)
media_link = 'https://%s.blob.core.chinacloudapi.cn/%s/%s.vhd' % (
storage_account_name, container_name, host_name)
os_hd = OSVirtualHardDisk(image_name, media_link)
# set linux_config and endpoint_config
customdata = self.__get_customdata_from_local_file()
linux_config = LinuxConfigurationSet(
host_name,
self.util.safe_get_config('dockerhostserver.vm.username',
'opentech'),
self.util.safe_get_config('dockerhostserver.vm.password',
'Password01!'),
False,
custom_data=customdata)
endpoint_config = self.__set_endpoint_config(service_name,
hackathon_id)
deployment_exist = True
deployment_slot = ServiceDeploymentSlot.PRODUCTION
try:
deployment_exist = self.__deployment_exists(
service_name, deployment_slot, hackathon_id)
except Exception as e:
self.log.error(e)
return False
if not deployment_exist:
try:
result = sms.create_virtual_machine_deployment(
service_name=service_name,
network_config=endpoint_config,
deployment_name=service_name,
deployment_slot=deployment_slot,
label=service_name,
role_name=host_name,
system_config=linux_config,
os_virtual_hard_disk=os_hd,
role_size=0)
self.log.debug('To create VM:%s in service:%s.(deployment)' %
(host_name, service_name))
except Exception as e:
self.log.error(e)
return False
else:
try:
result = sms.add_role(service_name=service_name,
deployment_name=service_name,
role_name=host_name,
system_config=linux_config,
os_virtual_hard_disk=os_hd,
network_config=endpoint_config,
role_size=0)
self.log.debug('To create VM:%s in service:%s.(add role)' %
(host_name, service_name))
except Exception as e:
self.log.error(e)
return False
# storage parameters in context
context = Context(hackathon_id=hackathon_id,
request_id=result.request_id,
service_name=service_name,
role_name=host_name,
deployment_name=service_name,
deployment_slot=deployment_slot,
host_name=host_name)
# start schedule
self.sche.add_once('docker_host_manager',
'check_vm_status',
context=context,
minutes=5)
return True
def __generate_base_context(self, azure_key_id, experiment_id,
template_unit):
return Context(azure_key_id=azure_key_id,
experiment_id=experiment_id,
template_unit=template_unit)
def start_virtual_machine(self, context):
"""
0. Prerequisites: a. virtual machine exist in both azure and database
b. input parameters are correct
:param context : contains azure_key_id, experiment_id, template_unit
:return:
"""
commit_azure_log(context.experiment_id,
ALOperation.START_VIRTUAL_MACHINE, ALStatus.START)
cloud_service_name = context.template_unit.get_cloud_service_name()
deployment_slot = context.template_unit.get_deployment_slot()
deployment_name = self.azure_adapter.get_deployment_name(
cloud_service_name, deployment_slot)
deployment = self.azure_adapter.get_deployment_by_name(
cloud_service_name, deployment_name)
virtual_machine_name = '%s-%d' % (
context.template_unit.get_virtual_machine_name(),
context.experiment_id)
status = self.azure_adapter.get_virtual_machine_instance_status(
deployment, virtual_machine_name)
if status == AVMStatus.READY_ROLE:
db_status = get_azure_virtual_machine_status(
cloud_service_name, deployment_name, virtual_machine_name)
if db_status == status:
m = '%s [%s] started by %s before' % (
AZURE_RESOURCE_TYPE.VIRTUAL_MACHINE, virtual_machine_name,
AZURE_FORMATION)
commit_azure_log(context.experiment_id,
ALOperation.START_VIRTUAL_MACHINE,
ALStatus.END, m, 1)
else:
m = '%s [%s] started but not by %s before' % (
AZURE_RESOURCE_TYPE.VIRTUAL_MACHINE, virtual_machine_name,
AZURE_FORMATION)
self.__start_virtual_machine_helper(context.experiment_id,
context.template_unit)
commit_azure_log(context.experiment_id,
ALOperation.START_VIRTUAL_MACHINE,
ALStatus.END, m, 2)
self.log.debug(m)
else:
try:
result = self.azure_adapter.start_virtual_machine(
cloud_service_name, deployment_name, virtual_machine_name)
except Exception as e:
m = '%s [%s] %s' % (AZURE_RESOURCE_TYPE.VIRTUAL_MACHINE,
virtual_machine_name, e.message)
commit_azure_log(context.experiment_id,
ALOperation.START_VIRTUAL_MACHINE,
ALStatus.FAIL, 0)
self.log.error(e)
self.log.error(m)
return False
# query async operation status
query_context = Context(
request_id=result.id,
azure_key_id=context.azure_key_id,
feature='azure_vm_service',
true_method='start_virtual_machine_async_true',
false_method='start_virtual_machine_async_false',
method_args_context=context)
self.scheduler.add_once(feature='azure_service',
method='query_async_operation_status',
context=query_context,
seconds=3)
return True
def stop_virtual_machine(self, context):
"""
0. Prerequisites: a. virtual machine exist in both azure and database
b. input parameters are correct
:param context :
azure_key_id,
experiment_id,
template_unit,
action: AVMStatus.STOPPED or AVMStatus.STOPPED_DEALLOCATED
:return:
"""
commit_azure_log(context.experiment_id,
ALOperation.STOP_VIRTUAL_MACHINE, ALStatus.START)
# need_status: AVMStatus.STOPPED_VM or AVMStatus.STOPPED_DEALLOCATED
need_status = AVMStatus.STOPPED_VM if context.action == AVMStatus.STOPPED else AVMStatus.STOPPED_DEALLOCATED
cloud_service_name = context.template_unit.get_cloud_service_name()
deployment_slot = context.template_unit.get_deployment_slot()
deployment_name = self.azure_adapter.get_deployment_name(
cloud_service_name, deployment_slot)
deployment = self.azure_adapter.get_deployment_by_name(
cloud_service_name, deployment_name)
virtual_machine_name = '%s-%d' % (
context.template_unit.get_virtual_machine_name(),
context.experiment_id)
now_status = self.azure_adapter.get_virtual_machine_instance_status(
deployment, virtual_machine_name)
if need_status == AVMStatus.STOPPED_VM and now_status == AVMStatus.STOPPED_DEALLOCATED:
m = '%s [%s] need status %s but now status %s' % (
AZURE_RESOURCE_TYPE.VIRTUAL_MACHINE, virtual_machine_name,
AVMStatus.STOPPED_VM, AVMStatus.STOPPED_DEALLOCATED)
commit_azure_log(context.experiment_id,
ALOperation.STOP_VIRTUAL_MACHINE, ALStatus.FAIL,
m, 1)
self.log.error(m)
return False
elif need_status == now_status:
db_status = get_azure_virtual_machine_status(
cloud_service_name, deployment_name, virtual_machine_name)
if db_status == need_status:
m = '%s [%s] %s and by %s before' % (
AZURE_RESOURCE_TYPE.VIRTUAL_MACHINE, virtual_machine_name,
need_status, AZURE_FORMATION)
commit_azure_log(context.experiment_id,
ALOperation.STOP_VIRTUAL_MACHINE,
ALStatus.END, m, 1)
else:
m = '%s [%s] %s but not by %s before' % (
AZURE_RESOURCE_TYPE.VIRTUAL_MACHINE, virtual_machine_name,
need_status, AZURE_FORMATION)
self.__stop_virtual_machine_helper(context.experiment_id,
context.template_unit,
need_status)
commit_azure_log(context.experiment_id,
ALOperation.STOP_VIRTUAL_MACHINE,
ALStatus.END, m, 2)
self.log.debug(m)
else:
try:
result = self.azure_adapter.stop_virtual_machine(
cloud_service_name, deployment_name, virtual_machine_name,
context.action)
except Exception as e:
m = '%s [%s] %s' % (AZURE_RESOURCE_TYPE.VIRTUAL_MACHINE,
virtual_machine_name, e.message)
commit_azure_log(context.experiment_id,
ALOperation.STOP_VIRTUAL_MACHINE,
ALStatus.FAIL, 0)
self.log.error(m)
self.log.error(e)
return False
# query async operation status
method_args_context = self.__generate_base_context(
context.azure_key_id, context.experiment_id,
context.template_unit)
method_args_context.need_status = need_status
query_context = Context(
request_id=result.id,
azure_key_id=context.azure_key_id,
feature='azure_vm_service',
true_method='stop_virtual_machine_async_true',
false_method='stop_virtual_machine_async_false',
method_args_context=method_args_context)
self.scheduler.add_once(feature='azure_service',
method='query_async_operation_status',
context=query_context,
seconds=3)
return True
def __setup_virtual_machine_done(self, sctx):
try:
self.log.debug("azure virtual environment %d vm setup done" %
sctx.current_job_index)
ctx = sctx.job_ctxs[sctx.current_job_index]
# update the status of virtual environment
expr = Experiment.objects(id=sctx.experiment_id).first()
ve = expr.virtual_environments[sctx.current_job_index]
adapter = self.__get_adapter_from_sctx(sctx, VirtualMachineAdapter)
ve.status = VEStatus.RUNNING
expr.save()
self._on_virtual_environment_success(
Context(experiment_id=expr.id))
azure_resource = AzureVirtualMachine(
name=ctx.virtual_machine_name,
label=ctx.virtual_machine_label,
dns="%s.chinacloudapp.cn" % ctx.cloud_service_name,
end_points=[])
# todo record AzureDeployment, AzureCloudService and so on in db for roll back
vm_role = adapter.get_virtual_machine_role(
ctx.cloud_service_name, ctx.deployment_name,
ctx.virtual_machine_name)
if (not vm_role) or (not vm_role.instance_endpoints):
self.log.warn(
"unable to find vm %s, cannot update virtual env config like guacamole"
% ctx.virtual_machine_name)
else:
for endpoint in vm_role.instance_endpoints:
azure_resource.public_ip = endpoint.vip
if endpoint.name == ctx.remote_endpoint_name: # endpoint for remote desktop
ve.remote_provider = VERemoteProvider.Guacamole
ve.remote_paras = get_remote_parameters(
ctx.raw_system_config, ctx.remote,
ctx.virtual_machine_name, endpoint.vip,
endpoint.public_port)
else:
try:
aep = self.__get_persistable_endpoint(
endpoint, ctx.raw_network_config)
azure_resource.end_points.append(aep)
except Exception as e:
self.log.error(e)
ve.azure_resource = azure_resource
azure_resource.save()
expr.save()
self.log.debug(
"azure virtual environment %d vm success callback done, step to next"
% sctx.current_job_index)
# step to config next unit
sctx.current_job_index += 1
self.__schedule_setup(sctx)
except Exception as e:
self.log.error(
"azure virtual environment %d failed on vm_done: %r" %
(sctx.current_job_index, e.message))
self._on_virtual_environment_failed(sctx)
def start_vm(self, resource_id, azure_key, template_units,
virtual_environments):
"""setup the resources needed by the template unit and start the virtual machine
this will create the missed VirtualMachine, CloudService and Storages
this function is not blocked, it will run in the background and return immediatly
:param resouce_id: a integer that can indentify the creation of azure resource, is reusable to checkout created virtual machine name
:param template_units: the azure template units, contains the data need by azure formation
:param azure_key: the azure_key object, use to access azure
:param virtual_environments: the virtual envrionments associated with this setup NOTE: this is a workaround
"""
# the setup of each unit must be SERLIALLY EXECUTED
# to avoid the creation of same resource in same time
# TODO: we still have't avoid the parrallel excution of the setup of same template
job_ctxs = []
ctx = Context(
job_ctxs=job_ctxs,
current_job_index=0,
resource_id=resource_id,
subscription_id=azure_key.subscription_id,
pem_url=azure_key.pem_url,
management_host=azure_key.management_host,
# remote_created is used to store the resources we create we create remote
# so we can do rollback
# TODO: if the user create a virtual machine with vm_image, we have to config the network of it
# but so far we have no way to rollback the network settings of it
remote_created=[])
assert len(template_units) == len(virtual_environments)
for i in xrange(0, len(template_units)):
unit = template_units[i]
ve = virtual_environments[i]
name = self.get_virtual_machine_name(
unit.get_virtual_machine_name(), resource_id)
# construct current virutal envrionment's context
job_ctxs.append(
Context(
cloud_service_name=unit.get_cloud_service_name(),
cloud_service_label=unit.get_cloud_service_label(),
cloud_service_host=unit.get_cloud_service_location(),
storage_account_name=unit.get_storage_account_name(),
storage_account_description=unit.
get_storage_account_description(),
storage_account_label=unit.get_storage_account_label(),
storage_account_location=unit.get_storage_account_location(
),
virtual_machine_name=name,
virtual_machine_label=unit.get_virtual_machine_label(),
deployment_name=unit.get_deployment_name(),
deployment_slot=unit.get_deployment_slot(),
system_config=unit.get_system_config(),
raw_system_config=unit.get_raw_system_config(),
os_virtual_hard_disk=unit.get_os_virtual_hard_disk(),
virtual_machine_size=unit.get_virtual_machine_size(),
image_name=unit.get_image_name(),
raw_network_config=unit.get_raw_network_config(),
is_vm_image=unit.is_vm_image(),
remote=unit.get_remote(),
endpoint_name=unit.get_remote_port_name(),
# NOTE: ONLY callback purpose functions can depend on virutal_environment_id
virtual_environment_id=ve.id,
# TODO: this part of info should move to super context
subscription_id=azure_key.subscription_id,
pem_url=azure_key.pem_url,
management_host=azure_key.management_host))
# execute from first job context
self.__schedule_setup(ctx)
