def wait_for_async(self, request_id, second_per_loop, loop): """ Wait for async operation, up to second_per_loop * loop :param request_id: :return: """ count = 0 result = self.get_operation_status(request_id) while result.status == self.IN_PROGRESS: log.debug('wait for async [%s] loop count [%d]' % (request_id, count)) count += 1 if count > loop: log.error('Timed out waiting for async operation to complete.') return False time.sleep(second_per_loop) result = self.get_operation_status(request_id) if result.status != self.SUCCEEDED: log.error(vars(result)) if result.error: log.error(result.error.code) log.error(vars(result.error)) log.error('Asynchronous operation did not succeed.') return False return True
def load_template(url): try: template = json.load(file(url)) except Exception as e: log.error(e) return None return template
def get_deployment_dns(self, cloud_service_name, deployment_slot): try: props = self.get_deployment_by_slot(cloud_service_name, deployment_slot) except Exception as e: log.error(e) return None return None if props is None else props.url
def deployment_exists(self, cloud_service_name, deployment_slot): try: props = self.get_deployment_by_slot(cloud_service_name, deployment_slot) except Exception as e: if e.message != self.NOT_FOUND: log.error(e) return False return props is not None
def create_virtual_machine_async_false_3(self, experiment_id, template_unit): deployment_slot = template_unit.get_deployment_slot() virtual_machine_name = self.VIRTUAL_MACHINE_NAME_BASE % (template_unit.get_virtual_machine_name(), experiment_id) m = self.CREATE_DEPLOYMENT_ERROR[2] % (DEPLOYMENT, deployment_slot) commit_azure_log(experiment_id, ALOperation.CREATE_DEPLOYMENT, ALStatus.FAIL, m, 2) log.error(m) m = self.CREATE_VIRTUAL_MACHINE_ERROR[2] % (VIRTUAL_MACHINE, virtual_machine_name) commit_azure_log(experiment_id, ALOperation.CREATE_VIRTUAL_MACHINE, ALStatus.FAIL, m, 2) log.error(m)
def virtual_machine_exists(self, cloud_service_name, deployment_name, virtual_machine_name): try: props = self.get_virtual_machine(cloud_service_name, deployment_name, virtual_machine_name) except Exception as e: if e.message != self.NOT_FOUND: log.error(e) return False return props is not None
def get_available_core_count(self): """ Get available count of core Return -1 if failed :return: """ try: result = self.service.get_subscription() except Exception as e: log.error(e) return self.ERROR_RESULT return result.max_core_count - result.current_core_count
def cloud_service_exists(self, name): """ Check whether specific cloud service exist in specific azure subscription :param name: :return: """ try: props = self.get_hosted_service_properties(name) except Exception as e: if e.message != self.NOT_FOUND: log.error(e) return False return props is not None
def storage_account_exists(self, name): """ Check whether specific storage account exist in specific azure subscription :param name: :return: """ try: props = self.get_storage_account_properties(name) except Exception as e: if e.message != self.NOT_FOUND: log.error(e) return False return props is not None
def get_certificates(self, hackathon_name): hackathon_id = db_adapter.find_first_object_by(Hackathon, name=hackathon_name).id hackathon_azure_keys = db_adapter.find_all_objects_by( HackathonAzureKey, hackathon_id=hackathon_id) if hackathon_azure_keys is None: log.error('hackathon [%s] has no certificates' % hackathon_id) return None certificates = [] for hackathon_azure_key in hackathon_azure_keys: dic = db_adapter.get_object( AzureKey, hackathon_azure_key.azure_key_id).dic() certificates.append(dic) return certificates
def get_virtual_machine_network_config(self, cloud_service_name, deployment_name, virtual_machine_name): try: virtual_machine = self.get_virtual_machine(cloud_service_name, deployment_name, virtual_machine_name) except Exception as e: log.error(e) return None if virtual_machine is not None: for configuration_set in virtual_machine.configuration_sets.configuration_sets: if configuration_set.configuration_set_type == self.NETWORK_CONFIGURATION: return configuration_set return None
def delete_certificate(self, certificate_id, hackathon_name): certificate_id = int(certificate_id) hackathon_id = db_adapter.find_first_object_by(Hackathon, name=hackathon_name).id hackathon_azure_keys = db_adapter.find_all_objects_by( HackathonAzureKey, hackathon_id=hackathon_id) if hackathon_azure_keys is None: log.error('hackathon [%d] has no certificates' % hackathon_id) return False azure_key_ids = map(lambda x: x.azure_key_id, hackathon_azure_keys) if certificate_id not in azure_key_ids: log.error('hackathon [%d] has no certificate [%d]' % (hackathon_id, certificate_id)) return False certificate = db_adapter.get_object(AzureKey, certificate_id) db_adapter.delete_object(certificate) db_adapter.commit() return True
def start_virtual_machine(self, experiment_id, template_unit): """ 0. Prerequisites: a. virtual machine exist in both azure and database b. input parameters are correct :param experiment_id: :param template_unit: :return: """ commit_azure_log(experiment_id, ALOperation.START_VIRTUAL_MACHINE, ALStatus.START) cloud_service_name = template_unit.get_cloud_service_name() deployment_slot = template_unit.get_deployment_slot() deployment_name = self.service.get_deployment_name(cloud_service_name, deployment_slot) deployment = self.service.get_deployment_by_name(cloud_service_name, deployment_name) virtual_machine_name = self.VIRTUAL_MACHINE_NAME_BASE % (template_unit.get_virtual_machine_name(), experiment_id) status = self.service.get_virtual_machine_instance_status(deployment, virtual_machine_name) if status == AVMStatus.READY_ROLE: db_status = get_azure_virtual_machine_status(cloud_service_name, deployment_name, virtual_machine_name) if db_status == status: m = self.START_VIRTUAL_MACHINE_INFO[1] % (VIRTUAL_MACHINE, virtual_machine_name, AZURE_FORMATION) commit_azure_log(experiment_id, ALOperation.START_VIRTUAL_MACHINE, ALStatus.END, m, 1) else: m = self.START_VIRTUAL_MACHINE_INFO[2] % (VIRTUAL_MACHINE, virtual_machine_name, AZURE_FORMATION) self.__start_virtual_machine_helper(experiment_id, template_unit) commit_azure_log(experiment_id, ALOperation.START_VIRTUAL_MACHINE, ALStatus.END, m, 2) log.debug(m) else: try: result = self.service.start_virtual_machine(cloud_service_name, deployment_name, virtual_machine_name) except Exception as e: m = self.START_VIRTUAL_MACHINE_ERROR[0] % (VIRTUAL_MACHINE, virtual_machine_name, e.message) commit_azure_log(experiment_id, ALOperation.START_VIRTUAL_MACHINE, ALStatus.FAIL, 0) log.error(e) return False # query async operation status run_job(MDL_CLS_FUNC[2], (self.azure_key_id, ), (result.request_id, MDL_CLS_FUNC[22], (self.azure_key_id, ), (experiment_id, template_unit), MDL_CLS_FUNC[23], (self.azure_key_id, ), (experiment_id, template_unit))) return True
def wait_for_virtual_machine(self, cloud_service_name, deployment_name, virtual_machine_name, second_per_loop, loop, status): count = 0 props = self.get_deployment_by_name(cloud_service_name, deployment_name) while self.get_virtual_machine_instance_status( props, virtual_machine_name) != status: log.debug('wait for virtual machine [%s] loop count: %d' % (virtual_machine_name, count)) count += 1 if count > loop: log.error('Timed out waiting for role instance status.') return False time.sleep(second_per_loop) props = self.get_deployment_by_name(cloud_service_name, deployment_name) return self.get_virtual_machine_instance_status( props, virtual_machine_name) == status
def wait_for_deployment(self, cloud_service_name, deployment_name, second_per_loop, loop, status=ADStatus.RUNNING): count = 0 props = self.get_deployment_by_name(cloud_service_name, deployment_name) if props is None: return False while props.status != status: log.debug('wait for deployment [%s] loop count: %d' % (deployment_name, count)) count += 1 if count > loop: log.error('Timed out waiting for deployment status.') return False time.sleep(second_per_loop) props = self.get_deployment_by_name(cloud_service_name, deployment_name) if props is None: return False return props.status == status
def assign_public_endpoints(self, cloud_service_name, deployment_slot, virtual_machine_name, private_endpoints): """ Assign public endpoints of cloud service for private endpoints of virtual machine Return None if failed :param cloud_service_name: :param deployment_slot: :param virtual_machine_name: :param private_endpoints: a list of int or str :return: public_endpoints: a list of int """ log.debug('private_endpoints: %s' % private_endpoints) assigned_endpoints = self.service.get_assigned_endpoints( cloud_service_name) log.debug('assigned_endpoints: %s' % assigned_endpoints) if assigned_endpoints is None: return self.ERROR_RESULT # duplicate detection for public endpoint public_endpoints = find_unassigned_endpoints(private_endpoints, assigned_endpoints) log.debug('public_endpoints: %s' % public_endpoints) deployment_name = self.service.get_deployment_name( cloud_service_name, deployment_slot) network_config = self.service.get_virtual_machine_network_config( cloud_service_name, deployment_name, virtual_machine_name) # compose new network config to update new_network_config = add_endpoint_to_network_config( network_config, public_endpoints, private_endpoints) if new_network_config is None: return self.ERROR_RESULT try: result = self.service.update_virtual_machine_network_config( cloud_service_name, deployment_name, virtual_machine_name, new_network_config) except Exception as e: log.error(e) return self.ERROR_RESULT if not self.service.wait_for_async(result.request_id, self.TICK, self.LOOP): log.error('wait for async fail') return self.ERROR_RESULT if not self.service.wait_for_virtual_machine( cloud_service_name, deployment_name, virtual_machine_name, self.TICK, self.LOOP, AVMStatus.READY_ROLE): log.error('%s [%s] not ready' % (VIRTUAL_MACHINE, virtual_machine_name)) return self.ERROR_RESULT return public_endpoints
def release_public_endpoints(self, cloud_service_name, deployment_slot, virtual_machine_name, private_endpoints): """ Release public endpoints of cloud service according to private endpoints of virtual machine Return False if failed :param cloud_service_name: :param deployment_slot: :param virtual_machine_name: :param private_endpoints: a list of int or str :return: """ log.debug('private_endpoints: %s' % private_endpoints) deployment_name = self.service.get_deployment_name( cloud_service_name, deployment_slot) network_config = self.service.get_virtual_machine_network_config( cloud_service_name, deployment_name, virtual_machine_name) new_network_config = delete_endpoint_from_network_config( network_config, private_endpoints) if new_network_config is None: return False try: result = self.service.update_virtual_machine_network_config( cloud_service_name, deployment_name, virtual_machine_name, new_network_config) except Exception as e: log.error(e) return False if not self.service.wait_for_async(result.request_id, self.TICK, self.LOOP): log.error('wait for async fail') return False if not self.service.wait_for_virtual_machine( cloud_service_name, deployment_name, virtual_machine_name, self.TICK, self.LOOP, AVMStatus.READY_ROLE): log.error('%s [%s] not ready' % (VIRTUAL_MACHINE, virtual_machine_name)) return False return True
def create_cloud_service(self, experiment_id, template_unit): """ If cloud service not exist in azure subscription, then create it Else reuse cloud service in azure subscription :return: """ name = template_unit.get_cloud_service_name() label = template_unit.get_cloud_service_label() location = template_unit.get_cloud_service_location() commit_azure_log(experiment_id, ALOperation.CREATE_CLOUD_SERVICE, ALStatus.START) # avoid duplicate cloud service in azure subscription if not self.service.cloud_service_exists(name): # avoid name already taken by other azure subscription if not self.service.check_hosted_service_name_availability( name).result: m = self.CREATE_CLOUD_SERVICE_ERROR[1] % (CLOUD_SERVICE, name) commit_azure_log(experiment_id, ALOperation.CREATE_CLOUD_SERVICE, ALStatus.FAIL, m, 1) log.error(m) return False # avoid no available subscription remained if self.subscription.get_available_cloud_service_count( ) < self.NEED_COUNT: m = self.CREATE_CLOUD_SERVICE_ERROR[2] % (CLOUD_SERVICE, name) commit_azure_log(experiment_id, ALOperation.CREATE_CLOUD_SERVICE, ALStatus.FAIL, m, 2) log.error(m) return False # delete old azure cloud service in database, cascade delete old azure deployment, # old azure virtual machine and old azure end point delete_azure_cloud_service(name) try: self.service.create_hosted_service(name=name, label=label, location=location) except Exception as e: m = self.CREATE_CLOUD_SERVICE_ERROR[0] % (CLOUD_SERVICE, name, e.message) commit_azure_log(experiment_id, ALOperation.CREATE_CLOUD_SERVICE, ALStatus.FAIL, m, 0) log.error(e) return False # make sure cloud service is created if not self.service.cloud_service_exists(name): m = self.CREATE_CLOUD_SERVICE_ERROR[3] % (CLOUD_SERVICE, name) commit_azure_log(experiment_id, ALOperation.CREATE_CLOUD_SERVICE, ALStatus.FAIL, m, 3) log.error(m) return False else: m = self.CREATE_CLOUD_SERVICE_INFO[0] % (CLOUD_SERVICE, name) commit_azure_cloud_service(name, label, location, ACSStatus.CREATED, experiment_id) commit_azure_log(experiment_id, ALOperation.CREATE_CLOUD_SERVICE, ALStatus.END, m, 0) log.debug(m) else: # check whether cloud service created by azure formation before if contain_azure_cloud_service(name): m = self.CREATE_CLOUD_SERVICE_INFO[1] % (CLOUD_SERVICE, name, AZURE_FORMATION) commit_azure_log(experiment_id, ALOperation.CREATE_CLOUD_SERVICE, ALStatus.END, m, 1) else: m = self.CREATE_CLOUD_SERVICE_INFO[2] % (CLOUD_SERVICE, name, AZURE_FORMATION) commit_azure_cloud_service(name, label, location, ACSStatus.CREATED, experiment_id) commit_azure_log(experiment_id, ALOperation.CREATE_CLOUD_SERVICE, ALStatus.END, m, 2) log.debug(m) # create virtual machine run_job(MDL_CLS_FUNC[5], (self.azure_key_id, ), (experiment_id, template_unit)) return True
def start_virtual_machine_async_false(self, experiment_id, template_unit): virtual_machine_name = self.VIRTUAL_MACHINE_NAME_BASE % (template_unit.get_virtual_machine_name(), experiment_id) m = self.START_VIRTUAL_MACHINE_ERROR[1] % (VIRTUAL_MACHINE, virtual_machine_name) commit_azure_log(experiment_id, ALOperation.START_VIRTUAL_MACHINE, ALStatus.FAIL, 1) log.error(m)
def stop_virtual_machine(self, experiment_id, template_unit, action): """ 0. Prerequisites: a. virtual machine exist in both azure and database b. input parameters are correct :param experiment_id: :param template_unit: :param action: AVMStatus.STOPPED or AVMStatus.STOPPED_DEALLOCATED :return: """ commit_azure_log(experiment_id, ALOperation.STOP_VIRTUAL_MACHINE, ALStatus.START) # need_status: AVMStatus.STOPPED_VM or AVMStatus.STOPPED_DEALLOCATED need_status = AVMStatus.STOPPED_VM if action == AVMStatus.STOPPED else AVMStatus.STOPPED_DEALLOCATED cloud_service_name = template_unit.get_cloud_service_name() deployment_slot = template_unit.get_deployment_slot() deployment_name = self.service.get_deployment_name(cloud_service_name, deployment_slot) deployment = self.service.get_deployment_by_name(cloud_service_name, deployment_name) virtual_machine_name = self.VIRTUAL_MACHINE_NAME_BASE % (template_unit.get_virtual_machine_name(), experiment_id) now_status = self.service.get_virtual_machine_instance_status(deployment, virtual_machine_name) if need_status == AVMStatus.STOPPED_VM and now_status == AVMStatus.STOPPED_DEALLOCATED: m = self.STOP_VIRTUAL_MACHINE_ERROR[1] % (VIRTUAL_MACHINE, virtual_machine_name, AVMStatus.STOPPED_VM, AVMStatus.STOPPED_DEALLOCATED) commit_azure_log(experiment_id, ALOperation.STOP_VIRTUAL_MACHINE, ALStatus.FAIL, m, 1) log.error(m) return False elif need_status == now_status: db_status = get_azure_virtual_machine_status(cloud_service_name, deployment_name, virtual_machine_name) if db_status == need_status: m = self.STOP_VIRTUAL_MACHINE_INFO[1] % (VIRTUAL_MACHINE, virtual_machine_name, need_status, AZURE_FORMATION) commit_azure_log(experiment_id, ALOperation.STOP_VIRTUAL_MACHINE, ALStatus.END, m, 1) else: m = self.STOP_VIRTUAL_MACHINE_INFO[2] % (VIRTUAL_MACHINE, virtual_machine_name, need_status, AZURE_FORMATION) self.__stop_virtual_machine_helper(experiment_id, template_unit, need_status) commit_azure_log(experiment_id, ALOperation.STOP_VIRTUAL_MACHINE, ALStatus.END, m, 2) log.debug(m) else: try: result = self.service.stop_virtual_machine(cloud_service_name, deployment_name, virtual_machine_name, action) except Exception as e: m = self.STOP_VIRTUAL_MACHINE_ERROR[0] % (VIRTUAL_MACHINE, virtual_machine_name, e.message) commit_azure_log(experiment_id, ALOperation.STOP_VIRTUAL_MACHINE, ALStatus.FAIL, 0) log.error(e) return False # query async operation status run_job(MDL_CLS_FUNC[2], (self.azure_key_id, ), (result.request_id, MDL_CLS_FUNC[18], (self.azure_key_id, ), (experiment_id, template_unit, need_status), MDL_CLS_FUNC[19], (self.azure_key_id, ), (experiment_id, template_unit, need_status))) return True
def create_virtual_machine_async_false_2(self, experiment_id, template_unit): virtual_machine_name = self.VIRTUAL_MACHINE_NAME_BASE % (template_unit.get_virtual_machine_name(), experiment_id) m = self.CREATE_VIRTUAL_MACHINE_ERROR[3] % (VIRTUAL_MACHINE, virtual_machine_name) commit_azure_log(experiment_id, ALOperation.CREATE_VIRTUAL_MACHINE, ALStatus.FAIL, m, 3) log.error(m)
def create_virtual_machine(self, experiment_id, template_unit): """ 0. Prerequisites: a. storage account and cloud service exist in both azure and database; b. input parameters are correct; 1. If deployment not exist in azure subscription, then create virtual machine with deployment Else reuse deployment in azure subscription 2. If virtual machine not exist in azure subscription, then add virtual machine to deployment Else reuse virtual machine in azure subscription :return: """ commit_azure_log(experiment_id, ALOperation.CREATE_DEPLOYMENT, ALStatus.START) commit_azure_log(experiment_id, ALOperation.CREATE_VIRTUAL_MACHINE, ALStatus.START) deployment_slot = template_unit.get_deployment_slot() # avoid virtual machine name conflict on same name in template virtual_machine_name = self.VIRTUAL_MACHINE_NAME_BASE % (template_unit.get_virtual_machine_name(), experiment_id) virtual_machine_size = template_unit.get_virtual_machine_size() if self.subscription.get_available_core_count() < self.SIZE_CORE_MAP[virtual_machine_size.lower()]: m = self.CREATE_DEPLOYMENT_ERROR[1] % (DEPLOYMENT, deployment_slot) commit_azure_log(experiment_id, ALOperation.CREATE_DEPLOYMENT, ALStatus.FAIL, m, 1) log.error(m) m = self.CREATE_VIRTUAL_MACHINE_ERROR[1] % (VIRTUAL_MACHINE, virtual_machine_name) commit_azure_log(experiment_id, ALOperation.CREATE_VIRTUAL_MACHINE, ALStatus.FAIL, m, 1) log.error(m) return False cloud_service_name = template_unit.get_cloud_service_name() vm_image_name = template_unit.get_vm_image_name() system_config = template_unit.get_system_config() os_virtual_hard_disk = template_unit.get_os_virtual_hard_disk() # avoid duplicate deployment in azure subscription if self.service.deployment_exists(cloud_service_name, deployment_slot): # use deployment name from azure subscription deployment_name = self.service.get_deployment_name(cloud_service_name, deployment_slot) if contain_azure_deployment(cloud_service_name, deployment_slot): m = self.CREATE_DEPLOYMENT_INFO[1] % (DEPLOYMENT, deployment_name, AZURE_FORMATION) commit_azure_log(experiment_id, ALOperation.CREATE_DEPLOYMENT, ALStatus.END, m, 1) else: m = self.CREATE_DEPLOYMENT_INFO[2] % (DEPLOYMENT, deployment_name, AZURE_FORMATION) commit_azure_deployment(deployment_name, deployment_slot, ADStatus.RUNNING, cloud_service_name, experiment_id) commit_azure_log(experiment_id, ALOperation.CREATE_DEPLOYMENT, ALStatus.END, m, 2) log.debug(m) # avoid duplicate virtual machine in azure subscription if self.service.virtual_machine_exists(cloud_service_name, deployment_name, virtual_machine_name): if contain_azure_virtual_machine(cloud_service_name, deployment_name, virtual_machine_name): m = self.CREATE_VIRTUAL_MACHINE_INFO[1] % (VIRTUAL_MACHINE, virtual_machine_name, AZURE_FORMATION) commit_azure_log(experiment_id, ALOperation.CREATE_VIRTUAL_MACHINE, ALStatus.END, m, 1) log.debug(m) else: m = self.CREATE_VIRTUAL_MACHINE_ERROR[4] % (VIRTUAL_MACHINE, virtual_machine_name, AZURE_FORMATION) commit_azure_log(experiment_id, ALOperation.CREATE_VIRTUAL_MACHINE, ALStatus.FAIL, m, 4) log.error(m) return False else: # delete old azure virtual machine, cascade delete old azure endpoint delete_azure_virtual_machine(cloud_service_name, deployment_name, virtual_machine_name) network_config = template_unit.get_network_config(self.service, False) try: result = self.service.add_virtual_machine(cloud_service_name, deployment_name, virtual_machine_name, system_config, os_virtual_hard_disk, network_config, virtual_machine_size, vm_image_name) except Exception as e: m = self.CREATE_VIRTUAL_MACHINE_ERROR[0] % (VIRTUAL_MACHINE, virtual_machine_name, e.message) commit_azure_log(experiment_id, ALOperation.CREATE_VIRTUAL_MACHINE, ALStatus.FAIL, m, 0) log.error(e) return False # query async operation status run_job(MDL_CLS_FUNC[2], (self.azure_key_id, ), (result.request_id, MDL_CLS_FUNC[6], (self.azure_key_id, ), (experiment_id, template_unit), MDL_CLS_FUNC[7], (self.azure_key_id, ), (experiment_id, template_unit))) else: # delete old azure deployment, cascade delete old azure virtual machine and azure endpoint delete_azure_deployment(cloud_service_name, deployment_slot) # use deployment name from template deployment_name = template_unit.get_deployment_name() virtual_machine_label = template_unit.get_virtual_machine_label() network_config = template_unit.get_network_config(self.service, False) try: result = self.service.create_virtual_machine_deployment(cloud_service_name, deployment_name, deployment_slot, virtual_machine_label, virtual_machine_name, system_config, os_virtual_hard_disk, network_config, virtual_machine_size, vm_image_name) except Exception as e: m = self.CREATE_DEPLOYMENT_ERROR[0] % (DEPLOYMENT, deployment_slot, e.message) commit_azure_log(experiment_id, ALOperation.CREATE_DEPLOYMENT, ALStatus.FAIL, m, 0) m = self.CREATE_VIRTUAL_MACHINE_ERROR[0] % (VIRTUAL_MACHINE, virtual_machine_name, e.message) commit_azure_log(experiment_id, ALOperation.CREATE_VIRTUAL_MACHINE, ALStatus.FAIL, m, 0) log.error(e) return False # query async operation status run_job(MDL_CLS_FUNC[2], (self.azure_key_id, ), (result.request_id, MDL_CLS_FUNC[13], (self.azure_key_id, ), (experiment_id, template_unit), MDL_CLS_FUNC[14], (self.azure_key_id, ), (experiment_id, template_unit))) return True