def score_within_confidence_interval(bot_eval: Box, past_bot_scores: Box) -> Tuple[bool, Box]: """ Compare with current mean score and check within acceptable_score_deviation range. If only 1 score, roughly double the acceptable range, since we could have gone from min to max. Also, follow the 2-sided CI for a t-student distribution that gives ~2x the acceptable_score_deviation with infinite samples (i.e. ± acceptable_score_deviation) https://en.wikipedia.org/wiki/Student%27s_t-distribution#Table_of_selected_values https://stats.stackexchange.com/a/230185/18187 n Confidence Level Multiplicative Factor 2 0.95 12.71 3 0.95 4.30 4 0.95 3.18 5 0.95 2.78 infinity 0.95 1.96 """ info = Box(mean=None, ci_high=None, ci_low=None, acceptable_score_deviation=None) if bot_eval.eval_key in [p.eval_key for p in past_bot_scores.scores]: log.warning('Score already recorded, this should not happen!') return True, info score = bot_eval.results.score acceptable_score_deviation = bot_eval.problem_def.acceptable_score_deviation if not past_bot_scores.scores: # If no previous scores, then we are the mean of the CI return True, info score_values = [b.score for b in past_bot_scores.scores] multiplier = { 2: 12.71, 3: 4.30, 4: 3.18, 5: 2.78, }.get(len(score_values) + 1, 1.96) diff_max = acceptable_score_deviation * multiplier / 2 ci_low = past_bot_scores.mean - diff_max ci_high = past_bot_scores.mean + diff_max info.high = ci_high info.low = ci_low info.mean = past_bot_scores.mean info.acceptable_score_deviation = acceptable_score_deviation if math.nan in [ci_high, ci_low]: ret = True elif ci_low <= score <= ci_high: ret = True else: ret = False return ret, info
def assign_job(self, job) -> Optional[Box]: if dbox(job).run_local_debug: log.warning(f'Run local debug is true, setting instance id to ' f'{constants.LOCAL_INSTANCE_ID}') self.assign_job_to_instance(constants.LOCAL_INSTANCE_ID, job) return job worker_instances = self.get_worker_instances() self.prune_terminated_instances(worker_instances) available_running_instances, available_stopped_instances = \ self.get_available_instances(worker_instances) if available_running_instances: self.assign_to_running_instance(available_running_instances, job) elif available_stopped_instances: self.start_instance_and_assign(available_stopped_instances, job) else: if len(worker_instances) < MAX_WORKER_INSTANCES: self.create_instance_and_assign(job, worker_instances) else: log.warning( f'Over instance limit, waiting for instances to become ' f'available to run job {job.id}') return job # TODO(Challenge): For network separation: Set DEEPDRIVE_SIM_HOST # TODO(Challenge): For network separation: Set network tags between # bot and problem container for port 5557 return job
def execute(self, context=None, namespace=None, arguments=None): if namespace is None: namespace = {} package = self.package if package: __import__(package) namespace["__package__"] = package normally = False managers.script_manager.watch() try: self.bytecode.execute(context, namespace, arguments) normally = True # complete before exception was raised managers.script_manager.ignore() # and cancel exception except ScriptTimeoutError: if not normally: message = "Terminate %s due to timeout" % self log.warning(message) raise Exception(message) except (RenderTermination, CompilationError, RequirePrecompileError): raise except Exception: show_exception_trace(caption="Unhandled exception in %s" % self, locals=True) raise finally: managers.script_manager.leave()
def _discover(self, full=False, load=False): if managers.file_manager.exists(file_access.FILE, None, settings.INDEX_LOCATION): data = managers.file_manager.read(file_access.FILE, None, settings.INDEX_LOCATION) for line in data.splitlines(): try: uuid, name = parse_index_line(line) except ValueError: log.warning("Ignore erroneous index entry: %s" % line) continue self._items.setdefault(uuid, NOT_LOADED) self._index.setdefault(name, uuid) # additionally check directories listing = managers.file_manager.list(file_access.TYPE) for uuid in listing: try: verificators.uuid(uuid) except ValueError: log.warning("Ignore non-type directory: %s" % uuid) continue self._items.setdefault(uuid, NOT_LOADED) # load ONLY types that has no record in index if full and len(self._items) > len(self._index): unknown = set(self._items.keys()) - set(self._index.values()) for uuid in unknown: self._load(uuid) # load all types that's not loaded yet if load: for uuid, item in self._items.iteritems(): if item is NOT_LOADED: self._load(uuid) self._lazy = False
def save_to_bot_scores(eval_data, eval_key, new_score: Box): db = get_bot_scores_db() score_id = get_scores_id(eval_data) orig = db.get(score_id) bot_scores = db.get(score_id) or dbox(Box(scores=[])) recorded = bot_scores and \ any([b.eval_key == eval_key for b in bot_scores.scores]) if not recorded: bot_scores.scores.append(new_score) score_values = [s.score for s in bot_scores.scores] if len(bot_scores) < 2: score_stdev = None else: score_stdev = stdev(score_values) new_bot_scores = Box(scores=bot_scores.scores, id=score_id, botname=eval_data.botname, username=eval_data.username, problem_id=eval_data.problem_id, updated_at=SERVER_TIMESTAMP, mean=mean(score_values), max=max(score_values), min=min(score_values), median=median(score_values), stdev=score_stdev) if not orig: new_bot_scores.created_at = SERVER_TIMESTAMP if not db.cas(score_id, orig, new_bot_scores): log.warning('Race condition saving bot scores! Trying again.') save_to_bot_scores(eval_data, eval_key, new_score) else: log.success(f'Saved new bot scores ' f'{box2json(new_bot_scores)}')
def start_instance(self, inst): if in_test(): log.warning('Not starting instance in test') else: op = self.gce.instances().start(project=self.project, zone=self.zone, instance=inst.name).execute() self.gce_ops_in_progress.append(op)
def work(self): now = time() with self._lock: for thread, (overall, counter, deadline) in list(self._threads.items()): if counter and now > deadline: log.warning("Terminate %s due to timeout" % describe_thread(thread)) ctypes.pythonapi.PyThreadState_SetAsyncExc(thread.ident, ctypes.py_object(ScriptTimeoutError)) self._threads.pop(thread, None)
def install_type(self, value=None, file=None, filename=None, into=None): def cleanup(uuid): if uuid: self.cleanup_type_infrastructure(uuid) if uuid in self._types: del self._types[uuid] def on_information(type): if managers.file_manager.exists(file_access.TYPE, type.id): raise AlreadyExistsError("Type already installed") context.uuid = type.id self.prepare_type_infrastructure(type.id) if value: description = "string" elif filename: description = os.path.basename(filename) else: try: description = file.name except AttributeError: description = "file" log.write("Install type from %s" % description) parser = Parser(builder=type_builder, notify=True, options=on_information) context = Structure(uuid=None) try: if value: type = parser.parse(value.encode("utf8") if isinstance(value, unicode) else value) elif filename: type = parser.parse(filename=filename) else: type = parser.parse(file=file) if parser.report: if into is None: log.warning("Install type notifications") for lineno, message in parser.report: log.warning(" %s, line %s" % (message, lineno)) else: for lineno, message in parser.report: into.append((lineno, message)) type.save() self._types.save() return type except AlreadyExistsError as error: raise except ParsingException as error: cleanup(context.uuid) raise Exception("Unable to parse %s, line %s: %s" % (description, error.lineno, error)) except IOError as error: cleanup(context.uuid) raise Exception("Unable to read from %s: %s" % (description, error.strerror)) except: # noqa cleanup(context.uuid) raise
def _load(self, uuid): try: item = self._owner.load_type(uuid, silently=True) except IOError: log.warning("Unable to load type: %s" % uuid) return None else: self._items[uuid] = item self._index[item.name] = uuid return item
def loads(vdomxml, origin, profile=None): parser = Parser(builder=vdomxml_builder, lower=True, options=origin, notify=True) try: object = parser.parse(vdomxml) if parser.report: log.warning("Loads VDOM XML notifications") for lineno, message in parser.report: log.warning(" %s at line %s" % (message, lineno)) return object except ParsingException as error: raise Exception("Unable to parse VDOM XML, line %s: %s" % (error.lineno, error))
def make_instance_available(self, instance_id): # TODO: Move this into problem-constants and rename # problem-helpers as it's shared with problem-worker instance = self.instances_db.get(instance_id) if not instance: log.warning('Instance does not exist, perhaps it was terminated.') elif instance.status != constants.INSTANCE_STATUS_AVAILABLE: instance.status = constants.INSTANCE_STATUS_AVAILABLE instance.time_last_available = SERVER_TIMESTAMP self.instances_db.set(instance_id, instance) log.info(f'Made instance {instance_id} available') else: log.warning(f'Instance {instance_id} already available')
def trigger_job(instances_db, job_id, jobs_db, botleague_liaison_host, docker_tag=None, job_type=JOB_TYPE_EVAL): docker_tag = docker_tag or 'deepdriveio/problem-worker-test' eval_mgr = JobManager(jobs_db=jobs_db, instances_db=instances_db) eval_mgr.check_for_finished_jobs() test_job = Box(botleague_liaison_host=botleague_liaison_host, status=JOB_STATUS_CREATED, id=job_id, job_type=job_type, eval_spec=Box(docker_tag=docker_tag, eval_id=utils.generate_rand_alphanumeric(32), eval_key='fake_eval_key', seed=1, problem='domain_randomization', pull_request=None, max_seconds=20)) # Make a copy of prod instances prod_instances_db = get_worker_instances_db(force_firestore_db=True) for inst in prod_instances_db.where('id', '>', ''): instances_db.set(inst.id, inst) try: eval_mgr.jobs_db.set(job_id, test_job) new_jobs, exceptions = eval_mgr.assign_jobs() assert not exceptions if new_jobs: # We don't actually start instances but we act like we did. assert new_jobs[0].status == JOB_STATUS_CREATED or \ new_jobs[0].instance_id if 'instance_id' in new_jobs[0]: instance_meta = eval_mgr.instances_db.get( new_jobs[0].instance_id) # So we have real instance meta, but inserted the job into a # test collection that the instance is not watching. # So the job will not actually run. assert instance_meta.status == INSTANCE_STATUS_USED else: log.warning('Test did not find an instance to run. TODO: use' ' test instance data.') finally: if jobs_db is not None: jobs_db.delete_all_test_data() if instances_db is not None: instances_db.delete_all_test_data()
def volume_list(): try: cmd = "nova volume-list" cmd_result = shell.shell_cmd(cmd) with open(list_dir+list_file_volume,'w') as f: f.writelines(cmd_result) if cmd_result: logging.info("%s %s" %(tenant_name,cmd_result)) else: logging.warning("%s:Instace information is empty,please check the relevant configuration" %tenant_name) except IOError: logging.error("Tenant:%s %s" % (tenant_name)) raise return cmd_result
def obtain_semaphore(self, timeout=None) -> bool: start = time.time() # TODO: Avoid polling by creating a Firestore watch and using a # mutex to avoid multiple threads processing the watch. if self.db.get(STATUS) == Box(): log.warning('No semaphore document found, creating one!') self.db.set(STATUS, RUNNING + self.id) return True elif self.db.get(STATUS) in [Box(), STOPPED]: self.db.set(STATUS, RUNNING + self.id) return True self.request_semaphore() # TODO: Check for a third loop that requested access and alert, die, # or re-request. As-is we just zombie. while not self.granted_semaphore(): log.info('Waiting for other loop to end') if self.started_waiting_for_other_loop_time is None: self.started_waiting_for_other_loop_time = time.time() if time.time() - self.started_waiting_for_other_loop_time > 5: log.error('Assuming previous loop died without releasing ' 'semaphore. Setting to stopped.') self.db.set(STATUS, STOPPED) self.started_waiting_for_other_loop_time = None if self.kill_now: log.warning('Killing loop while requesting semaphore, ' 'here be dragons!') if self.db.compare_and_swap(STATUS, REQUESTED + self.id, self.previous_status): # Other loop never saw us, good! return False else: # We have problems if self.db.get(STATUS) == GRANTED + self.id: # Other loop beat us in a race to set status # and released so thinks we own the semaphore. self.release_semaphore() # TODO: Create an alert from this log. raise RuntimeError(f'No {self.id} running! ' f'Needs manual start') else: # Could be that a third loop requested. self.release_semaphore() # TODO: Create an alert from this log. raise RuntimeError(f'Race condition encountered in ' f'{self.id} Needs manual start') elif timeout is not None and time.time() - start > timeout: return False else: time.sleep(1) return True
def create_instance(self, current_instances): if in_test(): log.warning('Not creating instance in test') return None instance_name = self.get_next_instance_name(current_instances) config_path = os.path.join(ROOT, INSTANCE_CONFIG_PATH) config = Box.from_json(filename=config_path) # TODO: If job is CI, no GPU needed, but maybe more CPU config.name = instance_name config.disks[0].deviceName = instance_name create_op = Box( self.gce.instances().insert(project=self.project, zone=self.zone, body=config.to_dict()).execute()) return create_op
def handle_pr_request(payload): action = payload['action'] if action in ['opened', 'synchronize', 'reopened']: pr_processor = get_pr_processor() pr_event = Box(payload) pull_request = pr_event.pull_request pr_processor.pr_event = pr_event if get_liaison_host_override(pull_request) and ON_GAE: log.warning(f'DEBUG local set on pull request ' f'{pr_event.to_json(indent=2)} ' f'Skipping!') else: log.info(f'Processing pull request event') log.trace(f'{pr_event.to_json(indent=2)}') return pr_processor.process_changes()
def load_application(self, uuid, silently=False): log.write("Load application %s" % uuid) location = managers.file_manager.locate(file_access.APPLICATION, uuid, settings.APPLICATION_FILENAME) parser = Parser(builder=application_builder, notify=True) try: application = parser.parse(filename=location) if not silently or parser.report: log.write("Load %s" % application) for lineno, message in parser.report: log.warning(" %s at line %s" % (message, lineno)) return application except IOError as error: raise Exception("Unable to read from \"%s\": %s" % (os.path.basename(location), error.strerror)) except ParsingException as error: raise Exception("Unable to parse \"%s\", line %s: %s" % (os.path.basename(location), error.lineno, error))
def process_results(result_payload: Box, db: DB) -> Tuple[Error, Box, Box, Optional[str], bool]: eval_key = result_payload.get('eval_key', '') results = result_payload.get('results', Box()) results.finished = time.time() error = Error() eval_data = Box() gist = None should_skip = False # Note that 200, 400, and 500 are the ONLY expected status codes. # Other codes will be retried by the worker in post_results_with_retries. # This is due to App Engine returning 409's on the rare occasion. # https://voyage.slack.com/archives/CJLS63AMD/p1571773377003400 if not eval_key: error.http_status_code = 400 error.message = 'eval_key must be in JSON data payload' else: eval_data = get_eval_data(eval_key, db) if not eval_data: error.http_status_code = 400 error.message = 'Could not find evaluation with that key' elif eval_data.botleague_liaison_host != constants.HOST and \ constants.ON_GAE: log.warning('Not processing results due to botleague liaison ' 'host being overridden') should_skip = True elif eval_data.status == constants.EVAL_STATUS_STARTED: error.http_status_code = 400 error.message = 'This evaluation has not been confirmed' elif eval_data.status == constants.EVAL_STATUS_COMPLETE: error.http_status_code = 400 error.message = 'This evaluation has already been processed' elif eval_data.status == constants.EVAL_STATUS_CONFIRMED: if 'results' not in result_payload: error.http_status_code = 400 error.message = 'No "results" found in request' elif dbox(results).errors: error.http_status_code = 500 error.message = box2json(Box(results.errors)) add_eval_data_to_results(eval_data, results) gist = post_results_to_gist(db, results) gist = gist.html_url if gist else None trigger_leaderboard_generation() else: error.http_status_code = 400 error.message = 'Eval data status unknown %s' % eval_data.status return error, results, eval_data, gist, should_skip
def get_next_instance_name(current_instances): current_instance_names = [i.name for i in current_instances] current_instance_indexes = [] for name in current_instance_names: index = name[name.rindex('-') + 1:] if index.isdigit(): current_instance_indexes.append(int(index)) else: log.warning('Instance with non-numeric index in name found ' + name) if not current_instance_indexes: next_index = 0 else: next_index = max(current_instance_indexes) + 1 instance_name = INSTANCE_NAME_PREFIX + str(next_index) return instance_name
def load_type(self, uuid, silently=False): log.write("Load type %s" % uuid) location = managers.file_manager.locate(file_access.TYPE, uuid, settings.TYPE_FILENAME) parser = Parser(builder=type_builder, notify=True) try: type = parser.parse(filename=location) if not silently: log.write("Load %s as %s" % (type, type.name)) if parser.report: log.warning("Load %s notifications" % type) for lineno, message in parser.report: log.warning(" %s at line %s" % (message, lineno)) return type except IOError as error: raise Exception("Unable to read from \"%s\": %s" % (os.path.basename(location), error.strerror)) except ParsingException as error: raise Exception("Unable to parse \"%s\", line %s: %s" % (os.path.basename(location), error.lineno, error))
def granted_semaphore(self): granted = self.db.compare_and_swap( key=STATUS, expected_current_value=GRANTED + self.id, new_value=RUNNING + self.id) found_orphan = self.db.compare_and_swap( key=STATUS, expected_current_value=STOPPED, new_value=RUNNING + self.id) if found_orphan: # TODO: Create an alert from this log log.warning(f'Found orphaned semaphore: {self.id}. ' f'This can happen when this process is forcefully ' f'killed. ' f'i.e. stopping the instance from the UI can cause ' f'this. You can run `make stop` to properly shut ' f'down the server.') ret = granted or found_orphan return ret
def write(self, data, action_name=None): log.write(u"Write action %s data to client: %d characters" % (action_name, len(data))) render_type = managers.request_manager.current.render_type if render_type != "e2vdom": log.warning("Perform write when render type is \"%s\"" % render_type) from utils.tracing import format_thread_trace log.warning(format_thread_trace(statements=False, skip=("write", "action"), until="scripting.executable")) # else: # from utils.tracing import format_thread_trace # log.debug(format_thread_trace(statements=False, skip=("write", "action"), until="scripting.executable")) # log.debug( # u"- - - - - - - - - - - - - - - - - - - -\n" # u"%s\n" # u"- - - - - - - - - - - - - - - - - - - -\n" % # data, module=False) # self._compute_state = STATE_AVOID_RECOMPUTE managers.request_manager.current.add_client_action(self._id, data)
def start_container(self, docker_tag, cmd=None, env=None, volumes=None, name=None): def start(**options): return self.docker.containers.run(docker_tag, command=cmd, detach=True, stdout=False, stderr=False, environment=env, volumes=volumes, **options) try: container = start(**CONTAINER_RUN_OPTIONS) except Exception as e: if '"nvidia-container-runtime": executable file not found' in str( e): # TODO: Remove patch when # https://github.com/docker/docker-py/pull/2471 is merged from docker_gpu_patch import DeviceRequest log.warning('nvidia docker runtime not found, trying gpus=all') self.docker = docker.from_env(version='1.40') # Danger! Mutating global - okay for now as single threaded and # fallback is to just change again after exception del CONTAINER_RUN_OPTIONS['runtime'] CONTAINER_RUN_OPTIONS['device_requests'] = [ DeviceRequest(count=-1, capabilities=[['gpu']]) ] container = start(**CONTAINER_RUN_OPTIONS) else: raise e return container
def get_image(self, tag): log.info('Pulling docker image %s ...' % tag) try: result = self.docker.images.pull(tag) except: log.exception(f'Could not pull {tag}') ret = None else: log.info('Finished pulling docker image %s' % tag) if isinstance(result, list): ret = self.get_latest_docker_image(result) if ret is None: raise RuntimeError( f'Could not get pull latest image for {tag}. ' f'tags found were: {result}') elif isinstance(result, Image): ret = result else: log.warning(f'Got unexpected result when pulling {tag} ' f'of {result}') ret = result return ret
def action(self, action_name, arguments=[], source_id=None): render_type = managers.request_manager.current.render_type if render_type != "e2vdom": log.warning("Invoke client action when render type is \"%s\"" % render_type) if source_id: information = "SRC_ID=\"o_%s\" " % source_id.replace('-', '_') else: information = "" information += "DST_ID=\"%s\" ACT_NAME=\"%s\"" % (self._id.replace('-', '_'), action_name) if not isinstance(arguments, (tuple, list)): arguments = (arguments,) data = (("str", unicode(argument)) if isinstance(argument, basestring) else ("obj", unicode(json.dumps(argument)).encode("xml")) for argument in arguments) if arguments: self.write("<EXECUTE %s>\n%s\n</EXECUTE>" % (information, "\n".join(" <PARAM type=\"%s\">%s</PARAM>" % (kind, value.encode("xml")) for kind, value in data)), action_name) else: self.write("<EXECUTE %s/>" % information, action_name)
def inner(args): temp = func(args) file_name = path + io_generate_name() if temp == '4k': try: print "生成文件为:%s 大小为:4k" % file_name logging.info("生成文件为:%s 大小为:4k" % file_name) with open(file_name + '_4k', 'wb') as f: f.write(os.urandom(1024 * 4)) except IOError: print "写入4k文件失败" logging.error("写入4k文件失败") return 4 elif temp == '1m': try: print "生成文件为:%s 大小为:1m" % file_name logging.info("生成文件为:%s 大小为:1m" % file_name) with open(file_name + '_1m', 'wb') as f: f.write(os.urandom(1024 * 1024)) except IOError: print "写入1m文件失败" logging.error("写入1m文件失败") return 1000 elif temp == '10m': try: print "生成文件为:%s 大小为:10m" % file_name logging.info("生成文件为:%s 大小为:10m" % file_name) with open(file_name + '_10m', 'wb') as f: f.write(os.urandom(1024 * 1024 * 10)) except IOError: print "写入10m文件失败" logging.error("写入10m文件失败") return 10000 else: print "请输入写入的文件大小...<4k|1m|10m>" logging.warning("请输入写入的文件大小...<4k|1m|10m>") return temp
def inner(args): temp = func(args) file_name=path+io_generate_name() if temp == '4k': try: print "生成文件为:%s 大小为:4k" % file_name logging.info("生成文件为:%s 大小为:4k" % file_name) with open(file_name+'_4k','wb') as f: f.write(os.urandom(1024*4)) except IOError: print "写入4k文件失败" logging.error("写入4k文件失败") return 4 elif temp == '1m': try: print "生成文件为:%s 大小为:1m" % file_name logging.info("生成文件为:%s 大小为:1m" % file_name) with open(file_name+'_1m','wb') as f: f.write(os.urandom(1024*1024)) except IOError: print "写入1m文件失败" logging.error( "写入1m文件失败") return 1000 elif temp == '10m': try: print "生成文件为:%s 大小为:10m" % file_name logging.info("生成文件为:%s 大小为:10m" % file_name) with open(file_name+'_10m','wb') as f: f.write(os.urandom(1024*1024*10)) except IOError: print "写入10m文件失败" logging.error("写入10m文件失败") return 10000 else: print "请输入写入的文件大小...<4k|1m|10m>" logging.warning("请输入写入的文件大小...<4k|1m|10m>") return temp
def exit(self): self.release_semaphore() if in_test(): log.info('Not exiting in test') return elif self.caught_exception: log.error('Exiting due to caught exception') status = 100 elif self.caught_sigint: log.warning('Exiting due to caught sigint') status = 101 elif self.caught_sigterm: log.warning('Exiting due to caught sigterm') status = 102 else: log.error('Unexpected reason for exit') status = 1 log.warning(f'Exiting with status {status}') # http://tldp.org/LDP/abs/html/exitcodes.html sys.exit(status)
def handle_sigint(self, signum=None, frame=None): log.warning(f'Sigint caught {signum} {frame}') self.kill_now = True self.caught_sigint = True
def instance_start(): check_old_id = [] check_id = [] server_shutdown_id = [] cmd_stop = "nova list|grep Shutdown |awk '{print $2}'" result_stop = shell.shell_cmd(cmd_stop) for j in xrange(len(result_stop)): pstop = result_stop[j].split()[0] server_shutdown_id.append(pstop) ######################## # instnace_id_list ################################# instance_id_list = server_shutdown_id ##################### # volume_id_list ########################## volume_id_list = [] volume_list = list_dir+list_file_volume cmd_volume = "grep in-use %s |awk '{print $12}'" %volume_list result_volume = shell.shell_cmd(cmd_volume) for v in xrange(len(result_volume)): v_volume = result_volume[v].split()[0] volume_id_list.append(v_volume) ############################# # instnace_id_list and volume_id_list difference ######################### difference=set(instance_id_list).difference(set(volume_id_list)) # No instances of mount cloud drive for di in difference: cmd_start = "nova start %s" % di result_start = shell.shell_cmd(cmd_start) logging.debug("instance[%s no mount cloud drive] already starting..." %di ) print "instance[%s no mount cloud drive] already starting... " %di if server_shutdown_id: if os.path.exists(volume_list) and not os.path.getsize(volume_list): for s_start in server_shutdown_id: # old volume id check_cmd_old = "grep %s |awk '{print $2}'" % s_start check_old_result = shell.shell_cmd(check_cmd_old) for x in xrange(len(check_old_result)): old = check_old_result[x].split()[0] check_old_id.append(old) # current volume id check_cmd = "nova volume-list |grep %s|awk '{print $2}'" % s_start check_result = shell.shell_cmd(check_cmd) for y in xrange(len(check_result)): current = check_result[y].split()[0] check_id.append(current) # intersection check_intersection = set(check_old_oid).intersection(set(check_id)) for id in check_intersection: cmd = "nova start %s" % s_start result = shell.shell_cmd(cmd) logging.debug("instance[%s] already starting... " %s_start ) print "instance[%s] already starting... " %s_start else: logging.warning("%s:%s file does not exit or is empty,without the basis of a instance to start!!!" %(tenant_name,volume_list)) else: logging.warning("%s:Instace information is empty,please check the relevant configuration" %tenant_name)
def install_application(self, value=None, file=None, filename=None, into=None): def cleanup(uuid): if uuid: self.cleanup_application_infrastructure(uuid) if uuid in self._applications: del self._applications[uuid] def on_information(application): if managers.file_manager.exists(file_access.APPLICATION, application.id): raise AlreadyExistsError("Application already installed") context.uuid = application.id self.prepare_application_infrastructure(application.id) # TODO: ENABLE THIS LATER # if application.server_version and VDOM_server_version < application.server_version: # raise Exception("Server version %s is unsuitable for this application %s" % (VDOM_server_version, application.server_version)) # TODO: License key... if value: description = "string" elif filename: description = os.path.basename(filename) else: try: description = file.name except AttributeError: description = "file" log.write("Install application from %s" % description) parser = Parser(builder=application_builder, notify=True, options=on_information) context = Structure(uuid=None) try: if value: application = parser.parse(value.encode("utf8") if isinstance(value, unicode) else value) elif filename: application = parser.parse(filename=filename) else: application = parser.parse(file=file) if parser.report: if into is None: log.warning("Install application notifications") for lineno, message in parser.report: log.warning(" %s, line %s" % (message, lineno)) else: for lineno, message in parser.report: into.append((lineno, message)) # TODO: check this later - why? application.unimport_libraries() application.save() return application except AlreadyExistsError as error: raise except ParsingException as error: cleanup(context.uuid) raise Exception("Unable to parse %s, line %s: %s" % (description, error.lineno, error)) except IOError as error: cleanup(context.uuid) raise Exception("Unable to read %s: %s" % (description, error.strerror)) except: # noqa cleanup(context.uuid) raise