def score_within_confidence_interval(bot_eval: Box,
past_bot_scores: Box) -> Tuple[bool, Box]:
"""
Compare with current mean score and check within
acceptable_score_deviation range.
If only 1 score, roughly
double the acceptable range, since we could
have gone from min to max.
Also, follow the 2-sided CI for a t-student distribution
that gives ~2x the acceptable_score_deviation with infinite
samples (i.e. ± acceptable_score_deviation)
https://en.wikipedia.org/wiki/Student%27s_t-distribution#Table_of_selected_values
https://stats.stackexchange.com/a/230185/18187
n Confidence Level Multiplicative Factor
2 0.95 12.71
3 0.95 4.30
4 0.95 3.18
5 0.95 2.78
infinity 0.95 1.96
"""
info = Box(mean=None,
ci_high=None,
ci_low=None,
acceptable_score_deviation=None)
if bot_eval.eval_key in [p.eval_key for p in past_bot_scores.scores]:
log.warning('Score already recorded, this should not happen!')
return True, info
score = bot_eval.results.score
acceptable_score_deviation = bot_eval.problem_def.acceptable_score_deviation
if not past_bot_scores.scores:
# If no previous scores, then we are the mean of the CI
return True, info
score_values = [b.score for b in past_bot_scores.scores]
multiplier = {
2: 12.71,
3: 4.30,
4: 3.18,
5: 2.78,
}.get(len(score_values) + 1, 1.96)
diff_max = acceptable_score_deviation * multiplier / 2
ci_low = past_bot_scores.mean - diff_max
ci_high = past_bot_scores.mean + diff_max
info.high = ci_high
info.low = ci_low
info.mean = past_bot_scores.mean
info.acceptable_score_deviation = acceptable_score_deviation
if math.nan in [ci_high, ci_low]:
ret = True
elif ci_low <= score <= ci_high:
ret = True
else:
ret = False
return ret, info
def assign_job(self, job) -> Optional[Box]:
if dbox(job).run_local_debug:
log.warning(f'Run local debug is true, setting instance id to '
f'{constants.LOCAL_INSTANCE_ID}')
self.assign_job_to_instance(constants.LOCAL_INSTANCE_ID, job)
return job
worker_instances = self.get_worker_instances()
self.prune_terminated_instances(worker_instances)
available_running_instances, available_stopped_instances = \
self.get_available_instances(worker_instances)
if available_running_instances:
self.assign_to_running_instance(available_running_instances, job)
elif available_stopped_instances:
self.start_instance_and_assign(available_stopped_instances, job)
else:
if len(worker_instances) < MAX_WORKER_INSTANCES:
self.create_instance_and_assign(job, worker_instances)
else:
log.warning(
f'Over instance limit, waiting for instances to become '
f'available to run job {job.id}')
return job
# TODO(Challenge): For network separation: Set DEEPDRIVE_SIM_HOST
# TODO(Challenge): For network separation: Set network tags between
# bot and problem container for port 5557
return job
def execute(self, context=None, namespace=None, arguments=None):
if namespace is None:
namespace = {}
package = self.package
if package:
__import__(package)
namespace["__package__"] = package
normally = False
managers.script_manager.watch()
try:
self.bytecode.execute(context, namespace, arguments)
normally = True # complete before exception was raised
managers.script_manager.ignore() # and cancel exception
except ScriptTimeoutError:
if not normally:
message = "Terminate %s due to timeout" % self
log.warning(message)
raise Exception(message)
except (RenderTermination, CompilationError, RequirePrecompileError):
raise
except Exception:
show_exception_trace(caption="Unhandled exception in %s" % self, locals=True)
raise
finally:
managers.script_manager.leave()
def _discover(self, full=False, load=False):
if managers.file_manager.exists(file_access.FILE, None, settings.INDEX_LOCATION):
data = managers.file_manager.read(file_access.FILE, None, settings.INDEX_LOCATION)
for line in data.splitlines():
try:
uuid, name = parse_index_line(line)
except ValueError:
log.warning("Ignore erroneous index entry: %s" % line)
continue
self._items.setdefault(uuid, NOT_LOADED)
self._index.setdefault(name, uuid)
# additionally check directories
listing = managers.file_manager.list(file_access.TYPE)
for uuid in listing:
try:
verificators.uuid(uuid)
except ValueError:
log.warning("Ignore non-type directory: %s" % uuid)
continue
self._items.setdefault(uuid, NOT_LOADED)
# load ONLY types that has no record in index
if full and len(self._items) > len(self._index):
unknown = set(self._items.keys()) - set(self._index.values())
for uuid in unknown:
self._load(uuid)
# load all types that's not loaded yet
if load:
for uuid, item in self._items.iteritems():
if item is NOT_LOADED:
self._load(uuid)
self._lazy = False
def save_to_bot_scores(eval_data, eval_key, new_score: Box):
db = get_bot_scores_db()
score_id = get_scores_id(eval_data)
orig = db.get(score_id)
bot_scores = db.get(score_id) or dbox(Box(scores=[]))
recorded = bot_scores and \
any([b.eval_key == eval_key for b in bot_scores.scores])
if not recorded:
bot_scores.scores.append(new_score)
score_values = [s.score for s in bot_scores.scores]
if len(bot_scores) < 2:
score_stdev = None
else:
score_stdev = stdev(score_values)
new_bot_scores = Box(scores=bot_scores.scores,
id=score_id,
botname=eval_data.botname,
username=eval_data.username,
problem_id=eval_data.problem_id,
updated_at=SERVER_TIMESTAMP,
mean=mean(score_values),
max=max(score_values),
min=min(score_values),
median=median(score_values),
stdev=score_stdev)
if not orig:
new_bot_scores.created_at = SERVER_TIMESTAMP
if not db.cas(score_id, orig, new_bot_scores):
log.warning('Race condition saving bot scores! Trying again.')
save_to_bot_scores(eval_data, eval_key, new_score)
else:
log.success(f'Saved new bot scores ' f'{box2json(new_bot_scores)}')
def start_instance(self, inst):
if in_test():
log.warning('Not starting instance in test')
else:
op = self.gce.instances().start(project=self.project,
zone=self.zone,
instance=inst.name).execute()
self.gce_ops_in_progress.append(op)
def work(self):
now = time()
with self._lock:
for thread, (overall, counter, deadline) in list(self._threads.items()):
if counter and now > deadline:
log.warning("Terminate %s due to timeout" % describe_thread(thread))
ctypes.pythonapi.PyThreadState_SetAsyncExc(thread.ident, ctypes.py_object(ScriptTimeoutError))
self._threads.pop(thread, None)
def install_type(self, value=None, file=None, filename=None, into=None):
def cleanup(uuid):
if uuid:
self.cleanup_type_infrastructure(uuid)
if uuid in self._types:
del self._types[uuid]
def on_information(type):
if managers.file_manager.exists(file_access.TYPE, type.id):
raise AlreadyExistsError("Type already installed")
context.uuid = type.id
self.prepare_type_infrastructure(type.id)
if value:
description = "string"
elif filename:
description = os.path.basename(filename)
else:
try:
description = file.name
except AttributeError:
description = "file"
log.write("Install type from %s" % description)
parser = Parser(builder=type_builder, notify=True, options=on_information)
context = Structure(uuid=None)
try:
if value:
type = parser.parse(value.encode("utf8") if isinstance(value, unicode) else value)
elif filename:
type = parser.parse(filename=filename)
else:
type = parser.parse(file=file)
if parser.report:
if into is None:
log.warning("Install type notifications")
for lineno, message in parser.report:
log.warning(" %s, line %s" % (message, lineno))
else:
for lineno, message in parser.report:
into.append((lineno, message))
type.save()
self._types.save()
return type
except AlreadyExistsError as error:
raise
except ParsingException as error:
cleanup(context.uuid)
raise Exception("Unable to parse %s, line %s: %s" % (description, error.lineno, error))
except IOError as error:
cleanup(context.uuid)
raise Exception("Unable to read from %s: %s" % (description, error.strerror))
except: # noqa
cleanup(context.uuid)
raise
def _load(self, uuid):
try:
item = self._owner.load_type(uuid, silently=True)
except IOError:
log.warning("Unable to load type: %s" % uuid)
return None
else:
self._items[uuid] = item
self._index[item.name] = uuid
return item
def loads(vdomxml, origin, profile=None):
parser = Parser(builder=vdomxml_builder, lower=True, options=origin, notify=True)
try:
object = parser.parse(vdomxml)
if parser.report:
log.warning("Loads VDOM XML notifications")
for lineno, message in parser.report:
log.warning(" %s at line %s" % (message, lineno))
return object
except ParsingException as error:
raise Exception("Unable to parse VDOM XML, line %s: %s" % (error.lineno, error))
def make_instance_available(self, instance_id):
# TODO: Move this into problem-constants and rename
# problem-helpers as it's shared with problem-worker
instance = self.instances_db.get(instance_id)
if not instance:
log.warning('Instance does not exist, perhaps it was terminated.')
elif instance.status != constants.INSTANCE_STATUS_AVAILABLE:
instance.status = constants.INSTANCE_STATUS_AVAILABLE
instance.time_last_available = SERVER_TIMESTAMP
self.instances_db.set(instance_id, instance)
log.info(f'Made instance {instance_id} available')
else:
log.warning(f'Instance {instance_id} already available')
def trigger_job(instances_db,
job_id,
jobs_db,
botleague_liaison_host,
docker_tag=None,
job_type=JOB_TYPE_EVAL):
docker_tag = docker_tag or 'deepdriveio/problem-worker-test'
eval_mgr = JobManager(jobs_db=jobs_db, instances_db=instances_db)
eval_mgr.check_for_finished_jobs()
test_job = Box(botleague_liaison_host=botleague_liaison_host,
status=JOB_STATUS_CREATED,
id=job_id,
job_type=job_type,
eval_spec=Box(docker_tag=docker_tag,
eval_id=utils.generate_rand_alphanumeric(32),
eval_key='fake_eval_key',
seed=1,
problem='domain_randomization',
pull_request=None,
max_seconds=20))
# Make a copy of prod instances
prod_instances_db = get_worker_instances_db(force_firestore_db=True)
for inst in prod_instances_db.where('id', '>', ''):
instances_db.set(inst.id, inst)
try:
eval_mgr.jobs_db.set(job_id, test_job)
new_jobs, exceptions = eval_mgr.assign_jobs()
assert not exceptions
if new_jobs:
# We don't actually start instances but we act like we did.
assert new_jobs[0].status == JOB_STATUS_CREATED or \
new_jobs[0].instance_id
if 'instance_id' in new_jobs[0]:
instance_meta = eval_mgr.instances_db.get(
new_jobs[0].instance_id)
# So we have real instance meta, but inserted the job into a
# test collection that the instance is not watching.
# So the job will not actually run.
assert instance_meta.status == INSTANCE_STATUS_USED
else:
log.warning('Test did not find an instance to run. TODO: use'
' test instance data.')
finally:
if jobs_db is not None:
jobs_db.delete_all_test_data()
if instances_db is not None:
instances_db.delete_all_test_data()
def volume_list():
try:
cmd = "nova volume-list"
cmd_result = shell.shell_cmd(cmd)
with open(list_dir+list_file_volume,'w') as f:
f.writelines(cmd_result)
if cmd_result:
logging.info("%s %s" %(tenant_name,cmd_result))
else:
logging.warning("%s:Instace information is empty,please check the relevant configuration" %tenant_name)
except IOError:
logging.error("Tenant:%s %s" % (tenant_name))
raise
return cmd_result
def volume_list():
try:
cmd = "nova volume-list"
cmd_result = shell.shell_cmd(cmd)
with open(list_dir+list_file_volume,'w') as f:
f.writelines(cmd_result)
if cmd_result:
logging.info("%s %s" %(tenant_name,cmd_result))
else:
logging.warning("%s:Instace information is empty,please check the relevant configuration" %tenant_name)
except IOError:
logging.error("Tenant:%s %s" % (tenant_name))
raise
return cmd_result
def obtain_semaphore(self, timeout=None) -> bool:
start = time.time()
# TODO: Avoid polling by creating a Firestore watch and using a
# mutex to avoid multiple threads processing the watch.
if self.db.get(STATUS) == Box():
log.warning('No semaphore document found, creating one!')
self.db.set(STATUS, RUNNING + self.id)
return True
elif self.db.get(STATUS) in [Box(), STOPPED]:
self.db.set(STATUS, RUNNING + self.id)
return True
self.request_semaphore()
# TODO: Check for a third loop that requested access and alert, die,
# or re-request. As-is we just zombie.
while not self.granted_semaphore():
log.info('Waiting for other loop to end')
if self.started_waiting_for_other_loop_time is None:
self.started_waiting_for_other_loop_time = time.time()
if time.time() - self.started_waiting_for_other_loop_time > 5:
log.error('Assuming previous loop died without releasing '
'semaphore. Setting to stopped.')
self.db.set(STATUS, STOPPED)
self.started_waiting_for_other_loop_time = None
if self.kill_now:
log.warning('Killing loop while requesting semaphore, '
'here be dragons!')
if self.db.compare_and_swap(STATUS, REQUESTED + self.id,
self.previous_status):
# Other loop never saw us, good!
return False
else:
# We have problems
if self.db.get(STATUS) == GRANTED + self.id:
# Other loop beat us in a race to set status
# and released so thinks we own the semaphore.
self.release_semaphore()
# TODO: Create an alert from this log.
raise RuntimeError(f'No {self.id} running! '
f'Needs manual start')
else:
# Could be that a third loop requested.
self.release_semaphore()
# TODO: Create an alert from this log.
raise RuntimeError(f'Race condition encountered in '
f'{self.id} Needs manual start')
elif timeout is not None and time.time() - start > timeout:
return False
else:
time.sleep(1)
return True
def create_instance(self, current_instances):
if in_test():
log.warning('Not creating instance in test')
return None
instance_name = self.get_next_instance_name(current_instances)
config_path = os.path.join(ROOT, INSTANCE_CONFIG_PATH)
config = Box.from_json(filename=config_path)
# TODO: If job is CI, no GPU needed, but maybe more CPU
config.name = instance_name
config.disks[0].deviceName = instance_name
create_op = Box(
self.gce.instances().insert(project=self.project,
zone=self.zone,
body=config.to_dict()).execute())
return create_op
def handle_pr_request(payload):
action = payload['action']
if action in ['opened', 'synchronize', 'reopened']:
pr_processor = get_pr_processor()
pr_event = Box(payload)
pull_request = pr_event.pull_request
pr_processor.pr_event = pr_event
if get_liaison_host_override(pull_request) and ON_GAE:
log.warning(f'DEBUG local set on pull request '
f'{pr_event.to_json(indent=2)} '
f'Skipping!')
else:
log.info(f'Processing pull request event')
log.trace(f'{pr_event.to_json(indent=2)}')
return pr_processor.process_changes()
def load_application(self, uuid, silently=False):
log.write("Load application %s" % uuid)
location = managers.file_manager.locate(file_access.APPLICATION, uuid, settings.APPLICATION_FILENAME)
parser = Parser(builder=application_builder, notify=True)
try:
application = parser.parse(filename=location)
if not silently or parser.report:
log.write("Load %s" % application)
for lineno, message in parser.report:
log.warning(" %s at line %s" % (message, lineno))
return application
except IOError as error:
raise Exception("Unable to read from \"%s\": %s" % (os.path.basename(location), error.strerror))
except ParsingException as error:
raise Exception("Unable to parse \"%s\", line %s: %s" % (os.path.basename(location), error.lineno, error))
def process_results(result_payload: Box,
db: DB) -> Tuple[Error, Box, Box, Optional[str], bool]:
eval_key = result_payload.get('eval_key', '')
results = result_payload.get('results', Box())
results.finished = time.time()
error = Error()
eval_data = Box()
gist = None
should_skip = False
# Note that 200, 400, and 500 are the ONLY expected status codes.
# Other codes will be retried by the worker in post_results_with_retries.
# This is due to App Engine returning 409's on the rare occasion.
# https://voyage.slack.com/archives/CJLS63AMD/p1571773377003400
if not eval_key:
error.http_status_code = 400
error.message = 'eval_key must be in JSON data payload'
else:
eval_data = get_eval_data(eval_key, db)
if not eval_data:
error.http_status_code = 400
error.message = 'Could not find evaluation with that key'
elif eval_data.botleague_liaison_host != constants.HOST and \
constants.ON_GAE:
log.warning('Not processing results due to botleague liaison '
'host being overridden')
should_skip = True
elif eval_data.status == constants.EVAL_STATUS_STARTED:
error.http_status_code = 400
error.message = 'This evaluation has not been confirmed'
elif eval_data.status == constants.EVAL_STATUS_COMPLETE:
error.http_status_code = 400
error.message = 'This evaluation has already been processed'
elif eval_data.status == constants.EVAL_STATUS_CONFIRMED:
if 'results' not in result_payload:
error.http_status_code = 400
error.message = 'No "results" found in request'
elif dbox(results).errors:
error.http_status_code = 500
error.message = box2json(Box(results.errors))
add_eval_data_to_results(eval_data, results)
gist = post_results_to_gist(db, results)
gist = gist.html_url if gist else None
trigger_leaderboard_generation()
else:
error.http_status_code = 400
error.message = 'Eval data status unknown %s' % eval_data.status
return error, results, eval_data, gist, should_skip
def get_next_instance_name(current_instances):
current_instance_names = [i.name for i in current_instances]
current_instance_indexes = []
for name in current_instance_names:
index = name[name.rindex('-') + 1:]
if index.isdigit():
current_instance_indexes.append(int(index))
else:
log.warning('Instance with non-numeric index in name found ' +
name)
if not current_instance_indexes:
next_index = 0
else:
next_index = max(current_instance_indexes) + 1
instance_name = INSTANCE_NAME_PREFIX + str(next_index)
return instance_name
def load_type(self, uuid, silently=False):
log.write("Load type %s" % uuid)
location = managers.file_manager.locate(file_access.TYPE, uuid, settings.TYPE_FILENAME)
parser = Parser(builder=type_builder, notify=True)
try:
type = parser.parse(filename=location)
if not silently:
log.write("Load %s as %s" % (type, type.name))
if parser.report:
log.warning("Load %s notifications" % type)
for lineno, message in parser.report:
log.warning(" %s at line %s" % (message, lineno))
return type
except IOError as error:
raise Exception("Unable to read from \"%s\": %s" % (os.path.basename(location), error.strerror))
except ParsingException as error:
raise Exception("Unable to parse \"%s\", line %s: %s" % (os.path.basename(location), error.lineno, error))
def granted_semaphore(self):
granted = self.db.compare_and_swap(
key=STATUS,
expected_current_value=GRANTED + self.id,
new_value=RUNNING + self.id)
found_orphan = self.db.compare_and_swap(
key=STATUS,
expected_current_value=STOPPED,
new_value=RUNNING + self.id)
if found_orphan:
# TODO: Create an alert from this log
log.warning(f'Found orphaned semaphore: {self.id}. '
f'This can happen when this process is forcefully '
f'killed. '
f'i.e. stopping the instance from the UI can cause '
f'this. You can run `make stop` to properly shut '
f'down the server.')
ret = granted or found_orphan
return ret
def write(self, data, action_name=None):
log.write(u"Write action %s data to client: %d characters" % (action_name, len(data)))
render_type = managers.request_manager.current.render_type
if render_type != "e2vdom":
log.warning("Perform write when render type is \"%s\"" % render_type)
from utils.tracing import format_thread_trace
log.warning(format_thread_trace(statements=False, skip=("write", "action"), until="scripting.executable"))
# else:
# from utils.tracing import format_thread_trace
# log.debug(format_thread_trace(statements=False, skip=("write", "action"), until="scripting.executable"))
# log.debug(
# u"- - - - - - - - - - - - - - - - - - - -\n"
# u"%s\n"
# u"- - - - - - - - - - - - - - - - - - - -\n" %
# data, module=False)
# self._compute_state = STATE_AVOID_RECOMPUTE
managers.request_manager.current.add_client_action(self._id, data)
def start_container(self,
docker_tag,
cmd=None,
env=None,
volumes=None,
name=None):
def start(**options):
return self.docker.containers.run(docker_tag,
command=cmd,
detach=True,
stdout=False,
stderr=False,
environment=env,
volumes=volumes,
**options)
try:
container = start(**CONTAINER_RUN_OPTIONS)
except Exception as e:
if '"nvidia-container-runtime": executable file not found' in str(
e):
# TODO: Remove patch when
# https://github.com/docker/docker-py/pull/2471 is merged
from docker_gpu_patch import DeviceRequest
log.warning('nvidia docker runtime not found, trying gpus=all')
self.docker = docker.from_env(version='1.40')
# Danger! Mutating global - okay for now as single threaded and
# fallback is to just change again after exception
del CONTAINER_RUN_OPTIONS['runtime']
CONTAINER_RUN_OPTIONS['device_requests'] = [
DeviceRequest(count=-1, capabilities=[['gpu']])
]
container = start(**CONTAINER_RUN_OPTIONS)
else:
raise e
return container
def get_image(self, tag):
log.info('Pulling docker image %s ...' % tag)
try:
result = self.docker.images.pull(tag)
except:
log.exception(f'Could not pull {tag}')
ret = None
else:
log.info('Finished pulling docker image %s' % tag)
if isinstance(result, list):
ret = self.get_latest_docker_image(result)
if ret is None:
raise RuntimeError(
f'Could not get pull latest image for {tag}. '
f'tags found were: {result}')
elif isinstance(result, Image):
ret = result
else:
log.warning(f'Got unexpected result when pulling {tag} '
f'of {result}')
ret = result
return ret
def action(self, action_name, arguments=[], source_id=None):
render_type = managers.request_manager.current.render_type
if render_type != "e2vdom":
log.warning("Invoke client action when render type is \"%s\"" % render_type)
if source_id:
information = "SRC_ID=\"o_%s\" " % source_id.replace('-', '_')
else:
information = ""
information += "DST_ID=\"%s\" ACT_NAME=\"%s\"" % (self._id.replace('-', '_'), action_name)
if not isinstance(arguments, (tuple, list)):
arguments = (arguments,)
data = (("str", unicode(argument)) if isinstance(argument, basestring) else
("obj", unicode(json.dumps(argument)).encode("xml")) for argument in arguments)
if arguments:
self.write("<EXECUTE %s>\n%s\n</EXECUTE>" % (information,
"\n".join(" <PARAM type=\"%s\">%s</PARAM>" % (kind, value.encode("xml")) for kind, value in data)), action_name)
else:
self.write("<EXECUTE %s/>" % information, action_name)
def inner(args):
temp = func(args)
file_name = path + io_generate_name()
if temp == '4k':
try:
print "生成文件为:%s 大小为:4k" % file_name
logging.info("生成文件为:%s 大小为:4k" % file_name)
with open(file_name + '_4k', 'wb') as f:
f.write(os.urandom(1024 * 4))
except IOError:
print "写入4k文件失败"
logging.error("写入4k文件失败")
return 4
elif temp == '1m':
try:
print "生成文件为:%s 大小为:1m" % file_name
logging.info("生成文件为:%s 大小为:1m" % file_name)
with open(file_name + '_1m', 'wb') as f:
f.write(os.urandom(1024 * 1024))
except IOError:
print "写入1m文件失败"
logging.error("写入1m文件失败")
return 1000
elif temp == '10m':
try:
print "生成文件为:%s 大小为:10m" % file_name
logging.info("生成文件为:%s 大小为:10m" % file_name)
with open(file_name + '_10m', 'wb') as f:
f.write(os.urandom(1024 * 1024 * 10))
except IOError:
print "写入10m文件失败"
logging.error("写入10m文件失败")
return 10000
else:
print "请输入写入的文件大小...<4k|1m|10m>"
logging.warning("请输入写入的文件大小...<4k|1m|10m>")
return temp
def inner(args):
temp = func(args)
file_name=path+io_generate_name()
if temp == '4k':
try:
print "生成文件为:%s 大小为:4k" % file_name
logging.info("生成文件为:%s 大小为:4k" % file_name)
with open(file_name+'_4k','wb') as f:
f.write(os.urandom(1024*4))
except IOError:
print "写入4k文件失败"
logging.error("写入4k文件失败")
return 4
elif temp == '1m':
try:
print "生成文件为:%s 大小为:1m" % file_name
logging.info("生成文件为:%s 大小为:1m" % file_name)
with open(file_name+'_1m','wb') as f:
f.write(os.urandom(1024*1024))
except IOError:
print "写入1m文件失败"
logging.error( "写入1m文件失败")
return 1000
elif temp == '10m':
try:
print "生成文件为:%s 大小为:10m" % file_name
logging.info("生成文件为:%s 大小为:10m" % file_name)
with open(file_name+'_10m','wb') as f:
f.write(os.urandom(1024*1024*10))
except IOError:
print "写入10m文件失败"
logging.error("写入10m文件失败")
return 10000
else:
print "请输入写入的文件大小...<4k|1m|10m>"
logging.warning("请输入写入的文件大小...<4k|1m|10m>")
return temp
def exit(self):
self.release_semaphore()
if in_test():
log.info('Not exiting in test')
return
elif self.caught_exception:
log.error('Exiting due to caught exception')
status = 100
elif self.caught_sigint:
log.warning('Exiting due to caught sigint')
status = 101
elif self.caught_sigterm:
log.warning('Exiting due to caught sigterm')
status = 102
else:
log.error('Unexpected reason for exit')
status = 1
log.warning(f'Exiting with status {status}')
# http://tldp.org/LDP/abs/html/exitcodes.html
sys.exit(status)
def handle_sigint(self, signum=None, frame=None):
log.warning(f'Sigint caught {signum} {frame}')
self.kill_now = True
self.caught_sigint = True
def instance_start():
check_old_id = []
check_id = []
server_shutdown_id = []
cmd_stop = "nova list|grep Shutdown |awk '{print $2}'"
result_stop = shell.shell_cmd(cmd_stop)
for j in xrange(len(result_stop)):
pstop = result_stop[j].split()[0]
server_shutdown_id.append(pstop)
########################
# instnace_id_list
#################################
instance_id_list = server_shutdown_id
#####################
# volume_id_list
##########################
volume_id_list = []
volume_list = list_dir+list_file_volume
cmd_volume = "grep in-use %s |awk '{print $12}'" %volume_list
result_volume = shell.shell_cmd(cmd_volume)
for v in xrange(len(result_volume)):
v_volume = result_volume[v].split()[0]
volume_id_list.append(v_volume)
#############################
# instnace_id_list and volume_id_list difference
#########################
difference=set(instance_id_list).difference(set(volume_id_list))
# No instances of mount cloud drive
for di in difference:
cmd_start = "nova start %s" % di
result_start = shell.shell_cmd(cmd_start)
logging.debug("instance[%s no mount cloud drive] already starting..." %di )
print "instance[%s no mount cloud drive] already starting... " %di
if server_shutdown_id:
if os.path.exists(volume_list) and not os.path.getsize(volume_list):
for s_start in server_shutdown_id:
# old volume id
check_cmd_old = "grep %s |awk '{print $2}'" % s_start
check_old_result = shell.shell_cmd(check_cmd_old)
for x in xrange(len(check_old_result)):
old = check_old_result[x].split()[0]
check_old_id.append(old)
# current volume id
check_cmd = "nova volume-list |grep %s|awk '{print $2}'" % s_start
check_result = shell.shell_cmd(check_cmd)
for y in xrange(len(check_result)):
current = check_result[y].split()[0]
check_id.append(current)
# intersection
check_intersection = set(check_old_oid).intersection(set(check_id))
for id in check_intersection:
cmd = "nova start %s" % s_start
result = shell.shell_cmd(cmd)
logging.debug("instance[%s] already starting... " %s_start )
print "instance[%s] already starting... " %s_start
else:
logging.warning("%s:%s file does not exit or is empty,without the basis of a instance to start!!!" %(tenant_name,volume_list))
else:
logging.warning("%s:Instace information is empty,please check the relevant configuration" %tenant_name)
def instance_start():
check_old_id = []
check_id = []
server_shutdown_id = []
cmd_stop = "nova list|grep Shutdown |awk '{print $2}'"
result_stop = shell.shell_cmd(cmd_stop)
for j in xrange(len(result_stop)):
pstop = result_stop[j].split()[0]
server_shutdown_id.append(pstop)
########################
# instnace_id_list
#################################
instance_id_list = server_shutdown_id
#####################
# volume_id_list
##########################
volume_id_list = []
volume_list = list_dir+list_file_volume
cmd_volume = "grep in-use %s |awk '{print $12}'" %volume_list
result_volume = shell.shell_cmd(cmd_volume)
for v in xrange(len(result_volume)):
v_volume = result_volume[v].split()[0]
volume_id_list.append(v_volume)
#############################
# instnace_id_list and volume_id_list difference
#########################
difference=set(instance_id_list).difference(set(volume_id_list))
# No instances of mount cloud drive
for di in difference:
cmd_start = "nova start %s" % di
result_start = shell.shell_cmd(cmd_start)
logging.debug("instance[%s no mount cloud drive] already starting..." %di )
print "instance[%s no mount cloud drive] already starting... " %di
if server_shutdown_id:
if os.path.exists(volume_list) and not os.path.getsize(volume_list):
for s_start in server_shutdown_id:
# old volume id
check_cmd_old = "grep %s |awk '{print $2}'" % s_start
check_old_result = shell.shell_cmd(check_cmd_old)
for x in xrange(len(check_old_result)):
old = check_old_result[x].split()[0]
check_old_id.append(old)
# current volume id
check_cmd = "nova volume-list |grep %s|awk '{print $2}'" % s_start
check_result = shell.shell_cmd(check_cmd)
for y in xrange(len(check_result)):
current = check_result[y].split()[0]
check_id.append(current)
# intersection
check_intersection = set(check_old_oid).intersection(set(check_id))
for id in check_intersection:
cmd = "nova start %s" % s_start
result = shell.shell_cmd(cmd)
logging.debug("instance[%s] already starting... " %s_start )
print "instance[%s] already starting... " %s_start
else:
logging.warning("%s:%s file does not exit or is empty,without the basis of a instance to start!!!" %(tenant_name,volume_list))
else:
logging.warning("%s:Instace information is empty,please check the relevant configuration" %tenant_name)
def install_application(self, value=None, file=None, filename=None, into=None):
def cleanup(uuid):
if uuid:
self.cleanup_application_infrastructure(uuid)
if uuid in self._applications:
del self._applications[uuid]
def on_information(application):
if managers.file_manager.exists(file_access.APPLICATION, application.id):
raise AlreadyExistsError("Application already installed")
context.uuid = application.id
self.prepare_application_infrastructure(application.id)
# TODO: ENABLE THIS LATER
# if application.server_version and VDOM_server_version < application.server_version:
# raise Exception("Server version %s is unsuitable for this application %s" % (VDOM_server_version, application.server_version))
# TODO: License key...
if value:
description = "string"
elif filename:
description = os.path.basename(filename)
else:
try:
description = file.name
except AttributeError:
description = "file"
log.write("Install application from %s" % description)
parser = Parser(builder=application_builder, notify=True, options=on_information)
context = Structure(uuid=None)
try:
if value:
application = parser.parse(value.encode("utf8") if isinstance(value, unicode) else value)
elif filename:
application = parser.parse(filename=filename)
else:
application = parser.parse(file=file)
if parser.report:
if into is None:
log.warning("Install application notifications")
for lineno, message in parser.report:
log.warning(" %s, line %s" % (message, lineno))
else:
for lineno, message in parser.report:
into.append((lineno, message))
# TODO: check this later - why?
application.unimport_libraries()
application.save()
return application
except AlreadyExistsError as error:
raise
except ParsingException as error:
cleanup(context.uuid)
raise Exception("Unable to parse %s, line %s: %s" % (description, error.lineno, error))
except IOError as error:
cleanup(context.uuid)
raise Exception("Unable to read %s: %s" % (description, error.strerror))
except: # noqa
cleanup(context.uuid)
raise
