def __init__(self, jobs_db=None, instances_db=None): self.gce_ops_in_progress = BoxList() self.instances_db: DB = instances_db or get_worker_instances_db() self.jobs_db: DB = jobs_db or get_jobs_db() self.gce = googleapiclient.discovery.build('compute', 'v1') self.project: str = GCP_PROJECT self.zone: str = GCP_ZONE
def submit_eval_job(docker_tag, eval_id, eval_key, problem_name: str, pull_request, seed, max_seconds, problem_def, full_eval_request, botleague_liaison_host=None): messages = [] start_job_submit = time.time() db = get_jobs_db() if not max_seconds: messages.append(f'max_seconds not set in problem definition, ' f'defaulting to ' f'{constants.MAX_EVAL_SECONDS_DEFAULT} seconds') max_seconds = constants.MAX_EVAL_SECONDS_DEFAULT job_id = f'{datetime.utcnow().strftime(DIR_DATE_FORMAT)}_{eval_id}' botleague_liaison_host = botleague_liaison_host or \ constants.BOTLEAGUE_LIAISON_HOST job = Box( id=job_id, status=constants.JOB_STATUS_CREATED, job_type=constants.JOB_TYPE_EVAL, botleague_liaison_host=botleague_liaison_host, created_at=SERVER_TIMESTAMP, eval_spec=dict( problem=problem_name, eval_id=eval_id, eval_key=eval_key, seed=seed, docker_tag=docker_tag, pull_request=pull_request, max_seconds=max_seconds, problem_def=problem_def, full_eval_request=full_eval_request, # TODO: Clean this up. )) log.info(f'Submitting job {eval_id}: {box2json(job)}') submitted = db.compare_and_swap(key=job_id, expected_current_value=None, new_value=job.to_dict()) if not submitted: ret = make_error(f'eval_id {eval_id} has already been processed', 403) else: for msg in messages: log.info(msg) ret = jsonify({'success': True, 'message': ' '.join(messages)}) log.info(f'Save submitted job took {time.time() - start_job_submit} ' f'seconds') return ret
def handle_job_status_request(): db = common.get_jobs_db() job_id = request.json['job_id'] job = dbox(db.get(job_id)) ret = dict( id=job_id, status=job.status, created_at=job.created_at, started_at=job.started_at, finished_at=job.finished_at, results=job.results, ) return jsonify(ret)
def start_build(build_type, job_abbr): db = common.get_jobs_db() commit = request.json['commit'] job_id = f'{datetime.utcnow().strftime(DIR_DATE_FORMAT)}_{job_abbr}_{commit}' run_local_debug = dbox(request.json).run_local_debug or False job = Box(id=job_id, commit=commit, branch=request.json['branch'], build_id=request.json['build_id'], status=constants.JOB_STATUS_CREATED, job_type=build_type, created_at=SERVER_TIMESTAMP, run_local_debug=run_local_debug) db.set(job_id, job) log.success(f'Created job {box2json(job)}') return jsonify({'job_id': job_id})
def handle_jobs_request(): from_local = request.remote_addr == '127.0.0.1' to_local = request.host in ['0.0.0.0:8000', 'localhost:8000'] in_whitelist = sys.executable in LOCAL_EXE_WHITELIST if not (from_local and to_local and in_whitelist): # TODO: Add user-auth or here to protect eval keys # https://cloud.google.com/appengine/docs/standard/python/users/ return 'Only available on localhost' db = common.get_jobs_db() jobs_ref = db.collection query = jobs_ref.order_by( 'created_at', direction=firestore.Query.DESCENDING).order_by( 'id', direction=firestore.Query.DESCENDING).limit(100) jobs = [j.to_dict() for j in list(query.stream())] [find_replace(j, math.inf, replace="Infinity") for j in jobs] [find_replace(j, -math.inf, replace="-Infinity") for j in jobs] ret = json.dumps(jobs, indent=2, default=str, sort_keys=True) ret = current_app.response_class(ret, mimetype="application/json") return ret
def __init__(self, jobs_db=None, instances_db=None, run_problem_only=False): """ :param jobs_db: Job status, etc... in Firestore :param instances_db: Instance status, etc... in Firestore :param run_problem_only: If True, will not run the bot container. This is not relevant to sim-build jobs. """ self.instance_id, self.is_on_gcp = fetch_instance_id() self.docker = docker.from_env() self.jobs_db = jobs_db or get_jobs_db() # Use this sparingly. Event loop should do most of the management # of instances so as to avoid race conditions. self.instances_db = instances_db or get_worker_instances_db() self.auto_updater = AutoUpdater(self.is_on_gcp) self.run_problem_only = run_problem_only self.loggedin_to_docker = False add_stackdriver_sink( log, f'{STACKDRIVER_LOG_NAME}-inst-{self.instance_id}') self.docker_creds = None