def __init__(self, jobs_db=None, instances_db=None):
self.gce_ops_in_progress = BoxList()
self.instances_db: DB = instances_db or get_worker_instances_db()
self.jobs_db: DB = jobs_db or get_jobs_db()
self.gce = googleapiclient.discovery.build('compute', 'v1')
self.project: str = GCP_PROJECT
self.zone: str = GCP_ZONE
def submit_eval_job(docker_tag,
eval_id,
eval_key,
problem_name: str,
pull_request,
seed,
max_seconds,
problem_def,
full_eval_request,
botleague_liaison_host=None):
messages = []
start_job_submit = time.time()
db = get_jobs_db()
if not max_seconds:
messages.append(f'max_seconds not set in problem definition, '
f'defaulting to '
f'{constants.MAX_EVAL_SECONDS_DEFAULT} seconds')
max_seconds = constants.MAX_EVAL_SECONDS_DEFAULT
job_id = f'{datetime.utcnow().strftime(DIR_DATE_FORMAT)}_{eval_id}'
botleague_liaison_host = botleague_liaison_host or \
constants.BOTLEAGUE_LIAISON_HOST
job = Box(
id=job_id,
status=constants.JOB_STATUS_CREATED,
job_type=constants.JOB_TYPE_EVAL,
botleague_liaison_host=botleague_liaison_host,
created_at=SERVER_TIMESTAMP,
eval_spec=dict(
problem=problem_name,
eval_id=eval_id,
eval_key=eval_key,
seed=seed,
docker_tag=docker_tag,
pull_request=pull_request,
max_seconds=max_seconds,
problem_def=problem_def,
full_eval_request=full_eval_request, # TODO: Clean this up.
))
log.info(f'Submitting job {eval_id}: {box2json(job)}')
submitted = db.compare_and_swap(key=job_id,
expected_current_value=None,
new_value=job.to_dict())
if not submitted:
ret = make_error(f'eval_id {eval_id} has already been processed', 403)
else:
for msg in messages:
log.info(msg)
ret = jsonify({'success': True, 'message': ' '.join(messages)})
log.info(f'Save submitted job took {time.time() - start_job_submit} '
f'seconds')
return ret
def handle_job_status_request():
db = common.get_jobs_db()
job_id = request.json['job_id']
job = dbox(db.get(job_id))
ret = dict(
id=job_id,
status=job.status,
created_at=job.created_at,
started_at=job.started_at,
finished_at=job.finished_at,
results=job.results,
)
return jsonify(ret)
def start_build(build_type, job_abbr):
db = common.get_jobs_db()
commit = request.json['commit']
job_id = f'{datetime.utcnow().strftime(DIR_DATE_FORMAT)}_{job_abbr}_{commit}'
run_local_debug = dbox(request.json).run_local_debug or False
job = Box(id=job_id,
commit=commit,
branch=request.json['branch'],
build_id=request.json['build_id'],
status=constants.JOB_STATUS_CREATED,
job_type=build_type,
created_at=SERVER_TIMESTAMP,
run_local_debug=run_local_debug)
db.set(job_id, job)
log.success(f'Created job {box2json(job)}')
return jsonify({'job_id': job_id})
def handle_jobs_request():
from_local = request.remote_addr == '127.0.0.1'
to_local = request.host in ['0.0.0.0:8000', 'localhost:8000']
in_whitelist = sys.executable in LOCAL_EXE_WHITELIST
if not (from_local and to_local and in_whitelist):
# TODO: Add user-auth or here to protect eval keys
# https://cloud.google.com/appengine/docs/standard/python/users/
return 'Only available on localhost'
db = common.get_jobs_db()
jobs_ref = db.collection
query = jobs_ref.order_by(
'created_at', direction=firestore.Query.DESCENDING).order_by(
'id', direction=firestore.Query.DESCENDING).limit(100)
jobs = [j.to_dict() for j in list(query.stream())]
[find_replace(j, math.inf, replace="Infinity") for j in jobs]
[find_replace(j, -math.inf, replace="-Infinity") for j in jobs]
ret = json.dumps(jobs, indent=2, default=str, sort_keys=True)
ret = current_app.response_class(ret, mimetype="application/json")
return ret
def __init__(self,
jobs_db=None,
instances_db=None,
run_problem_only=False):
"""
:param jobs_db: Job status, etc... in Firestore
:param instances_db: Instance status, etc... in Firestore
:param run_problem_only: If True, will not run the bot container. This
is not relevant to sim-build jobs.
"""
self.instance_id, self.is_on_gcp = fetch_instance_id()
self.docker = docker.from_env()
self.jobs_db = jobs_db or get_jobs_db()
# Use this sparingly. Event loop should do most of the management
# of instances so as to avoid race conditions.
self.instances_db = instances_db or get_worker_instances_db()
self.auto_updater = AutoUpdater(self.is_on_gcp)
self.run_problem_only = run_problem_only
self.loggedin_to_docker = False
add_stackdriver_sink(
log, f'{STACKDRIVER_LOG_NAME}-inst-{self.instance_id}')
self.docker_creds = None