def prune_terminated_instances(self, worker_instances):
worker_ids = [w.id for w in worker_instances]
db_instances = self.instances_db.where('id', '>', '')
term_db = get_db('deepdrive_worker_instances_terminated')
for dbinst in db_instances:
if dbinst.id not in worker_ids:
term_db.set(dbinst.id, dbinst)
self.instances_db.delete(dbinst.id)
def rand_str_get_set(collection_name):
db = get_db(collection_name, force_firestore_db=True)
rand_str = 'test_data_can_delete_' + ''.join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(12))
db.set(rand_str, rand_str)
assert db.get(rand_str) == rand_str
db.delete_all_test_data()
def test_job_trigger():
# Mark test job as to start
test_id = utils.generate_rand_alphanumeric(32)
test_jobs_collection = 'test_jobs_' + test_id
test_instances_collection = 'test_instances_' + test_id
jobs_db = get_db(test_jobs_collection,
use_boxes=True,
force_firestore_db=True)
instances_db = get_db(test_instances_collection,
use_boxes=True,
force_firestore_db=True)
job_id = 'TEST_JOB_' + utils.generate_rand_alphanumeric(32)
trigger_job(instances_db,
job_id,
jobs_db,
botleague_liaison_host=constants.BOTLEAGUE_LIAISON_HOST)
def encrypt_db_key(unencrypted_value, key, db=None):
from botleague_helpers.db import get_db
db = db or get_db(DEFAULT_DB_NAME, force_firestore_db=True)
key = f'{key}{POSTFIX}'
if isinstance(unencrypted_value, dict):
encrypted_value = dict()
for k, v in unencrypted_value.items():
encrypted_value[k] = encrypt_symmetric(v)
db.set(key, encrypted_value)
else:
db.set(key, encrypt_symmetric(unencrypted_value))
def test_reduce():
test_id = ''.join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(32))
db_name = f'test_data_reduce_can_delete_{test_id}'
db = get_db(db_name, force_firestore_db=True)
reduce.create_reduce(test_id, db=db)
a = True
b = False
def ready_fn():
return a and b
def reduce_fn():
return 'asdf'
result = reduce.try_reduce_async(test_id,
ready_fn,
reduce_fn,
db,
max_attempts=1)
assert not result
# Wait for other reviewer
db.set(test_id, reduce.REVIEWING)
result = reduce.try_reduce_async(test_id,
ready_fn,
reduce_fn,
db,
max_attempts=1)
assert not result
assert db.get(test_id) == reduce.REVIEWING
# Reduce
db.set(test_id, reduce.WAITING)
b = True
result = reduce.try_reduce_async(test_id,
ready_fn,
reduce_fn,
db,
max_attempts=1)
assert result == 'asdf'
assert db.get(test_id) == reduce.FINISHED
# Don't allow double reduce
result = reduce.try_reduce_async(test_id,
ready_fn,
reduce_fn,
db,
max_attempts=1)
assert not result
assert db.get(test_id) == reduce.FINISHED
db.delete_all_test_data()
def test_compare_and_swap_live_db():
db = get_db(TEST_DB_NAME, force_firestore_db=True)
db.set('yo', 1)
should_be_false = db.compare_and_swap('yo', 2, 2)
assert should_be_false is False
assert db.get('yo') == 1
x = db.get('doesnotexist')
y = Box(a=1)
should_be_true = db.compare_and_swap('doesnotexist', x, y)
assert should_be_true
assert db.get('doesnotexist') == y
db.delete_all_test_data()
def __init__(self, loop_name, fn, force_firestore_db=False):
self.fn = fn
self.loop_name = loop_name
self.db = get_db(loop_name + '_semaphore', use_boxes=True,
force_firestore_db=force_firestore_db)
self.kill_now = False
self.caught_exception = False
self.id = datetime.now().strftime(
f'%Y-%m-%d__%I-%M-%S%p#'
f'{utils.generate_rand_alphanumeric(3)}')
self.previous_status = None
self.caught_sigterm = False
self.caught_sigint = False
self.started_waiting_for_other_loop_time = None
signal.signal(signal.SIGINT, self.handle_sigint)
signal.signal(signal.SIGTERM, self.handle_sigterm)
def collect_bot_scores(
docker_tag='deepdriveio/deepdrive:bot_domain_randomization'):
"""
Catches up bot scores using deepdrive_jobs. This is a violation of
data boundaries across deepdrive and botleague, and won't be possible
for future independent problem providers. We are now storing results
in the bot_eval data as well, to avoid such problems in the future.
Alternatively, we could have just downloaded all results from
gist/botleague-results which is a source of truth, but this was easier.
"""
job_db = get_db('deepdrive_jobs')
ldb = get_liaison_db_store()
for job in job_db.where('eval_spec.docker_tag', '==', docker_tag):
eval_key = job.eval_spec.eval_key
eval_data = ldb.get(get_eval_db_key(eval_key))
score = Box(score=job.results.score, eval_key=eval_key)
save_to_bot_scores(eval_data, eval_key, score)
def decrypt_db_key(key, db=None):
from botleague_helpers.db import get_db
db = db or get_db(DEFAULT_DB_NAME, force_firestore_db=True)
if not key.endswith(POSTFIX):
key = f'{key}{POSTFIX}'
encrypted_value = db.get(key)
if isinstance(encrypted_value, Box):
if 'token' in encrypted_value:
encrypted_value = encrypted_value.token
ret = decrypt_symmetric(encrypted_value)
else:
ret = Box()
for k, v in encrypted_value.items():
ret[k] = decrypt_symmetric(v)
else:
ret = decrypt_symmetric(encrypted_value)
return ret
def run_test_job(job, run_problem_only=False):
test_id = utils.generate_rand_alphanumeric(32)
test_jobs_collection = 'test_jobs_' + test_id
jobs_db = get_db(test_jobs_collection,
use_boxes=True,
force_firestore_db=True)
try:
jobs_db.set(job.id, job)
worker = Worker(jobs_db=jobs_db, run_problem_only=run_problem_only)
job = worker.loop(max_iters=1)
assert job
assert job.results
assert job.results.logs
assert not job.results.errors
assert job.status.lower() == JOB_STATUS_FINISHED
assert not utils.dbox(job).coordinator_error
del os.environ['FORCE_FIRESTORE_DB']
assert 'FORCE_FIRESTORE_DB' not in os.environ
finally:
jobs_db.delete_all_test_data()
def watch_collection_play():
db = get_db(TEST_DB_NAME, force_firestore_db=True)
# Create a callback on_snapshot function to capture changes
def on_snapshot(col_snapshot, changes, read_time):
for change in changes:
if change.type.name == 'ADDED':
print(u'New city: {}'.format(change.document.id))
elif change.type.name == 'MODIFIED':
print(u'Modified city: {}'.format(change.document.id))
elif change.type.name == 'REMOVED':
print(u'Removed city: {}'.format(change.document.id))
col_query = db.collection.where('b', '>=', '')
# Watch the collection query
query_watch = col_query.on_snapshot(on_snapshot)
db.set('a3', {'b': 'c'})
db.set('a4', {'b': 'd'})
input('press any key to exit')
db.delete_all_test_data()
def get_worker_instances_db(force_firestore_db=False):
return get_db(constants.WORKER_INSTANCES_COLLECTION_NAME,
force_firestore_db=force_firestore_db)
def get_instances_db():
return get_db(constants.WORKER_INSTANCES_COLLECTION_NAME)
def get_secrets_db():
return get_db('secrets', force_firestore_db=should_force_firestore_db())
def get_worker_instances_db():
return get_db(WORKER_INSTANCES_COLLECTION_NAME,
force_firestore_db=should_force_firestore_db())
def get_jobs_db():
return get_db(JOBS_COLLECTION_NAME,
force_firestore_db=should_force_firestore_db())
def get_config_db():
return get_db(constants.EVAL_CONFIG_COLLECTION_NAME)
def get_jobs_db():
return get_db(constants.JOBS_COLLECTION_NAME)
def get_bot_scores_db():
return get_db('botleague_liaison_bot_scores')
def get_liaison_db_store():
# TODO: Deprecate this in favor of botleague_helpers version
ret = get_db(collection_name='botleague_liaison')
return ret
def trigger_leaderboard_generation():
db = get_db(collection_name=blconfig.botleague_collection_name)
db.set(blconfig.should_gen_key, True)
def get_botleague_db():
return get_db(blconfig.botleague_collection_name)
def get_liaison_db_store():
ret = get_db(collection_name='botleague_liaison')
return ret