def _clear_job_results(*, jobs, incomplete_only=True):
for job in jobs:
val = ca.getValue(key=job)
if val:
if (not incomplete_only) or (val.startswith('in-process')) or (
val.startswith('error')):
print('Clearing job: ' + job['label'])
ca.setValue(key=job, value=None)
def stop_batch(*, batch_name):
status0 = get_batch_status(batch_name=batch_name)
if status0 is not None:
if status0['status'] not in ['finished', 'error']:
batch_code = 'batch_code_force_stop'
ca.setValue(key=dict(name='batcho_batch_code',
batch_name=batch_name), value=batch_code)
_set_batch_status(batch_name=batch_name, status=dict(
status='error', error='force stopped'))
def _clear_batch_names_for_compute_resource(compute_resource):
print('Clearing batch names for compute resource: '+compute_resource)
key0 = dict(
name='compute_resource_batch_names',
compute_resource=compute_resource
)
obj = ca.setValue(key=key0, subkey='-', value=None)
print('-----------------------',
get_batch_names_for_compute_resource(compute_resource))
def _run_job(job):
val = ca.getValue(key=job)
if val:
return
code = ''.join(random.choice(string.ascii_uppercase) for x in range(10))
if not ca.setValue(key=job, value='in-process-' + code, overwrite=False):
return
status = dict(time_started=_make_timestamp(), status='running')
_set_job_status(job, status)
print('Running job: ' + job['label'])
try:
result = _do_run_job(job)
except:
status['time_finished'] = _make_timestamp()
status['status'] = 'error'
status['error'] = 'Exception in _do_run_job'
val = ca.getValue(key=job)
if val == 'in-process-' + code:
_set_job_status(job, status)
raise
val = ca.getValue(key=job)
if val != 'in-process-' + code:
print(
'Not saving result because in-process code does not match {} <> {}.'
.format(val, 'in-process-' + code))
return
status['time_finished'] = _make_timestamp()
status['result'] = result
if 'error' in result:
print('Error running job: ' + result['error'])
status['status'] = 'error'
status['error'] = result['error']
_set_job_status(job, status)
ca.setValue(key=job, value='error-' + code)
return
status['status'] = 'finished'
ca.saveObject(
key=job, object=result
) # Not needed in future, because we should instead use the status object
def _take_next_batch_job_index_to_run(*, batch_name):
key = dict(name='batcho_next_batch_job_index_to_run',
batch_name=batch_name)
last_attempted_job_index = -1
last_attempted_job_index_timestamp = time.time()
while True:
val = ca.getValue(key=key)
if val is None:
return None
job_index = int(val)
if _acquire_job_lock(batch_name=batch_name, job_index=job_index):
ca.setValue(key=key, value=str(job_index+1))
return job_index
else:
if job_index == last_attempted_job_index:
elapsed0 = time.time()-last_attempted_job_index_timestamp
if elapsed0 > 10:
raise Exception('Unexpected problem where we cannot obtain the job lock, and yet the current job index remains at {} for {} seconds.'.format(
job_index, elapsed0))
last_attempted_job_index = job_index
last_attempted_job_index_timestamp = time.time()
time.sleep(random.uniform(0, 2))
def _set_job_status(*, batch_name, job_index, status):
# if job_code:
# code = _get_job_lock_code(batch_name=batch_name, job_index=job_index)
# if code != job_code:
# print('Not setting job status because lock code does not match batch code')
# return
status_string = None
if status:
status_string = status.get('status', None)
key = dict(name='batcho_job_statuses',
batch_name=batch_name)
subkey = str(job_index)
if not ca.saveObject(key=key, subkey=subkey, object=status):
return False
key = dict(name='batcho_job_status_strings',
batch_name=batch_name)
subkey = str(job_index)
if not ca.setValue(key=key, subkey=subkey, value=status_string):
print('WARNING: problem setting batch job status for subkey {}: {}'.format(
subkey, status_string))
return False
return True
def _clear_job_lock(*, batch_name, job_index):
key = dict(name='batcho_job_lock',
batch_name=batch_name, job_index=job_index)
ca.setValue(key=key, value=None, overwrite=True)
def _acquire_job_lock(*, batch_name, job_index):
key = dict(name='batcho_job_lock',
batch_name=batch_name, job_index=job_index)
code = 'lock_'+_random_string(10)
return ca.setValue(key=key, value=code, overwrite=False)
def remove_batch_name_for_compute_resource(compute_resource, batch_name):
key0 = dict(
name='compute_resource_batch_names',
compute_resource=compute_resource
)
return ca.setValue(key=key0, subkey=batch_name, value=None)
def add_batch_name_for_compute_resource(compute_resource, batch_name):
key0 = dict(
name='compute_resource_batch_names',
compute_resource=compute_resource
)
return ca.setValue(key=key0, subkey=batch_name, value='dummy_val')
def _set_batch_code(batch_name, batch_code):
ca.setValue(key=dict(name='batcho_batch_code',
batch_name=batch_name), value=batch_code)
def _init_next_batch_job_index_to_run(*, batch_name):
key = dict(name='batcho_next_batch_job_index_to_run',
batch_name=batch_name)
ca.setValue(key=key, value=str(0))
from cairio import client as ca
print('------------------------------------------------')
# Local key/value store for associating relatively short strings (<=80 characters) with arbitrary keys (strings or dicts)
# Setting values (these should be short strings, <=80 characters)
ca.setValue(key='some-key1', value='hello 1')
ca.setValue(key=dict(name='some_name', number=2), value='hello 2')
# Getting values
val1 = ca.getValue(key='some-key1')
val2 = ca.getValue(key=dict(name='some_name', number=2))
print(val1)
print(val2)
print('------------------------------------------------')
# Setting password-protected values
ca.setValue(key='some_key2', password='******', value='the-secret-*y$#a')
# Retrieving password-protected values
print(ca.getValue(key='some_key2', password='******'))
print('------------------------------------------------')
# Local storage of data and files, retrievable by SHA-1 hash
path = ca.saveText('This is some text', basename='test.txt')
print(path)
# Output: sha1://482cb0cfcbed6740a2bcb659c9ccc22a4d27b369/test.txt
def setBatchStatus(*, batch_name, status):
ca.setValue(key=dict(name='spikeforest_batch_status',
batch_name=batch_name),
value=status)