def execute(self): print('Dataset: '+self._dataset['name']) val=pa.get(self._key) if val: if val.startswith('in-process'): print('In progress, skipping...') else: print('Completed, skipping...') return if not pa.set(self._key,'in-process-'+self._code,overwrite=False): print('Problem setting in-process value skipping...') return try: result=self.run() except: if pa.get(self._key)=='in-process-'+self._code: pa.set(self._key,'error') else: print('Unexpected: not setting error value because existing value does not match') raise if pa.get(self._key)=='in-process-'+self._code: print('Saving result object.') kb.saveObject(key=self._key,object=result) else: print('Unexpected: not setting result because existing value does not match.')
def clear_job_result(job, *, incomplete_only=True): val = pa.get(key=job) if val: if (not incomplete_only) or (val.startswith('in-process')) or ( val.startswith('error')): print('Clearing job: ' + job['label']) pa.set(key=job, value=None)
def process(self): for dataset in self._datasets: for sorter in self._sorters: print ('SORTER: {} DATASET: {}'.format(sorter['processor'].NAME,dataset['name'])) lock_obj=self._get_lock_object(sorter,dataset) if pa.set(key=lock_obj,value='running',overwrite=False): try: print ('Running...') result=sf.sortDataset( sorter = sorter, dataset = dataset ) result['comparison_with_truth'] = sf.compareWithTruth(result) result['summary'] = sf.summarizeSorting(result) kb.saveObject(key=lock_obj,object=result) except: pa.set(key=lock_obj,value='error',overwrite=True) raise else: val0=pa.get(key=lock_obj) if val0 == 'running': print ('Skipping (result is running)...') else: print ('Skipping (result is locked)...')
def saveFile(self,fname,*,key=None,share_id=None,upload_token=None,basename=None,remote=None): ret=self._save_file_helper(fname,share_id=share_id,upload_token=upload_token,basename=basename,remote=remote) if key: sha1=self.computeFileSha1(fname) pairio.set(key,sha1) return ret
def test_001(self): key0='testkey' val0='testval000' pa.set(key0,val0) val=pa.get(key0) self.assertEqual(val,val0) pa.set(key0,val0+'abc') val=pa.get(key0) self.assertEqual(val,val0+'abc')
def clear_result_for_key(*, key, in_process_only=False): val = pa.get(key=key) if val: if in_process_only: do_clear = ((val.startswith('in-process')) or (val.startswith('error'))) else: do_clear = True if do_clear: print('Clearing results for: {}' + json.dumps(key)) pa.set(key=key, value=None)
def sf_batch_run(config): login(config) study_obj = kb.loadObject(key=dict(name='spikeforest_recordings')) recordings = select_recordings(study_obj, config) sorters = config['sorters'] code = ''.join(random.choice(string.ascii_uppercase) for x in range(10)) for i, ds in enumerate(recordings): if config.get('summarize_recordings', None): key = dict(name='summarize_recording', batch_name=config['name'], study_name=ds['study'], recording_name=ds['name']) if acquire_lock_for_key(key=key, code=code): try: print( '========= Summarizing recording {}/{}: {}/{}'.format( i, len(recordings), ds['study'], ds['name'])) result0 = sf_summarize_recording(ds) except: if check_consistent_code(key=key, code=code): pa.set(key=key, value='error-' + code) raise if check_consistent_code(key=key, code=code): kb.saveObject(key=key, object=result0) else: print('Warning: inconsistent code for {}'.format( json.dumps(key))) for sorter in sorters: key = dict(name='sort_recording', batch_name=config['name'], study_name=ds['study'], recording_name=ds['name'], sorter_name=sorter['name'], sorter_params=sorter['params']) if acquire_lock_for_key(key=key, code=code): try: print( '========= Sorting recording {}/{}: {} - {}/{}'.format( i, len(recordings), sorter['name'], ds['study'], ds['name'])) result0 = sf_sort_recording(sorter, ds) except: if check_consistent_code(key=key, code=code): pa.set(key=key, value='error-' + code) raise if check_consistent_code(key=key, code=code): kb.saveObject(key=key, object=result0) else: print('Warning: inconsistent code for {}'.format( json.dumps(key)))
def run_job(job): val = pa.get(key=job) if val: return code = ''.join(random.choice(string.ascii_uppercase) for x in range(10)) if not pa.set(key=job, value='in-process-' + code, overwrite=False): return print('Running job: ' + job['label']) result = do_run_job(job) val = pa.get(key=job) if val != 'in-process-' + code: return if 'error' in result: print('Error running job: ' + result['error']) pa.set(key=job, value='error-' + code) kb.save(key=dict(job=job, name='error'), value=result) return kb.saveObject(key=job, object=result)
def _run_job(job): val = pa.get(key=job) if val: return code = ''.join(random.choice(string.ascii_uppercase) for x in range(10)) if not pa.set(key=job, value='in-process-' + code, overwrite=False): return status = dict(time_started=_make_timestamp(), status='running') _set_job_status(job, status) print('Running job: ' + job['label']) try: result = _do_run_job(job) except: status['time_finished'] = _make_timestamp() status['status'] = 'error' status['error'] = 'Exception in _do_run_job' val = pa.get(key=job) if val == 'in-process-' + code: _set_job_status(job, status) raise val = pa.get(key=job) if val != 'in-process-' + code: print( 'Not saving result because in-process code does not match {} <> {}.' .format(val, 'in-process-' + code)) return status['time_finished'] = _make_timestamp() status['result'] = result if 'error' in result: print('Error running job: ' + result['error']) status['status'] = 'error' status['error'] = result['error'] _set_job_status(job, status) pa.set(key=job, value='error-' + code) return status['status'] = 'finished' kb.saveObject( key=job, object=result ) # Not needed in future, because we should instead use the status object
def acquire_lock_for_key(*, key, code): val = pa.get(key=key) if val: if val.startswith('in-process'): return False if val.startswith('error'): return False return False if not pa.set(key, 'in-process-' + code, overwrite=False): return False return True
def clearResults(self,*,in_process_only): val=pa.get(self._key) if val: if (not in_process_only) or (val.startswith('in-process')) or (val.startswith('error')): print('Clearing results for: '+self._key['dataset_name']) pa.set(key=self._key,value=None)
def clearResults(self): for dataset in self._datasets: for sorter in self._sorters: lock_obj=self._get_lock_object(sorter,dataset) pa.set(key=lock_obj,value=None)
def setBatchStatus(*,batch_name,status): pa.set(key=dict(name='spikeforest_batch_status',batch_name=batch_name),value=status)