def execute(self): print('Dataset: '+self._dataset['name']) val=pa.get(self._key) if val: if val.startswith('in-process'): print('In progress, skipping...') else: print('Completed, skipping...') return if not pa.set(self._key,'in-process-'+self._code,overwrite=False): print('Problem setting in-process value skipping...') return try: result=self.run() except: if pa.get(self._key)=='in-process-'+self._code: pa.set(self._key,'error') else: print('Unexpected: not setting error value because existing value does not match') raise if pa.get(self._key)=='in-process-'+self._code: print('Saving result object.') kb.saveObject(key=self._key,object=result) else: print('Unexpected: not setting result because existing value does not match.')
def test_001(self): key0='testkey' val0='testval000' pa.set(key0,val0) val=pa.get(key0) self.assertEqual(val,val0) pa.set(key0,val0+'abc') val=pa.get(key0) self.assertEqual(val,val0+'abc')
def clear_job_result(job, *, incomplete_only=True): val = pa.get(key=job) if val: if (not incomplete_only) or (val.startswith('in-process')) or ( val.startswith('error')): print('Clearing job: ' + job['label']) pa.set(key=job, value=None)
def _find_file_helper(self, *, path, sha1=None, share_ids=None, key=None, collection=None, local=None, remote=None): if local is None: local = self._config['load_local'] if remote is None: remote = self._config['load_remote'] if share_ids is None: share_ids = self._config['share_ids'] if key is not None: sha1 = pairio.get(key=key, collection=collection) if not sha1: return (None, None, None) if path is not None: if sha1 is not None: raise Exception( 'Cannot specify both path and sha1 in find file') if path.startswith('sha1://'): list = path.split('/') sha1 = list[2] ### continue to below elif path.startswith('kbucket://'): list = path.split('/') share_ids = [_filter_share_id(list[2])] path0 = '/'.join(list[3:]) prv = self._get_prv_for_file(share_id=share_ids[0], path=path0) if not prv: return (None, None, None) sha1 = prv['original_checksum'] remote = True ### continue to below else: if os.path.exists(path): ## Todo: also check if it is file return (path, None, os.path.getsize(path)) else: return (None, None, None) # search locally if local: path = self._sha1_cache.findFile(sha1=sha1) else: path = '' if path: return (path, sha1, os.path.getsize(path)) if remote: for id in share_ids: url, size = self._find_in_share(sha1=sha1, share_id=id) if url: return (url, sha1, size) return (None, None, None)
def process(self): for dataset in self._datasets: for sorter in self._sorters: print ('SORTER: {} DATASET: {}'.format(sorter['processor'].NAME,dataset['name'])) lock_obj=self._get_lock_object(sorter,dataset) if pa.set(key=lock_obj,value='running',overwrite=False): try: print ('Running...') result=sf.sortDataset( sorter = sorter, dataset = dataset ) result['comparison_with_truth'] = sf.compareWithTruth(result) result['summary'] = sf.summarizeSorting(result) kb.saveObject(key=lock_obj,object=result) except: pa.set(key=lock_obj,value='error',overwrite=True) raise else: val0=pa.get(key=lock_obj) if val0 == 'running': print ('Skipping (result is running)...') else: print ('Skipping (result is locked)...')
def getResults(self): results=[] for dataset in self._datasets: for sorter in self._sorters: lock_obj=self._get_lock_object(sorter,dataset) result=pa.get(key=lock_obj) results.append(result) return results
def run_job(job): val = pa.get(key=job) if val: return code = ''.join(random.choice(string.ascii_uppercase) for x in range(10)) if not pa.set(key=job, value='in-process-' + code, overwrite=False): return print('Running job: ' + job['label']) result = do_run_job(job) val = pa.get(key=job) if val != 'in-process-' + code: return if 'error' in result: print('Error running job: ' + result['error']) pa.set(key=job, value='error-' + code) kb.save(key=dict(job=job, name='error'), value=result) return kb.saveObject(key=job, object=result)
def loadResult(self): val=pa.get(self._key) if val: if val.startswith('in-process'): print('Inable to load result... it is in process.') return None else: return kb.loadObject(key=self._key) else: return None
def _download_recordings(*,jobs): for ii,job in enumerate(jobs): val=pa.get(key=job) if not val: if 'recording' in job: if 'directory' in job['recording']: dsdir=job['recording']['directory'] fname=dsdir+'/raw.mda' print('REALIZING FILE: '+fname) kb.realizeFile(fname)
def _run_job(job): val = pa.get(key=job) if val: return code = ''.join(random.choice(string.ascii_uppercase) for x in range(10)) if not pa.set(key=job, value='in-process-' + code, overwrite=False): return status = dict(time_started=_make_timestamp(), status='running') _set_job_status(job, status) print('Running job: ' + job['label']) try: result = _do_run_job(job) except: status['time_finished'] = _make_timestamp() status['status'] = 'error' status['error'] = 'Exception in _do_run_job' val = pa.get(key=job) if val == 'in-process-' + code: _set_job_status(job, status) raise val = pa.get(key=job) if val != 'in-process-' + code: print( 'Not saving result because in-process code does not match {} <> {}.' .format(val, 'in-process-' + code)) return status['time_finished'] = _make_timestamp() status['result'] = result if 'error' in result: print('Error running job: ' + result['error']) status['status'] = 'error' status['error'] = result['error'] _set_job_status(job, status) pa.set(key=job, value='error-' + code) return status['status'] = 'finished' kb.saveObject( key=job, object=result ) # Not needed in future, because we should instead use the status object
def acquire_lock_for_key(*, key, code): val = pa.get(key=key) if val: if val.startswith('in-process'): return False if val.startswith('error'): return False return False if not pa.set(key, 'in-process-' + code, overwrite=False): return False return True
def clear_result_for_key(*, key, in_process_only=False): val = pa.get(key=key) if val: if in_process_only: do_clear = ((val.startswith('in-process')) or (val.startswith('error'))) else: do_clear = True if do_clear: print('Clearing results for: {}' + json.dumps(key)) pa.set(key=key, value=None)
def _filter_share_id(id): if id in _filter_share_id_cache: return _filter_share_id_cache[id] if '.' in id: list = id.split('.') if len(list) != 2: return id ret = pairio.get(list[1], collection=list[0]) if ret: _filter_share_id_cache[id] = ret return ret else: return id
def kbucketConfigRemoteOld(*, user='******', share_id='spikeforest.spikeforest2', password=None, write=False): pa.setConfig(collections=[user], user='', token='', read_local=False, write_local=False, read_remote=True, write_remote=False) kb.setConfig(share_ids=[share_id], upload_share_id='', upload_token='', load_local=True, load_remote=True, save_remote=False) if write: if password is None: password = getpass.getpass( 'Enter the spikeforest password (or leave blank for read-only)' ) if not password: return pa.setConfig(user=user, token=pa.get(collection='spikeforest', key=dict(name='pairio_token', user=user, password=password)), write_remote=True) kb.setConfig(upload_share_id=share_id, upload_token=pa.get(collection='spikeforest', key=dict(name='kbucket_token', share_id=share_id, password=password)), save_remote=True)
def clearResults(self,*,in_process_only): val=pa.get(self._key) if val: if (not in_process_only) or (val.startswith('in-process')) or (val.startswith('error')): print('Clearing results for: '+self._key['dataset_name']) pa.set(key=self._key,value=None)
def check_consistent_code(*, key, code): val = pa.get(key=key) if not val: return False return (val == 'in-process-' + code)