def random_annotation(self): p = gen_id(self.namespace) i = gen_id(self.namespace) b = [[123,234], [345,456]] # tuples do not survive JSON roundtripping t = gen_id(self.namespace) a = gen_id(self.namespace) ts = iso8601() return structs(timestamp=ts, pid=p, image=i, geometry=b, category=t, annotator=a)
def create_work(): print 'creating work' for n in range(5): ping_pid = gen_id() job_pid = gen_id() print 'pinging: %s <- %s' % (job_pid, ping_pid) assert isok(client.depend(job_pid, ping_pid, PING_ROLE)) print 'waking up' assert isok(client.wakeup(WAKEUP_KEY))
def timestamped_output_of(cmd,stdout=True,jobid=None): issued = iso8601ms() if jobid is None: jobid = gen_id() def log_struct(message): return { 'command': cmd, 'issued': issued, 'jobid': jobid, 'message': message, 'timestamp': iso8601ms() } yield log_struct('executing shell command %s, jobid %s' % (cmd,jobid)) if stdout: p = Popen(cmd, shell=True, stdout=PIPE) else: p = Popen(cmd, shell=True, stderr=PIPE) while True: if stdout: line = p.stdout.readline() else: line = p.stderr.readline() if not line: p.wait() if p.returncode != 0: message = 'process exited abnormally with exit code %d: %s' % (p.returncode, cmd) yield log_struct(message) raise RuntimeError(message) yield log_struct('process closed output stream: '+cmd) return yield log_struct(line.rstrip('\n'))
def run_callback(self,message): jobid = gen_id()[:5] def selflog(line): self.log('%s %s' % (jobid, line)) def self_check_log(line,bin_pid): selflog(line) self.output_check -= 1 if self.output_check <= 0: if self.exists(bin_pid): selflog('STOPPING JOB - %s completed by another worker' % bin_pid) raise JobExit(bin_pid, SKIP) self.output_check = CHECK_EVERY bin_pid = message dest_file = self.storage.dest(bin_pid) if self.exists(bin_pid): selflog('SKIPPING %s - already completed' % bin_pid) return SKIP job_dir = os.path.join(self.config.tmp_dir, gen_id()) try: os.makedirs(job_dir) except: selflog('WARNING cannot create temporary directory %s' % job_dir) tmp_file = os.path.join(job_dir, self.storage.zipname(bin_pid)) matlab = Matlab(self.config.matlab_exec_path,self.config.matlab_path,output_callback=lambda l: self_check_log(l, bin_pid)) cmd = 'bin_blobs(\'%s\',\'%s\')' % (bin_pid, job_dir) try: self.output_check = CHECK_EVERY matlab.run(cmd) if not os.path.exists(tmp_file): selflog('WARNING bin_blobs succeeded but produced no output for %s' % bin_pid) elif not self.exists(bin_pid): # check to make sure another worker hasn't finished it in the meantime if self.deposit is not None: selflog('DEPOSITING blob zip for %s to deposit service' % bin_pid) self.deposit.deposit(bin_pid,tmp_file) else: selflog('SAVING completed blob zip for %s to %s' % (bin_pid, dest_file)) local_deposit(bin_pid,tmp_file) else: selflog('NOT SAVING - blobs for %s already present at output destination' % bin_pid) except KeyboardInterrupt: selflog('KeyboardInterrupt, requeueing job before exit') return DIE finally: try: shutil.rmtree(job_dir) except: selflog('WARNING cannot remove temporary directory %s' % job_dir)
def annotations_for(file): for raw in csv.DictReader(fin,['bin','roi','category','annotator']): yield { PID: gen_id(ANNOTATION_NAMESPACE), IMAGE: DATA_NAMESPACE + raw['bin'].replace('.mat','_') + raw['roi'], TIMESTAMP: iso8601(), CATEGORY: raw['category'], ANNOTATOR: raw['annotator'], }
def add_user(self,username, password, **kvs): salt = gen_id() # generate salt kvs['annotator_id'] = username kvs['passwd'] = md5_string(salt + password) # md5 salt + password is the encrypted credential kvs['salt'] = salt kvs = kvs.items() ks = ','.join([k for k,_ in kvs]) ss = ','.join(['%s' for _ in kvs]) vs = [v for _,v in kvs] with xa(self.config.psql_connect) as (connection,cursor): cursor.execute('insert into auth (' + ks + ') values (' + ss + ')',vs) connection.commit()
def preflight(self): for p in self.config.matlab_path: assert os.path.exists(p) assert os.path.exists(self.config.tmp_dir) try: tempfile = os.path.join(self.config.tmp_dir,gen_id()+'.txt') with open(tempfile,'w') as tf: tf.write('test') tf.flush() assert os.path.exists(tempfile) os.remove(tempfile) assert not os.path.exists(tempfile) except: raise
def test_list_annotations(self): with app.test_request_context(): ns = [0,1,2,3,100] image_pids = [gen_id(self.namespace) for _ in range(len(ns))] for (n,image_pid) in zip(ns,image_pids): # create n annotations for this image pid for _ in range(n): ann = self.random_annotation() ann.image = image_pid self.app.post(url_for('create_annotations', pid=ann.pid), data=jsons([ann])) for (n,image_pid) in zip(ns,image_pids): ann_list = structs(self.app.get(url_for('list_annotations', image_pid=image_pid)).data) # FIXME don't just check the length of the result, check the contents assert len(ann_list) == n
def preflight(self): for p in self.config.matlab_path: if not os.path.exists(p): raise if not os.path.exists(self.config.tmp_dir): raise tempfile = os.path.join(self.config.tmp_dir,gen_id()+'.txt') with open(tempfile,'w') as tf: tf.write('test') tf.flush() if not os.path.exists(tempfile): raise os.remove(tempfile) if os.path.exists(tempfile): raise
def win_callback(self,params): start_time = params['start_time'] end_time = iso8601() img_in = params['img_in'] tmp_out = params['img_out'] # temporary output file img_out = params['final_out'] # final output file process_id = gen_id() if not os.path.exists(tmp_out): self.log('FAIL temporary output file does not exist: %s' % tmp_out) return FAIL in_md5 = md5_file(img_in) out_md5 = md5_file(tmp_out) in_length = os.stat(img_in).st_size out_length = os.stat(tmp_out).st_size used = { 'process_id': process_id, 'algorithm_id': '201203_ic', # FIXME uncontrolled 'direction': 'used', # FIXME local id 'pathname': img_in, # FIXME local pathname 'no_earlier_than': start_time, 'no_later_than': end_time, 'fixity_md5': in_md5, 'fixity_length': in_length } generated_by = { 'process_id': process_id, 'algorithm_id': '201203_ic', # FIXME uncontrolled 'direction': 'generated by', # FIXME local id 'pathname': img_out, # FIXME local pathname 'no_earlier_than': start_time, 'no_later_than': end_time, 'fixity_md5': out_md5, 'fixity_length': out_length } # FIXME emit provenance record prov_qname = '%s_prov' % self.qname try: self.enqueue(json.dumps(used), prov_qname) self.enqueue(json.dumps(generated_by), prov_qname) except: raise JobExit('Failed to enqueue provenance records', FAIL) try: os.rename(tmp_out, img_out) except: raise JobExit('Cannot move temporary file into place: %s -> %s' % (tmp_out, img_out), FAIL) return WIN
def deposit_impl(product,pid): destpath = get_destpath(product, pid) product_data = request.data destpath_part = '%s_%s.part' % (destpath, gen_id()) try: os.makedirs(os.path.dirname(destpath)) except: pass with open(destpath_part,'w') as out: shutil.copyfileobj(StringIO(product_data), out) os.rename(destpath_part, destpath) utcnow = iso8601utcnow() message = '%s wrote %d bytes to %s' % (utcnow, len(product_data), destpath) return jsonr(dict( status='OK', time=utcnow, message=message, pid=pid, path=destpath ))
def deposit(pid): req = DashboardRequest(pid, request) try: destpath = files.get_product_destination(session, pid) except NotFound: abort(404) product_data = request.data destpath_part = '%s_%s.part' % (destpath, gen_id()) try: os.makedirs(os.path.dirname(destpath)) except: pass with open(destpath_part,'w') as out: shutil.copyfileobj(StringIO(product_data), out) os.rename(destpath_part, destpath) utcnow = iso8601() message = '%s wrote %d bytes to %s' % (utcnow, len(product_data), destpath) return Response(json.dumps(dict( status='OK', time=utcnow, message=message, pid=pid, path=destpath )), mimetype=MIME_JSON)
def extract_blobs(self,bin_pid): try: jobid = self.config.task_id except: jobid = gen_id()[:5] def selflog(line): self.log('[%s] %s' % (jobid, line)) def self_check_log(line,bin_pid): selflog(line) now = time.time() elapsed = now - self.last_check self.last_check = now if elapsed > CHECK_EVERY: if self.exists(bin_pid): msg = 'STOPPING JOB - %s completed by another worker' % bin_pid selflog(msg) raise JobExit(msg, SKIP) if self.exists(bin_pid): selflog('SKIPPING %s - already completed' % bin_pid) return SKIP job_dir = os.path.join(self.config.tmp_dir, gen_id()) zip_dir = os.path.join(self.config.tmp_dir, gen_id()) bin_zip_path = os.path.join(zip_dir, binzipname(bin_pid)) try: os.makedirs(job_dir) selflog('CREATED temporary directory %s for %s' % (job_dir, bin_pid)) except: selflog('WARNING cannot create temporary directory %s for %s' % (job_dir, bin_pid)) try: os.makedirs(zip_dir) selflog('CREATED temporary directory %s for %s' % (zip_dir, bin_pid)) except: selflog('WARNING cannot create temporary directory %s for %s' % (zip_dir, bin_pid)) try: selflog('LOADING and STITCHING %s' % bin_pid) with open(bin_zip_path,'wb') as binzip: targets = represent.binpid2zip(bin_pid, binzip, resolver=self.resolver) if len(targets)==0: selflog('SKIPPING %s - no targets in bin' % bin_pid) return SKIP tmp_file = os.path.join(job_dir, zipname(bin_pid)) matlab = Matlab(self.config.matlab_exec_path,self.config.matlab_path,output_callback=lambda l: self_check_log(l, bin_pid)) cmd = 'bin_blobs(\'%s\',\'%s\',\'%s\')' % (bin_pid, bin_zip_path, job_dir) self.output_check = CHECK_EVERY matlab.run(cmd) if not os.path.exists(tmp_file): selflog('WARNING bin_blobs succeeded but no output file found at %s' % tmp_file) elif not self.exists(bin_pid): # check to make sure another worker hasn't finished it in the meantime selflog('DEPOSITING blob zip for %s to deposit service at %s' % (bin_pid, self.config.blob_deposit)) self.deposit.deposit(bin_pid,tmp_file) selflog('DEPOSITED blob zip for %s ' % bin_pid) else: selflog('NOT SAVING - blobs for %s already present at output destination' % bin_pid) except KeyboardInterrupt: selflog('KeyboardInterrupt, exiting') return DIE except JobExit: pass finally: try: shutil.rmtree(job_dir) selflog('DELETED temporary directory %s for %s' % (job_dir, bin_pid)) except: selflog('WARNING cannot remove temporary directory %s for %s' % (job_dir, bin_pid)) try: shutil.rmtree(zip_dir) selflog('DELETED temporary directory %s for %s' % (zip_dir, bin_pid)) except: selflog('WARNING cannot remove temporary directory %s for %s' % (zip_dir, bin_pid)) selflog('DONE - no more actions for %s' % bin_pid)
from oii.ifcb2.orm import User, Role from oii.ifcb2.dashboard.flasksetup import user_manager if __name__=='__main__': print 'Searching for default admin user in database ...' user = session.query(User).filter_by(email='*****@*****.**').first() if user is not None: print 'Found user. Deleting ...' session.delete(user) session.commit() else: print 'Not found.' print 'Creating recovery user/password ...' username = '******' password = gen_id()[:8] u = User( first_name='Admin', last_name='User', email=username, username=username, password=user_manager.hash_password(password), is_enabled=True ) r = session.query(Role).filter_by(name='Admin').first() u.roles.append(r) session.add(u) session.commit() print 'User %s created with password %s' % (username, password)
def __init__(self,qname,host='localhost'): self.host = host self.qname = qname self.log_channel = None self.work_channels = {} self.workerid = ('%s_%s' % (gen_id()[:4], platform.node()))
import sys from oii.utils import gen_id from oii.ifcb2.session import dbengine, session from oii.ifcb2.orm import User, Role from oii.ifcb2.dashboard.flasksetup import user_manager if __name__ == '__main__': try: password = sys.argv[1] except IndexError: password = gen_id()[:8] try: email = sys.argv[2] except IndexError: email = '*****@*****.**' user = session.query(User).filter_by(email=email).first() if user is not None: session.delete(user) session.commit() u = User(first_name='Admin', last_name='User', email=email, username=email, password=user_manager.hash_password(password), is_enabled=True) r = session.query(Role).filter_by(name='Admin').first() u.roles.append(r) session.add(u)
def extract_features(self,bin_pid): jobid = gen_id()[:5] def selflog(line): self.log('[%s] %s' % (jobid, line)) def self_check_log(line,bin_pid): selflog(line) now = time.time() elapsed = now - self.last_check self.last_check = now if elapsed > CHECK_EVERY: if self.complete(bin_pid): msg = 'STOPPING JOB - %s completed by another worker' % bin_pid selflog(msg) raise JobExit(msg, SKIP) if self.complete(bin_pid): selflog('SKIPPING %s - already completed' % bin_pid) return SKIP job_dir = os.path.join(self.config.tmp_dir, gen_id()) zip_dir = os.path.join(self.config.tmp_dir, gen_id()) bin_zip_path = os.path.join(zip_dir, binzipname(bin_pid)) try: os.makedirs(job_dir) selflog('CREATED temporary directory %s for %s' % (job_dir, bin_pid)) except: selflog('WARNING cannot create temporary directory %s for %s' % (job_dir, bin_pid)) try: os.makedirs(zip_dir) selflog('CREATED temporary directory %s for %s' % (zip_dir, bin_pid)) except: selflog('WARNING cannot create temporary directory %s for %s' % (zip_dir, bin_pid)) selflog('LOADING and STITCHING %s' % bin_pid) with open(bin_zip_path,'wb') as binzip: represent.binpid2zip(bin_pid, binzip, resolver=self.resolver) blobzipurl = bin_pid + '_blob.zip' blobzipfile = re.sub(r'\.zip','_blob.zip',bin_zip_path) selflog('LOADING blob zip from %s -> %s' % (blobzipurl, blobzipfile)) drain(UrlSource(blobzipurl), LocalFileSink(blobzipfile)) feature_csv = os.path.join(job_dir, csvname(bin_pid)) multiblob_csv = os.path.join(job_dir, 'multiblob', multiblobname(bin_pid)) matlab = Matlab(self.config.matlab_exec_path,self.config.matlab_path,output_callback=lambda l: self_check_log(l, bin_pid)) namespace = os.path.dirname(bin_zip_path) + '/' lid = os.path.basename(bin_zip_path) cmd = 'bin_features(\'%s\',\'%s\',\'%s\',\'chatty\')' % (namespace, lid, job_dir + '/') selflog('RUNNING %s' % cmd) try: self.output_check = CHECK_EVERY matlab.run(cmd) if not os.path.exists(feature_csv): msg = 'WARNING bin_features succeeded but no output file found at %s' % feature_csv selflog(msg) raise JobExit(msg,FAIL) if not self.complete(bin_pid): # check to make sure another worker hasn't finished it in the meantime selflog('DEPOSITING features csv for %s to deposit service at %s' % (bin_pid, self.config.features_deposit)) self.deposit.deposit(bin_pid,feature_csv) selflog('DEPOSITED features csv for %s ' % bin_pid) if os.path.exists(multiblob_csv): selflog('DEPOSITING multiblob csv for %s to deposit service at %s' % (bin_pid, self.config.features_deposit)) self.multiblob_deposit.deposit(bin_pid,multiblob_csv) selflog('DEPOSITED multiblob csv for %s ' % bin_pid) else: selflog('NOT SAVING - features for %s already present at output destination' % bin_pid) except KeyboardInterrupt: selflog('KeyboardInterrupt, exiting') return DIE except JobExit: pass finally: try: shutil.rmtree(job_dir) selflog('DELETED temporary directory %s for %s' % (job_dir, bin_pid)) except: selflog('WARNING cannot remove temporary directory %s for %s' % (job_dir, bin_pid)) try: shutil.rmtree(zip_dir) selflog('DELETED temporary directory %s for %s' % (zip_dir, bin_pid)) except: selflog('WARNING cannot remove temporary directory %s for %s' % (zip_dir, bin_pid)) selflog('DONE - no more actions for %s' % bin_pid)
def generate_ids(n,ns=''): return json.dumps([ns + gen_id() for _ in range(n)])
def new_process_id(self): try: return gen_id(self.config.namespace) except KeyError: return gen_id()
def new_process_id(self): try: return gen_id(self.config.namespace) except KeyError: return gen_id() return (connection,cursor)