def queue_bin(bin_lid,config): config = get_config('alt_celery.conf') R = parse_stream(config.resolver) psql_connect = config.psql_connect for hit in R['list_images'].resolve_all(pid=bin_lid): (imagename, cfa_LR_path) = hit.lid, hit.p do_alt.s(imagename, cfa_LR_path, psql_connect).apply_async(queue='alt') # FIXME hardcoded queue name
def enqueue_blobs(time_series,queue): """config needs psql_connect, resolver""" config = get_config(CONFIG_FILE, time_series) feed = IfcbFeed(config.psql_connect) r = parse_stream(config.resolver) blob_resolver = r['mvco_blob'] pid_resolver = r['pid'] for lid in feed.latest_bins(n=10000): if blob_resolver.resolve(pid=lid,time_series=time_series) is None: pid = pid_resolver.resolve(pid=lid,time_series=time_series).bin_pid print 'No blobs found for %s, enqueuing' % pid extract_blobs.apply_async(args=[time_series, pid],queue=queue)
def enqueue_features(time_series,queue): """config needs psql_connect, resolver""" config = get_config(CONFIG_FILE, time_series) feed = IfcbFeed(config.psql_connect) r = parse_stream(config.resolver) blob_resolver = r['mvco_blob'] feature_resolver = r['features'] pid_resolver = r['pid'] for lid in feed.latest_bins(n=5000): if blob_resolver.resolve(pid=lid,time_series=time_series) is not None: pid = pid_resolver.resolve(pid=lid,time_series=time_series).bin_pid if feature_resolver.resolve(pid=lid,time_series=time_series) is None: print 'found blobs but no features for %s' % pid extract_features.apply_async(args=[time_series, pid],queue=queue)
def accede(config_file, time_series): config = get_config(config_file, time_series) logging.info('parsed config file %s:%s' % (config_file, time_series)) fx = IfcbFixity(config.psql_connect) feed = IfcbFeed(config.psql_connect) try: year_pattern = config.year_pattern except: year_pattern = '....' with xa(config.psql_connect) as (c, db): for s in list_new_filesets(time_series,config.psql_connect,config.resolver,year_pattern=year_pattern): # FIXME hardcoded try: check_integrity(s.pid, s.hdr_path, s.adc_path, s.roi_path, s.schema_version) except Exception, e: logger.warn('%s FAIL integrity checks: %s' % (s.pid, e)) continue # hot diggity, we've got some good data # compute fixity try: fx.fix(s.pid, s.hdr_path, cursor=db, filetype='hdr') logger.info('%s FIXITY computed for %s' % (s.pid, s.hdr_path)) fx.fix(s.pid, s.adc_path, cursor=db, filetype='adc') logger.info('%s FIXITY computed for %s' % (s.pid, s.adc_path)) fx.fix(s.pid, s.roi_path, cursor=db, filetype='roi') logger.info('%s FIXITY computed for %s' % (s.pid, s.roi_path)) except: logger.error('%s FAIL fixity cannot be computed!' % s.pid) c.rollback() continue # register bin try: ts = text2utcdatetime(s.date, s.date_format) feed.create(s.pid, ts, cursor=db) c.commit() logger.info('%s DONE' % s.pid) except: logger.error('%s FAILED' % s.pid) continue
from oii.workflow.deposit import app, DirectoryStorage, STORAGE from oii.config import get_config import re if __name__=='__main__': config = get_config('./features_deposit.conf') app.config[STORAGE] = DirectoryStorage(config) (h,p) = re.match(r'http://(.*):(\d+)',config.features_deposit).groups() app.run(host=h, port=int(p))
def extract_features(time_series, bin_pid): """config needs matlab_base, matlab_exec_path, tmp_dir, blob_deposit""" be = FeatureExtraction(get_config(CONFIG_FILE, time_series)) be.extract_features(bin_pid)
return DIE except JobExit: pass finally: try: shutil.rmtree(job_dir) selflog('DELETED temporary directory %s for %s' % (job_dir, bin_pid)) except: selflog('WARNING cannot remove temporary directory %s for %s' % (job_dir, bin_pid)) try: shutil.rmtree(zip_dir) selflog('DELETED temporary directory %s for %s' % (zip_dir, bin_pid)) except: selflog('WARNING cannot remove temporary directory %s for %s' % (zip_dir, bin_pid)) selflog('DONE - no more actions for %s' % bin_pid) CONFIG_FILE = './features.conf' # FIXME hardcoded @celery.task def extract_features(time_series, bin_pid): """config needs matlab_base, matlab_exec_path, tmp_dir, blob_deposit""" be = FeatureExtraction(get_config(CONFIG_FILE, time_series)) be.extract_features(bin_pid) if __name__=='__main__': config_file = sys.argv[1] time_series = sys.argv[2] bin_pid = sys.argv[3] be = FeatureExtraction(get_config(config_file, time_series)) be.extract_features(bin_pid)
import psycopg2 from oii.psql import xa from oii.config import get_config from oii.ifcb.classification import class_scores_mat2class_label_score, load_class_scores from oii.resolver import parse_stream from oii.ifcb.db import IfcbFeed from oii.times import text2utcdatetime, ISO_8601_FORMAT try: time_series = sys.argv[1] except: time_series = 'mvco' config = get_config('./db.conf',time_series) outdir = config.outdir psql_connect = '%s dbname=%s' % (config.psql_connect, config.dbname) R = parse_stream(config.resolver) NAMESPACE='http://demi.whoi.edu/mvco/' feed = IfcbFeed(psql_connect) start=strptime('2005-01-01T00:00:00Z',ISO_8601_FORMAT); end=strptime('2014-01-01T00:00:00Z',ISO_8601_FORMAT); with xa(psql_connect) as (c, db): bin_lids = list(feed.between(start,end))
# Start simple with making the existing annotation tool work. from os import urandom from sys import argv from oii.config import get_config from oii.webapi.annotation import app from oii.webapi.annotation import ANNOTATION_STORE, CATEGORIES, ASSIGNMENT_STORE from oii.seabed.annotation.assignments import SeabedAssignmentStore from oii.seabed.annotation.categories import SeabedCategories from oii.seabed.annotation.annotation import SeabedAnnotationStore if __name__ == '__main__': config = get_config(argv[1]) app.config[ANNOTATION_STORE] = SeabedAnnotationStore(config) app.config[ASSIGNMENT_STORE] = SeabedAssignmentStore(config) app.config[CATEGORIES] = SeabedCategories(config) app.secret_key = urandom(24) app.run(host=config.interface,port=int(config.port))
logging.warn('%s UNRESOLVABLE cannot find raw files' % s.pid) else: yield fs def check_integrity(pid, hdr_path, adc_path, roi_path, schema_version): integrity.check_hdr(LocalFileSource(hdr_path)) logging.info('%s PASS integrity check %s' % (pid, hdr_path)) targets = list(integrity.check_adc(LocalFileSource(adc_path), schema_version=schema_version)) logging.info('%s PASS integrity check %s' % (pid, adc_path)) integrity.check_roi(LocalFileSource(roi_path), targets) logging.info('%s PASS integrity check %s' % (pid, roi_path)) if __name__=='__main__': try: time_series=sys.argv[2] config = get_config(sys.argv[1], time_series) except: sys.stderr.write('usage: [python] oii/ifcb/accession.py [config file] [time series name]\n') sys.exit(-1) logging.basicConfig(level=logging.INFO) fx = IfcbFixity(config.psql_connect) feed = IfcbFeed(config.psql_connect) with xa(config.psql_connect) as (c, db): for s in list_new_filesets(time_series,config.psql_connect,config.resolver,after_year=2005): # FIXME hardcoded try: check_integrity(s.pid, s.hdr_path, s.adc_path, s.roi_path, s.schema_version) except Exception, e: logging.warn('%s FAIL integrity checks: %s' % (s.pid, e)) continue # hot diggity, we've got some good data # compute fixity
def extract_blobs(time_series, bin_pid): """config needs matlab_base, matlab_exec_path, tmp_dir, blob_deposit""" c = get_config(CONFIG_FILE, time_series) c.task_id = extract_blobs.request.id be = BlobExtraction(c) be.extract_blobs(bin_pid)
finally: try: shutil.rmtree(job_dir) except: selflog('WARNING cannot remove temporary directory %s' % job_dir) def enqueue_feed(self,n=4): feed = client.list_bins(namespace=self.config.namespace,n=n) for bin in feed: bin_pid = bin['pid'] #if not should_skip(bin_pid): print 'queueing %s' % bin_pid self.enqueue(bin_pid) if __name__=='__main__': timeseries = sys.argv[1] config = get_config('./blob.conf',timeseries) job = BlobExtraction(config) job.preflight() command = sys.argv[2] if command == 'q': for pid in sys.argv[3:]: job.enqueue(pid) elif command == 'log': job.consume_log() elif command == 'w': job.work(True) elif command == 'r': job.retry_failed(filter=lambda x: len(x)>3) elif command == 'cron': job.enqueue_feed(n=25)
TRIM='-evalchannels "[1]+[2] > 0 ? val : 0" "val" "val" -crop' return ' '.join([self.config.imagestack_exec_path, '-load %(img_in)s', DEBAYER, '-dup', CROP_L, ILLUM_CORRECT, CONTRAST, '-pull 1', CROP_R, ILLUM_CORRECT, CONTRAST, ALIGN, GRAYSCALE, CYAN_ONLY, '-pull 1', GRAYSCALE, RED_ONLY, '-add', TRIM, '-save %(img_out)s' ]) # usage: # python stereo.py {config file} {command} {arguments} # commands: # q (file1, file2, file3, ... filen) # enqueue files for processing # q - # read a list of files to process from stdin # r # requeue failed processing jobs # w # run as a worker # log # show logging messages as they come in if __name__=='__main__': hl = HabcamRedCyan(get_config(sys.argv[1])) cli(hl, sys.argv)
)) def configure(config=None): app.config[RESOLVER] = config.resolver try: if config.debug in ['True', 'true', 'T', 't', 'Yes', 'yes', 'debug']: app.debug = True except: pass try: app.config[PORT] = int(config.port) except: app.config[PORT] = 5063 if __name__=='__main__': if len(sys.argv) > 1: configure(get_config(sys.argv[1])) else: configure() else: configure(get_config(os.environ['DEPOSIT_CONFIG_FILE'])) rs = parse_stream(app.config[RESOLVER]) blob_resolver = rs['mvco_blob'] blob_destination = rs['blobs'] features_destination = rs['features_destination'] multiblob_destination = rs['multiblob_destination'] if __name__=='__main__': app.run(host='0.0.0.0',port=app.config[PORT])
"""Serve a stitched ROI image given the output of the pid resolver""" roi_image = get_stitched_roi(hit.bin_pid, hit.target_no, mask=mask, stitch_version=stitch_version) if roi_image is None: abort(404) # now determine PIL format and MIME type (pil_format, mimetype) = image_types(hit) # return the image data return image_response(roi_image,pil_format,mimetype) app.secret_key = os.urandom(24) if __name__=='__main__': """First argument is a config file which must at least have psql_connect in it to support feed arguments. Filesystem config is in the resolver.""" if len(sys.argv) > 1: configure(get_config(sys.argv[1])) else: configure() else: configure(get_config(os.environ['IFCB_CONFIG_FILE'])) # FIXME don't use globals # FIXME do this in config rs = parse_stream(app.config[RESOLVER]) binpid2path = rs['binpid2path'] pid_resolver = rs['pid'] blob_resolver = rs['mvco_blob'] fea_resolver = rs['features'] class_scores_resolver = rs['class_scores'] ts_resolver = rs['time_series'] all_series = rs['all_series']
params += [offset] cursor.execute('select imagename from imagelist where assignment_id=%s '+status_clause+' order by imagename '+limitclause+'offset %s', tuple(params)) for row in cursor.fetchall(): d = {} d['pid'] = self.pid(row[0], self.config.image_namespace) d['image'] = d['pid'] yield d def find_image(self,pid,offset,status,post_status=None): with xa(self.config.psql_connect) as (connection,cursor): cursor.execute('select imagename,status from imagelist where assignment_id=%s order by imagename offset %s for update of imagelist',(self.lid(pid),offset)) i = offset for row in cursor.fetchall(): if row[1]==status: if post_status is not None: cursor.execute('update imagelist set status=%s where assignment_id=%s and imagename=%s',(post_status,self.lid(pid),row[0])) connection.commit() return i+1 i += 1 return offset def set_status(self,assignment_id,image_id,status): with xa(self.config.psql_connect) as (connection,cursor): #print 'gonna set status to %s for assignment %s image %s' % (status,self.lid(assignment_id),self.lid(image_id)) cursor.execute('update imagelist set status=%s where assignment_id=%s and imagename=%s',(status,int(self.lid(assignment_id)),self.lid(image_id))) connection.commit() if __name__=='__main__': config = get_config('habcam_annotation.conf') has = HabcamAssignmentStore(config) for ass in has.list_assignments(): print ass
message=message, pid=pid, path=destpath )) # client code class BlobDeposit(object): def __init__(self,url_base='http://localhost:5000'): self.url_base = url_base def exists(self,pid): req = urllib.Request('%s/exists/blobs/%s' % (self.url_base, pid)) resp = json.loads(urllib.urlopen(req).read()) return resp['exists'] def deposit(self,pid,zipfile): with open(zipfile,'r') as inzip: zipdata = inzip.read() req = urllib.Request('%s/deposit/blobs/%s' % (self.url_base, pid), zipdata) req.add_header('Content-type','application/x-ifcb-blobs') resp = json.loads(urllib.urlopen(req).read()) return resp if __name__=='__main__': config = get_config('./blob.conf', sys.argv[1]) blob_storage = BlobStorage(config) app.config['BLOB_STORAGE'] = blob_storage (h,p) = re.match(r'http://(.*):(\d+)',config.blob_deposit).groups() app.run(host=h, port=int(p))
def configure(config=None): app.config[CACHE] = SimpleCache() app.config[CACHE_TTL] = 120 try: if config.debug in ['True', 'true', 'T', 't', 'Yes', 'yes', 'debug']: app.debug = True except AttributeError: pass try: app.config[RESOLVER] = config.resolver except AttributeError: app.config[RESOLVER] = 'oii/habcam/image_resolver.xml' try: app.config[PORT] = int(config.port) except: app.config[PORT] = 5061 # utilities if __name__=='__main__': """First argument is a config file""" if len(sys.argv) > 1: configure(get_config(sys.argv[1])) else: configure() app.secret_key = os.urandom(24) app.run(host='0.0.0.0',port=app.config[PORT]) else: config = get_config(os.environ['IMAGE_SERVICE_CONFIG_FILE']) configure(config)
for hit in R['list_images'].resolve_all(pid=bin_lid): (imagename, cfa_LR_path) = hit.lid, hit.p do_alt.s(imagename, cfa_LR_path, psql_connect).apply_async(queue='alt') # FIXME hardcoded queue name # when running this as a worker use a queue name like leg1_alt # and set concurrency to number of hardware threads # so: # celery -A oii.habcam.alt_celery worker -c 24 --queue=leg1_alt # to enqueue all images from bin do this: # # celery --config={celery config file} call oii.habcam.alt_celery.queue_bin --args='["{bin lid}" "{alt config file}"]' --queue=leg1_alt if __name__=='__main__': config = get_config('alt_celery.conf') R = parse_stream(config.resolver) psql_connect = config.psql_connect with xa(psql_connect) as (c,db): db.execute(""" select imageid || '.tif' from burton_alts where parallax_alt is null """) while True: rows = db.fetchmany(100) if len(rows) == 0: break else: for row in rows: imagename = row[0]
with xa(self.psql_connect) as (c,db): try: db.execute('select lat,lon,description from bin_props where lid=%s',(bin_lid,)) (lat,lon,description) = db.fetchone() d = dict(lat=lat,lon=lon,description=description) for k in d.keys(): if d[k] is None: del d[k] return d except: return {} import sys from oii.config import get_config, Configuration if __name__=='__main__': if len(sys.argv) > 1: config = get_config(sys.argv[1]) else: config = Configuration() try: resolvers = resolver.parse_stream(config.resolver) except: resolvers = None #fixity = IfcbFixity(config.psql_connect, resolvers) #fixity.check_all() autoclass = IfcbAutoclass(config.psql_connect) for roc in autoclass.rois_of_class('tintinnid'): print roc
pfx = term[:i] for r in structs(self.app.get(url_for('taxonomy_autocomplete', term=pfx)).data): assert r.label[:i] == pfx assert r.pid == url def config_backend(config): try: app.config[CONFIG] = config app.config[ANNOTATION_STORE] = HabcamAnnotationStore(config) app.config[ASSIGNMENT_STORE] = HabcamAssignmentStore(config) app.config[CATEGORIES] = HabcamCategories(config) app.config[AUTHENTICATION] = HabcamAuthentication(config) except KeyError: pass # FIXME log error app.secret_key = os.urandom(24) if __name__=='__main__': if len(sys.argv) > 1: config = get_config(sys.argv[1]) config_backend(config) try: port = int(config.port) except KeyError: port = 5000 app.run(host='0.0.0.0',port=port) else: config = get_config(os.environ['ANNOTATOR_CONFIG_FILE']) config_backend(config)