def queue_bin(bin_lid,config):
config = get_config('alt_celery.conf')
R = parse_stream(config.resolver)
psql_connect = config.psql_connect
for hit in R['list_images'].resolve_all(pid=bin_lid):
(imagename, cfa_LR_path) = hit.lid, hit.p
do_alt.s(imagename, cfa_LR_path, psql_connect).apply_async(queue='alt') # FIXME hardcoded queue name
def enqueue_blobs(time_series,queue):
"""config needs psql_connect, resolver"""
config = get_config(CONFIG_FILE, time_series)
feed = IfcbFeed(config.psql_connect)
r = parse_stream(config.resolver)
blob_resolver = r['mvco_blob']
pid_resolver = r['pid']
for lid in feed.latest_bins(n=10000):
if blob_resolver.resolve(pid=lid,time_series=time_series) is None:
pid = pid_resolver.resolve(pid=lid,time_series=time_series).bin_pid
print 'No blobs found for %s, enqueuing' % pid
extract_blobs.apply_async(args=[time_series, pid],queue=queue)
def enqueue_features(time_series,queue):
"""config needs psql_connect, resolver"""
config = get_config(CONFIG_FILE, time_series)
feed = IfcbFeed(config.psql_connect)
r = parse_stream(config.resolver)
blob_resolver = r['mvco_blob']
feature_resolver = r['features']
pid_resolver = r['pid']
for lid in feed.latest_bins(n=5000):
if blob_resolver.resolve(pid=lid,time_series=time_series) is not None:
pid = pid_resolver.resolve(pid=lid,time_series=time_series).bin_pid
if feature_resolver.resolve(pid=lid,time_series=time_series) is None:
print 'found blobs but no features for %s' % pid
extract_features.apply_async(args=[time_series, pid],queue=queue)
def accede(config_file, time_series):
config = get_config(config_file, time_series)
logging.info('parsed config file %s:%s' % (config_file, time_series))
fx = IfcbFixity(config.psql_connect)
feed = IfcbFeed(config.psql_connect)
try:
year_pattern = config.year_pattern
except:
year_pattern = '....'
with xa(config.psql_connect) as (c, db):
for s in list_new_filesets(time_series,config.psql_connect,config.resolver,year_pattern=year_pattern): # FIXME hardcoded
try:
check_integrity(s.pid, s.hdr_path, s.adc_path, s.roi_path, s.schema_version)
except Exception, e:
logger.warn('%s FAIL integrity checks: %s' % (s.pid, e))
continue
# hot diggity, we've got some good data
# compute fixity
try:
fx.fix(s.pid, s.hdr_path, cursor=db, filetype='hdr')
logger.info('%s FIXITY computed for %s' % (s.pid, s.hdr_path))
fx.fix(s.pid, s.adc_path, cursor=db, filetype='adc')
logger.info('%s FIXITY computed for %s' % (s.pid, s.adc_path))
fx.fix(s.pid, s.roi_path, cursor=db, filetype='roi')
logger.info('%s FIXITY computed for %s' % (s.pid, s.roi_path))
except:
logger.error('%s FAIL fixity cannot be computed!' % s.pid)
c.rollback()
continue
# register bin
try:
ts = text2utcdatetime(s.date, s.date_format)
feed.create(s.pid, ts, cursor=db)
c.commit()
logger.info('%s DONE' % s.pid)
except:
logger.error('%s FAILED' % s.pid)
continue
from oii.workflow.deposit import app, DirectoryStorage, STORAGE
from oii.config import get_config
import re
if __name__=='__main__':
config = get_config('./features_deposit.conf')
app.config[STORAGE] = DirectoryStorage(config)
(h,p) = re.match(r'http://(.*):(\d+)',config.features_deposit).groups()
app.run(host=h, port=int(p))
def extract_features(time_series, bin_pid):
"""config needs matlab_base, matlab_exec_path, tmp_dir, blob_deposit"""
be = FeatureExtraction(get_config(CONFIG_FILE, time_series))
be.extract_features(bin_pid)
return DIE
except JobExit:
pass
finally:
try:
shutil.rmtree(job_dir)
selflog('DELETED temporary directory %s for %s' % (job_dir, bin_pid))
except:
selflog('WARNING cannot remove temporary directory %s for %s' % (job_dir, bin_pid))
try:
shutil.rmtree(zip_dir)
selflog('DELETED temporary directory %s for %s' % (zip_dir, bin_pid))
except:
selflog('WARNING cannot remove temporary directory %s for %s' % (zip_dir, bin_pid))
selflog('DONE - no more actions for %s' % bin_pid)
CONFIG_FILE = './features.conf' # FIXME hardcoded
@celery.task
def extract_features(time_series, bin_pid):
"""config needs matlab_base, matlab_exec_path, tmp_dir, blob_deposit"""
be = FeatureExtraction(get_config(CONFIG_FILE, time_series))
be.extract_features(bin_pid)
if __name__=='__main__':
config_file = sys.argv[1]
time_series = sys.argv[2]
bin_pid = sys.argv[3]
be = FeatureExtraction(get_config(config_file, time_series))
be.extract_features(bin_pid)
import psycopg2
from oii.psql import xa
from oii.config import get_config
from oii.ifcb.classification import class_scores_mat2class_label_score, load_class_scores
from oii.resolver import parse_stream
from oii.ifcb.db import IfcbFeed
from oii.times import text2utcdatetime, ISO_8601_FORMAT
try:
time_series = sys.argv[1]
except:
time_series = 'mvco'
config = get_config('./db.conf',time_series)
outdir = config.outdir
psql_connect = '%s dbname=%s' % (config.psql_connect, config.dbname)
R = parse_stream(config.resolver)
NAMESPACE='http://demi.whoi.edu/mvco/'
feed = IfcbFeed(psql_connect)
start=strptime('2005-01-01T00:00:00Z',ISO_8601_FORMAT);
end=strptime('2014-01-01T00:00:00Z',ISO_8601_FORMAT);
with xa(psql_connect) as (c, db):
bin_lids = list(feed.between(start,end))
# Start simple with making the existing annotation tool work.
from os import urandom
from sys import argv
from oii.config import get_config
from oii.webapi.annotation import app
from oii.webapi.annotation import ANNOTATION_STORE, CATEGORIES, ASSIGNMENT_STORE
from oii.seabed.annotation.assignments import SeabedAssignmentStore
from oii.seabed.annotation.categories import SeabedCategories
from oii.seabed.annotation.annotation import SeabedAnnotationStore
if __name__ == '__main__':
config = get_config(argv[1])
app.config[ANNOTATION_STORE] = SeabedAnnotationStore(config)
app.config[ASSIGNMENT_STORE] = SeabedAssignmentStore(config)
app.config[CATEGORIES] = SeabedCategories(config)
app.secret_key = urandom(24)
app.run(host=config.interface,port=int(config.port))
logging.warn('%s UNRESOLVABLE cannot find raw files' % s.pid)
else:
yield fs
def check_integrity(pid, hdr_path, adc_path, roi_path, schema_version):
integrity.check_hdr(LocalFileSource(hdr_path))
logging.info('%s PASS integrity check %s' % (pid, hdr_path))
targets = list(integrity.check_adc(LocalFileSource(adc_path), schema_version=schema_version))
logging.info('%s PASS integrity check %s' % (pid, adc_path))
integrity.check_roi(LocalFileSource(roi_path), targets)
logging.info('%s PASS integrity check %s' % (pid, roi_path))
if __name__=='__main__':
try:
time_series=sys.argv[2]
config = get_config(sys.argv[1], time_series)
except:
sys.stderr.write('usage: [python] oii/ifcb/accession.py [config file] [time series name]\n')
sys.exit(-1)
logging.basicConfig(level=logging.INFO)
fx = IfcbFixity(config.psql_connect)
feed = IfcbFeed(config.psql_connect)
with xa(config.psql_connect) as (c, db):
for s in list_new_filesets(time_series,config.psql_connect,config.resolver,after_year=2005): # FIXME hardcoded
try:
check_integrity(s.pid, s.hdr_path, s.adc_path, s.roi_path, s.schema_version)
except Exception, e:
logging.warn('%s FAIL integrity checks: %s' % (s.pid, e))
continue
# hot diggity, we've got some good data
# compute fixity
def extract_blobs(time_series, bin_pid):
"""config needs matlab_base, matlab_exec_path, tmp_dir, blob_deposit"""
c = get_config(CONFIG_FILE, time_series)
c.task_id = extract_blobs.request.id
be = BlobExtraction(c)
be.extract_blobs(bin_pid)
finally:
try:
shutil.rmtree(job_dir)
except:
selflog('WARNING cannot remove temporary directory %s' % job_dir)
def enqueue_feed(self,n=4):
feed = client.list_bins(namespace=self.config.namespace,n=n)
for bin in feed:
bin_pid = bin['pid']
#if not should_skip(bin_pid):
print 'queueing %s' % bin_pid
self.enqueue(bin_pid)
if __name__=='__main__':
timeseries = sys.argv[1]
config = get_config('./blob.conf',timeseries)
job = BlobExtraction(config)
job.preflight()
command = sys.argv[2]
if command == 'q':
for pid in sys.argv[3:]:
job.enqueue(pid)
elif command == 'log':
job.consume_log()
elif command == 'w':
job.work(True)
elif command == 'r':
job.retry_failed(filter=lambda x: len(x)>3)
elif command == 'cron':
job.enqueue_feed(n=25)
TRIM='-evalchannels "[1]+[2] > 0 ? val : 0" "val" "val" -crop'
return ' '.join([self.config.imagestack_exec_path,
'-load %(img_in)s',
DEBAYER,
'-dup',
CROP_L, ILLUM_CORRECT, CONTRAST,
'-pull 1',
CROP_R, ILLUM_CORRECT, CONTRAST,
ALIGN, GRAYSCALE, CYAN_ONLY, '-pull 1', GRAYSCALE, RED_ONLY, '-add',
TRIM,
'-save %(img_out)s'
])
# usage:
# python stereo.py {config file} {command} {arguments}
# commands:
# q (file1, file2, file3, ... filen)
# enqueue files for processing
# q -
# read a list of files to process from stdin
# r
# requeue failed processing jobs
# w
# run as a worker
# log
# show logging messages as they come in
if __name__=='__main__':
hl = HabcamRedCyan(get_config(sys.argv[1]))
cli(hl, sys.argv)
))
def configure(config=None):
app.config[RESOLVER] = config.resolver
try:
if config.debug in ['True', 'true', 'T', 't', 'Yes', 'yes', 'debug']:
app.debug = True
except:
pass
try:
app.config[PORT] = int(config.port)
except:
app.config[PORT] = 5063
if __name__=='__main__':
if len(sys.argv) > 1:
configure(get_config(sys.argv[1]))
else:
configure()
else:
configure(get_config(os.environ['DEPOSIT_CONFIG_FILE']))
rs = parse_stream(app.config[RESOLVER])
blob_resolver = rs['mvco_blob']
blob_destination = rs['blobs']
features_destination = rs['features_destination']
multiblob_destination = rs['multiblob_destination']
if __name__=='__main__':
app.run(host='0.0.0.0',port=app.config[PORT])
"""Serve a stitched ROI image given the output of the pid resolver"""
roi_image = get_stitched_roi(hit.bin_pid, hit.target_no, mask=mask, stitch_version=stitch_version)
if roi_image is None:
abort(404)
# now determine PIL format and MIME type
(pil_format, mimetype) = image_types(hit)
# return the image data
return image_response(roi_image,pil_format,mimetype)
app.secret_key = os.urandom(24)
if __name__=='__main__':
"""First argument is a config file which must at least have psql_connect in it
to support feed arguments. Filesystem config is in the resolver."""
if len(sys.argv) > 1:
configure(get_config(sys.argv[1]))
else:
configure()
else:
configure(get_config(os.environ['IFCB_CONFIG_FILE']))
# FIXME don't use globals
# FIXME do this in config
rs = parse_stream(app.config[RESOLVER])
binpid2path = rs['binpid2path']
pid_resolver = rs['pid']
blob_resolver = rs['mvco_blob']
fea_resolver = rs['features']
class_scores_resolver = rs['class_scores']
ts_resolver = rs['time_series']
all_series = rs['all_series']
params += [offset]
cursor.execute('select imagename from imagelist where assignment_id=%s '+status_clause+' order by imagename '+limitclause+'offset %s', tuple(params))
for row in cursor.fetchall():
d = {}
d['pid'] = self.pid(row[0], self.config.image_namespace)
d['image'] = d['pid']
yield d
def find_image(self,pid,offset,status,post_status=None):
with xa(self.config.psql_connect) as (connection,cursor):
cursor.execute('select imagename,status from imagelist where assignment_id=%s order by imagename offset %s for update of imagelist',(self.lid(pid),offset))
i = offset
for row in cursor.fetchall():
if row[1]==status:
if post_status is not None:
cursor.execute('update imagelist set status=%s where assignment_id=%s and imagename=%s',(post_status,self.lid(pid),row[0]))
connection.commit()
return i+1
i += 1
return offset
def set_status(self,assignment_id,image_id,status):
with xa(self.config.psql_connect) as (connection,cursor):
#print 'gonna set status to %s for assignment %s image %s' % (status,self.lid(assignment_id),self.lid(image_id))
cursor.execute('update imagelist set status=%s where assignment_id=%s and imagename=%s',(status,int(self.lid(assignment_id)),self.lid(image_id)))
connection.commit()
if __name__=='__main__':
config = get_config('habcam_annotation.conf')
has = HabcamAssignmentStore(config)
for ass in has.list_assignments():
print ass
message=message,
pid=pid,
path=destpath
))
# client code
class BlobDeposit(object):
def __init__(self,url_base='http://localhost:5000'):
self.url_base = url_base
def exists(self,pid):
req = urllib.Request('%s/exists/blobs/%s' % (self.url_base, pid))
resp = json.loads(urllib.urlopen(req).read())
return resp['exists']
def deposit(self,pid,zipfile):
with open(zipfile,'r') as inzip:
zipdata = inzip.read()
req = urllib.Request('%s/deposit/blobs/%s' % (self.url_base, pid), zipdata)
req.add_header('Content-type','application/x-ifcb-blobs')
resp = json.loads(urllib.urlopen(req).read())
return resp
if __name__=='__main__':
config = get_config('./blob.conf', sys.argv[1])
blob_storage = BlobStorage(config)
app.config['BLOB_STORAGE'] = blob_storage
(h,p) = re.match(r'http://(.*):(\d+)',config.blob_deposit).groups()
app.run(host=h, port=int(p))
def configure(config=None):
app.config[CACHE] = SimpleCache()
app.config[CACHE_TTL] = 120
try:
if config.debug in ['True', 'true', 'T', 't', 'Yes', 'yes', 'debug']:
app.debug = True
except AttributeError:
pass
try:
app.config[RESOLVER] = config.resolver
except AttributeError:
app.config[RESOLVER] = 'oii/habcam/image_resolver.xml'
try:
app.config[PORT] = int(config.port)
except:
app.config[PORT] = 5061
# utilities
if __name__=='__main__':
"""First argument is a config file"""
if len(sys.argv) > 1:
configure(get_config(sys.argv[1]))
else:
configure()
app.secret_key = os.urandom(24)
app.run(host='0.0.0.0',port=app.config[PORT])
else:
config = get_config(os.environ['IMAGE_SERVICE_CONFIG_FILE'])
configure(config)
for hit in R['list_images'].resolve_all(pid=bin_lid):
(imagename, cfa_LR_path) = hit.lid, hit.p
do_alt.s(imagename, cfa_LR_path, psql_connect).apply_async(queue='alt') # FIXME hardcoded queue name
# when running this as a worker use a queue name like leg1_alt
# and set concurrency to number of hardware threads
# so:
# celery -A oii.habcam.alt_celery worker -c 24 --queue=leg1_alt
# to enqueue all images from bin do this:
#
# celery --config={celery config file} call oii.habcam.alt_celery.queue_bin --args='["{bin lid}" "{alt config file}"]' --queue=leg1_alt
if __name__=='__main__':
config = get_config('alt_celery.conf')
R = parse_stream(config.resolver)
psql_connect = config.psql_connect
with xa(psql_connect) as (c,db):
db.execute("""
select imageid || '.tif'
from burton_alts
where parallax_alt is null
""")
while True:
rows = db.fetchmany(100)
if len(rows) == 0:
break
else:
for row in rows:
imagename = row[0]
with xa(self.psql_connect) as (c,db):
try:
db.execute('select lat,lon,description from bin_props where lid=%s',(bin_lid,))
(lat,lon,description) = db.fetchone()
d = dict(lat=lat,lon=lon,description=description)
for k in d.keys():
if d[k] is None:
del d[k]
return d
except:
return {}
import sys
from oii.config import get_config, Configuration
if __name__=='__main__':
if len(sys.argv) > 1:
config = get_config(sys.argv[1])
else:
config = Configuration()
try:
resolvers = resolver.parse_stream(config.resolver)
except:
resolvers = None
#fixity = IfcbFixity(config.psql_connect, resolvers)
#fixity.check_all()
autoclass = IfcbAutoclass(config.psql_connect)
for roc in autoclass.rois_of_class('tintinnid'):
print roc
pfx = term[:i]
for r in structs(self.app.get(url_for('taxonomy_autocomplete', term=pfx)).data):
assert r.label[:i] == pfx
assert r.pid == url
def config_backend(config):
try:
app.config[CONFIG] = config
app.config[ANNOTATION_STORE] = HabcamAnnotationStore(config)
app.config[ASSIGNMENT_STORE] = HabcamAssignmentStore(config)
app.config[CATEGORIES] = HabcamCategories(config)
app.config[AUTHENTICATION] = HabcamAuthentication(config)
except KeyError:
pass # FIXME log error
app.secret_key = os.urandom(24)
if __name__=='__main__':
if len(sys.argv) > 1:
config = get_config(sys.argv[1])
config_backend(config)
try:
port = int(config.port)
except KeyError:
port = 5000
app.run(host='0.0.0.0',port=port)
else:
config = get_config(os.environ['ANNOTATOR_CONFIG_FILE'])
config_backend(config)
