def serve_imgdata(imagename=None):
resolver = app.config[RESOLVER]
hit = resolver[IMGDATA].resolve(pid=imagename)
imagename = remove_extension(imagename)
if hit is None:
abort(404)
with open(hit.value,'r') as csvin:
for line in csvin:
imgname = remove_extension(re.split(',',line)[0].strip())
if imgname == imagename:
return Response(line, mimetype='text/plain')
abort(404)
def metadata2eic(bin_lid,parallax,merge=False):
"""Convert bin image metadata to CSV"""
#imagename,lat,lon,head,pitch,roll,alt1,alt2,depth,s,t,o2,cdom,chlorophyll,backscatter,therm
if merge:
(PITCH_COL, ROLL_COL) = (4, 5)
fields = ['imagename', 'alt', 'pitch','roll']
imgdata = resolver['img'].resolve(pid=bin_lid).value
logging.info('reading CSV data from .img file %s' % imgdata)
(pitch,roll) = ({}, {})
for row in read_csv(LocalFileSource(imgdata)):
imagename = remove_extension(row[0])
try:
pitch[imagename] = row[PITCH_COL]
roll[imagename] = row[ROLL_COL]
except KeyError:
pass
logging.info('merging with parallax-based altitude from %s' % parallax)
else:
logging.info('reading parallax-based altitude from %s' % parallax)
for l in open(parallax):
(imagename, _, _, alt) = re.split(r',',l.rstrip())
try:
if merge:
yield [imagename, alt, pitch[imagename], roll[imagename]]
else:
yield [imagename, alt, '0.0', '0.0']
except KeyError:
pass
def alt(bin_lid):
tmp = mkdirs(scratch(bin_lid))
csv_filename = os.path.join(tmp,bin_lid+'_alt.csv')
logging.info('listing images for %s' % bin_lid)
imagenames = [remove_extension(i) for i in list_images(bin_lid)]
logging.info('looking for existing altitude data...')
already_done = []
if os.path.exists(csv_filename):
for row in read_csv(LocalFileSource(csv_filename)):
already_done += [remove_extension(row[0])]
logging.info('found %d existing altitude records' % len(already_done))
if len(already_done) == -1:
logging.info('emptying CSV file ...')
with open(csv_filename,'w') as csv_out:
pass
pids = []
for n in range(NUM_PROCS):
pid = os.fork()
if pid == 0:
for imagename in imagenames[n::NUM_PROCS]:
if imagename not in already_done:
tif = img_as_float(read_image(imagename+'.tif'))
logging.info('[%d] START aligning %s' % (n, imagename))
x,y,m = stereo2altitude(tif)
line = '%s,%d,%d,%.2f' % (imagename,x,y,m)
logging.info('[%d] DONE aligned %s' % (n, line))
with open(csv_filename,'a') as csv_out:
print >>csv_out, line
csv_out.flush()
os._exit(0)
else:
logging.info('spawned process %d' % pid)
pids += [pid]
for pid in pids:
logging.info('waiting for process %d' % pid)
os.waitpid(pid,0)
logging.info('joined alignment process %d' % pid)
# now sort file
logging.info('sorting CSV data...')
rows = list(read_csv(LocalFileSource(csv_filename)))
rows = sorted(rows, key=lambda r: r[0])
csv_out = csv_filename # same as in
with open(csv_out,'w') as co:
for row in rows:
print >>co, ','.join([row[n] for n in range(4)])
logging.info('wrote CSV data to %s' % csv_out)
def check_fileset(hdr_file, schema_version=SCHEMA_VERSION_2):
lid = remove_extension(os.path.basename(hdr_file))
hdr_source = LocalFileSource(hdr_file)
adc_file = change_extension(hdr_file, 'adc')
adc_source = LocalFileSource(adc_file)
roi_source = LocalFileSource(change_extension(hdr_file, 'roi'))
try:
check_all(lid, hdr_source, adc_source, roi_source, schema_version=schema_version)
logging.info('%s PASS ALL' % lid)
except IntegrityException, e:
logging.info('%s FAIL %s' % (lid, e))
raise
def fetch_eic(bin_lid,suffix='',tmp=None,skip=[]):
"""Fetch bin image metadata and write CSV to file"""
if tmp is None:
tmp = mkdirs(os.path.join(scratch(bin_lid),'tmp'))
parallax = os.path.join(scratch(bin_lid),bin_lid+'_alt.csv')
eic = os.path.join(tmp,'%s%s.eic' % (bin_lid, suffix))
with open(eic,'w') as fout:
for tup in metadata2eic(bin_lid,parallax):
imagename = remove_extension(tup[0])
if imagename not in skip:
tup[0] = imagename
print >> fout, ' '.join(tup)
return eic
def check_fileset(fileset, schema_version=SCHEMA_VERSION_2):
hdr_path = fileset[HDR]
adc_path = fileset[ADC]
roi_path = fileset[ROI]
lid = remove_extension(os.path.basename(hdr_path))
try:
check_hdr(hdr_path)
logging.info('PASS %s hdr %s' % (lid, hdr_path))
targets = check_adc(adc_path, schema_version=schema_version)
logging.info('PASS %s adc %s' % (lid, adc_path))
check_roi(roi_path, targets)
logging.info('PASS %s roi %s' % (lid, roi_path))
except IntegrityException, e:
logging.info('%s FAIL %s' % (lid, e))
raise
def list_images(bin_pid):
for image in json.loads(urlopen('%s%s.json' % (BASE_URL, bin_pid)).read()):
yield remove_extension(image['imagename'])
def correct(bin_lid,learn_lid=None):
(h,w),LR_dir = stage(bin_lid)
if learn_lid is None:
learn_lid = bin_lid
# provision temp space
tmp = mkdirs(scratch(bin_lid,'tmp'))
for LR in 'LR':
# check for lightmap existence
lightmap_dir = mkdirs(scratch(learn_lid,learn_lid + '_lightmap_' + LR))
lightmap = os.path.join(lightmap_dir,learn_lid+'_lightmap_' + LR)
if not os.path.exists(lightmap):
logging.info('requested lightmap does not exist, skipping correct phase')
return
# check for existing output
logging.info('checking for existing corrected images...')
outdir = mkdirs(scratch(bin_lid,bin_lid + '_cfa_illum_' + LR))
skip = []
for fn in os.listdir(outdir):
imagename = re.sub('_cfa_illum_' + LR + '.tif','',fn)
skip += [remove_extension(imagename)]
mergedir = mkdirs(scratch(bin_lid,bin_lid + '_cfa_illum_LR'))
for fn in os.listdir(mergedir):
imagename = re.sub('_cfa_illum_LR.tif','',fn)
skip += [remove_extension(imagename)]
logging.info('found %d existing corrected images ...' % len(skip))
outdir = mkdirs(scratch(bin_lid,bin_lid + '_cfa_illum_' + LR))
# fetch eic
eic = fetch_eic(bin_lid,suffix='_'+LR,skip=skip)
# now see if that file is empty
if os.stat(eic)[6] == 0:
logging.info('no images to correct, skipping %s/%s' % (bin_lid, LR))
continue
# construct param file
correct_tmp = mkdirs(scratch(bin_lid,bin_lid + '_correct_tmp_' + LR))
param = os.path.join(tmp,bin_lid + '_correct.txt')
# produce param file
logging.info('writing param file %s' % param)
with open(param,'w') as fout:
print >> fout, 'imagedir %s' % LR_dir
print >> fout, 'outdir %s' % outdir
print >> fout, 'metafile %s' % re.sub(r'/([^/]+)$',r'/ \1',eic)
print >> fout, 'tmpdir %s' % correct_tmp
print >> fout, 'load %s' % lightmap
print >> fout, 'num_to_process %d' % NUM_CORRECT
for k,v in DEFAULT_IC_CONFIG.items():
print >> fout, '%s %s' % (k,str(v))
print >> fout, 'binary_format'
print >> fout, 'num_rows %d' % h
print >> fout, 'num_cols %d' % (w/2)
print >> fout, 'top %d' % 0
if LR == 'L':
print >> fout, 'left %d' % 0
else:
print >> fout, 'left %d' % (w/2)
print >> fout, PATTERN
print >> fout, 'scallop_eic'
print >> fout, 'correct'
# now correct
logging.info('correcting %s' % bin_lid)
correct = Process('"%s" "%s"' % (IC_EXEC, param))
for line in correct.run():
logging.info(line['message'])
