def __init__(self, pid, request=None):
try:
self.parsed = parse_pid(pid)
except StopIteration:
abort(404)
self.time_series = self.parsed['ts_label']
self.bin_lid = self.parsed['bin_lid']
self.lid = self.parsed[LID]
self.schema_version = self.parsed[SCHEMA_VERSION]
self.adc_cols = re.split(' ', self.parsed['adc_cols'])
if request is not None:
self.url_root = get_url_root()
self.canonical_pid = canonicalize(self.url_root, self.time_series, self.lid)
self.extension = 'json' # default
if 'extension' in self.parsed:
self.extension = self.parsed['extension']
self.timestamp = get_timestamp(self.parsed)
self.product = self.parsed[PRODUCT]
def serve_after_before(ts_label,after_before,n=1,pid=None):
if not after_before in ['before','after']:
abort(400)
try:
parsed = next(ifcb().pid(pid))
except StopIteration:
abort(404)
bin_lid = parsed['bin_lid']
with Feed(session, ts_label) as feed:
if after_before=='before':
bins = list(feed.before(bin_lid, n))
else:
bins = list(feed.after(bin_lid, n))
resp = []
for bin in bins:
sample_time_str = iso8601(bin.sample_time.timetuple())
pid = canonicalize(get_url_root(), ts_label, bin.lid)
resp.append(dict(pid=pid, date=sample_time_str))
return Response(json.dumps(resp), mimetype=MIME_JSON)
def acc_wakeup(wakeup_key):
"""- wake up and expire the session
- acquire a mutex on the acquisition key
- query for the instrument
- run the copy job
- schedule the accession job
- wakeup accession workers"""
# figure out if this wakeup matters to us
if not is_acc_key(wakeup_key):
return
time_series = get_time_series(wakeup_key)
# attempt to acquire mutex. if fails, that's fine,
# that means batch accession is already underway
try:
then = time.time()
count = 0
with Mutex(wakeup_key,ttl=45) as mutex:
session.expire_all() # don't be stale!
accession = Accession(session, time_series)
logging.warn('START BATCH %s' % time_series)
for fs in accession.list_filesets(): # FIXME debug, do all
lid = fs[LID]
if accession.bin_exists(lid):
continue # don't schedule accession if bin exists
pid = canonicalize(URL_PREFIX, time_series, fs[LID])
count += 1
if count % 100 == 0:
logging.warn('batch %s: scheduled %d bins' % (time_series, count))
schedule_accession(client,pid)
elapsed = time.time() - then
if elapsed > 25: # don't send heartbeats too often
mutex.heartbeat() # retain mutex
then = time.time()
client.wakeup() # wakeup workers
logging.warn('END BATCH %s: %d bins scheduled' % (time_series,count))
client.wakeup()
except Busy:
logging.warn('BATCH not waking up')
pass
def serve_mosaic_image(time_series=None, pid=None, params='/'):
"""Generate a mosaic of ROIs from a sample bin.
params include the following, with default values
- series (mvco) - time series (FIXME: handle with resolver, clarify difference between namespace, time series, pid, lid)
- size (1024x1024) - size of the mosaic image
- page (1) - page. for typical image sizes the entire bin does not fit and so is split into pages.
- scale - scaling factor for image dimensions """
# parse params
size, scale, page = parse_mosaic_params(params)
(w,h) = size
parsed = parse_pid(pid)
schema_version = parsed['schema_version']
try:
paths = get_fileset(parsed)
except NotFound:
abort(404)
adc_path = paths['adc_path']
roi_path = paths['roi_path']
bin_pid = canonicalize(get_url_root(), time_series, parsed['bin_lid'])
# perform layout operation
scaled_size = (int(w/scale), int(h/scale))
layout = list(get_mosaic_layout(adc_path, schema_version, bin_pid, scaled_size, page))
extension = parsed['extension']
# serve JSON on request
if extension == 'json':
return Response(json.dumps(list(layout2json(layout, scale))), mimetype=MIME_JSON)
mimetype = mimetypes.types_map['.' + extension]
# read all images needed for compositing and inject into Tiles
image_layout = []
with open(roi_path,'rb') as roi_file:
for tile in layout:
target = tile.image # in mosaic API, the record is called 'image'
# FIXME 1. replace PIL
image = get_target_image(parsed, target, file=roi_file, raw_stitch=True)
image_layout.append(Tile(PIL.Image.fromarray(image), tile.size, tile.position))
# produce and serve composite image
mosaic_image = thumbnail(mosaic.composite(image_layout, scaled_size, mode='L', bgcolor=160), (w,h))
#pil_format = filename2format('foo.%s' % extension)
return Response(as_bytes(mosaic_image), mimetype=mimetype)
def serve_pid(pid):
req = DashboardRequest(pid, request)
try:
paths = get_fileset(req.parsed)
except NotFound:
abort(404)
hdr_path = paths['hdr_path']
adc_path = paths['adc_path']
roi_path = paths['roi_path']
adc = Adc(adc_path, req.schema_version)
heft = 'full' # heft is either short, medium, or full
if 'extension' in req.parsed:
extension = req.parsed['extension']
if 'target' in req.parsed:
canonical_bin_pid = canonicalize(req.url_root, req.time_series, req.bin_lid)
target_no = int(req.parsed['target'])
# pull three targets, then find any stitched pair
targets = adc.get_some_targets(target_no-1, 3)
targets = list_stitched_targets(targets)
for t in targets:
if t[TARGET_NUMBER] == target_no:
target = t
add_pid(target, canonical_bin_pid)
# check for image
mimetype = mimetypes.types_map['.' + extension]
if mimetype.startswith('image/'):
if req.product=='raw':
img = get_target_image(req.parsed, target, roi_path)
return Response(as_bytes(img,mimetype),mimetype=mimetype)
if req.product=='blob':
return serve_blob_image(req.parsed, mimetype)
if req.product=='blob_outline':
img = get_target_image(req.parsed, target, roi_path)
return serve_blob_image(req.parsed, mimetype, outline=True, target_img=img)
# not an image, so get more metadata
targets = get_targets(adc, canonical_bin_pid)
# not an image, check for JSON
if extension == 'json':
return Response(json.dumps(target),mimetype=MIME_JSON)
target = get_target_metadata(target,targets)
# more metadata representations. we'll need the header
hdr = parse_hdr_file(hdr_path)
if extension == 'xml':
return Response(target2xml(req.canonical_pid, target, req.timestamp, canonical_bin_pid), mimetype='text/xml')
if extension == 'rdf':
return Response(target2rdf(req.canonical_pid, target, req.timestamp, canonical_bin_pid), mimetype='text/xml')
if extension in ['html', 'htm']:
template = {
'static': STATIC,
'target_pid': req.canonical_pid,
'bin_pid': canonical_bin_pid,
'properties': target,
'target': target.items(), # FIXME use order_keys
'date': req.timestamp
}
return template_response('target.html',**template)
else: # bin
if req.extension in ['hdr', 'adc', 'roi']:
path = dict(hdr=hdr_path, adc=adc_path, roi=roi_path)[req.extension]
mimetype = dict(hdr='text/plain', adc='text/csv', roi='application/octet-stream')[req.extension]
return Response(file(path,'rb'), direct_passthrough=True, mimetype=mimetype)
try:
if req.product in ['blobs','blob']: # accept old url pattern
return serve_blob_bin(req.parsed)
if req.product=='features':
return serve_features_bin(req.parsed)
if req.product=='class_scores':
return serve_class_scores_bin(req.parsed)
except NotFound:
abort(404)
# gonna need targets unless heft is medium or below
targets = []
if req.product != 'short':
targets = get_targets(adc, req.canonical_pid)
# end of views
# not a special view, handle representations of targets
if req.extension=='csv':
adc_cols = req.parsed[ADC_COLS].split(' ')
lines = targets2csv(targets,adc_cols)
return Response('\n'.join(lines)+'\n',mimetype='text/csv')
# we'll need the header for the other representations
hdr = parse_hdr_file(hdr_path)
if req.extension in ['html', 'htm']:
targets = list(targets)
context, props = split_hdr(hdr)
template = {
'static': STATIC,
'bin_pid': req.canonical_pid,
'time_series': req.time_series,
'context': context,
'properties': props,
'targets': targets,
'target_pids': [t['pid'] for t in targets],
'date': req.timestamp,
'files': get_files(req.parsed,check=True) # note: ORM call!
}
print get_files(req.parsed)
return template_response('bin.html', **template)
if req.extension=='json':
if req.product=='short':
return Response(bin2json_short(req.canonical_pid,hdr,req.timestamp),mimetype=MIME_JSON)
if req.product=='medium':
return Response(bin2json_medium(req.canonical_pid,hdr,targets,req.timestamp),mimetype=MIME_JSON)
return Response(bin2json(req.canonical_pid,hdr,targets,req.timestamp),mimetype=MIME_JSON)
if req.extension=='xml':
return Response(bin2xml(req.canonical_pid,hdr,targets,req.timestamp),mimetype='text/xml')
if req.extension=='rdf':
return Response(bin2rdf(req.canonical_pid,hdr,targets,req.timestamp),mimetype='text/xml')
if req.extension=='zip':
# look to see if the zipfile is resolvable
try:
zip_path = get_product_file(req.parsed, 'binzip')
if os.path.exists(zip_path):
return Response(file(zip_path), direct_passthrough=True, mimetype='application/zip')
except NotFound:
pass
except:
raise
buffer = BytesIO()
bin2zip(req.parsed,req.canonical_pid,targets,hdr,req.timestamp,roi_path,buffer)
return Response(buffer.getvalue(), mimetype='application/zip')
return 'unimplemented'
def canonicalize_bin(ts_label, b):
return {
'pid': canonicalize(get_url_root(), ts_label, b.lid),
'date': iso8601(b.sample_time.timetuple())
}