def _update_layer_tile_coords(self, layer=0):
from PYME.IO import clusterIO
tiles = []
for xdir in clusterIO.cglob('/'.join(
[self.base_dir, '%d' % layer, '*'])):
for fn in clusterIO.cglob('/'.join([xdir, '*_%s' % self.suff])):
tiles.append(
tuple(
[int(s) for s in os.path.basename(fn).split('_')[:2]]))
self._coords[layer] = tiles
def fileTasksForInputs(self, **kwargs):
from PYME.IO import clusterIO
input_names = kwargs.keys()
inputs = {k : kwargs[k] if isinstance(kwargs[k], list) else clusterIO.cglob(kwargs[k], include_scheme=True) for k in input_names}
numTotalFrames = len(list(inputs.values())[0])
self.currentFrameNum = 0
logger.debug('numTotalFrames = %d' % numTotalFrames)
logger.debug('inputs = %s' % inputs)
while numTotalFrames > (self.currentFrameNum + 1):
logging.debug('we have unpublished frames - push them')
newFrameNum = min(self.currentFrameNum + 1000, numTotalFrames)
#create task definitions for each frame
tasks = [self._generate_task(**{k : inputs[k][frameNum] for k in inputs.keys()}) for frameNum in range(self.currentFrameNum, newFrameNum)]
task_list = tasks #json.dumps(tasks)
#print tasks
threading.Thread(target=self._postTasks, args=(task_list,)).start()
self.currentFrameNum = newFrameNum
def fileTasksForInputs(self, **kwargs):
from PYME.IO import clusterIO
input_names = kwargs.keys()
inputs = {k : kwargs[k] if isinstance(kwargs[k], list) else clusterIO.cglob(kwargs[k], include_scheme=True) for k in input_names}
numTotalFrames = len(list(inputs.values())[0])
self.currentFrameNum = 0
logger.debug('numTotalFrames = %d' % numTotalFrames)
logger.debug('inputs = %s' % inputs)
inputs_by_task = {frameNum: {k : inputs[k][frameNum] for k in inputs.keys()} for frameNum in range(numTotalFrames)}
rule = {'template': self._taskTemplate, 'inputsByTask' : inputs_by_task}
s = clusterIO._getSession(self.taskQueueURI)
r = s.post('%s/add_integer_id_rule?max_tasks=%d&release_start=%d&release_end=%d' % (self.taskQueueURI,numTotalFrames, 0, numTotalFrames), data=json.dumps(rule),
headers = {'Content-Type': 'application/json'})
if r.status_code == 200:
resp = r.json()
self._ruleID = resp['ruleID']
logging.debug('Successfully created rule')
else:
logging.error('Failed creating rule with status code: %d' % r.status_code)
def _to_input_list(v):
# TODO - move this fcn definition??
if isinstance(v, list):
return v
else:
# value is a string glob
name, serverfilter = unifiedIO.split_cluster_url(v)
return clusterIO.cglob(name, serverfilter)
def get_input_glob(request):
from PYME.IO import clusterIO
filepaths = clusterIO.cglob(request.GET.get('glob').lstrip('/'))
return render(request, 'recipes/input_list.html', {
'filepaths': filepaths,
'serverfilter': server_filter
})
def requeue_missed(self, n_x, n_y, x_spacing, y_spacing, start_pos, protocol_name, nice=20, sleep=1000):
from PYME.Acquire.actions import FunctionAction
from PYME.IO import clusterIO
import posixpath
import time
logger.debug('requeuing missed wells')
time.sleep(sleep)
x_wells, y_wells, names = self._get_positions(n_x, n_y, x_spacing, y_spacing, start_pos)
x_wells, y_wells, names = self._pop_wells(x_wells, y_wells, names, self._drop_wells)
# if a node dies we might lose the detections file, but likely won't
# lose the entire subdirectory
spooldir = self.scope.spoolController.dirname
to_pop = set()
for name in names:
if clusterIO.isdir(posixpath.join(spooldir, name)):
to_pop.add(name)
else:
for shame in range(1, self._shame_index):
if clusterIO.isdir(posixpath.join(spooldir, name + '_%d' % shame)):
to_pop.add(name)
logger.debug('subdirectories present for %d wells' % (len(names) - len(to_pop)))
# Look for h5 detections too - if we just tiled the last well and hit
# this call we might have a detection file after the sleep but this
# call blocks the acquisition task queue, so we can't have a subdir yet
imaged = clusterIO.cglob(posixpath.join(spooldir,
'[A-Z][0-9]*_detections.h5'))
imaged_wells = [im.split('/')[-1].split('_detections.h5')[0] for im in imaged]
detected = set([fn.split('_')[0] for fn in imaged_wells])
logger.debug('detection h5 present for an additional %d wells' % (len(detected) - len(to_pop.intersection(detected))))
to_pop = to_pop.union(detected)
x_wells, y_wells, names = self._pop_wells(x_wells, y_wells, names, list(to_pop))
if len(names) < 1:
return
self._shame_index += 1
shame_suffix = '_%d' % self._shame_index
names = [name + shame_suffix for name in names]
actions = self._get_action_list(x_wells, y_wells, names, protocol_name)
actions.append(FunctionAction('turnAllLasersOff', {}))
# lets just make it recursive for fun
if self.cb_get_it_done.GetValue():
actions.append(FunctionAction('multiwellpanel.requeue_missed',
{'n_x': n_x, 'n_y': n_y,
'x_spacing': x_spacing, 'y_spacing': y_spacing,
'start_pos': start_pos,
'protocol_name': protocol_name,
'nice': nice}))
logger.debug('requeuing %d wells' % len(names))
self.scope.actions.queue_actions(actions, nice)
def requeue_missed(self, n_x, n_y, x_spacing, y_spacing, start_pos, protocol_name, nice=20, sleep=5):
from PYME.Acquire.actions import FunctionAction
from PYME.IO import clusterIO
import posixpath
import time
logger.debug('requeuing missed wells')
time.sleep(sleep)
spooldir = self.scope.spoolController.dirname
detections_pattern = posixpath.join(spooldir, '[A-Z][0-9]*_detections.h5')
imaged = clusterIO.cglob(detections_pattern)
imaged_wells = [im.split('/')[-1].split('_detections.h5')[0] for im in imaged]
logger.debug('imaged %d wells' % len(imaged_wells))
x_wells, y_wells, names = self._get_positions(n_x, n_y, x_spacing, y_spacing, start_pos)
x_wells, y_wells, names = self._pop_wells(x_wells, y_wells, names, self._drop_wells)
to_pop = [fn.split('_')[0] for fn in imaged_wells]
x_wells, y_wells, names = self._pop_wells(x_wells, y_wells, names, to_pop)
if len(names) < 1:
return
self._shame_index += 1
shame_suffix = '_%d' % self._shame_index
names = [name + shame_suffix for name in names]
actions = self._get_action_list(x_wells, y_wells, names, protocol_name)
actions.append(FunctionAction('turnAllLasersOff', {}))
# lets just make it recursive for fun
if self.cb_get_it_done.GetValue():
actions.append(FunctionAction('multiwellpanel.requeue_missed',
{'n_x': n_x, 'n_y': n_y,
'x_spacing': x_spacing, 'y_spacing': y_spacing,
'start_pos': start_pos,
'protocol_name': protocol_name,
'nice': nice}))
logger.debug('requeuing %d wells' % len(names))
self.scope.actions.queue_actions(actions, nice)