def event(self, doctuple):
dbname = self.dbname
remote_load = self.remote_load
# parent_uid, self_uid, doc
descriptor_uid, event_uid, doc = doctuple
# descriptor_uid = doc['descriptor']
if descriptor_uid not in self.descriptors:
msg = "Missing descriptor for event"
msg += "\nEvent uid : {}".format(event_uid)
msg += "\nDescriptor uid: {}".format(descriptor_uid)
raise Exception(msg)
start_uid, descriptor = self.descriptors[descriptor_uid]
_, start = self.start_docs[start_uid]
kwargs = self.kwargs.copy()
if self.remote:
# run but don't return result
#msg = 'submitting function {} to server'.format(self.func.__name__)
#print(msg)
#print("about to submit function {}".format(self.func))
#print("start : {}".format(start))
#print("descriptor : {}".format(descriptor))
#print("doc : {}".format(doc))
#res = client.submit(wraps(self.func)(eval_func), self.func, start,
# TODO find out why I can't use "wraps" (scheduler complains about pickling)
res = client.submit(eval_func,
self.func,
start,
descriptor,
doc,
*self.args,
fill=self.fill,
remote_load=remote_load,
dbname=dbname,
**kwargs)
# the client may not submit remotely but return a value
# (depending on the client setup)
if isinstance(res, Future):
config.futures_cache_sinks.append(res)
config.futures_total.inc()
else:
# don't do things remotely, so block if things are Futures
#print("not remote")
if isinstance(doc, Future):
doc = doc.result()
if isinstance(descriptor, Future):
descriptor = descriptor.result()
if isinstance(start, Future):
start = start.result()
#print("Running function {}".format(self.func))
eval_func(self.func,
start,
descriptor,
doc,
*self.args,
fill=self.fill,
dbname=dbname,
remote_load=remote_load,
**kwargs)
def fill_events(sdoc, dbname=None, remote=True):
''' fill events in place'''
# the subset of self.fields that are (1) in the doc and (2) unfilled
# print("Filled events")
# print(doc['filled'])
non_filled = sdoc.get('_unfilled', [])
if remote:
# submit remotely
#print("remotely filling to client {}".format(client))
#print("data : {}".format(sdoc['kwargs']))
sdoc['kwargs'] = client.submit(_fill_events1,
data=sdoc['kwargs'],
non_filled=non_filled,
dbname="cms:data")
#futures_cache.append(sdoc['kwargs'])
#sdoc['kwargs'] = client.submit(lambda *x, **x2 : print('foo'))
#print("done remotely filling")
else:
sdoc['kwargs'] = _fill_events1(sdoc['kwargs'],
non_filled,
dbname="cms:data")
#print("kwargs of sdoc now : {}".format(sdoc['kwargs'].result()))
# assume it's filled now
sdoc['_unfilled'] = []
return sdoc
def pick_allowed_detectors(sdoc):
''' Only pass through 2d array events, ignore rest.
It will output a list of sdocs, each with different detector.
'''
sdocs = list()
md = sdoc.attributes
kwargs = sdoc.kwargs
#print(kwargs)
has_futures = False
for key in allowed_detector_keys:
if key in kwargs:
data = kwargs[key]
else:
continue
#if hasattr(data, 'ndim') and data.ndim > 0:
if True:
# this picks data of dimensions 2 only
# but also outputs some hints (in case we get a different
# detector that outputs different data. for ex: time series etc)
#if data.ndim == 1:
if False:
# TODO : return an sdoc that can raise something if needed
print("Found a 1D line of data? Ignoring...")
continue
elif False:
#elif data.ndim == 3:
msg = "Found 3D array data. Ignoring (but make sure this"
msg += " is not something you want to analyze"
continue
#elif data.ndim > 3:
elif False:
continue
else:
continue
# everything is good, output this in list
new_md = dict(md)
new_md.update(detector_key=key)
new_kwargs = dict(image=data)
def make_dict(key, data):
return {'image': data}
if isinstance(data, Future):
new_kwargs = client.submit(make_dict, key, data)
# the key is now image
sdoc_new = sd.StreamDoc(kwargs=new_kwargs,
attributes=new_md,
unfilled=['image'])
sdocs.append(sdoc_new)
return sdocs
def stop(self, doc):
if isinstance(doc, Future):
client.submit(super().stop, doc)
else:
super().stop(doc)
def descriptor(self, doc):
if isinstance(doc, Future):
client.submit(super().descriptor, doc)
else:
super().descriptor(doc)
def event(self, doc):
if isinstance(doc, Future):
client.submit(super().event, doc)
else:
super().event(doc)
def submittoclient(x):
print("Submitting to client : {}".format(client))
return client.submit(test_func, x)
def f_new(*args, **kwargs):
return client.submit(f, *args, **kwargs)