def from_json(cls, json_str, regs, reg_params):
"""Instantiate a gmm object from a json object.
"""
gmm_dict = json.loads(json_str)
prior_params_pattern = \
get_prior_params_pattern(
gmm_dict['obs_dim'], gmm_dict['num_components'])
prior_params = prior_params_pattern.fold(gmm_dict['prior_params_flat'],
free=False)
transform_mat = json_tricks.loads(gmm_dict['transform_mat_json'])
unrotate_transform_mat = json_tricks.loads(
gmm_dict['unrotate_transform_mat_json'])
gmm = cls(num_components=gmm_dict['num_components'],
prior_params=prior_params,
regs=regs,
reg_params=reg_params,
transform_mat=transform_mat,
unrotate_transform_mat=unrotate_transform_mat,
cov_regularization=gmm_dict['cov_regularization'],
inflate_coef_cov=gmm_dict['inflate_coef_cov'])
preconditioner = json_tricks.loads(gmm_dict['preconditioner_json'])
gmm.get_kl_conditioned.set_preconditioner_matrix(preconditioner)
return gmm
def read_log(self):
"""
reading log with results and calculation of average importance
:return:
"""
feature_scores = {}
data = pd.read_csv(self._log_path, sep=';')
for row in data.iterrows():
features = json.loads(row[1].algorithm_features)
ranking = json.loads(row[1].algorithm_ranking)
for feature, rank in zip(features, ranking):
if feature in feature_scores.keys():
feature_scores[feature].append(rank)
else:
feature_scores[feature] = [rank]
feature_mean_scores = {}
for feature_name in feature_scores.keys():
feature_mean_scores[feature_name] = np.array(
feature_scores[feature_name]).mean()
self._feature_scores = feature_mean_scores
def coo_matrix_from_json(spdict_json):
spd = json.loads(spdict_json)
assert spd['type'] == 'coo_matrix'
data = json_tricks.loads(spd['data'])
row = json_tricks.loads(spd['row'])
col = json_tricks.loads(spd['col'])
return sp.sparse.coo_matrix(
( data, (row, col)), shape = spd['shape'])
def test_tzaware_date_time():
json = dumps(DTOBJ)
back = loads(json)
assert DTOBJ == back
for orig, bck in zip(DTOBJ, back):
assert orig == bck
assert type(orig) == type(bck)
txt = '{"__datetime__": null, "year": 1988, "month": 3, "day": 15, "hour": 8, "minute": 3, ' \
'"second": 59, "microsecond": 7, "tzinfo": "Europe/Amsterdam"}'
obj = loads(txt)
assert obj == pytz.timezone('Europe/Amsterdam').localize(datetime(year=1988, month=3, day=15, hour=8, minute=3, second=59, microsecond=7))
def test_path():
json = dumps(PATHS)
back = loads(json)
assert PATHS == back
for orig, bck in zip(PATHS, back):
assert orig == bck
txt = '{"__pathlib__": "."}'
obj = loads(txt)
assert obj == Path()
def exceptionReplay(parameters):
"""
Replay world playsback position and orientation, can be additionally used for screen capturing
Servo instantiates the servo object
next check for if ROS is running in the background, else start ros and the kinefly for WBAD
"""
# If replay world for playback old data, copy existing params to prevent overwrite from the saved state of previous run
if parameters["replayWorld"]:
# save params from current instance as backup toload back into the playback file
replay = parameters["replayWorld"]
scale = parameters["captureScale"]
start = parameters["captureStart"]
increment = parameters["playbackIncrement"]
# # replayPath = easygui.fileopenbox(multiple=False, filetypes=["*.pickle"])
# replayPath = "/home/behaviour/catkin/src/beginner/scripts/panda/world/bags/fly4/fly4_quad_rg_gain7.0_speed_3.5_" \
# "trial_1_2016-04-13__23:31:35.bag_df.pickle"
print parameters["replayPath"]
# repExt=parameters["replayPath"].split('.')[-1]
# if repExt=='.bag':
# from analyseWorld.ipy_notebooks import ipyimports
# df,parameters,p=bag2pickle(parameters["replayPath"])
df = pd.read_pickle(parameters["replayPath"])
# slice pos and orientation and remove nans heading,pitch,roll, x,y,z and drop na which is from camera message time
traj = df.loc[:, "trajectory__orientation_x":
"trajectory__position_z"].dropna()
cols = traj.columns.tolist() # list of colums
cols = cols[
-3:] + cols[:
-3] # reorder colums. hprpos to poshpr by python splicing
traj = traj[cols] # reassign with this order
# //todo change current parameters to parameters saved in dataframe
parameters = None
try:
parameters = json.loads(df.metadata__data.values[1])
except:
parameters = json.loads(df.metadata__data.values[0])
print "using exceprion to load params"
# dump back params from vars
parameters["replayWorld"] = replay
parameters["captureScale"] = scale
parameters["captureStart"] = start
parameters["playbackIncrement"] = increment
return parameters
def test_memory_order():
arrC = array([[1., 2.], [3., 4.]], order='C')
json = dumps(arrC)
arr = loads(json)
assert array_equal(arrC, arr)
assert arrC.flags['C_CONTIGUOUS'] == arr.flags['C_CONTIGUOUS'] and \
arrC.flags['F_CONTIGUOUS'] == arr.flags['F_CONTIGUOUS']
arrF = array([[1., 2.], [3., 4.]], order='F')
json = dumps(arrF)
arr = loads(json)
assert array_equal(arrF, arr)
assert arrF.flags['C_CONTIGUOUS'] == arr.flags['C_CONTIGUOUS'] and \
arrF.flags['F_CONTIGUOUS'] == arr.flags['F_CONTIGUOUS']
def test_pandas_series():
for name, col in COLUMNS.items():
ds = Series(data=col, name=name)
txt = dumps(ds, allow_nan=True)
back = loads(txt)
assert (ds.equals(back))
assert ds.dtype == back.dtype
for name, col in COLUMNS.items():
ds = Series(data=col, name=name)
txt = dumps(ds, primitives=True, allow_nan=True)
back = loads(txt)
assert isinstance(back, dict)
assert_equal(ds.index.values, back['index'])
assert_equal(ds.values, back['data'])
def exceptionReplay(parameters):
"""
Replay world playsback position and orientation, can be additionally used for screen capturing
Servo instantiates the servo object
next check for if ROS is running in the background, else start ros and the kinefly for WBAD
"""
# If replay world for playback old data, copy existing params to prevent overwrite from the saved state of previous run
if parameters["replayWorld"]:
# save params from current instance as backup toload back into the playback file
replay = parameters["replayWorld"]
scale = parameters["captureScale"]
start = parameters["captureStart"]
increment = parameters["playbackIncrement"]
# # replayPath = easygui.fileopenbox(multiple=False, filetypes=["*.pickle"])
# replayPath = "/home/behaviour/catkin/src/beginner/scripts/panda/world/bags/fly4/fly4_quad_rg_gain7.0_speed_3.5_" \
# "trial_1_2016-04-13__23:31:35.bag_df.pickle"
print parameters["replayPath"]
# repExt=parameters["replayPath"].split('.')[-1]
# if repExt=='.bag':
# from analyseWorld.ipy_notebooks import ipyimports
# df,parameters,p=bag2pickle(parameters["replayPath"])
df = pd.read_pickle(parameters["replayPath"])
# slice pos and orientation and remove nans heading,pitch,roll, x,y,z and drop na which is from camera message time
traj = df.loc[:, "trajectory__orientation_x":"trajectory__position_z"].dropna()
cols = traj.columns.tolist() # list of colums
cols = cols[-3:] + cols[:-3] # reorder colums. hprpos to poshpr by python splicing
traj = traj[cols] # reassign with this order
# //todo change current parameters to parameters saved in dataframe
parameters = None
try:
parameters = json.loads(df.metadata__data.values[1])
except:
parameters = json.loads(df.metadata__data.values[0])
print "using exceprion to load params"
# dump back params from vars
parameters["replayWorld"] = replay
parameters["captureScale"] = scale
parameters["captureStart"] = start
parameters["playbackIncrement"] = increment
return parameters
def test_with_value():
obj = [
CombineComplexTypesEnum.class_inst, CombineComplexTypesEnum.timepoint
]
encoder = partial(enum_instance_encode, with_enum_value=True)
txt = dumps(obj, extra_obj_encoders=(encoder, ))
assert '"value":' in txt
back = loads(txt, obj_pairs_hooks=())
class_inst_encoding = loads(dumps(
CombineComplexTypesEnum.class_inst.value),
obj_pairs_hooks=())
timepoint_encoding = loads(dumps(CombineComplexTypesEnum.timepoint.value),
obj_pairs_hooks=())
assert back[0]['__enum__']['value'] == class_inst_encoding
assert back[1]['__enum__']['value'] == timepoint_encoding
def plot_auto_scores(cfg, subject, session):
"""Plot automated bad channel detection scores.
"""
import json_tricks
fname_scores = BIDSPath(subject=subject,
session=session,
task=cfg.task,
acquisition=cfg.acq,
run=None,
processing=cfg.proc,
recording=cfg.rec,
space=cfg.space,
suffix='scores',
extension='.json',
datatype=cfg.datatype,
root=cfg.deriv_root,
check=False)
all_figs = []
all_captions = []
for run in cfg.runs:
fname_scores.update(run=run)
auto_scores = json_tricks.loads(
fname_scores.fpath.read_text(encoding='utf-8-sig'))
figs = config.plot_auto_scores(auto_scores)
all_figs.extend(figs)
# Could be more than 1 fig, e.g. "grad" and "mag"
captions = [f'Run {run}'] * len(figs)
all_captions.extend(captions)
return all_figs, all_captions
def load(s):
with open(s,'r') as f:
json=f.read()
global D
D=loads(json)
if len(D["CHROMOSOMES"])==0:#empty list doesnt make sense, so we just generate the whole chr list
D["CHROMOSOMES"]=[str(i) for i in xrange(1,23)] + ["X","Y","XY"]
def handle_request(self, request):
command, data = request
_logger.debug('handle request: command: [{}], data: [{}]'.format(
command, data))
if command is CommandType.Terminate:
# if receive Terminate command, exit process
_logger.info(
'Receive Terminate command from NNI manager, terminating')
exit(0)
data = json_tricks.loads(data)
command_handlers = {
# Tunner commands:
CommandType.Initialize: self.handle_initialize,
CommandType.RequestTrialJobs: self.handle_request_trial_jobs,
CommandType.UpdateSearchSpace: self.handle_update_search_space,
CommandType.AddCustomizedTrialJob:
self.handle_add_customized_trial,
# Tunner/Assessor commands:
CommandType.ReportMetricData: self.handle_report_metric_data,
CommandType.TrialEnd: self.handle_trial_end,
}
if command not in command_handlers:
raise AssertionError('Unsupported command: {}'.format(command))
return command_handlers[command](data)
def from_json(data: str):
"""
Load and recreate a ``QuestPlus`` instance from a JSON string.
Parameters
----------
data
The JSON string, generated via :meth:`to_json`.
Returns
-------
QuestPlus
A ``QuestPlus`` instance, generated from the JSON string.
See Also
--------
to_json
"""
loaded = json_tricks.loads(data)
loaded.prior = xr.DataArray.from_dict(loaded.prior)
loaded.posterior = xr.DataArray.from_dict(loaded.posterior)
loaded.likelihoods = xr.DataArray.from_dict(loaded.likelihoods)
if loaded._rng is not None:
state = deepcopy(loaded._rng)
loaded._rng = np.random.RandomState()
loaded._rng.set_state(state)
return loaded
def handle_request(self, request):
command, data = request
_logger.debug('handle request: command: [{}], data: [{}]'.format(
command, data))
if data:
data = json_tricks.loads(data)
command_handlers = {
# Tunner commands:
CommandType.Initialize: self.handle_initialize,
CommandType.RequestTrialJobs: self.handle_request_trial_jobs,
CommandType.UpdateSearchSpace: self.handle_update_search_space,
CommandType.AddCustomizedTrialJob:
self.handle_add_customized_trial,
# Tunner/Assessor commands:
CommandType.ReportMetricData: self.handle_report_metric_data,
CommandType.TrialEnd: self.handle_trial_end,
CommandType.Ping: self.handle_ping,
}
if command not in command_handlers:
raise AssertionError('Unsupported command: {}'.format(command))
return command_handlers[command](data)
def read_log(self):
"""
reading log with results and calculation of average score
:return:
"""
data = pd.read_csv(self._log_path, sep=';')
feature_scores = {}
for row in data.iterrows():
features = json.loads(row[1].features)
score = row[1].score
for feature in features:
if feature in feature_scores.keys():
feature_scores[feature].append(score)
else:
feature_scores[feature] = [score]
feature_mean_scores = {}
for feature_name in feature_scores.keys():
feature_mean_scores[feature_name] = np.array(
feature_scores[feature_name]).mean()
self._feature_scores = feature_mean_scores
def load(string: str = None,
fp: Optional[Any] = None,
**json_tricks_kwargs) -> Any:
"""
Load the string or from file, and convert it to a complex data structure.
At least one of string or fp has to be not none.
Parameters
----------
"""
assert string is not None or fp is not None
# see encoders for explanation
hooks = [
json_tricks.pathlib_hook, json_tricks.pandas_hook,
json_tricks.json_numpy_obj_hook,
json_tricks.decoders.EnumInstanceHook(),
json_tricks.json_date_time_hook, json_tricks.json_complex_hook,
json_tricks.json_set_hook, json_tricks.numeric_types_hook,
_json_tricks_serializable_object_decode,
_json_tricks_func_or_cls_decode, _json_tricks_any_object_decode
]
if string is not None:
return json_tricks.loads(string,
obj_pairs_hooks=hooks,
**json_tricks_kwargs)
else:
return json_tricks.load(fp,
obj_pairs_hooks=hooks,
**json_tricks_kwargs)
def handle_request(self):
_logger.debug('waiting receive_message')
command, data = receive()
if command is None:
return False
_logger.debug('handle request: command: [{}], data: [{}]'.format(
command, data))
if command is CommandType.Terminate:
return False
data = json_tricks.loads(data)
command_handlers = {
# Tunner commands:
CommandType.RequestTrialJobs: self.handle_request_trial_jobs,
CommandType.UpdateSearchSpace: self.handle_update_search_space,
CommandType.AddCustomizedTrialJob:
self.handle_add_customized_trial,
# Tunner/Assessor commands:
CommandType.ReportMetricData: self.handle_report_metric_data,
CommandType.TrialEnd: self.handle_trial_end,
}
if command not in command_handlers:
raise AssertionError('Unsupported command: {}'.format(command))
return command_handlers[command](data)
def test_enum():
member = MyEnum.member1
txt = dumps(member)
back = loads(txt)
assert isinstance(back, MyEnum)
assert back == member
def verify():
print("verification request recieved")
if request.method == 'POST':
f = request.get_json(force=True)
filename = f['filename']
print(filename)
print(path)
registeredImg = f['face_vector']
registeredImg = loads(registeredImg, conv_str_byte=True)
for python_filename in os.listdir(path):
if python_filename == filename:
print('file found')
else:
print('file not found')
val = verifyFaceComparedToRegisteredFace(registeredImg,
os.path.join(path, filename))
print(os.path.join(path, filename))
os.remove(os.path.join(path, filename))
return jsonify(val)
else:
print("no request")
return jsonify(valid=False)
def test_pandas_dataframe():
df = DataFrame(COLUMNS, columns=tuple(COLUMNS.keys()))
txt = dumps(df, allow_nan=True)
back = loads(txt)
assert isnan(back.ix[0, -1])
assert (df.equals(back))
assert (df.dtypes == back.dtypes).all()
df = DataFrame(COLUMNS, columns=tuple(COLUMNS.keys()))
txt = dumps(df, primitives=True, allow_nan=True)
back = loads(txt)
assert isinstance(back, dict)
assert isnan(back['special'][0])
assert all(df.index.values == tuple(back.pop('index')))
for name, col in back.items():
assert name in COLUMNS
assert_equal(list(COLUMNS[name]), col)
def test_json_dump_after_iteration(self):
s = data.StairHandler(5)
s.__next__()
dump = s.saveAsJson()
s.origin = ''
assert s == json_tricks.loads(dump)
def refresh_config(self):
'''refresh to get latest config'''
sql = 'select params from ExperimentProfile where id=? order by revision DESC'
args = (self.experiment_id, )
self.config = config_v0_to_v1(
json_tricks.loads(self.conn.cursor().execute(sql,
args).fetchone()[0]))
def handle_trial_end(self, data):
'''
data: it has three keys: trial_job_id, event, hyper_params
trial_job_id: the id generated by training service
event: the job's state
hyper_params: the hyperparameters (a string) generated and returned by tuner
'''
hyper_params = json_tricks.loads(data['hyper_params'])
bracket_id, i, _ = hyper_params['parameter_id'].split('_')
hyper_configs = self.brackets[int(bracket_id)].inform_trial_end(int(i))
if hyper_configs is not None:
_logger.debug('bracket %s next round %s, hyper_configs: %s',
bracket_id, i, hyper_configs)
self.generated_hyper_configs = self.generated_hyper_configs + hyper_configs
for _ in range(self.credit):
if not self.generated_hyper_configs:
break
params = self.generated_hyper_configs.pop()
ret = {
'parameter_id': params[0],
'parameter_source': 'algorithm',
'parameters': params[1]
}
send(CommandType.NewTrialJob, json_tricks.dumps(ret))
self.credit -= 1
return True
def run(self):
"""Run the tuner.
This function will never return unless raise.
"""
_logger.info('Start dispatcher')
mode = os.getenv('NNI_MODE')
if mode == 'resume':
self.load_checkpoint()
while True:
command, data = receive()
if data:
data = json_tricks.loads(data)
if command is None or command is CommandType.Terminate:
break
if multi_thread_enabled():
result = self.pool.map_async(self.process_command_thread, [(command, data)])
self.thread_results.append(result)
if any([thread_result.ready() and not thread_result.successful() for thread_result in self.thread_results]):
_logger.debug('Caught thread exception')
break
else:
self.enqueue_command(command, data)
_logger.info('Dispatcher exiting...')
self.stopping = True
if multi_thread_enabled():
self.pool.close()
self.pool.join()
else:
self.default_worker.join()
self.assessor_worker.join()
_logger.info('Terminated by NNI manager')
def handle_report_metric_data(self, data):
"""
data: a dict received from nni_manager, which contains:
- 'parameter_id': id of the trial
- 'value': metric value reported by nni.report_final_result()
- 'type': report type, support {'FINAL', 'PERIODICAL'}
"""
# metrics value is dumped as json string in trial, so we need to decode it here
if 'value' in data:
data['value'] = json_tricks.loads(data['value'])
if data['type'] == MetricType.FINAL:
self._handle_final_metric_data(data)
elif data['type'] == MetricType.PERIODICAL:
if self.assessor is not None:
self._handle_intermediate_metric_data(data)
elif data['type'] == MetricType.REQUEST_PARAMETER:
assert multi_phase_enabled()
assert data['trial_job_id'] is not None
assert data['parameter_index'] is not None
param_id = _create_parameter_id()
try:
param = self.tuner.generate_parameters(
param_id, trial_job_id=data['trial_job_id'])
except NoMoreTrialError:
param = None
send(
CommandType.SendTrialJobParameter,
_pack_parameter(param_id,
param,
trial_job_id=data['trial_job_id'],
parameter_index=data['parameter_index']))
else:
raise ValueError('Data type not supported: {}'.format(
data['type']))
def test_dump_np_scalars():
data = [
int8(-27),
complex64(exp(1) + 37j),
(
{
'alpha': float64(-exp(10)),
'str-only': complex64(-1 - 1j),
},
uint32(123456789),
float16(exp(-1)),
set((
int64(37),
uint64(-0),
)),
),
]
replaced = encode_scalars_inplace(deepcopy(data))
json = dumps(replaced)
rec = loads(json)
print(data)
print(rec)
assert data[0] == rec[0]
assert data[1] == rec[1]
assert data[2][0] == rec[2][0]
assert data[2][1] == rec[2][1]
assert data[2][2] == rec[2][2]
assert data[2][3] == rec[2][3]
assert data[2] == tuple(rec[2])
def test_json_dump_after_iteration(self):
s = data.StairHandler(5)
s.__next__()
dump = s.saveAsJson()
s.origin = ''
assert s == json_tricks.loads(dump)
def from_json(s):
"""
desc: |
*Requires json_tricks*
Creates a DataMatrix from a `json` string.
arguments:
s:
desc: A json string.
type: str
returns:
desc: A DataMatrix.
type: DataMatrix.
"""
import json_tricks
d = json_tricks.loads(s)
dm = DataMatrix(length=len(d['rowid']))
for name, (coltype, seq) in d['columns'].items():
if coltype == '_SeriesColumn':
dm[name] = SeriesColumn(depth=seq.shape[1])
dm[name]._seq = seq
else:
dm[name] = globals()[coltype]
dm[name]._seq = seq
return dm
def cluster_analysis(input_json):
json = loads(input_json, preserve_order=True)
csv = pd.read_csv(StringIO(json['Csv']))
countCluster = json['CountCluster']
raw1 = csv['X']
raw2 = csv['Y']
x = [[] for i in range(countCluster)]
y = [[] for i in range(countCluster)]
X = np.array(list(zip(raw1, raw2)))
kmeans = KMeans(n_clusters=countCluster, random_state=0).fit(X)
centers = kmeans.cluster_centers_
Y = X[:, 1]
X = X[:, 0]
for i in range(len(Y)):
distances = np.zeros(countCluster)
for j in range(countCluster):
distances[j] = dist(X[i], centers[j][0], Y[i], centers[j][1])
indx = np.argmin(distances)
x[indx].append(X[i])
y[indx].append(Y[i])
return dumps(Cluster(x, y, centers), primitives=True)
def handle_trial_end(self, data):
"""receive the information of trial end and generate next configuaration.
Parameters
----------
data: dict()
it has three keys: trial_job_id, event, hyper_params
trial_job_id: the id generated by training service
event: the job's state
hyper_params: the hyperparameters (a string) generated and returned by tuner
"""
logger.debug('Tuner handle trial end, result is %s', data)
hyper_params = json_tricks.loads(data['hyper_params'])
s, i, _ = hyper_params['parameter_id'].split('_')
hyper_configs = self.brackets[int(s)].inform_trial_end(int(i))
if hyper_configs is not None:
logger.debug(
'bracket %s next round %s, hyper_configs: %s', s, i, hyper_configs)
self.generated_hyper_configs = self.generated_hyper_configs + hyper_configs
for _ in range(self.credit):
self._request_one_trial_job()
# Finish this bracket and generate a new bracket
elif self.brackets[int(s)].no_more_trial:
self.curr_s -= 1
self.generate_new_bracket()
for _ in range(self.credit):
self._request_one_trial_job()
def load_json(path_or_buf, use_pandas=True):
"""Load a compressed JSON object, by default using Pandas"""
def loader(path_or_buf):
"""Load either from a buffer or a file path."""
if isinstance(path_or_buf, str):
try:
exists = os.path.exists(path_or_buf)
except (TypeError, ValueError):
exists = False
# If it's a filepath, open and read it, else treat as bytes
if exists:
return open(path_or_buf, 'rb').read()
else:
return bytes(path_or_buf, 'ascii')
# If it's a bytes object, just return it
if isinstance(path_or_buf, bytes):
return path_or_buf
# Buffer-like
if hasattr(path_or_buf, 'read') and callable(path_or_buf.read):
return path_or_buf.read()
raise ValueError("Could not load `path_or_buf`")
elem = zlib.decompress(loader(path_or_buf))
if use_pandas:
return pd.read_json(elem)
else:
return jt.loads(str(elem, 'ascii'))
def test_json_dump(self):
q = data.QuestHandler(0.5, 0.2, pThreshold=0.63, gamma=0.01,
nTrials=20, minVal=0, maxVal=1)
dump = q.saveAsJson()
q.origin = ''
assert q == json_tricks.loads(dump)
def test_json_dump_with_data(self):
s = data.StairHandler(5)
s.addResponse(1)
s.addOtherData('foo', 'bar')
dump = s.saveAsJson()
s.origin = ''
assert s == json_tricks.loads(dump)
def loadFeaturesForFrame(self, features, timeframe):
"""
loads feature data
"""
assert(self.server_address is not None)
assert(self.uuid is not None)
node_service = DVIDNodeService(self.server_address, self.uuid)
node_service.create_keyvalue(self.keyvalue_store)
return json.loads(node_service.get(self.keyvalue_store, "frame-{}".format(timeframe)))
def test_json_dump_with_data(self):
q = data.QuestHandler(0.5, 0.2, pThreshold=0.63, gamma=0.01,
nTrials=20, minVal=0, maxVal=1)
q.addResponse(1)
q.addOtherData('foo', 'bar')
dump = q.saveAsJson()
q.origin = ''
assert q == json_tricks.loads(dump)
def getTimeRange(self, Resource, PathInResource):
"""
Count Label images to derive the total number of frames
Loads label image data from local resource file in hdf5 format.
PathInResource provides the internal image path
Return tuple of (first frame, last frame)
"""
node_service = DVIDNodeService(Resource, PathInResource)
config = json.loads(node_service.get("config", "imageInfo"))
return config["time_range"]
def getImageShape(self, Resource, PathInResource):
"""
Derive Image Shape from label image.
Loads label image data from local resource file in hdf5 format.
PathInResource provides the internal image path
Return list with image dimensions
"""
node_service = DVIDNodeService(Resource, PathInResource)
config = json.loads(node_service.get("config", "imageInfo"))
self.shape = config["shape"]
return self.shape
def test_json_dump(self):
t = data.TrialHandler2(self.conditions, nReps=5)
dump = t.saveAsJson()
t.origin = ''
t_loaded = json_tricks.loads(dump)
t_loaded._rng = np.random.RandomState()
t_loaded._rng.set_state(t_loaded._rng_state)
del t_loaded._rng_state
assert t == t_loaded
def test_json_dump(self):
if _travisTesting:
pytest.skip()
p = data.PsiHandler(nTrials=10, intensRange=[0.1, 10],
alphaRange=[0.1, 10], betaRange=[0.1, 3],
intensPrecision=1, alphaPrecision=1,
betaPrecision=0.5, delta=0.01)
dump = p.saveAsJson()
p.origin = ''
assert p == json_tricks.loads(dump)
def test_json_dump_with_data(self):
if _travisTesting:
pytest.skip()
p = data.PsiHandler(nTrials=10, intensRange=[0.1, 10],
alphaRange=[0.1, 10], betaRange=[0.1, 3],
intensPrecision=1, alphaPrecision=1,
betaPrecision=0.5, delta=0.01)
p.addResponse(1)
p.addOtherData('foo', 'bar')
dump = p.saveAsJson()
p.origin = ''
assert p == json_tricks.loads(dump)
def test_json_dump_with_data_after_iteration(self):
t = data.TrialHandler2(self.conditions, nReps=5)
t.addData('foo', 'bar')
t.__next__()
dump = t.saveAsJson()
t.origin = ''
t_loaded = json_tricks.loads(dump)
t_loaded._rng = np.random.RandomState()
t_loaded._rng.set_state(t_loaded._rng_state)
del t_loaded._rng_state
assert t == t_loaded
async def read_packets(self):
_L().debug('start listening for packets')
while not self.stop_event.is_set():
_L().debug('waiting for packet')
try:
packet = await self.read_packet()
except Exception:
if self.stop_event.is_set():
break
_L().debug('error reading packet', exc_info=True)
await asyncio.sleep(.01)
continue
if packet.type_ == PACKET_TYPES.STREAM:
try:
# XXX Use `json_tricks` rather than standard `json` to
# support serializing [Numpy arrays and scalars][1].
#
# [1]: http://json-tricks.readthedocs.io/en/latest/#numpy-arrays
message = json_tricks.loads(packet.data().decode('utf8'))
self.signals.signal(message['event']).send(message)
# Do not add event packets to a queue. This prevents the
# `stream` queue from filling up with rapidly occurring
# events.
continue
except Exception:
_L().debug('Stream packet contents do not describe an '
'event: %s', packet.data().decode('utf8'),
exc_info=True)
if packet.type_ == PACKET_TYPES.DATA:
await self._request_queue.put(packet)
for packet_type_i in ('data', 'ack', 'stream', 'id_response'):
if packet.type_ == getattr(PACKET_TYPES, packet_type_i.upper()):
self.signals.signal('%s-received' %
packet_type_i).send(packet)
_L().debug('stop listening for packets')
def parse(self, data):
'''
Parse data.
For each complete packet contained in the parsed data (or a packet
started on previous read that is completed), push the packet on a queue
according to the type of packet: ``data``, ``ack``, or ``stream``.
.. versionchanged:: 0.30
Add handling for :attr:`nadamq.NadaMq.PACKET_TYPES.ID_RESPONSE`
packets.
.. versionchanged:: 0.41
Send signal when packet is received, queue is full, or whenever a
JSON encoded message containing an ``"event"`` key received in a
:attr:`nadamq.NadaMq.PACKET_TYPES.STREAM` packet. See
:attr:`signals`.
.. versionchanged:: 0.41.1
Do not add event packets to a queue. This prevents the ``stream``
queue from filling up with rapidly occurring events.
Parameters
----------
data : str or bytes
'''
packets = []
for c in data:
result = self._packet_parser.parse(np.fromstring(c, dtype='uint8'))
if result is not False:
# A full packet has been parsed.
packet_str = np.fromstring(result.tostring(), dtype='uint8')
# Add parsed packet to list of packets parsed during this
# method call.
packets.append((datetime.now(),
cPacketParser().parse(packet_str)))
# Reset the state of the packet parser to prepare for next
# packet.
self._packet_parser.reset()
elif self._packet_parser.error:
# A parsing error occurred.
# Reset the state of the packet parser to prepare for next
# packet.
self._packet_parser.reset()
# Filter packets parsed during this method call and queue according to
# packet type.
for t, p in packets:
if p.type_ == PACKET_TYPES.STREAM:
try:
# XXX Use `json_tricks` rather than standard `json` to
# support serializing [Numpy arrays and scalars][1].
#
# [1]: http://json-tricks.readthedocs.io/en/latest/#numpy-arrays
message = json_tricks.loads(p.data().decode('utf8'))
self.signals.signal(message['event']).send(message)
# Do not add event packets to a queue. This prevents the
# `stream` queue from filling up with rapidly occurring
# events.
continue
except Exception:
logger.debug('Stream packet contents do not describe an '
'event: %s', p.data(), exc_info=True)
for packet_type_i in ('data', 'ack', 'stream', 'id_response'):
if p.type_ == getattr(PACKET_TYPES, packet_type_i.upper()):
self.signals.signal('%s-received' % packet_type_i).send(p)
if self.queue_full(packet_type_i):
self.signals.signal('%s-full' % packet_type_i).send()
else:
self.packet_queues[packet_type_i].put((t, p))
def test_json_dump(self):
s = data.StairHandler(5)
dump = s.saveAsJson()
s.origin = ''
assert s == json_tricks.loads(dump)
