def test_agent():
env0 = MyTask()
env1 = MyTask()
env2 = MyTask()
env3 = MyTask()
hps_dict = {'hp_layer': [5, 10, 15], 'hp_epoch': [128, 256, 512]}
hps = Hyperparameters(hps_dict)
task_scheduler = TaskScheduler(['step0', 'step1'])
task_scheduler.add_subtask('step0', 'task0', env=env0, hps=hps)
task_scheduler.add_subtask('step0', 'task1', env=env1)
task_scheduler.add_subtask('step1', 'task2', env=env2, hps=hps)
task_scheduler.add_subtask('step1', 'task3', env=env3)
task_scheduler.show_info()
task_scheduler.get_sorted_task_ids()
assert task_scheduler.get_sorted_task_ids() == ['step0', 'step1']
assert task_scheduler.get_subtask_ids('step0') == ['task0', 'task1']
assert task_scheduler.get_children_task_ids('step0') == ['step1']
assert task_scheduler.get_parents_task_ids('step1') == ['step0']
assert task_scheduler.get_subtask('step0', 'task0').env == env0
storegate = StoreGate(backend='numpy', data_id='test_agent')
saver = Saver()
metric = RandomMetric()
logger.set_level(logger.DEBUG)
agent = GridSearchAgent(saver=saver,
storegate=storegate,
task_scheduler=task_scheduler,
metric=metric,
dump_all_results=True)
assert agent._storegate is storegate
assert agent._saver is saver
assert agent._task_scheduler is task_scheduler
assert agent._metric is metric
agent.storegate = storegate
agent.saver = saver
agent.task_scheduler = task_scheduler
agent.metric = metric
assert agent.storegate is storegate
assert agent.saver is saver
assert agent.task_scheduler is task_scheduler
assert agent.metric is metric
agent.execute()
agent.finalize()
best_result = agent.get_best_result()
assert best_result.metric_value > 0
def get_storegate(data_path='/tmp/onlyDiTau/', max_events=50000):
# Index for signal/background shuffle
cur_seed = np.random.get_state()
np.random.seed(1)
permute = np.random.permutation(2 * max_events)
np.random.set_state(cur_seed)
storegate = StoreGate(backend='numpy', data_id='')
for path, var_names in [
("jet.npy",
('1stRecoJetPt', '1stRecoJetEta', '1stRecoJetPhi', '1stRecoJetMass',
'2ndRecoJetPt', '2ndRecoJetEta', '2ndRecoJetPhi', '2ndRecoJetMass')),
("tau.npy",
('1stTruthTauJetPt', '1stTruthTauJetEta', '1stTruthTauJetPhi',
'1stTruthTauJetMass', '2ndTruthTauJetPt', '2ndTruthTauJetEta',
'2ndTruthTauJetPhi', '2ndTruthTauJetMass')),
("istau.npy", ('tauFlag1stJet', 'tauFlag2ndJet')),
("energy.npy", ('1stRecoJetEnergyMap', '2ndRecoJetEnergyMap')),
]:
data_list = []
for label in ['Htautau', 'Zpure_tau']:
data_loaded = np.load(data_path + f"{label}_{path}")
data_loaded = data_loaded[:max_events]
data_list.append(data_loaded)
data_loaded = np.concatenate(data_list)
data_loaded = data_loaded[permute]
storegate.update_data(data=data_loaded,
var_names=var_names,
phase=(0.6, 0.2, 0.2))
# # Added TauMass
# tau_mass = np.full(shape=2 * max_events, fill_value=1.777)
# storegate.update_data(
# data_id='',
# data=tau_mass,
# var_names=['TauMass'],
# )
# storegate._num_events['TauMass'] += len(tau_mass)
# Setting labels
labels = np.concatenate([
np.ones(max_events),
np.zeros(max_events),
])[permute]
storegate.update_data(data=labels,
var_names='label',
phase=(0.6, 0.2, 0.2))
storegate.compile()
storegate.show_info()
return storegate
def build_storegate():
# Toy data
storegate = StoreGate(backend='numpy', data_id='test_keras_mlp')
data0 = np.random.normal(size=(100, 2))
label = np.random.binomial(n=1, p=0.5, size=(100, ))
phase = (0.8, 0.1, 0.1)
storegate.add_data(var_names=['var0', 'var1'], data=data0, phase=phase)
storegate.add_data(var_names='label', data=label, phase=phase)
storegate.compile()
storegate.show_info()
return storegate
def get_storegate(max_events=50000):
from my_tasks import reco_tau_4vec, truth_tau_4vec
fourvec_var_list = tuple(reco_tau_4vec + truth_tau_4vec)
# Toy data
storegate = StoreGate(backend='numpy', data_id='')
data0 = np.random.normal(size=(max_events, len(fourvec_var_list)))
data1 = np.random.uniform(size=(max_events, 2, 16, 16, 3))
label = np.random.binomial(n=1, p=0.5, size=(max_events, ))
phase = (0.6, 0.2, 0.2)
storegate.add_data(var_names=fourvec_var_list, data=data0, phase=phase)
storegate.add_data(var_names=('1stRecoJetEnergyMap',
'2ndRecoJetEnergyMap'),
data=data1,
phase=phase)
storegate.add_data(var_names='label', data=label, phase=phase)
storegate.compile()
storegate.show_info()
return storegate
def test_agent_basic_grid_scan():
saver = Saver()
storegate = StoreGate(backend='numpy', data_id='test_agent')
task_scheduler = TaskScheduler()
metric = RandomMetric()
subtask0 = BaseTask()
subtask1 = BaseTask()
task_scheduler.add_task('step0')
task_scheduler.add_subtask('step0', 'task0', env=subtask0)
task_scheduler.add_task('step1', parents=['step0'])
task_scheduler.add_subtask('step1',
'task1',
env=subtask1,
hps=Hyperparameters({'job_id': [0, 1]}))
for metric_type in ['min', 'max']:
agent = GridSearchAgent(saver=saver,
storegate=storegate,
task_scheduler=task_scheduler,
metric=metric,
metric_type=metric_type,
dump_all_results=True)
agent.execute()
agent.finalize()
agent.get_best_result()
with pytest.raises(NotImplementedError):
agent = GridSearchAgent(
saver=saver,
storegate=storegate,
task_scheduler=task_scheduler,
metric=metric,
metric_type='dummy',
dump_all_results=True,
)
agent.execute()
agent.finalize()
def test_agent_basic_base():
saver = Saver()
storegate = StoreGate(backend='numpy', data_id='test_agent')
task_scheduler = TaskScheduler()
metric = RandomMetric()
agent = BaseAgent()
agent.storegate = storegate
agent.saver = saver
agent.task_scheduler = task_scheduler
agent.metric = metric
assert agent.storegate is storegate
assert agent.saver is saver
assert agent.task_scheduler is task_scheduler
assert agent.metric is metric
agent.execute()
agent.finalize()
print(agent)
def test_agent_basic_simple():
saver = Saver()
storegate = StoreGate(backend='numpy', data_id='test_agent')
task_scheduler = TaskScheduler()
metric = RandomMetric()
subtask0 = BaseTask()
subtask1 = BaseTask()
task_scheduler.add_task('step0')
task_scheduler.add_subtask('step0', 'task0', env=subtask0)
task_scheduler.add_task('step1', parents=['step0'])
task_scheduler.add_subtask('step1',
'task1',
env=subtask1,
hps=Hyperparameters({'job_id': [0, 1]}))
agent = RandomSearchAgent(saver=saver,
storegate=storegate,
task_scheduler=task_scheduler,
metric=metric)
agent.execute()
agent.finalize()
def get_storegate(data_path='/tmp/onlyDiTau/', max_events=50000):
# Index for signal/background shuffle
cur_seed = np.random.get_state()
np.random.seed(1)
permute = np.random.permutation(2 * max_events)
np.random.set_state(cur_seed)
storegate = StoreGate(
backend='numpy',
data_id=''
)
for path, var_names in [
("jet.npy", (
'1stRecoJetPt', '1stRecoJetEta', '1stRecoJetPhi', '1stRecoJetMass',
'2ndRecoJetPt', '2ndRecoJetEta', '2ndRecoJetPhi', '2ndRecoJetMass'
)),
("tau.npy", (
'1stTruthTauJetPt', '1stTruthTauJetEta', '1stTruthTauJetPhi',
'1stTruthTauJetMass', '2ndTruthTauJetPt', '2ndTruthTauJetEta',
'2ndTruthTauJetPhi', '2ndTruthTauJetMass'
)),
("istau.npy", ('tauFlag1stJet', 'tauFlag2ndJet')),
("energy.npy", ('1stRecoJetEnergyMap', '2ndRecoJetEnergyMap')),
]:
data_list = []
for label in ['Htautau', 'Zpure_tau']:
data_loaded = np.load(data_path + f"{label}_{path}")
data_loaded = data_loaded[:max_events]
data_list.append(data_loaded)
data_loaded = np.concatenate(data_list)
data_loaded = data_loaded[permute]
if path == "energy.npy":
# for Pytorch image axis
data_loaded = np.transpose(data_loaded, (0, 1, 4, 2, 3))
storegate.update_data(
data=data_loaded,
var_names=var_names,
phase=(0.6, 0.2, 0.2)
)
# Setting labels
labels = np.concatenate([
np.ones(max_events),
np.zeros(max_events),
])[permute]
storegate.update_data(
data=labels,
var_names='label',
phase=(0.6, 0.2, 0.2)
)
storegate.compile()
# storegate.show_info()
return storegate
def __init__(self, max_events, data_path, phase=None):
from multiml.storegate import StoreGate
from multiml.data.numpy import NumpyFlatData
cur_seed = np.random.get_state()
np.random.seed(1)
permute = np.random.permutation(2 * max_events)
np.random.set_state(cur_seed)
self.storegate = StoreGate(backend='numpy', data_id='')
self.jet_vals = [
'1stRecoJetPt', '1stRecoJetEta', '1stRecoJetPhi', '1stRecoJetMass',
'2ndRecoJetPt', '2ndRecoJetEta', '2ndRecoJetPhi', '2ndRecoJetMass'
]
self.tau_vals = [
'1stTruthTauJetPt', '1stTruthTauJetEta', '1stTruthTauJetPhi',
'1stTruthTauJetMass', '2ndTruthTauJetPt', '2ndTruthTauJetEta',
'2ndTruthTauJetPhi', '2ndTruthTauJetMass'
]
self.tau_vals_wo_mass = [
'1stTruthTauJetPt',
'1stTruthTauJetEta',
'1stTruthTauJetPhi',
'2ndTruthTauJetPt',
'2ndTruthTauJetEta',
'2ndTruthTauJetPhi',
]
self.istau_vals = ['tauFlag1stJet', 'tauFlag2ndJet']
self.label_vals = ['label']
self.energy_vals = ['1stRecoJetEnergyMap', '2ndRecoJetEnergyMap']
for path, var_names in [
("jet.npy", self.jet_vals),
("tau.npy", self.tau_vals),
("istau.npy", self.istau_vals),
("energy.npy", self.energy_vals),
]:
data_list = []
for label in ['Htautau', 'Zpure_tau']:
data_loaded = NumpyFlatData().load_file(data_path +
f"{label}_{path}")
data_loaded = data_loaded[:max_events]
data_list.append(data_loaded)
data_loaded = np.concatenate(data_list)
data_loaded = data_loaded[permute]
self.storegate.update_data(data_id='',
data=data_loaded,
var_names=var_names,
phase=(0.6, 0.2, 0.2))
labels = np.concatenate([
np.ones(max_events),
np.zeros(max_events),
])[permute]
self.storegate.update_data(data_id='',
data=labels,
var_names='label',
phase=(0.6, 0.2, 0.2))
self.storegate.compile()
if phase is None:
phase = 'train'
self.phase = phase
self._swich_phase()
def test_storegate_zarr():
storegate = StoreGate(backend='hybrid',
backend_args={'mode': 'w'},
data_id=data_id)
assert storegate.data_id == data_id
storegate.set_data_id(data_id)
assert storegate.data_id == data_id
phase = (0.8, 0.1, 0.1)
# add new variables (bind_vars=True)
storegate.add_data(var_names=var_names01, data=data01_bind, phase=phase)
assert storegate.get_data_ids() == [data_id]
# change hybrid mode
storegate.set_mode('numpy')
# add new variables (bind_vars=False)
storegate.add_data(var_names=var_names23,
data=data23,
phase=phase,
bind_vars=False)
storegate.to_storage(var_names=var_names23, phase='all')
storegate.to_memory(var_names=var_names23, phase='train')
# update existing variables and data (bind_vars=True)
storegate.update_data(var_names=var_names23, data=data23_bind, phase=phase)
# update existing variables and data (bind_vars=False)
storegate.update_data(var_names=var_names23,
data=data23,
phase=phase,
bind_vars=False)
# compile for multiai
storegate.compile()
storegate.get_metadata()
storegate.show_info()
# update data by auto mode
storegate.update_data(var_names=var_names23,
data=data23,
phase='all',
bind_vars=False)
storegate['all'][var_names23][:] = data23_bind
storegate.compile()
# get data
storegate.get_data(var_names=var_names23, phase='train', index=0)
storegate.get_data(var_names=var_names23, phase='all')
storegate['all'][var_names23][:]
# tests
total_events = len(data0)
train_events = total_events * phase[0]
valid_events = total_events * phase[1]
test_events = total_events * phase[2]
assert len(storegate['train']) == total_events * phase[0]
assert len(storegate['valid']) == total_events * phase[1]
assert len(storegate['test']) == total_events * phase[2]
assert var_names23[0] in storegate['train']
assert var_names23[1] in storegate['train']
storegate_train = storegate['train']
assert storegate_train[var_names23][:].shape == (train_events,
len(var_names23))
assert storegate_train[var_names23][0].shape == (len(var_names23), )
assert storegate_train[var_names23[0]][:].shape == (train_events, )
assert storegate_train[var_names23[0]][0] == 20
# delete data
storegate.delete_data(var_names='var2', phase='train')
del storegate['train'][['var0', 'var1']]
assert storegate.get_var_names(phase='train') == ['var3']
# convert dtype
storegate.astype(var_names='var3', dtype=np.int)
storegate.onehot(var_names='var3', num_classes=40)
storegate.argmax(var_names='var3', axis=1)
assert storegate['train']['var3'][0] == 30
# test shuffle with numpy mode
storegate = StoreGate(backend='numpy', data_id=data_id)
storegate.add_data(var_names=var_names01,
data=data01_bind,
phase=phase,
shuffle=True)
storegate.add_data(var_names=var_names23,
data=data01_bind,
phase=phase,
shuffle=True)
storegate.compile()
storegate_train = storegate['train']
assert storegate_train[var_names01][0][0] == storegate_train[var_names23][
0][0]