def experiment_get_template(self, experiment_id=None):
"""Retrieve the template for an experiment.
Can be used to duplicate the experiment on another machine.
"""
util.check_par('experiment id', experiment_id, _type=int, _min=0)
exp = self._rest_session.get(self._rest_prefix + '/experiments/%d'%experiment_id)
try:
exp = exp.json()
exp['experiment']['id']
except:
raise Exception('Failed to get template for experiment id %d'%experiment_id)
# build the template protocol
ret_proto = {}
ret_proto['lid_temperature'] = exp['experiment']['protocol']['lid_temperature']
for stage in exp['experiment']['protocol']['stages']:
stage['stage'].pop('id')
for step in stage['stage']['steps']:
step['step'].pop('id')
ret_proto['stages'] = exp['experiment']['protocol']['stages']
ret = {'experiment':{'name':exp['experiment']['name'], 'protocol':ret_proto}}
return ret
def experiment_delete(self, experiment_id=None):
util.check_par('experiment id', experiment_id, _type=int, _min=0)
ret = self._rest_session.delete(self._rest_prefix + '/experiments/%d'%experiment_id)
if 'experiment' in ret.json() and not ret.json()['experiment']:
return True
else:
raise Exception('Failed to delete experiment id %d'%experiment_id)
def experiment_info(self, experiment_id=None):
"""Get status of an experiment."""
util.check_par('experiment id', experiment_id, _type=int, _min=0)
ret = self._rest_session.get(self._rest_prefix + '/experiments/%d'%experiment_id)
try:
return ret.json()
except:
raise Exception('Failed to get experiment id %d'%experiment_id)
def experiment_info(self, experiment_id=None):
"""Get status of an experiment."""
util.check_par('experiment id', experiment_id, _type=int, _min=0)
ret = self._rest_session.get(self._rest_prefix +
'/experiments/%d' % experiment_id)
try:
return ret.json()
except:
raise Exception('Failed to get experiment id %d' % experiment_id)
def set_heat_block_temp(self, temp, tol=0.5):
util.check_par('temperature', temp, _type=float, _min=0, _max=100)
util.check_par('tolerance', tol, _type=float, _min=0, _max=5)
self.test_control('heat_block_target_temp','%d'%temp)
while True:
time.sleep(1)
if abs(temp - float(self.status()['heat_block']['temperature'])) <= tol:
break
def set_heat_block_temp(self, temp, tol=0.5):
util.check_par('temperature', temp, _type=float, _min=0, _max=100)
util.check_par('tolerance', tol, _type=float, _min=0, _max=5)
self.test_control('heat_block_target_temp', '%d' % temp)
while True:
time.sleep(1)
if abs(temp -
float(self.status()['heat_block']['temperature'])) <= tol:
break
def experiment_delete(self, experiment_id=None):
util.check_par('experiment id', experiment_id, _type=int, _min=0)
ret = self._rest_session.delete(self._rest_prefix +
'/experiments/%d' % experiment_id)
if 'experiment' in ret.json() and not ret.json()['experiment']:
return True
else:
raise Exception('Failed to delete experiment id %d' %
experiment_id)
def experiment_start(self, experiment_id=None):
util.check_par('experiment id', experiment_id, _type=int, _min=0)
ret = self._rest_session.post(self._rest_prefix + ':8000/control/start', data='{"experiment_id":"%d"}'%experiment_id)
status = True
try:
if ret.json()['status']['status'] == 'true':
return True
else:
raise Exception('Failed to start experiment %d. (%s)'%(experiment_id, ret.json()['status']['error']))
except KeyError:
raise Exception('Invalid reply from instrument')
def experiment_start(self, experiment_id=None):
util.check_par('experiment id', experiment_id, _type=int, _min=0)
ret = self._rest_session.post(
self._rest_prefix + ':8000/control/start',
data='{"experiment_id":"%d"}' % experiment_id)
status = True
try:
if ret.json()['status']['status'] == 'true':
return True
else:
raise Exception('Failed to start experiment %d. (%s)' %
(experiment_id, ret.json()['status']['error']))
except KeyError:
raise Exception('Invalid reply from instrument')
def data_logger_start(self, cnt_pre, cnt_post):
"""Setup and start the data logger"""
util.check_par('pre-trigger sample count', cnt_pre, int, _min = 0)
util.check_par('post-trigger sample count', cnt_post, int, _min = 0)
ret = self._rest_session.post(
self._rest_prefix + ':8000/test/data_logger/start',
headers={'Content-Type':'application/json'},
data=json.dumps({'pre_samples':'%d'%cnt_pre, 'post_samples':'%d'%cnt_post,})
)
try:
if ret.json()['status']['status'] == 'true':
return True
else:
raise Exception('Failed to start data logger (%s)'%ret.json()['status']['error'])
except KeyError:
raise Exception('Invalid reply from instrument')
def data_logger_trigger(self, timeout_s=None):
"""Wait for logger data and retrieve it"""
if timeout_s != None:
util.check_par('timeout (seconds)', timeout_s, int, _min=1)
remote_output_file = '/tmp/data_logger.csv'
local_output_file = 'data.csv.%s' % self._config['host']
self.ssh_command('rm -f ' + remote_output_file)
ret = self._rest_session.post(
self._rest_prefix + ':8000/test/data_logger/trigger',
headers={'Content-Type': 'application/json'})
try:
if ret.json()['status']['status'] != 'true':
raise Exception('Failed to trigger data logger (%s)' %
ret.json()['status']['error'])
except KeyError:
raise Exception('Invalid reply from instrument')
ret = []
while True:
if timeout_s != None:
if timeout_s < 0:
return False
else:
timeout_s -= 10
time.sleep(10)
ret = self.ssh_command('stat ' + remote_output_file)
if ret[0] == 0:
# file exists
time.sleep(1)
break
self.sftp_get(remote_output_file, local_output_file)
ret = pd.read_csv(local_output_file, header=0)
ret = ret.set_index(['time_offset'])
return ret
def data_logger_trigger(self, timeout_s=None):
"""Wait for logger data and retrieve it"""
if timeout_s != None:
util.check_par('timeout (seconds)', timeout_s, int, _min=1)
remote_output_file = '/tmp/data_logger.csv'
local_output_file = 'data.csv.%s'%self._config['host']
self.ssh_command('rm -f ' + remote_output_file)
ret = self._rest_session.post(
self._rest_prefix + ':8000/test/data_logger/trigger',
headers={'Content-Type':'application/json'}
)
try:
if ret.json()['status']['status'] != 'true':
raise Exception('Failed to trigger data logger (%s)'%ret.json()['status']['error'])
except KeyError:
raise Exception('Invalid reply from instrument')
ret = []
while True:
if timeout_s != None:
if timeout_s < 0:
return False
else:
timeout_s -= 10
time.sleep(10)
ret = self.ssh_command('stat ' + remote_output_file)
if ret[0] == 0:
# file exists
time.sleep(1)
break
self.sftp_get(remote_output_file, local_output_file)
ret = pd.read_csv(local_output_file, header=0)
ret = ret.set_index(['time_offset'])
return ret
def experiment_copy(self, experiment_id=None, name=None):
"""Copy an existing experiment.
Returns the id of the new experiment.
"""
util.check_par('experiment id', experiment_id, _type=int, _min=0)
ret = self._rest_session.post(self._rest_prefix + '/experiments/%d/copy'%experiment_id)
new_experiment_id = -1
try:
new_experiment_id = int(ret.json()['experiment']['id'])
except:
raise Exception('Failed to copy experiment id %d'%experiment_id)
if name:
ret = self._rest_session.put(
self._rest_prefix + '/experiments/%d'%new_experiment_id,
headers={'Content-Type':'application/json'},
data=json.dumps({'experiment':{'name':name}})
)
return new_experiment_id
def data_logger_start(self, cnt_pre, cnt_post):
"""Setup and start the data logger"""
util.check_par('pre-trigger sample count', cnt_pre, int, _min=0)
util.check_par('post-trigger sample count', cnt_post, int, _min=0)
ret = self._rest_session.post(
self._rest_prefix + ':8000/test/data_logger/start',
headers={'Content-Type': 'application/json'},
data=json.dumps({
'pre_samples': '%d' % cnt_pre,
'post_samples': '%d' % cnt_post,
}))
try:
if ret.json()['status']['status'] == 'true':
return True
else:
raise Exception('Failed to start data logger (%s)' %
ret.json()['status']['error'])
except KeyError:
raise Exception('Invalid reply from instrument')
def experiment_copy(self, experiment_id=None, name=None):
"""Copy an existing experiment.
Returns the id of the new experiment.
"""
util.check_par('experiment id', experiment_id, _type=int, _min=0)
ret = self._rest_session.post(self._rest_prefix +
'/experiments/%d/copy' % experiment_id)
new_experiment_id = -1
try:
new_experiment_id = int(ret.json()['experiment']['id'])
except:
raise Exception('Failed to copy experiment id %d' % experiment_id)
if name:
ret = self._rest_session.put(
self._rest_prefix + '/experiments/%d' % new_experiment_id,
headers={'Content-Type': 'application/json'},
data=json.dumps({'experiment': {
'name': name
}}))
return new_experiment_id
def experiment_get_template(self, experiment_id=None):
"""Retrieve the template for an experiment.
Can be used to duplicate the experiment on another machine.
"""
util.check_par('experiment id', experiment_id, _type=int, _min=0)
exp = self._rest_session.get(self._rest_prefix +
'/experiments/%d' % experiment_id)
try:
exp = exp.json()
exp['experiment']['id']
except:
raise Exception('Failed to get template for experiment id %d' %
experiment_id)
# build the template protocol
ret_proto = {}
ret_proto['lid_temperature'] = exp['experiment']['protocol'][
'lid_temperature']
for stage in exp['experiment']['protocol']['stages']:
stage['stage'].pop('id')
for step in stage['stage']['steps']:
step['step'].pop('id')
ret_proto['stages'] = exp['experiment']['protocol']['stages']
ret = {
'experiment': {
'name': exp['experiment']['name'],
'protocol': ret_proto
}
}
return ret
def read_well(self, well, cnt, mode='status', source_on=True):
"""Get optical readings from the status page or from the csv dump"""
util.check_par('well', well, int, 0, 15)
util.check_par('sample count', cnt, int, 0)
util.check_par('mode', mode, str, _list=['status', 'dump'])
util.check_par('source on', source_on, bool, _list=[True, False])
self.test_control("photodiode_mux_channel", '%d' % well)
if source_on:
self.test_control("activate_led", '%d' % well)
else:
self.test_control("disable_leds", '')
time.sleep(0.5)
ret = []
if mode == 'status':
for i in range(cnt):
ret.append([
int(i) for i in self.status()['optics']['photodiode_value']
])
ret = pd.DataFrame(ret)
ret.columns = [
'optics_%d' % (i + 1) for i in range(ret.columns.size)
]
self.test_control("disable_leds", '')
elif mode == 'dump':
self.data_logger_start(10, cnt - 10)
ret = self.data_logger_trigger()
self.test_control("disable_leds", '')
ret = ret[[c for c in list(ret.columns) if c.startswith('optics')]]
else:
raise Exception('Unknown mode: %s' % mode)
return ret
def read_well(self, well, cnt, mode='status', source_on=True):
"""Get optical readings from the status page or from the csv dump"""
util.check_par('well', well, int, 0, 15)
util.check_par('sample count', cnt, int, 0)
util.check_par('mode', mode, str, _list=['status', 'dump'])
util.check_par('source on', source_on, bool, _list=[True, False])
self.test_control("photodiode_mux_channel",'%d'%well)
if source_on:
self.test_control("activate_led",'%d'%well)
else:
self.test_control("disable_leds",'')
time.sleep(0.5)
ret = []
if mode == 'status':
for i in range(cnt):
ret.append([int(i) for i in self.status()['optics']['photodiode_value']])
ret = pd.DataFrame(ret)
ret.columns = ['optics_%d'%(i+1) for i in range(ret.columns.size)]
self.test_control("disable_leds",'')
elif mode == 'dump':
self.data_logger_start(10, cnt-10)
ret = self.data_logger_trigger()
self.test_control("disable_leds",'')
ret = ret[[c for c in list(ret.columns) if c.startswith('optics')]]
else:
raise Exception('Unknown mode: %s'%mode)
return ret
def __init__(self,
wells=range(16),
temperatures_C=[10, 60, 80],
loops=300,
settle_s=300,
leds_on=True,
lid_temp_C=None):
self.wells = wells
self.temperatures_C = temperatures_C
self.loops = loops
self.settle_s = settle_s
self.leds_on = leds_on
self.lid_temp_C = lid_temp_C
super(ChaiNoiseTest, self).__init__()
[
util.check_par('well', well, _type=int, _min=0, _max=15)
for well in self.wells
]
[
util.check_par('temperature (degC)',
temp,
_type=int,
_min=0,
_max=90) for temp in [
temp_number
for temp_number in self.temperatures_C
if temp_number != None
]
]
util.check_par('loops', self.loops, _type=int, _min=0)
util.check_par('settle time (s)',
self.settle_s,
_type=int,
_min=0,
_max=3600)
self.logger_data = {}
self.parsed_data = None
self.results = None
self.meas_values = {
'avg_p2p': [None, None, 300, 500],
'avg_diff': [None, None, 100, 200],
'step_p2p': [None, None, 300, 500]
}
self.step_data_slice = slice(9, 16)
def sql_get_data(self, table, experiment_id, channel=None, well=None):
"""Get data for a specific experiment."""
util.check_par('experiment id', experiment_id, _type=int, _min=0)
channel_string = ''
if channel is not None:
util.check_par('channel', channel, _type=int, _min=1, _max=2)
channel_string = 'and channel = %d'%(int(channel))
well_string = ''
if well is not None:
util.check_par('well number', well, _type=int, _min=0, _max=15)
well_string = 'and well_num = %d'%(int(well))
data = self.sql_command(
"select * from %s where experiment_id = %d %s %s"%(table, experiment_id, channel_string, well_string)
)
return data
def sql_get_data(self, table, experiment_id, channel=None, well=None):
"""Get data for a specific experiment."""
util.check_par('experiment id', experiment_id, _type=int, _min=0)
channel_string = ''
if channel is not None:
util.check_par('channel', channel, _type=int, _min=1, _max=2)
channel_string = 'and channel = %d' % (int(channel))
well_string = ''
if well is not None:
util.check_par('well number', well, _type=int, _min=0, _max=15)
well_string = 'and well_num = %d' % (int(well))
data = self.sql_command(
"select * from %s where experiment_id = %d %s %s" %
(table, experiment_id, channel_string, well_string))
return data
def __init__(
self,
wells = range(16),
temperatures_C = [10, 60, 80],
loops = 300,
settle_s = 300,
leds_on = True,
lid_temp_C = None
):
self.wells = wells
self.temperatures_C = temperatures_C
self.loops = loops
self.settle_s = settle_s
self.leds_on = leds_on
self.lid_temp_C = lid_temp_C
super(ChaiNoiseTest, self).__init__()
[util.check_par('well', well, _type=int, _min=0, _max=15) for well in self.wells]
[util.check_par('temperature (degC)', temp, _type=int, _min=0, _max=90) for temp in
[temp_number for temp_number in self.temperatures_C if temp_number != None]
]
util.check_par('loops', self.loops, _type=int, _min=0)
util.check_par('settle time (s)', self.settle_s, _type=int, _min=0, _max=3600)
self.logger_data = {}
self.parsed_data = None
self.results = None
self.meas_values = {
'avg_p2p' : [None, None, 300, 500],
'avg_diff': [None, None, 100, 200],
'step_p2p': [None, None, 300, 500]
}
self.step_data_slice = slice(9,16)
def experiment_loop(self,
experiment_id=None,
loop_cnt=100,
data_log=False,
data_log_duration_s=None,
poll_seconds=60,
gap_minutes=1,
stop_on_error=True,
delete_loop_experiments=False):
"""Run the same experiment in a loop.
Copies of the experiments are created, run and optionally deleted.
"""
if self.state() != 'idle':
raise Exception('Device is running an experiment')
util.check_par('experiment id', experiment_id, _type=int, _min=0)
util.check_par('loop count', loop_cnt, _type=int, _min=1, _max=1000)
util.check_par('poll seconds',
poll_seconds,
_type=int,
_min=1,
_max=3600)
util.check_par('gap minutes',
gap_minutes,
_type=int,
_min=0,
_max=1440)
exp_info = self.experiment_info(experiment_id)
if data_log:
if data_log_duration_s != None:
util.check_par('data log duration (seconds)',
data_log_duration_s,
_type=int,
_min=1)
else:
if exp_info['experiment']['completion_status'] != 'success':
raise Exception(
"Template experiment did not complete successfully. Please specify the data_log_duration_s parameter"
)
try:
start_time = datetime.strptime(
exp_info['experiment']['started_at'],
'%Y-%m-%dT%H:%M:%S.000Z')
end_time = datetime.strptime(
exp_info['experiment']['completed_at'],
'%Y-%m-%dT%H:%M:%S.000Z')
except:
raise Exception("Failed to detect experiment duration.")
duration = end_time - start_time
if duration.days != 0:
raise Exception(
"Detected invalid experiment duration (days): %d" %
duration.days)
if duration.seconds < 0 or duration.seconds > 12 * 3600:
raise Exception(
"Detected invalid experiment duration (seconds): %d" %
duration.seconds)
data_log_duration_s = duration.seconds + 60
data = {}
for loop in range(1, loop_cnt + 1):
new_id = self.experiment_copy(experiment_id,
name=exp_info['experiment']['name'] +
'_loop_%d' % loop)
print("Starting loop %d with experiment id %d" % (loop, new_id))
if data_log:
self.data_logger_start(
int(1.2 * data_log_duration_s) *
self.data_logger_samplerate, 10)
self.experiment_start(new_id)
time.sleep(10)
if self.state() == 'idle':
raise Exception(
'Failed to start loop %d for experiment id %d' %
(loop, new_id))
while True:
time.sleep(poll_seconds)
if self.state() == 'idle':
break
if data_log:
data[new_id] = self.data_logger_trigger(timeout_s=60)
self.data_logger_stop()
pickle.dump(data, open('data_%s' % self._config['host'], 'wb'))
if stop_on_error and self.experiment_info(
new_id)['experiment']['completion_status'] != 'success':
raise Exception('Loop %d for experiment id %d failed' %
(loop, new_id))
time.sleep(gap_minutes * 60)
if delete_loop_experiments:
try:
self.experiment_delete(new_id)
except:
print('Failed to delete experiment id %d' % new_id)
time.sleep(10)
# renew connection authorization
self.connect_rest()
print("Finished %d loops for experiment id %d" % (loop, experiment_id))
return data
def experiment_loop(
self,
experiment_id=None,
loop_cnt=100,
data_log=False, data_log_duration_s=None,
poll_seconds=60, gap_minutes=1,
stop_on_error=True,
delete_loop_experiments=False
):
"""Run the same experiment in a loop.
Copies of the experiments are created, run and optionally deleted.
"""
if self.state() != 'idle':
raise Exception('Device is running an experiment')
util.check_par('experiment id', experiment_id, _type=int, _min=0)
util.check_par('loop count', loop_cnt, _type=int, _min=1, _max=1000)
util.check_par('poll seconds', poll_seconds, _type=int, _min=1, _max=3600)
util.check_par('gap minutes', gap_minutes, _type=int, _min=0, _max=1440)
exp_info = self.experiment_info(experiment_id)
if data_log:
if data_log_duration_s != None:
util.check_par('data log duration (seconds)', data_log_duration_s, _type=int, _min=1)
else:
if exp_info['experiment']['completion_status'] != 'success':
raise Exception("Template experiment did not complete successfully. Please specify the data_log_duration_s parameter")
try:
start_time = datetime.strptime(exp_info['experiment']['started_at'], '%Y-%m-%dT%H:%M:%S.000Z')
end_time = datetime.strptime(exp_info['experiment']['completed_at'], '%Y-%m-%dT%H:%M:%S.000Z')
except:
raise Exception("Failed to detect experiment duration.")
duration = end_time - start_time
if duration.days != 0:
raise Exception("Detected invalid experiment duration (days): %d"%duration.days)
if duration.seconds < 0 or duration.seconds > 12*3600:
raise Exception("Detected invalid experiment duration (seconds): %d"%duration.seconds)
data_log_duration_s = duration.seconds + 60
data={}
for loop in range(1, loop_cnt+1):
new_id = self.experiment_copy(experiment_id, name = exp_info['experiment']['name'] + '_loop_%d'%loop)
print("Starting loop %d with experiment id %d"%(loop, new_id))
if data_log:
self.data_logger_start(int(1.2 * data_log_duration_s) * self.data_logger_samplerate, 10)
self.experiment_start(new_id)
time.sleep(10)
if self.state() == 'idle':
raise Exception('Failed to start loop %d for experiment id %d'%(loop, new_id))
while True:
time.sleep(poll_seconds)
if self.state() == 'idle':
break;
if data_log:
data[new_id] = self.data_logger_trigger(timeout_s = 60)
self.data_logger_stop()
pickle.dump(data, open('data_%s'%self._config['host'], 'wb'))
if stop_on_error and self.experiment_info(new_id)['experiment']['completion_status'] != 'success':
raise Exception('Loop %d for experiment id %d failed'%(loop, new_id))
time.sleep(gap_minutes * 60)
if delete_loop_experiments:
try:
self.experiment_delete(new_id)
except:
print('Failed to delete experiment id %d'%new_id)
time.sleep(10)
# renew connection authorization
self.connect_rest()
print("Finished %d loops for experiment id %d"%(loop, experiment_id))
return data
