def main(input_dir, output_dir, positions=None, write_mode=0):
"""
Read metadata from all log files in the input data directory.
args:
input_dir (path): raw data experiment directory
output_dir (path): path to save pickled results
"""
expt = dict.fromkeys(('log', 'image', 'pumps', 'stage'))
# First read the main log.txt file
expt['log'] = read_log(input_dir)
# Transfer the log.txt file and rename it to experiment.log
shutil.copy(os.path.join(input_dir, 'log.txt'),
os.path.join(output_dir, 'experiment.log'))
# Read in any image metadata available
expt['image'] = read_img(input_dir)
# Read in the positions
expt['stage'] = read_posns(input_dir)
expt['log']['Total Positions'] = len(expt['stage'])
# Attempt to read the phase.txt and fluor.txt time files
total_time = 0
for k in expt['image'].keys():
if expt['image'][k]['include']:
expt[k] = read_time(input_dir, expt['log']['Start Date'], k)
total_time = max(total_time, expt[k].max())
expt['log']['Total Frames %s' % k.capitalize()] = expt[k].shape[1]
expt['log']['Total Time'] = total_time
# Then read in the pump rates and on/off values
expt['pumps'] = read_pumps(input_dir, expt['log'])
# Iterate over all positions in the experiment
if positions is None:
positions = read.listdirs(input_dir, read.PATTERN['posndir'])
for p in positions:
i = int(re.match(read.PATTERN['posndir'], p).group(2))
# Create a log dictionary specific to each position
posn = {}
for k, v in expt.iteritems():
if k in ('log', 'image', 'pumps'):
posn[k] = v
elif k in ('phase', 'fluor', 'stage'):
posn[k] = v[i]
# Add information about the specific position
posn['log']['Position Name'] = p
posn['log']['Position Number'] = i
# Save the results
read.rmkdir(os.path.join(output_dir, p))
output_file = os.path.join(output_dir, p, 'log.pickle')
if not os.path.isfile(output_file) or write_mode == 1:
pickle.dump(posn, open(output_file, 'wb'))
def read_img(input_dir):
"""
Read image metadata. Search for both phase-contrast and fluorescence files.
args:
input_dir (path): raw data experiment directory
"""
results = {k : {'include' : False, 'shape' : (0, 0), 'dtype' : None}
for k in ('phase', 'fluor')}
# Only look in the directory of the first position
p = read.listdirs(input_dir, read.PATTERN['posndir']).next()
# Try to read metadata about phase and fluor files (need one image each)
for k in ('phase', 'fluor'):
pattern = read.PATTERN['%stif' % k]
for v in read.listfiles(os.path.join(input_dir, p), pattern):
if os.path.splitext(v)[1] == '.tif':
phase_img = imread(os.path.join(input_dir, p, v))
results[k]['include'] = True
results[k]['shape'] = phase_img.shape
results[k]['dtype'] = phase_img.dtype
break
return results
def main(input_dir, output_dir, positions=None, write_mode=0):
"""
Read metadata from all log files in the input data directory.
args:
input_dir (path): raw data experiment directory
output_dir (path): path to save pickled results
"""
expt = dict.fromkeys(('log', 'image', 'pumps', 'stage'))
# First read the main log.txt file
expt['log'] = read_log(input_dir)
# Transfer the log.txt file and rename it to experiment.log
shutil.copy(os.path.join(input_dir, 'log.txt'),
os.path.join(output_dir, 'experiment.log'))
# Read in any image metadata available
expt['image'] = read_img(input_dir)
# Read in the positions
expt['stage'] = read_posns(input_dir)
expt['log']['Total Positions'] = len(expt['stage'])
# Attempt to read the phase.txt and fluor.txt time files
total_time = 0
for k in expt['image'].keys():
if expt['image'][k]['include']:
expt[k] = read_time(input_dir, expt['log']['Start Date'], k)
total_time = max(total_time, expt[k].max())
expt['log']['Total Frames %s' % k.capitalize()] = expt[k].shape[1]
expt['log']['Total Time'] = total_time
# Then read in the pump rates and on/off values
expt['pumps'] = read_pumps(input_dir, expt['log'])
# Iterate over all positions in the experiment
if positions is None:
positions = read.listdirs(input_dir, read.PATTERN['posndir'])
for p in positions:
i = int(re.match(read.PATTERN['posndir'], p).group(2))
# Create a log dictionary specific to each position
posn = {}
for k, v in expt.iteritems():
if k in ('log', 'image', 'pumps'):
posn[k] = v
elif k in ('phase', 'fluor', 'stage'):
posn[k] = v[i]
# Add information about the specific position
posn['log']['Position Name'] = p
posn['log']['Position Number'] = i
# Save the results
read.rmkdir(os.path.join(output_dir, p))
output_file = os.path.join(output_dir, p, 'log.pickle')
if not os.path.isfile(output_file) or write_mode == 1:
pickle.dump(posn, open(output_file, 'wb'))
def read_img(input_dir):
"""
Read image metadata. Search for both phase-contrast and fluorescence files.
args:
input_dir (path): raw data experiment directory
"""
results = {
k: {
'include': False,
'shape': (0, 0),
'dtype': None
}
for k in ('phase', 'fluor')
}
# Only look in the directory of the first position
p = read.listdirs(input_dir, read.PATTERN['posndir']).next()
# Try to read metadata about phase and fluor files (need one image each)
for k in ('phase', 'fluor'):
pattern = read.PATTERN['%stif' % k]
for v in read.listfiles(os.path.join(input_dir, p), pattern):
if os.path.splitext(v)[1] == '.tif':
phase_img = imread(os.path.join(input_dir, p, v))
results[k]['include'] = True
results[k]['shape'] = phase_img.shape
results[k]['dtype'] = phase_img.dtype
break
return results
def main(analysis_dir, posn_nums=None):
posns = [v for v in read.listdirs(analysis_dir, read.PATTERN['posndir'])]
if posn_nums is not None:
posns = [
v for v in posns
if int(re.match(read.PATTERN['posndir'], v).group(2)) in posn_nums
]
return TraceData(analysis_dir, posns)
def save_posn(self):
"""
Save data for the current position.
"""
# Update the edited file
self.data.to_pickle(self.data_file)
# Update the file of saved traces, keeping only relevant variables
df = self.data[self.data['Saved'] == True]
if len(df) > 0:
d = {k : list(df[k]) for k in ('Trace', 'Divns', 'Mother')}
# Restructure the labels into a DataFrame commensurate with data
num_saved = len(df)
num_frames = self.num_frames
frames = range(num_frames) * num_saved
traces = []
for v in xrange(num_saved):
traces.extend([v] * num_frames)
index = MultiIndex.from_arrays([traces, frames],
names=('Trace', 'Frame'))
data = np.hstack(df['Label'].values)
s = DataFrame(data, index=index, columns=('Label', ))['Label']
# Remove frames that one should not keep
d['Label'] = s[np.hstack(df['Keep'].values) & (data > 0)]
# Save the times directly as arrays
d['TimeP'] = self.time_phase
if hasattr(self, 'time_fluor'):
d['TimeF'] = self.time_fluor
# Make generations data from divisons
d['Divns'] = [np.asarray(v) for v in d['Divns']]
d['Gens'] = [[] for _ in d['Divns']]
d['Taus'] = [[] for _ in d['Divns']]
for i, v in enumerate(d['Divns']):
v += 1
for j1, j2 in zip(v[:-1], v[1:]):
d['Gens'][i].append(slice(j1, j2))
t1 = np.mean(d['TimeP'][j1-1:j1+1])
t2 = np.mean(d['TimeP'][j2-1:j2+1])
d['Taus'][i].append(t2 - t1)
d['Taus'] = [np.asarray(v) for v in d['Taus']]
with open(os.path.join(self.posn_dir, 'saved.pickle'), 'wb') as f:
pickle.dump(d, f)
# Delete unzipped directories
b = os.path.join(self.posn_dir, 'blocks')
for v in read.listdirs(b, read.PATTERN['blockdir']):
shutil.rmtree(os.path.join(b, v, 'PhaseSegment'))
# Update the log file
read.updatelog(self.expt_name, self.posns[self.posn_idx],
'edit', self.analyses_dir)
def save_posn(self):
"""
Save data for the current position.
"""
# Update the edited file
self.data.to_pickle(self.data_file)
# Update the file of saved traces, keeping only relevant variables
df = self.data[self.data['Saved'] == True]
if len(df) > 0:
d = {k: list(df[k]) for k in ('Trace', 'Divns', 'Mother')}
# Restructure the labels into a DataFrame commensurate with data
num_saved = len(df)
num_frames = self.num_frames
frames = range(num_frames) * num_saved
traces = []
for v in xrange(num_saved):
traces.extend([v] * num_frames)
index = MultiIndex.from_arrays([traces, frames],
names=('Trace', 'Frame'))
data = np.hstack(df['Label'].values)
s = DataFrame(data, index=index, columns=('Label', ))['Label']
# Remove frames that one should not keep
d['Label'] = s[np.hstack(df['Keep'].values) & (data > 0)]
# Save the times directly as arrays
d['TimeP'] = self.time_phase
if hasattr(self, 'time_fluor'):
d['TimeF'] = self.time_fluor
# Make generations data from divisons
d['Divns'] = [np.asarray(v) for v in d['Divns']]
d['Gens'] = [[] for _ in d['Divns']]
d['Taus'] = [[] for _ in d['Divns']]
for i, v in enumerate(d['Divns']):
v += 1
for j1, j2 in zip(v[:-1], v[1:]):
d['Gens'][i].append(slice(j1, j2))
t1 = np.mean(d['TimeP'][j1 - 1:j1 + 1])
t2 = np.mean(d['TimeP'][j2 - 1:j2 + 1])
d['Taus'][i].append(t2 - t1)
d['Taus'] = [np.asarray(v) for v in d['Taus']]
with open(os.path.join(self.posn_dir, 'saved.pickle'), 'wb') as f:
pickle.dump(d, f)
# Delete unzipped directories
b = os.path.join(self.posn_dir, 'blocks')
for v in read.listdirs(b, read.PATTERN['blockdir']):
shutil.rmtree(os.path.join(b, v, 'PhaseSegment'))
# Update the log file
read.updatelog(self.expt_name, self.posns[self.posn_idx], 'edit',
self.analyses_dir)
def init_expt(self, raw_data_dir, analyses_dir, posns=None):
"""
Get experiment information.
args:
input_dir (path): input directory
kwargs:
positions (list): list of positions to edit
"""
self.raw_data_dir = raw_data_dir
self.analyses_dir = analyses_dir
self.expt_name = os.path.basename(self.analyses_dir)
self.root.title(self.expt_name)
# Get list of valid positions if unspecified
if posns is None:
posns = []
for p in read.listdirs(self.analyses_dir, read.PATTERN['posndir']):
if os.path.isfile(
os.path.join(self.analyses_dir, p, 'edits.pickle')):
posns.append(p)
if not isinstance(posns, (list, tuple, np.ndarray)):
raise TypeError('list of positions required.')
elif not posns:
raise IOError('no valid positions found in %s' % self.analyses_dir)
self.posns = sorted(posns)
# Reset the position listbox
self.posn_listbox.delete(0, END)
[self.posn_listbox.insert(END, pos) for pos in self.posns]
# Variable for holding temporary area traces
self.temp = None
# Save the coordinates of each object and the RGB images
self.image = [
dict.fromkeys(['label', 'trace', 'raw', 'rgb']) for _ in (0, 1)
]
# These values will hold information about each plot
self.plots = dict.fromkeys(['data', 'divns', 'image'])
self.plots['image'] = [None, None]
self.connect()
# Initialize the GUI
self.posn_idx = 0
self.trace_idx = 0
self.frame_idx = 0
self.saved_idx = []
self.divn_idx = []
self.set_mother = None
self.set_posn()
def init_expt(self, raw_data_dir, analyses_dir, posns=None):
"""
Get experiment information.
args:
input_dir (path): input directory
kwargs:
positions (list): list of positions to edit
"""
self.raw_data_dir = raw_data_dir
self.analyses_dir = analyses_dir
self.expt_name = os.path.basename(self.analyses_dir)
self.root.title(self.expt_name)
# Get list of valid positions if unspecified
if posns is None:
posns = []
for p in read.listdirs(self.analyses_dir, read.PATTERN['posndir']):
if os.path.isfile(os.path.join(
self.analyses_dir, p, 'edits.pickle')):
posns.append(p)
if not isinstance(posns, (list, tuple, np.ndarray)):
raise TypeError('list of positions required.')
elif not posns:
raise IOError('no valid positions found in %s' % self.analyses_dir)
self.posns = sorted(posns)
# Reset the position listbox
self.posn_listbox.delete(0, END)
[self.posn_listbox.insert(END, pos) for pos in self.posns]
# Variable for holding temporary area traces
self.temp = None
# Save the coordinates of each object and the RGB images
self.image = [dict.fromkeys(['label', 'trace', 'raw', 'rgb'])
for _ in (0, 1)]
# These values will hold information about each plot
self.plots = dict.fromkeys(['data', 'divns', 'image'])
self.plots['image'] = [None, None]
self.connect()
# Initialize the GUI
self.posn_idx = 0
self.trace_idx = 0
self.frame_idx = 0
self.saved_idx = []
self.divn_idx = []
self.set_mother = None
self.set_posn()
def load_posn(self):
"""
Load data for the current position.
"""
self.posn_dir = os.path.join(self.analyses_dir,
self.posns[self.posn_idx])
b = os.path.join(self.posn_dir, 'blocks')
# Read in values for the current position
self.data_file = os.path.join(self.posn_dir, 'edits.pickle')
self.data = read_pickle(self.data_file)
# Read in values from the log file
log_file = os.path.join(self.posn_dir, 'log.pickle')
log_data = pickle.load(open(log_file, 'rb'))
self.img_shape = log_data['image']['phase']['shape']
self.img_dtype = log_data['image']['phase']['dtype']
self.pumps = log_data['pumps']
self.TraceList = list(self.data['Trace'])
self.SavedList = [
v for i, v in self.data['Trace'].iteritems()
if self.data['Saved'][i]
]
self.num_traces = len(self.TraceList)
if self.num_traces < 1:
return
self.num_frames = len(self.data.ix[0]['Label'])
self.frames = np.arange(self.num_frames)
self.time_phase = log_data['phase'][:self.num_frames]
self.time = self.time_phase / 60
if log_data.has_key('fluor'):
max_time = np.max(self.time_phase)
num_frames_fluor = np.argmin(
np.abs(log_data['fluor'] - max_time)) + 1
self.time_fluor = log_data['fluor'][:num_frames_fluor]
# Unzip phase-contrast image files and read in names of image files
old_dir = os.curdir
self.files = [''] * self.num_frames
for v in read.listdirs(b, read.PATTERN['blockdir']):
# Extract all .tif images in the input directory
os.chdir(os.path.join(b, v))
zipfile.ZipFile('PhaseSegment.zip').extractall()
for f in read.listfiles('PhaseSegment', read.PATTERN['phasetif']):
i = read.getframenum(f, read.PATTERN['phasetif'])
if i < self.num_frames:
self.files[i] = os.path.join(b, v, 'PhaseSegment', f)
os.chdir(old_dir)
def load_posn(self):
"""
Load data for the current position.
"""
self.posn_dir = os.path.join(self.analyses_dir,
self.posns[self.posn_idx])
b = os.path.join(self.posn_dir, 'blocks')
# Read in values for the current position
self.data_file = os.path.join(self.posn_dir, 'edits.pickle')
self.data = read_pickle(self.data_file)
# Read in values from the log file
log_file = os.path.join(self.posn_dir, 'log.pickle')
log_data = pickle.load(open(log_file, 'rb'))
self.img_shape = log_data['image']['phase']['shape']
self.img_dtype = log_data['image']['phase']['dtype']
self.pumps = log_data['pumps']
self.TraceList = list(self.data['Trace'])
self.SavedList = [v for i, v in self.data['Trace'].iteritems() if
self.data['Saved'][i]]
self.num_traces = len(self.TraceList)
if self.num_traces < 1:
return
self.num_frames = len(self.data.ix[0]['Label'])
self.frames = np.arange(self.num_frames)
self.time_phase = log_data['phase'][:self.num_frames]
self.time = self.time_phase / 60
if log_data.has_key('fluor'):
max_time = np.max(self.time_phase)
num_frames_fluor = np.argmin(np.abs(log_data['fluor']-max_time)) + 1
self.time_fluor = log_data['fluor'][:num_frames_fluor]
# Unzip phase-contrast image files and read in names of image files
old_dir = os.curdir
self.files = [''] * self.num_frames
for v in read.listdirs(b, read.PATTERN['blockdir']):
# Extract all .tif images in the input directory
os.chdir(os.path.join(b, v))
zipfile.ZipFile('PhaseSegment.zip').extractall()
for f in read.listfiles('PhaseSegment', read.PATTERN['phasetif']):
i = read.getframenum(f, read.PATTERN['phasetif'])
if i < self.num_frames:
self.files[i] = os.path.join(b, v, 'PhaseSegment', f)
os.chdir(old_dir)
def loadvar(self, key, convert=True):
"""
Load additional variables from file. These should be values that have been analyzed in blocks, but not sorted according to trace. This function will parse the requested variables for all saved traces.
args:
key (str): name of variable to load from file
"""
name = key + '.pickle'
for i, p in enumerate(self.posns):
b = os.path.join(self.analysis_dir, p, 'blocks')
# Load the original Series from file
s = None
for v in read.listdirs(b, read.PATTERN['blockdir']):
n = os.path.join(b, v, name)
try:
s = concat((s, read_pickle(n)))
except IOError:
raise IOError('No such file: %s' % n)
if s is None:
continue
# Reformat according to the saved traces
j = self.frame_data[i]['Label']
v = Series(index=j.index, dtype=s.dtype)
for (trace, frame), label in j.iteritems():
v[trace, frame] = s[frame, label]
self.frame_data[i][key] = v
# Convert to the correct units
if convert:
if 'Area' in key:
for i in range(self.num_posns):
self.frame_data[i][key] *= self.PX2UM**2
elif ('Centroid' in key or 'Length' in key or 'Perimeter' in key
or 'Pole' in key or 'Radius' in key or 'Width' in key):
for i in range(self.num_posns):
self.frame_data[i][key] *= self.PX2UM
elif 'Spline' in key:
for i in range(self.num_posns):
for k, v in self.frame_data[i][key].iteritems():
if len(v) == 3:
v[1][0] *= self.PX2UM
v[1][1] *= self.PX2UM
self.frame_data[i][key][k] = v
def loadvar(self, key, convert=True):
"""
Load additional variables from file. These should be values that have been analyzed in blocks, but not sorted according to trace. This function will parse the requested variables for all saved traces.
args:
key (str): name of variable to load from file
"""
name = key + '.pickle'
for i, p in enumerate(self.posns):
b = os.path.join(self.analysis_dir, p, 'blocks')
# Load the original Series from file
s = None
for v in read.listdirs(b, read.PATTERN['blockdir']):
n = os.path.join(b, v, name)
try:
s = concat((s, read_pickle(n)))
except IOError:
raise IOError('No such file: %s' % n)
if s is None:
continue
# Reformat according to the saved traces
j = self.frame_data[i]['Label']
v = Series(index=j.index, dtype=s.dtype)
for (trace, frame), label in j.iteritems():
v[trace, frame] = s[frame, label]
self.frame_data[i][key] = v
# Convert to the correct units
if convert:
if 'Area' in key:
for i in range(self.num_posns):
self.frame_data[i][key] *= self.PX2UM ** 2
elif ('Centroid' in key or 'Length' in key or 'Perimeter' in key or
'Pole' in key or 'Radius' in key or 'Width' in key):
for i in range(self.num_posns):
self.frame_data[i][key] *= self.PX2UM
elif 'Spline' in key:
for i in range(self.num_posns):
for k, v in self.frame_data[i][key].iteritems():
if len(v) == 3:
v[1][0] *= self.PX2UM
v[1][1] *= self.PX2UM
self.frame_data[i][key][k] = v
def loadinputs(input_dir):
"""
Load analysis input parameters from .txt file.
args:
input_dir (path): experiment analysis directory containing .yaml file
returns:
dict: input parameters
"""
input_file = os.path.join(input_dir, 'params.yaml')
try:
# Load experiment-specific parameters from file
inputs = yaml.load(open(input_file, 'r'))
except IOError:
# Load the default parameters and change the experiment name
inputs = yaml.load(
open(os.path.join(os.path.dirname(__file__), 'defaults.yaml'),
'r'))
for k in inputs['name'].keys():
inputs['name'][k] = os.path.basename(input_dir)
# Make experiment directory as necessary, and save parameters to file
read.rmkdir(input_dir)
yaml.dump(inputs,
open(input_file, 'w'),
explicit_start=True,
default_flow_style=False)
msg = '''
Loaded default parameters to %s.
Exit now and edit this file to use non-default values.
Starting automatic analysis in''' % input_file
# countdown(msg) # -BK
cls()
# Expand all directories
for k, v in inputs['paths'].iteritems():
root, pattern = os.path.split(os.path.abspath(os.path.expanduser(v)))
try:
sub_dir = read.listdirs(root, pattern).next()
except StopIteration:
sub_dir = pattern
inputs['paths'][k] = os.path.join(root, sub_dir)
return inputs
def loadinputs(input_dir):
"""
Load analysis input parameters from .txt file.
args:
input_dir (path): experiment analysis directory containing .yaml file
returns:
dict: input parameters
"""
input_file = os.path.join(input_dir, 'params.yaml')
try:
# Load experiment-specific parameters from file
inputs = yaml.load(open(input_file, 'r'))
except IOError:
# Load the default parameters and change the experiment name
inputs = yaml.load(open(os.path.join(os.path.dirname(__file__),
'defaults.yaml'), 'r'))
for k in inputs['name'].keys():
inputs['name'][k] = os.path.basename(input_dir)
# Make experiment directory as necessary, and save parameters to file
read.rmkdir(input_dir)
yaml.dump(inputs, open(input_file, 'w'),
explicit_start=True, default_flow_style=False)
msg = '''
Loaded default parameters to %s.
Exit now and edit this file to use non-default values.
Starting automatic analysis in''' % input_file
# countdown(msg) # -BK
cls()
# Expand all directories
for k, v in inputs['paths'].iteritems():
root, pattern = os.path.split(os.path.abspath(os.path.expanduser(v)))
try:
sub_dir = read.listdirs(root, pattern).next()
except StopIteration:
sub_dir = pattern
inputs['paths'][k] = os.path.join(root, sub_dir)
return inputs
def main(analysis_dir, posn_nums=None):
posns = [v for v in read.listdirs(analysis_dir, read.PATTERN['posndir'])]
if posn_nums is not None:
posns = [v for v in posns if int(re.match(read.PATTERN['posndir'],
v).group(2)) in posn_nums]
return TraceData(analysis_dir, posns)