output_path.mkdir(exist_ok=True)
output_file_path = (output_path / (output_name)).resolve()
if classifier.SAVES_INTERMEDIATES and not done:
output_file_path = intermediate_path
data.to_csv(str(output_file_path), header=True, encoding='utf-8', index=None)
if done and intermediate_path.exists():
# Clear the intermediate file
intermediate_path.unlink()
if not done:
print(
colorize("red",
"The classifier did not finish or there was an error."))
else:
event_file_path = (output_path / (event_summary_name)).resolve()
cell_file_path = (output_path / (cell_summary_name)).resolve()
# Generate event/cell summaries
event_summary = classifier.get_event_summary(data, conf)
if isinstance(event_summary, pd.DataFrame):
event_summary.to_csv(str(event_file_path),
header=True,
encoding='utf-8',
index=None)
cell_summary = classifier.get_cell_summary(data, conf)
if isinstance(cell_summary, pd.DataFrame):
cell_summary.to_csv(str(cell_file_path),
import subprocess
import re
### Constant for getting our base input dir
MAKE_SINGLE_VIDEO_PATH = (ROOT_PATH /
("validate/render-particle-video.py")).resolve()
MAKE_VIDEO_PATH = (ROOT_PATH / ("validate/render-full-video.py")).resolve()
MAX_PROCESSES = 4
### Arguments and inputs
arguments = docopt(__doc__, version=get_version())
classifier = re.sub(r'[^a-zA-Z0-9\-\_\.\+]', '', arguments['CLASSIFIER'])
if classifier != arguments['CLASSIFIER']:
print(
colorize("yellow",
"Classifier input has been sanitized to " + classifier))
r_graph_gen_path = (
ROOT_PATH / ("classifiers/" + classifier + "/make-graphs.R")).resolve()
if r_graph_gen_path and not r_graph_gen_path.is_file():
print(
colorize(
"red", "The supplied classifier cannot make graphs: \033[1m" +
str(classifier) + "\033[0m"))
exit(1)
conf_path = (ROOT_PATH /
("classifiers/" + classifier + "/conf.json")).resolve()
tiff_path = Path(
arguments['--img-dir']).resolve() if arguments['--img-dir'] else False
import subprocess
import math
import re
from tabulate import tabulate
from multiprocessing import Pool, cpu_count
### Constant for getting our base input dir
QA_PATH = (ROOT_PATH / ("validate/qa.py")).resolve()
### Arguments and inputs
arguments = docopt(__doc__, version=get_version())
classifier_name = re.sub(r'[^a-zA-Z0-9\-\_\.\+]', '', arguments['CLASSIFIER'])
if classifier_name != arguments['CLASSIFIER']:
print(
colorize("yellow",
"Classifier input has been sanitized to " + classifier_name))
input_path = Path(arguments['--input-path']).resolve(
) if arguments['--input-path'] else Path(
"validate/validation-data/input").resolve()
if not input_path.exists():
print(
colorize(
"red", "Data folder input cannot be found: \033[1m" +
str(input_path) + "\033[0m"))
exit(1)
data_file_path = input_path / (arguments['--input-name']) if arguments[
'--input-name'] else input_path / "data.csv"
tiff_path = input_path / (
arguments['--img-dir']) if arguments['--img-dir'] else (
def process_data(data_path, params):
frame_rate = params['frame_rate']
tiff_path = params['tiff_path']
""""
What Imaris CSV files we need and their columns
"""
data_files = {
'position': {
'file': 'Position.csv',
'data_cols': [ 'TrackID', 'Time', 'Position X', 'Position Y' ]
},
'area': {
'file': 'Area.csv',
'data_cols': [ 'TrackID', 'Time', 'Area' ]
},
'sum': {
'file': 'Intensity_Sum_Ch=1.csv',
'data_cols': [ 'TrackID', 'Time', 'Intensity Sum' ]
},
'median': {
'file': 'Intensity_Median_Ch=1.csv',
'data_cols': [ 'TrackID', 'Time', 'Intensity Median' ]
},
'oblate_ellipticity': {
'file': 'Ellipticity_Oblate.csv',
'data_cols': [ 'TrackID', 'Time', 'Ellipticity (oblate)' ]
}
}
"""
Constant for mappng the Imaris column names to easier ones to type
"""
COL_MAP = {
'TrackID': 'particle_id',
'Time' : 'frame',
'Position X' : 'x',
'Position Y' : 'y',
'Area': 'area',
'Intensity Median': 'median',
'Intensity Sum': 'sum',
'Ellipticity (oblate)': 'oblate_ellipticity'
}
### Collect our data
data = None
for name,csv_info in tqdm(data_files.items(), ncols=90, unit="files"):
file_path = data_path / csv_info['file']
if not file_path.exists():
print(colorize("yellow", str(file_path) + " not found; skipping"))
continue
with open(file_path, 'rb') as f:
# Find the actual start of CSV data
# Imaris outpus really silly headers
found_header = False
for line in f:
line = line.decode('utf-8').rstrip()
if line.find(",") != -1:
# Extract header
headers = line.split(",")
if(len(list(set(headers) & set(csv_info['data_cols']))) > 0):
found_header = True
break
if not found_header:
raise UnexpectedEOFException("Reached end of " + csv_info['file'] + " without finding headers.")
this_df = pd.read_csv(f, header=None, names=headers, dtype={ 'TrackID': str })
this_df = this_df[csv_info['data_cols']]
if(data is None):
data = this_df
else:
data = pd.merge(data, this_df, how='left', on=[ 'TrackID', 'Time' ])
data.rename(columns=COL_MAP, inplace=True)
data['particle_id'] = data['particle_id'].astype(str)
### Direct image processing
# Get pixel to micron conversion factor
frame_idx = np.min(data['frame'])
frame_file_name = str(frame_idx).zfill(4) + '.tif'
frame_path = (tiff_path / frame_file_name).resolve()
with Image.open(frame_path) as img:
resolution = img.info['resolution']
x_conversion = resolution[0]
y_conversion = resolution[1]
data['x_conversion'] = x_conversion
data['y_conversion'] = y_conversion
### Make particle IDs less… long.
data['particle_id'] = data['particle_id'].str.replace(r"^1[0]+", "")
data['particle_id'] = data['particle_id'].str.replace(r"^$", "000")
data = base_transform(data, params)
return data