def __init__(self, factory):
"""
:type factory: AbstractModelFactory
"""
self._factory = factory
self._logger = Logger(factory.__name__)
def __init__(self, fixture_settings):
"""
:type fixture_settings: scanomatic.io.fixtures.FixtureSettings
"""
self._logger = Logger("Gridding History {0}".format(
fixture_settings.model.name))
self._fixture_settings = fixture_settings
self._models_per_plate_pinning = defaultdict(dict)
def logger(cls):
"""
:rtype: scanomatic.io.logger.Logger
"""
if cls._LOGGER is None:
cls._LOGGER = Logger(cls.__name__)
return cls._LOGGER
def __init__(self, name, dir_path=None):
self._logger = Logger("Fixture {0}".format(name))
path_name = Paths().get_fixture_path(name, only_name=True)
if dir_path:
conf_rel_path = Paths().fixture_conf_file_rel_pattern.format(path_name)
self._conf_path = os.path.join(dir_path, conf_rel_path)
else:
self._conf_path = Paths().get_fixture_path(name)
self.model = self._load_model(name)
self.history = grid_history.GriddingHistory(self)
def __init__(self, analysis_model, scanning_meta_data):
"""
:param analysis_model: The model
:type analysis_model : scanomatic.models.analysis_model.AnalysisModel
:param scanning_meta_data: scanning info
:type scanning_meta_data : scanomatic.models.scanning_model.ScanningModel
:return:
"""
self._analysis_model = analysis_model
self._scanning_meta_data = scanning_meta_data
self._logger = Logger("Analysis Image")
self._im_loaded = False
self.im = None
self._grid_arrays = self._new_grid_arrays
""":type : dict[int|scanomatic.image_analysis.grid_array.GridArray]"""
self.features = _get_init_features(self._grid_arrays)
def __init__(self, fixture=None):
"""
:type fixture: scanomatic.io.fixtures.FixtureSettings
"""
self._logger = Logger("Fixture Image")
self._reference_fixture_settings = fixture
self._current_fixture_settings = None
""":type : scanomatic.io.fixtures.Fixture_Settings"""
if fixture:
self._history = GriddingHistory(fixture)
else:
self._history = None
self.im = None
self.im_path = None
self._original_dpi = None
self._name = "default"
def __init__(self,
compilation_path=None,
compile_instructions_path=None,
scanner_instructions_path=None,
sort_mode=FIRST_PASS_SORTING.Time):
self._logger = Logger("Compilation results")
self._compilation_path = compilation_path
self._compile_instructions = None
self._scanner_instructions = None
""":type : scanomatic.models.scanning_model.ScanningModel"""
self.load_scanner_instructions(scanner_instructions_path)
self._plates = None
self._plate_position_keys = None
self._image_models = []
self._used_models = []
self._current_model = None
self._loading_length = 0
if compile_instructions_path:
self._load_compile_instructions(compile_instructions_path)
if compilation_path:
self._load_compilation(self._compilation_path, sort_mode=sort_mode)
from scanomatic.image_analysis.image_grayscale import get_grayscale
from scanomatic.io.logger import Logger
from scanomatic.models.factories.fixture_factories import FixtureFactory
from scanomatic.models.fixture_models import GrayScaleAreaModel
from scanomatic.image_analysis.image_basics import Image_Transpose
from scanomatic.image_analysis.grid_cell import GridCell
from scanomatic.image_analysis.grid_array import get_calibration_polynomial_coeffs
from scanomatic.models.analysis_model import COMPARTMENTS, VALUES
from scanomatic.models.factories.analysis_factories import AnalysisFeaturesFactory
from .general import get_fixture_image_by_name, usable_markers, split_areas_into_grayscale_and_plates, \
get_area_too_large_for_grayscale, get_grayscale_is_valid, usable_plates, image_is_allowed, \
get_fixture_image, convert_url_to_path
_logger = Logger("Data API")
def _depth(arr, lvl=1):
if isinstance(arr, ListType) and len(arr) and isinstance(arr[0], ListType):
_depth(arr[0], lvl + 1)
else:
return lvl
def _validate_depth(data):
depth = _depth(data)
while depth < 4:
import os
from multiprocessing import Process
from time import sleep
from subprocess import call, STDOUT
from scanomatic.io.logger import Logger
_logger = Logger("Daemonizer")
def _daemon_process(path_to_exec, std_out_path, args, shell):
with open(std_out_path, 'w') as fh:
args = list(str(a) for a in args)
if shell:
fh.write("*** LAUNCHING IN SHELL: {0} ***\n\n".format(" ".join([path_to_exec] + list(args))))
retcode = call(" ".join([path_to_exec] + args), stderr=STDOUT, stdout=fh, shell=True)
else:
fh.write("*** LAUNCHING WITHOUT SHELL: {0} ***\n\n".format([path_to_exec] + list(args)))
retcode = call([path_to_exec] + args, stderr=STDOUT, stdout=fh, shell=False)
if retcode:
fh.write("\n*** DAEMON EXITED WITH CODE {0} ***\n".format(retcode))
else:
fh.write("\n*** DAEMON DONE ***\n")
def daemon(path_to_executable, std_out=os.devnull, daemon_args=tuple(), shell=True):
_logger.info("Launching daemon {0} (args={2}, {3}), outputting to {1} ".format(
import numpy as np
from enum import Enum
from scanomatic.data_processing import growth_phenotypes
from scanomatic.io.logger import Logger
from scanomatic.data_processing.phases.analysis import CurvePhasePhenotypes
from scanomatic.data_processing.phases.segmentation import CurvePhases, is_detected_non_linear
_l = Logger("Curve Phase Meta Phenotyping")
class CurvePhaseMetaPhenotypes(Enum):
"""Phenotypes of an entire growth-curve based on the phase segmentation.
Attributes:
CurvePhaseMetaPhenotypes.MajorImpulseYieldContribution:
The fraction of the total yield (in population doublings) that the
`CurvePhases.Impulse` that contribute most to the total yield is
responsible for (`CurvePhasePhenotypes.FractionYield`).
CurvePhaseMetaPhenotypes.FirstMinorImpulseYieldContribution:
As with `CurvePhaseMetaPhenotypes.MajorImpulseYieldContribution`
but for the second most important `CurvePhases.Impulse`
CurvePhaseMetaPhenotypes.MajorImpulseAveragePopulationDoublingTime:
The `CurvePhases.Impulse` that contribute most to the
total yield, its average population doubling time
(`CurvePhasePhenotypes.PopulationDoublingTime`).
CurvePhaseMetaPhenotypes.FirstMinorImpulseAveragePopulationDoublingTime:
The average population doubling time of
from itertools import chain
from flask import send_file
import numpy as np
from scanomatic.io.app_config import Config
from scanomatic.io.paths import Paths
from scanomatic.io.logger import Logger
from scanomatic.models.factories.scanning_factory import ScanningModelFactory
from scipy.misc import toimage
from scanomatic.image_analysis.first_pass_image import FixtureImage
from scanomatic.models.fixture_models import GrayScaleAreaModel, FixturePlateModel
from scanomatic.image_analysis.image_grayscale import is_valid_grayscale
_safe_dir = re.compile(r"^[A-Za-z_0-9./]*$")
_no_super = re.compile(r"/?\.{2}/")
_logger = Logger("UI API helpers")
_ALLOWED_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.tiff'}
_TOO_LARGE_GRAYSCALE_AREA = 300000
def image_is_allowed(ext):
"""Validates that the image extension is allowed
:param ext: The image file's extension
:type ext: str
:returns bool
"""
global _ALLOWED_EXTENSIONS
return ext.lower() in _ALLOWED_EXTENSIONS
from enum import Enum
import numpy as np
from scipy.optimize import leastsq
from itertools import izip
from scipy.stats import linregress
from scanomatic.io.logger import Logger
_logger = Logger("Growth Phenotypes")
def _linreg_helper(X, Y):
return linregress(X, Y)[0::4]
def get_derivative(curve_strided, times_strided):
linreg_values = []
log2_strided_curve = np.log2(curve_strided)
filters = np.isfinite(log2_strided_curve)
min_size = curve_strided.shape[-1] - 1
for times, value_segment, filt in izip(times_strided, log2_strided_curve,
filters):
if filt.sum() >= min_size:
linreg_values.append(
_linreg_helper(times[filt], value_segment[filt]))
else:
linreg_values.append((np.nan, np.nan))
class GridCell(object):
MAX_THRESHOLD = 4200
MIN_THRESHOLD = 0
_logger = Logger("Grid Cell")
def __init__(self, identifier, polynomial_coeffs, save_extra_data=False):
self._identifier = identifier
self.position = tuple(identifier[-1])
self.save_extra_data = save_extra_data
self._polynomial_coeffs = polynomial_coeffs
self._adjustment_warning = False
self.xy1 = []
self.xy2 = []
self.source = None
self.ready = False
self._previous_image = None
self.image_index = -1
self.features = AnalysisFeaturesFactory.create(index=tuple(
self.position),
data={})
self._analysis_items = {}
""":type: dict[scanomatic.models.analysis_model.COMPARTMENTS | scanomatic.image_analysis.grid_cell_extra.CellItem]"""
self._set_empty_analysis_items()
def _set_empty_analysis_items(self):
for item_name in COMPARTMENTS:
self._analysis_items[item_name] = None
def __str__(self):
s = "< {0}".format(self._identifier)
if self.source is None:
s += " No image set"
else:
s += " Image size: {0}".format(self.source.shape)
s += " Layers: {0} >".format(self._analysis_items.keys())
return s
def __repr__(self):
return self.__str__()
def set_grid_coordinates(self, grid_cell_corners):
flipped_long_axis_position = grid_cell_corners.shape[
2] - self.position[0] - 1
self.xy1 = grid_cell_corners[:, 0, flipped_long_axis_position,
self.position[1]].astype(np.int)
self.xy2 = grid_cell_corners[:, 1, flipped_long_axis_position,
self.position[1]].astype(np.int)
def set_new_data_source_space(self,
space=VALUES.Cell_Estimates,
bg_sub_source=None,
polynomial_coeffs=None):
if space is VALUES.Cell_Estimates:
if bg_sub_source is not None:
feature_array = self.source[np.where(bg_sub_source)]
# bg_sub = tmean(feature_array,
# mquantiles(feature_array, prob=[0.25, 0.75]))
bg_sub = iqr_mean(feature_array)
if not np.isfinite(bg_sub):
bg_sub = np.mean(feature_array)
GridCell._logger.warning(
"{0} caused background mean ({1}) due to inf".format(
self._identifier, bg_sub))
self.source -= bg_sub
self.source[self.source < self.MIN_THRESHOLD] = self.MIN_THRESHOLD
if polynomial_coeffs is not None:
self.source = np.polyval(polynomial_coeffs, self.source)
self._set_max_value_filter()
self.push_source_data_to_cell_items()
def _set_max_value_filter(self):
max_detect_filter = self.source > self.MAX_THRESHOLD
if self._adjustment_warning != max_detect_filter.any():
self._adjustment_warning = not self._adjustment_warning
if self._adjustment_warning:
self._logger.warning(
"{0} got {1} pixel-values overshooting {2}.".format(
self._identifier, max_detect_filter.sum(),
self.MAX_THRESHOLD) +
" Further warnings for this colony suppressed.")
else:
self._logger.info(
"{0} no longer have pixels that reach {1} depth.".format(
self._identifier, self.MAX_THRESHOLD))
def push_source_data_to_cell_items(self):
for item_names in self._analysis_items.keys():
self._analysis_items[item_names].grid_array = self.source
def get_item(self, item_name):
if item_name in self._analysis_items:
return self._analysis_items[item_name]
else:
return None
def analyse(self, detect=True, remember_filter=True):
"""get_analysis iterates through all possible cell items
and runs their detect and do_analysis if they are attached.
The cell items' features dictionaries are put inside a
dictionary with the items' names as keys.
If cell item is not attached, a None is put in the
dictionary to avoid key errors..
"""
background = self._analysis_items[COMPARTMENTS.Background]
if detect:
self.detect(remember_filter=remember_filter)
if background.filter_array.sum() == 0:
self.clear_features()
else:
self._analyse()
def get_save_data_path(self, base_path):
if base_path is None:
base_path = Paths().log
return os.path.join(
base_path, "grid_cell_{0}_{1}_{2}".format(
self.image_index, self._identifier[0][1],
"_".join(map(str, self._identifier[-1][::-1]))))
def save_data_image(self, suffix="", base_path=None):
base_path = self.get_save_data_path(base_path)
np.save(base_path + suffix + ".image.npy", self.source)
def save_data_detections(self, base_path=None):
base_path = self.get_save_data_path(base_path)
blob = self._analysis_items[COMPARTMENTS.Blob]
background = self._analysis_items[COMPARTMENTS.Background]
np.save(base_path + ".background.filter.npy", background.filter_array)
np.save(base_path + ".image.cells.npy", background.grid_array)
np.save(base_path + ".blob.filter.npy", blob.filter_array)
np.save(base_path + ".blob.trash.current.npy", blob.trash_array)
np.save(base_path + ".blob.trash.old.npy", blob.old_trash)
def clear_features(self):
for item in self._analysis_items.itervalues():
if item:
item.features.data.clear()
def _analyse(self):
background = self._analysis_items[COMPARTMENTS.Background]
self.set_new_data_source_space(
space=VALUES.Cell_Estimates,
bg_sub_source=background.filter_array,
polynomial_coeffs=self._polynomial_coeffs)
for item in self._analysis_items.itervalues():
if item:
item.set_data_source(self.source)
item.do_analysis()
def detect(self, remember_filter=True):
blob = self._analysis_items[COMPARTMENTS.Blob]
background = self._analysis_items[COMPARTMENTS.Background]
blob.detect(remember_filter=remember_filter)
background.detect()
def attach_analysis(self,
blob=True,
background=True,
cell=True,
blob_detect=grid_cell_extra.BlobDetectionTypes.DEFAULT,
run_detect=False,
center=None,
radius=None):
"""attach_analysis connects the analysis modules to the Grid_Cell.
Function has three optional boolean arguments:
@blob Attaches blob item (default)
@background Attaches background item (default)
Only possible if blob is attached
@cell Attaches cell item (default)
@use_fallback_detection Causes simple thresholding instead
of more sophisticated detection (default False)
@run_detect Causes the initiation to run detection
@center A manually set blob centrum (if set
radius must be set as well)
(if not supplied, blob will be detected
automatically)
@radius A manually set blob radus (if set
center must be set as well)
(if not supplied, blob will be detected
automatically)"""
if cell:
item = grid_cell_extra.Cell((self._identifier, COMPARTMENTS.Total),
self.source,
run_detect=run_detect)
self.features.data[item.features.index] = item.features
self._analysis_items[item.features.index] = item
if blob:
item = grid_cell_extra.Blob((self._identifier, COMPARTMENTS.Blob),
self.source,
blob_detect=blob_detect,
run_detect=run_detect,
center=center,
radius=radius)
self.features.data[item.features.index] = item.features
self._analysis_items[item.features.index] = item
if background and self._analysis_items[COMPARTMENTS.Blob]:
item = grid_cell_extra.Background(
(self._identifier, COMPARTMENTS.Background),
self.source,
self._analysis_items[COMPARTMENTS.Blob],
run_detect=run_detect)
self.features.data[item.features.index] = item.features
self._analysis_items[item.features.index] = item
self.features.shape = (len(self.features.data), )
self.set_ready_state()
def set_ready_state(self):
self.ready = any(self._analysis_items.values())
import numpy as np
from enum import Enum
import json
from itertools import izip
import os
import shutil
from scipy.optimize import curve_fit
import time
from datetime import datetime
from dateutil import tz
from scipy.stats import linregress
from scanomatic.io.logger import Logger
from scanomatic.io.paths import Paths
_logger = Logger("Calibration")
class CalibrationEntry(Enum):
image = 0
""":type : CalibrationEntry"""
colony_name = 1
""":type : CalibrationEntry"""
target_value = 2
""":type : CalibrationEntry"""
source_values = (3, 0)
""":type : CalibrationEntry"""
source_value_counts = (3, 1)
""":type : CalibrationEntry"""
import glob
import os
import numpy as np
from scanomatic.io.paths import Paths
from scanomatic.image_analysis.image_basics import load_image_to_numpy
from scanomatic.io.logger import Logger
from scanomatic.models.factories.compile_project_factory import CompileImageAnalysisFactory
from scanomatic.generics.purge_importing import ExpiringModule
_logger = Logger("Analysis Utils")
def produce_grid_images(path=".", image=None, mark_position=None):
project_path = os.path.join(os.path.dirname(os.path.abspath(path)))
compilations = glob.glob(os.path.join(os.path.dirname(os.path.abspath(path)),
Paths().project_compilation_pattern.format("*")))
if not compilations:
raise ValueError("There are no compilations in the parent directory")
compilation = compilations[0]
_logger.info("Using {0}".format(os.path.basename(compilation)))
compilation = CompileImageAnalysisFactory.serializer.load(compilation)
image_path = compilation[-1].image.path
plates = compilation[-1].fixture.plates
if image is not None:
for c in compilation:
if os.path.basename(c.image.path) == os.path.basename(image):
image_path = c.image.path
from scanomatic.models.factories.compile_project_factory import CompileProjectFactory
from scanomatic.models.factories.features_factory import FeaturesFactory
from scanomatic.models.factories.scanning_factory import ScanningModelFactory
from . import qc_api
from . import analysis_api
from . import compilation_api
from . import calibration_api
from . import scan_api
from . import management_api
from . import tools_api
from . import data_api
from .general import get_2d_list
_url = None
_logger = Logger("UI-server")
def launch_server(is_local=None, port=None, host=None, debug=False):
global _url
app = Flask("Scan-o-Matic UI", template_folder=Paths().ui_templates)
rpc_client = get_client(admin=True)
if rpc_client.local and rpc_client.online is False:
rpc_client.launch_local()
if port is None:
port = Config().ui_server.port
from flask import request, Flask, jsonify
from itertools import product, chain
import os
import glob
from scanomatic.ui_server.general import safe_directory_name
from scanomatic.io.app_config import Config
from scanomatic.io.logger import Logger
from scanomatic.data_processing.phenotyper import path_has_saved_project_state
_logger = Logger("Tools API")
def valid_range(settings):
return 'min' in settings and 'max' in settings
def add_routes(app):
"""
Args:
app (Flask): The flask app to decorate
"""
@app.route("/api/tools/selection", methods=['POST'])
@app.route("/api/tools/selection/<operation>", methods=['POST'])
def tools_create_selection(operation='rect'):
"""Converts selection ranges to api-understood selections.
_Note_ that the range-boundary uses inclusive min and
exclusive max indices.