def __init__(self, factory): """ :type factory: AbstractModelFactory """ self._factory = factory self._logger = Logger(factory.__name__)
def __init__(self, fixture_settings): """ :type fixture_settings: scanomatic.io.fixtures.FixtureSettings """ self._logger = Logger("Gridding History {0}".format( fixture_settings.model.name)) self._fixture_settings = fixture_settings self._models_per_plate_pinning = defaultdict(dict)
def logger(cls): """ :rtype: scanomatic.io.logger.Logger """ if cls._LOGGER is None: cls._LOGGER = Logger(cls.__name__) return cls._LOGGER
def __init__(self, name, dir_path=None): self._logger = Logger("Fixture {0}".format(name)) path_name = Paths().get_fixture_path(name, only_name=True) if dir_path: conf_rel_path = Paths().fixture_conf_file_rel_pattern.format(path_name) self._conf_path = os.path.join(dir_path, conf_rel_path) else: self._conf_path = Paths().get_fixture_path(name) self.model = self._load_model(name) self.history = grid_history.GriddingHistory(self)
def __init__(self, analysis_model, scanning_meta_data): """ :param analysis_model: The model :type analysis_model : scanomatic.models.analysis_model.AnalysisModel :param scanning_meta_data: scanning info :type scanning_meta_data : scanomatic.models.scanning_model.ScanningModel :return: """ self._analysis_model = analysis_model self._scanning_meta_data = scanning_meta_data self._logger = Logger("Analysis Image") self._im_loaded = False self.im = None self._grid_arrays = self._new_grid_arrays """:type : dict[int|scanomatic.image_analysis.grid_array.GridArray]""" self.features = _get_init_features(self._grid_arrays)
def __init__(self, fixture=None): """ :type fixture: scanomatic.io.fixtures.FixtureSettings """ self._logger = Logger("Fixture Image") self._reference_fixture_settings = fixture self._current_fixture_settings = None """:type : scanomatic.io.fixtures.Fixture_Settings""" if fixture: self._history = GriddingHistory(fixture) else: self._history = None self.im = None self.im_path = None self._original_dpi = None self._name = "default"
def __init__(self, compilation_path=None, compile_instructions_path=None, scanner_instructions_path=None, sort_mode=FIRST_PASS_SORTING.Time): self._logger = Logger("Compilation results") self._compilation_path = compilation_path self._compile_instructions = None self._scanner_instructions = None """:type : scanomatic.models.scanning_model.ScanningModel""" self.load_scanner_instructions(scanner_instructions_path) self._plates = None self._plate_position_keys = None self._image_models = [] self._used_models = [] self._current_model = None self._loading_length = 0 if compile_instructions_path: self._load_compile_instructions(compile_instructions_path) if compilation_path: self._load_compilation(self._compilation_path, sort_mode=sort_mode)
from scanomatic.image_analysis.image_grayscale import get_grayscale from scanomatic.io.logger import Logger from scanomatic.models.factories.fixture_factories import FixtureFactory from scanomatic.models.fixture_models import GrayScaleAreaModel from scanomatic.image_analysis.image_basics import Image_Transpose from scanomatic.image_analysis.grid_cell import GridCell from scanomatic.image_analysis.grid_array import get_calibration_polynomial_coeffs from scanomatic.models.analysis_model import COMPARTMENTS, VALUES from scanomatic.models.factories.analysis_factories import AnalysisFeaturesFactory from .general import get_fixture_image_by_name, usable_markers, split_areas_into_grayscale_and_plates, \ get_area_too_large_for_grayscale, get_grayscale_is_valid, usable_plates, image_is_allowed, \ get_fixture_image, convert_url_to_path _logger = Logger("Data API") def _depth(arr, lvl=1): if isinstance(arr, ListType) and len(arr) and isinstance(arr[0], ListType): _depth(arr[0], lvl + 1) else: return lvl def _validate_depth(data): depth = _depth(data) while depth < 4:
import os from multiprocessing import Process from time import sleep from subprocess import call, STDOUT from scanomatic.io.logger import Logger _logger = Logger("Daemonizer") def _daemon_process(path_to_exec, std_out_path, args, shell): with open(std_out_path, 'w') as fh: args = list(str(a) for a in args) if shell: fh.write("*** LAUNCHING IN SHELL: {0} ***\n\n".format(" ".join([path_to_exec] + list(args)))) retcode = call(" ".join([path_to_exec] + args), stderr=STDOUT, stdout=fh, shell=True) else: fh.write("*** LAUNCHING WITHOUT SHELL: {0} ***\n\n".format([path_to_exec] + list(args))) retcode = call([path_to_exec] + args, stderr=STDOUT, stdout=fh, shell=False) if retcode: fh.write("\n*** DAEMON EXITED WITH CODE {0} ***\n".format(retcode)) else: fh.write("\n*** DAEMON DONE ***\n") def daemon(path_to_executable, std_out=os.devnull, daemon_args=tuple(), shell=True): _logger.info("Launching daemon {0} (args={2}, {3}), outputting to {1} ".format(
import numpy as np from enum import Enum from scanomatic.data_processing import growth_phenotypes from scanomatic.io.logger import Logger from scanomatic.data_processing.phases.analysis import CurvePhasePhenotypes from scanomatic.data_processing.phases.segmentation import CurvePhases, is_detected_non_linear _l = Logger("Curve Phase Meta Phenotyping") class CurvePhaseMetaPhenotypes(Enum): """Phenotypes of an entire growth-curve based on the phase segmentation. Attributes: CurvePhaseMetaPhenotypes.MajorImpulseYieldContribution: The fraction of the total yield (in population doublings) that the `CurvePhases.Impulse` that contribute most to the total yield is responsible for (`CurvePhasePhenotypes.FractionYield`). CurvePhaseMetaPhenotypes.FirstMinorImpulseYieldContribution: As with `CurvePhaseMetaPhenotypes.MajorImpulseYieldContribution` but for the second most important `CurvePhases.Impulse` CurvePhaseMetaPhenotypes.MajorImpulseAveragePopulationDoublingTime: The `CurvePhases.Impulse` that contribute most to the total yield, its average population doubling time (`CurvePhasePhenotypes.PopulationDoublingTime`). CurvePhaseMetaPhenotypes.FirstMinorImpulseAveragePopulationDoublingTime: The average population doubling time of
from itertools import chain from flask import send_file import numpy as np from scanomatic.io.app_config import Config from scanomatic.io.paths import Paths from scanomatic.io.logger import Logger from scanomatic.models.factories.scanning_factory import ScanningModelFactory from scipy.misc import toimage from scanomatic.image_analysis.first_pass_image import FixtureImage from scanomatic.models.fixture_models import GrayScaleAreaModel, FixturePlateModel from scanomatic.image_analysis.image_grayscale import is_valid_grayscale _safe_dir = re.compile(r"^[A-Za-z_0-9./]*$") _no_super = re.compile(r"/?\.{2}/") _logger = Logger("UI API helpers") _ALLOWED_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.tiff'} _TOO_LARGE_GRAYSCALE_AREA = 300000 def image_is_allowed(ext): """Validates that the image extension is allowed :param ext: The image file's extension :type ext: str :returns bool """ global _ALLOWED_EXTENSIONS return ext.lower() in _ALLOWED_EXTENSIONS
from enum import Enum import numpy as np from scipy.optimize import leastsq from itertools import izip from scipy.stats import linregress from scanomatic.io.logger import Logger _logger = Logger("Growth Phenotypes") def _linreg_helper(X, Y): return linregress(X, Y)[0::4] def get_derivative(curve_strided, times_strided): linreg_values = [] log2_strided_curve = np.log2(curve_strided) filters = np.isfinite(log2_strided_curve) min_size = curve_strided.shape[-1] - 1 for times, value_segment, filt in izip(times_strided, log2_strided_curve, filters): if filt.sum() >= min_size: linreg_values.append( _linreg_helper(times[filt], value_segment[filt])) else: linreg_values.append((np.nan, np.nan))
class GridCell(object): MAX_THRESHOLD = 4200 MIN_THRESHOLD = 0 _logger = Logger("Grid Cell") def __init__(self, identifier, polynomial_coeffs, save_extra_data=False): self._identifier = identifier self.position = tuple(identifier[-1]) self.save_extra_data = save_extra_data self._polynomial_coeffs = polynomial_coeffs self._adjustment_warning = False self.xy1 = [] self.xy2 = [] self.source = None self.ready = False self._previous_image = None self.image_index = -1 self.features = AnalysisFeaturesFactory.create(index=tuple( self.position), data={}) self._analysis_items = {} """:type: dict[scanomatic.models.analysis_model.COMPARTMENTS | scanomatic.image_analysis.grid_cell_extra.CellItem]""" self._set_empty_analysis_items() def _set_empty_analysis_items(self): for item_name in COMPARTMENTS: self._analysis_items[item_name] = None def __str__(self): s = "< {0}".format(self._identifier) if self.source is None: s += " No image set" else: s += " Image size: {0}".format(self.source.shape) s += " Layers: {0} >".format(self._analysis_items.keys()) return s def __repr__(self): return self.__str__() def set_grid_coordinates(self, grid_cell_corners): flipped_long_axis_position = grid_cell_corners.shape[ 2] - self.position[0] - 1 self.xy1 = grid_cell_corners[:, 0, flipped_long_axis_position, self.position[1]].astype(np.int) self.xy2 = grid_cell_corners[:, 1, flipped_long_axis_position, self.position[1]].astype(np.int) def set_new_data_source_space(self, space=VALUES.Cell_Estimates, bg_sub_source=None, polynomial_coeffs=None): if space is VALUES.Cell_Estimates: if bg_sub_source is not None: feature_array = self.source[np.where(bg_sub_source)] # bg_sub = tmean(feature_array, # mquantiles(feature_array, prob=[0.25, 0.75])) bg_sub = iqr_mean(feature_array) if not np.isfinite(bg_sub): bg_sub = np.mean(feature_array) GridCell._logger.warning( "{0} caused background mean ({1}) due to inf".format( self._identifier, bg_sub)) self.source -= bg_sub self.source[self.source < self.MIN_THRESHOLD] = self.MIN_THRESHOLD if polynomial_coeffs is not None: self.source = np.polyval(polynomial_coeffs, self.source) self._set_max_value_filter() self.push_source_data_to_cell_items() def _set_max_value_filter(self): max_detect_filter = self.source > self.MAX_THRESHOLD if self._adjustment_warning != max_detect_filter.any(): self._adjustment_warning = not self._adjustment_warning if self._adjustment_warning: self._logger.warning( "{0} got {1} pixel-values overshooting {2}.".format( self._identifier, max_detect_filter.sum(), self.MAX_THRESHOLD) + " Further warnings for this colony suppressed.") else: self._logger.info( "{0} no longer have pixels that reach {1} depth.".format( self._identifier, self.MAX_THRESHOLD)) def push_source_data_to_cell_items(self): for item_names in self._analysis_items.keys(): self._analysis_items[item_names].grid_array = self.source def get_item(self, item_name): if item_name in self._analysis_items: return self._analysis_items[item_name] else: return None def analyse(self, detect=True, remember_filter=True): """get_analysis iterates through all possible cell items and runs their detect and do_analysis if they are attached. The cell items' features dictionaries are put inside a dictionary with the items' names as keys. If cell item is not attached, a None is put in the dictionary to avoid key errors.. """ background = self._analysis_items[COMPARTMENTS.Background] if detect: self.detect(remember_filter=remember_filter) if background.filter_array.sum() == 0: self.clear_features() else: self._analyse() def get_save_data_path(self, base_path): if base_path is None: base_path = Paths().log return os.path.join( base_path, "grid_cell_{0}_{1}_{2}".format( self.image_index, self._identifier[0][1], "_".join(map(str, self._identifier[-1][::-1])))) def save_data_image(self, suffix="", base_path=None): base_path = self.get_save_data_path(base_path) np.save(base_path + suffix + ".image.npy", self.source) def save_data_detections(self, base_path=None): base_path = self.get_save_data_path(base_path) blob = self._analysis_items[COMPARTMENTS.Blob] background = self._analysis_items[COMPARTMENTS.Background] np.save(base_path + ".background.filter.npy", background.filter_array) np.save(base_path + ".image.cells.npy", background.grid_array) np.save(base_path + ".blob.filter.npy", blob.filter_array) np.save(base_path + ".blob.trash.current.npy", blob.trash_array) np.save(base_path + ".blob.trash.old.npy", blob.old_trash) def clear_features(self): for item in self._analysis_items.itervalues(): if item: item.features.data.clear() def _analyse(self): background = self._analysis_items[COMPARTMENTS.Background] self.set_new_data_source_space( space=VALUES.Cell_Estimates, bg_sub_source=background.filter_array, polynomial_coeffs=self._polynomial_coeffs) for item in self._analysis_items.itervalues(): if item: item.set_data_source(self.source) item.do_analysis() def detect(self, remember_filter=True): blob = self._analysis_items[COMPARTMENTS.Blob] background = self._analysis_items[COMPARTMENTS.Background] blob.detect(remember_filter=remember_filter) background.detect() def attach_analysis(self, blob=True, background=True, cell=True, blob_detect=grid_cell_extra.BlobDetectionTypes.DEFAULT, run_detect=False, center=None, radius=None): """attach_analysis connects the analysis modules to the Grid_Cell. Function has three optional boolean arguments: @blob Attaches blob item (default) @background Attaches background item (default) Only possible if blob is attached @cell Attaches cell item (default) @use_fallback_detection Causes simple thresholding instead of more sophisticated detection (default False) @run_detect Causes the initiation to run detection @center A manually set blob centrum (if set radius must be set as well) (if not supplied, blob will be detected automatically) @radius A manually set blob radus (if set center must be set as well) (if not supplied, blob will be detected automatically)""" if cell: item = grid_cell_extra.Cell((self._identifier, COMPARTMENTS.Total), self.source, run_detect=run_detect) self.features.data[item.features.index] = item.features self._analysis_items[item.features.index] = item if blob: item = grid_cell_extra.Blob((self._identifier, COMPARTMENTS.Blob), self.source, blob_detect=blob_detect, run_detect=run_detect, center=center, radius=radius) self.features.data[item.features.index] = item.features self._analysis_items[item.features.index] = item if background and self._analysis_items[COMPARTMENTS.Blob]: item = grid_cell_extra.Background( (self._identifier, COMPARTMENTS.Background), self.source, self._analysis_items[COMPARTMENTS.Blob], run_detect=run_detect) self.features.data[item.features.index] = item.features self._analysis_items[item.features.index] = item self.features.shape = (len(self.features.data), ) self.set_ready_state() def set_ready_state(self): self.ready = any(self._analysis_items.values())
import numpy as np from enum import Enum import json from itertools import izip import os import shutil from scipy.optimize import curve_fit import time from datetime import datetime from dateutil import tz from scipy.stats import linregress from scanomatic.io.logger import Logger from scanomatic.io.paths import Paths _logger = Logger("Calibration") class CalibrationEntry(Enum): image = 0 """:type : CalibrationEntry""" colony_name = 1 """:type : CalibrationEntry""" target_value = 2 """:type : CalibrationEntry""" source_values = (3, 0) """:type : CalibrationEntry""" source_value_counts = (3, 1) """:type : CalibrationEntry"""
import glob import os import numpy as np from scanomatic.io.paths import Paths from scanomatic.image_analysis.image_basics import load_image_to_numpy from scanomatic.io.logger import Logger from scanomatic.models.factories.compile_project_factory import CompileImageAnalysisFactory from scanomatic.generics.purge_importing import ExpiringModule _logger = Logger("Analysis Utils") def produce_grid_images(path=".", image=None, mark_position=None): project_path = os.path.join(os.path.dirname(os.path.abspath(path))) compilations = glob.glob(os.path.join(os.path.dirname(os.path.abspath(path)), Paths().project_compilation_pattern.format("*"))) if not compilations: raise ValueError("There are no compilations in the parent directory") compilation = compilations[0] _logger.info("Using {0}".format(os.path.basename(compilation))) compilation = CompileImageAnalysisFactory.serializer.load(compilation) image_path = compilation[-1].image.path plates = compilation[-1].fixture.plates if image is not None: for c in compilation: if os.path.basename(c.image.path) == os.path.basename(image): image_path = c.image.path
from scanomatic.models.factories.compile_project_factory import CompileProjectFactory from scanomatic.models.factories.features_factory import FeaturesFactory from scanomatic.models.factories.scanning_factory import ScanningModelFactory from . import qc_api from . import analysis_api from . import compilation_api from . import calibration_api from . import scan_api from . import management_api from . import tools_api from . import data_api from .general import get_2d_list _url = None _logger = Logger("UI-server") def launch_server(is_local=None, port=None, host=None, debug=False): global _url app = Flask("Scan-o-Matic UI", template_folder=Paths().ui_templates) rpc_client = get_client(admin=True) if rpc_client.local and rpc_client.online is False: rpc_client.launch_local() if port is None: port = Config().ui_server.port
from flask import request, Flask, jsonify from itertools import product, chain import os import glob from scanomatic.ui_server.general import safe_directory_name from scanomatic.io.app_config import Config from scanomatic.io.logger import Logger from scanomatic.data_processing.phenotyper import path_has_saved_project_state _logger = Logger("Tools API") def valid_range(settings): return 'min' in settings and 'max' in settings def add_routes(app): """ Args: app (Flask): The flask app to decorate """ @app.route("/api/tools/selection", methods=['POST']) @app.route("/api/tools/selection/<operation>", methods=['POST']) def tools_create_selection(operation='rect'): """Converts selection ranges to api-understood selections. _Note_ that the range-boundary uses inclusive min and exclusive max indices.