def setRootLogger(quiet, debug): """Sets the root logger with the desired verbosity level The root logger logs to logs/twlog.txt and every single logging instruction is propagated to it (not really nice to read) :arg bool quiet: it tells if a quiet logger is needed :arg bool debug: it tells if needs a verbose logger :return logger: a logger with the appropriate logger level.""" createLogdir('logs') createLogdir('logs/processes') createLogdir('logs/tasks') if self.TEST: #if we are testing log to the console is easier logging.getLogger().addHandler(logging.StreamHandler()) else: logHandler = MultiProcessingLog('logs/twlog.txt', when='midnight') logFormatter = \ logging.Formatter("%(asctime)s:%(levelname)s:%(module)s:%(message)s") logHandler.setFormatter(logFormatter) logging.getLogger().addHandler(logHandler) loglevel = logging.INFO if quiet: loglevel = logging.WARNING if debug: loglevel = logging.DEBUG logging.getLogger().setLevel(loglevel) logger = setProcessLogger("master") logger.debug("PID %s.", os.getpid()) logger.debug("Logging level initialized to %s.", loglevel) return logger
def setRootLogger(quiet=False, debug=True, console=False): """Sets the root logger with the desired verbosity level The root logger logs to logs/twlog.txt and every single logging instruction is propagated to it (not really nice to read) :arg bool quiet: it tells if a quiet logger is needed :arg bool debug: it tells if needs a verbose logger :arg bool console: it tells if to direct all printoput to console rather then files, useful for debug :return logger: a logger with the appropriate logger level.""" createLogdir('Publisher/logs') createLogdir('Publisher/logs/processes') createLogdir('Publisher/logs/tasks') if console: #if we are testing log to the console is easier logging.getLogger().addHandler(logging.StreamHandler()) else: logHandler = MultiProcessingLog('logs/log.txt', when='midnight') logFormatter = logging.Formatter("%(asctime)s:%(levelname)s:%(module)s,%(lineno)d:%(message)s") logHandler.setFormatter(logFormatter) logging.getLogger().addHandler(logHandler) loglevel = logging.INFO if quiet: loglevel = logging.WARNING if debug: loglevel = logging.DEBUG logging.getLogger().setLevel(loglevel) logger = setProcessLogger("master") logger.debug("PID %s.", os.getpid()) logger.debug("Logging level initialized to %s.", loglevel) return logger
def getLogging(quiet, debug): """Retrieves a logger and set the proper level :arg bool quiet: it tells if a quiet logger is needed :arg bool debug: it tells if needs a verbose logger :return logger: a logger with the appropriate logger level.""" if self.TEST: #if we are testing log to the console is easier logging.getLogger().addHandler(logging.StreamHandler()) else: logHandler = MultiProcessingLog('twlog.log', when="midnight") logFormatter = \ logging.Formatter("%(asctime)s:%(levelname)s:%(module)s:%(message)s") logHandler.setFormatter(logFormatter) logging.getLogger().addHandler(logHandler) loglevel = logging.INFO if quiet: loglevel = logging.WARNING if debug: loglevel = logging.DEBUG logging.getLogger().setLevel(loglevel) logger = logging.getLogger() logger.debug("Logging level initialized to %s." % loglevel) return logger
def getLogging(quiet, debug): """Retrieves a logger and set the proper level :arg bool quiet: it tells if a quiet logger is needed :arg bool debug: it tells if needs a verbose logger :return logger: a logger with the appropriate logger level.""" if self.TEST: #if we are testing log to the console is easier logging.getLogger().addHandler(logging.StreamHandler()) else: logHandler = MultiProcessingLog('twlog.log', when="midnight") logFormatter = \ logging.Formatter("%(asctime)s:%(levelname)s:%(module)s:%(message)s") logHandler.setFormatter(logFormatter) logging.getLogger().addHandler(logHandler) loglevel = logging.INFO if quiet: loglevel = logging.WARNING if debug: loglevel = logging.DEBUG logging.getLogger().setLevel(loglevel) logger = logging.getLogger() logger.debug("Logging level initialized to %s." %loglevel) return logger
def main(args): # setup logging formatter = logging.Formatter( '%(asctime)s %(levelname)s | %(filename)s/%(funcName)s: %(message)s') mp_log = MultiProcessingLog( os.path.join( args.source_dir, 'find-empty-sites-' + time.strftime('%Y%m%d-%H%M%S') + '.log'), 'w', 0, 0) mp_log.setFormatter(formatter) logger.addHandler(mp_log) if args.move and (not os.path.exists(args.target_dir)): os.makedirs(args.target_dir) images = [ os.path.basename(full_path) for full_path in glob.glob(args.source_dir + '*C01.tif') ] logger.debug('found %d images in %s', len(images), args.source_dir) source_dirs = [args.source_dir for image in images] target_dirs = [args.target_dir for image in images] params = [args.move for image in images] function_args = zip(source_dirs, images, target_dirs, params) # use a multi-processing pool to get the work done pool = mp.Pool() pool.map(check_site_and_move_star, function_args) pool.close() pool.join() return
def setRootLogger(logWarning, logDebug, console): """Sets the root logger with the desired verbosity level The root logger logs to logsDir/twlog.txt and every single logging instruction is propagated to it (not really nice to read) :arg bool logWarning: it tells if a quiet logger is needed :arg bool logDebug: it tells if needs a verbose logger :arg bool console: it tells if to log to console :return logger: a logger with the appropriate logger level.""" logsDir = config.TaskWorker.logsDir createLogdir(logsDir) createLogdir(logsDir+'/processes') createLogdir(logsDir+'/tasks') if console: logging.getLogger().addHandler(logging.StreamHandler()) else: logHandler = MultiProcessingLog(logsDir+'/twlog.txt', when='midnight') logFormatter = \ logging.Formatter("%(asctime)s:%(levelname)s:%(module)s,%(lineno)d:%(message)s") logHandler.setFormatter(logFormatter) logging.getLogger().addHandler(logHandler) loglevel = logging.INFO if logWarning: loglevel = logging.WARNING if logDebug: loglevel = logging.DEBUG logging.getLogger().setLevel(loglevel) logger = setProcessLogger("master", logsDir) logger.debug("PID %s.", os.getpid()) logger.debug("Logging level initialized to %s.", loglevel) return logger
def setRootLogger(quiet, debug): """Sets the root logger with the desired verbosity level The root logger logs to logs/twlog.txt and every single logging instruction is propagated to it (not really nice to read) :arg bool quiet: it tells if a quiet logger is needed :arg bool debug: it tells if needs a verbose logger :return logger: a logger with the appropriate logger level.""" createLogdir('logs') createLogdir('logs/processes') createLogdir('logs/tasks') logHandler = MultiProcessingLog('logs/log.txt', when='midnight') logFormatter = logging.Formatter("%(asctime)s:%(levelname)s:%(module)s,%(lineno)d:%(message)s") logHandler.setFormatter(logFormatter) logging.getLogger().addHandler(logHandler) loglevel = logging.INFO if quiet: loglevel = logging.WARNING if debug: loglevel = logging.DEBUG logging.getLogger().setLevel(loglevel) logger = setProcessLogger("master") logger.debug("PID %s.", os.getpid()) logger.debug("Logging level initialized to %s.", loglevel) return logger
def setRootLogger(quiet, debug): """ Taken from CRABServer TaskWorker Sets the root logger with the desired verbosity level The root logger logs to logs/asolog.txt and every single logging instruction is propagated to it (not really nice to read) :arg bool quiet: it tells if a quiet logger is needed :arg bool debug: it tells if needs a verbose logger :return logger: a logger with the appropriate logger level.""" createLogdir('logs') if self.TEST: # if we are testing log to the console is easier logging.getLogger().addHandler(logging.StreamHandler()) else: logHandler = MultiProcessingLog('logs/monitor.txt', when='midnight') logFormatter = \ logging.Formatter("%(asctime)s:%(levelname)s:%(module)s:%(message)s") logHandler.setFormatter(logFormatter) logging.getLogger().addHandler(logHandler) loglevel = logging.INFO if quiet: loglevel = logging.WARNING if debug: loglevel = logging.DEBUG logging.getLogger().setLevel(loglevel) logger = setProcessLogger("master") logger.debug("PID %s.", os.getpid()) logger.debug("Logging level initialized to %s.", loglevel) return logger
def main(args): # setup logging formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') mp_log = MultiProcessingLog( path.join(args.output_dir, 'png_conversion-' + time.strftime('%Y%m%d-%H%M%S') + '.log'), 'w', 0, 0) mp_log.setFormatter(formatter) logger.addHandler(mp_log) tiff_paths = [] output_dirs = [] # find tiff files and keep track of the source directory structure for root, dirs, files in walk(args.source_dir): for fyle in files: if fyle.endswith('.tif'): tiff_paths.append(path.join(root, fyle)) # Hack to catch inputs ending with and without a / if args.source_dir.endswith('/'): soure_path_length = len(args.source_dir) else: soure_path_length = len(args.source_dir) + 1 output_dirs.append( path.join(args.output_dir, root[soure_path_length:])) # re-create the directory structure in output_dir unique_dirs = list(set(output_dirs)) logger.info('found %d TIFF files in subfolders: %s', len(tiff_paths), unique_dirs) for d in unique_dirs: if not path.exists(d): logger.info('creating output directory: %s', d) makedirs(d) else: logger.info('output directory %s already exists', d) # generate list of tuples containing input/output pairs convert_args = zip(tiff_paths, output_dirs) # use a multi-processing pool to get the work done pool = mp.Pool() pool.map(convert_single_site_star, convert_args) pool.close() pool.join() return
def main(args): if not os.path.exists(args.target_dir): os.makedirs(args.target_dir) # setup logging formatter = logging.Formatter( '%(asctime)s %(levelname)s | %(filename)s/%(funcName)s: %(message)s') mp_log = MultiProcessingLog( os.path.join( args.target_dir, 'modify-site-order-' + time.strftime('%Y%m%d-%H%M%S') + '.log'), 'w', 0, 0) mp_log.setFormatter(formatter) logger.addHandler(mp_log) # get all names for channel 01 all_filenames_C01 = sorted([ os.path.basename(full_path) for full_path in glob.glob(args.source_dir + '*C01.tif') ]) # get unique wells well_names = [ m.group('well') for f in all_filenames_C01 for m in [re.match(pattern, f)] if m ] unique_well_names = sorted(list(set(well_names))) # generate a list of tuples containing files to be linked files_to_link = [] for well in unique_well_names: # split list by well_name well_regex = re.compile('.*_' + well + '_') fnames_C01 = filter(well_regex.match, all_filenames_C01) # generate list of links per well files_to_link.extend(links_per_well(args.source_dir, fnames_C01)) for fname_pair in files_to_link: logger.info('creating %s as link to %s', os.path.join(args.target_dir, fname_pair[1]), os.path.join(args.source_dir, fname_pair[0])) os.link(os.path.join(args.source_dir, fname_pair[0]), os.path.join(args.target_dir, fname_pair[1])) return
def setup_logger(args): global logger logfile = os.path.join(os.getcwd(), 'popcon-' + time.strftime('%Y%m%d-%H%M%S') + '.log') mp_log = MultiProcessingLog(logfile, 'w', 0, 0) formatter = logging.Formatter( '%(asctime)s [%(thread)d] %(funcName)s %(levelname)s: %(message)s') mp_log.setFormatter(formatter) logger.addHandler(mp_log) logger.setLevel(logging.INFO) if args.verbose > 0: logger.setLevel(logging.DEBUG) print 'Logging to {} at level {}'.format( logfile, logging.getLevelName(logger.getEffectiveLevel())) return
def setRootLogger(logWarning, logDebug, console, name): """Sets the root logger with the desired verbosity level The root logger logs to logsDir/twlog.txt and every single logging instruction is propagated to it (not really nice to read) :arg bool logWarning: it tells if a quiet logger is needed :arg bool logDebug: it tells if needs a verbose logger :arg bool console: it tells if to log to console :arg string name: define a name for the log file of this master process :return logger: a logger with the appropriate logger level.""" # this must only done for real Master, not when it is used by TapeRecallStatus logsDir = config.TaskWorker.logsDir if name == 'master': createAndCleanLogDirectories(logsDir) if console: logging.getLogger().addHandler(logging.StreamHandler()) else: logHandler = MultiProcessingLog(logsDir + '/twlog.txt', when='midnight') logFormatter = \ logging.Formatter("%(asctime)s:%(levelname)s:%(module)s,%(lineno)d:%(message)s") logHandler.setFormatter(logFormatter) logging.getLogger().addHandler(logHandler) loglevel = logging.INFO if logWarning: loglevel = logging.WARNING if logDebug: loglevel = logging.DEBUG logging.getLogger().setLevel(loglevel) logger = setProcessLogger(name, logsDir) logger.info("PID %s.", os.getpid()) logger.info("Logging level initialized to %s.", loglevel) return logger
import warnings import logging import itertools import skimage.io import numpy as np from os import path, getcwd import multiprocessing as mp from MultiProcessingLog import MultiProcessingLog warnings.filterwarnings('ignore') logger = logging.getLogger() mp_log = MultiProcessingLog(path.join(getcwd(), 'StackToMaxSum.log'), 'a', 0, 0) formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') mp_log.setFormatter(formatter) logger.addHandler(mp_log) logger.setLevel(logging.INFO) def project_single_site(base_dir, fname_stub, well_name, timeline_name, field, l_name, action_name, input_channel_name, output_channel_names, z_planes, input_dir, output_dir): logger.info('Processing channel %s, well %s, site %d', input_channel_name, well_name, field) field_name = 'F' + str(field).zfill(3) image_names = [] for z in range(1, z_planes + 1): z_name = 'Z' + str(z).zfill(2)
def main(args): if not os.path.exists(args.target_dir): os.makedirs(args.target_dir) # setup logging formatter = logging.Formatter( '%(asctime)s %(levelname)s | %(filename)s/%(funcName)s: %(message)s') mp_log = MultiProcessingLog( os.path.join( args.target_dir, 'extend-wells-to-max-site-number' + time.strftime('%Y%m%d-%H%M%S') + '.log'), 'w', 0, 0) mp_log.setFormatter(formatter) logger.addHandler(mp_log) # get all names for channel 01 all_filenames_C01 = sorted([ os.path.basename(full_path) for full_path in glob.glob(args.source_dir + '*C01.png') ]) # get unique well and sites well_site_names = [(m.group('well'), int(m.group('site'))) for f in all_filenames_C01 for m in [re.match(pattern, f)] if m] # find number of sites per well unique_wells = sorted(list(set([i[0] for i in well_site_names]))) logger.info('found %d wells in %s', len(unique_wells), args.source_dir) max_sites_per_well = [] for well in unique_wells: sites = [x for x in well_site_names if x[0] == well] n = max(sites, key=itemgetter(1)) max_sites_per_well.append(n) logger.debug('well %s has maximum site number of %d', well, n[1]) # find maximum number of sites and select the next largest value from list required_sites = max(max_sites_per_well, key=itemgetter(1))[1] n_sites = next(v for i, v in enumerate(possible_site_numbers) if v >= required_sites) logger.info( '%d sites are required, generating links for %d sites per well', required_sites, n_sites) # generate a list of (well, site) pairs to add # also include possibly missing sites < max number of sites per well well_site_names_to_add = [] for well in unique_wells: existing_sites = set([x[1] for x in well_site_names if x[0] == well]) all_required_sites = set(range(1, n_sites + 1)) well_site_names_to_add.extend([ (well, s) for s in all_required_sites.difference(existing_sites) ]) # convert this list to filenames for all channels # 1. use the first site as a template for new filenames # 2. randomly select a site from the empty directory and # link the corresponding channels to that site filenames_single_site = sorted( list_all_files_same_site(args.source_dir, all_filenames_C01[0])) files_to_link = [] empty_files = [ os.path.basename(full_path) for full_path in glob.glob(args.empty_dir + '*.png') ] logger.debug('selecting random sites from %s', args.empty_dir) for well, site in well_site_names_to_add: empty_site = re.match(pattern, random.choice(empty_files)) for basefile in filenames_single_site: new_site = re.match(pattern, basefile) link_name = (new_site.group('stem') + '_' + well + '_T' + new_site.group('t') + 'F' + str(site).zfill(3) + 'L' + new_site.group('l') + 'A' + new_site.group('a') + 'Z' + new_site.group('z') + new_site.group('c') + '.png') # add a couple of catches to account for differing dimensions between sites if (int(new_site.group('z')) < 20) and (new_site.group('c') == 'C04'): z_plane = new_site.group('z') elif (int(new_site.group('z')) < 40) and (new_site.group('c') == 'C03'): z_plane = new_site.group('z') else: z_plane = '01' source_file = (empty_site.group('stem') + '_' + empty_site.group('well') + '_T' + new_site.group('t') + 'F' + empty_site.group('site') + 'L' + new_site.group('l') + 'A' + new_site.group('a') + 'Z' + z_plane + new_site.group('c') + '.png') files_to_link.append((source_file, link_name)) for source, link in files_to_link: logger.info('creating hard link %s to file %s', os.path.join(args.target_dir, link), os.path.join(args.empty_dir, source)) try: os.link(os.path.join(args.empty_dir, source), os.path.join(args.target_dir, link)) except OSError: logger.warning('could not link %s to %s', os.path.join(args.target_dir, link), os.path.join(args.empty_dir, source)) pass return