def init(config): """ This function initializes global variables. It gets values from the configuration file, evaluates and processes the values. If mandatory file or directory is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance dep : dictionary a dictionary containing dependency values read from the configured 'dependencies' file """ conf = utils.get_config(config) if conf is None: print ('configuration file is missing') exit(-1) logger = utils.get_logger(__name__, conf) dependencies = utils.get_file(conf, 'dependencies', logger) if dependencies is None: sys.exit(-1) with open(dependencies) as file: dep = json.loads(file.read()) return logger, dep
def init(config): """ This function initializes global variables. It gets values from the configuration file, evaluates and processes the values. If mandatory file or directory is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance pvs : dictionary a dictionary containing pvs values and attributes read from the configured 'pv_file' file """ conf = utils.get_config(config) if conf is None: print ('configuration file is missing') exit(-1) logger = utils.get_logger(__name__, conf) pvfile = utils.get_file(conf, 'pv_file', logger) if pvfile is None: sys.exit(-1) with open(pvfile) as file: pvs = json.loads(file.read())['required_pvs'] return logger, pvs
def init(config): """ This function initializes global variables. It gets values from the configuration file, evaluates and processes the values. If mandatory file or directory is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance pvs : dictionary a dictionary containing pvs values and attributes read from the configured 'pv_file' file """ conf = utils.get_config(config) if conf is None: print('configuration file is missing') exit(-1) logger = utils.get_logger(__name__, conf) pvfile = utils.get_file(conf, 'pv_file', logger) if pvfile is None: sys.exit(-1) with open(pvfile) as file: pvs = json.loads(file.read())['required_pvs'] return logger, pvs
def init(config): """ This function initializes global variables. It gets values from the configuration file, evaluates and processes the values. If mandatory file or directory is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance dep : dictionary a dictionary containing dependency values read from the configured 'dependencies' file """ conf = utils.get_config(config) if conf is None: print('configuration file is missing') exit(-1) logger = utils.get_logger(__name__, conf) dependencies = utils.get_file(conf, 'dependencies', logger) if dependencies is None: sys.exit(-1) with open(dependencies) as file: dep = json.loads(file.read()) return logger, dep
def init(config): """ This function initializes global variables. It gets values from the configuration file, evaluates and processes the values. If mandatory file or directory is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance limits : dictionary a dictionary containing limit values read from the configured 'limit' file quality_checks : dict a dictionary containing quality check functions ids extensions : list a list containing extensions of files to be monitored read from the configuration file report_type : int report type; currently supporting 'none', 'error', and 'full' """ conf = utils.get_config(config) logger = utils.get_logger(__name__, conf) limitsfile = utils.get_file(conf, 'limits', logger) if limitsfile is None: sys.exit(-1) with open(limitsfile) as limits_file: limits = json.loads(limits_file.read()) report_file = utils.get_file(conf, 'report_file', logger) try: extensions = conf['extensions'] except KeyError: logger.warning('no file extension specified. Monitoring for all files.') extensions = [''] qcfile = utils.get_file(conf, 'quality_checks', logger) if qcfile is None: sys.exit(-1) with open(qcfile) as qc_file: dict = json.loads(qc_file.read()) quality_checks = utils.get_quality_checks(dict) try: report_type = conf['report_type'] except KeyError: report_type = const.REPORT_FULL return logger, limits, quality_checks, extensions, report_type
def init(config): """ This function initializes global variables. It gets values from the configuration file, evaluates and processes the values. If mandatory file or directory is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance tags : dictionary a dictionary containing tag and attributes values read from the configured 'schema' file """ conf = utils.get_config(config) if conf is None: print('configuration file is missing') exit(-1) logger = utils.get_logger(__name__, conf) schema = utils.get_file(conf, 'schema', logger) if schema is None: sys.exit(-1) with open(schema) as file: tags = json.loads(file.read()) try: type = conf['verification_type'] except KeyError: logger.error('config error: verification type not configured') sys.exit(-1) if type != 'hdf_structure' and type != 'hdf_tags': logger.error('configured verification type ' + type + ' is not supported') sys.exit(-1) return logger, tags, type
def init(config): """ This function initializes global variables. It gets values from the configuration file, evaluates and processes the values. If mandatory file or directory is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance tags : dictionary a dictionary containing tag and attributes values read from the configured 'schema' file """ conf = utils.get_config(config) if conf is None: print ('configuration file is missing') exit(-1) logger = utils.get_logger(__name__, conf) schema = utils.get_file(conf, 'schema', logger) if schema is None: sys.exit(-1) with open(schema) as file: tags = json.loads(file.read()) try: type = conf['verification_type'] except KeyError: logger.error('config error: verification type not configured') sys.exit(-1) if type != 'hdf_structure' and type != 'hdf_tags': logger.error('configured verification type ' + type + ' is not supported') sys.exit(-1) return logger, tags, type
def parse_config(config): """ This function parses the configuration file. It must return the specified variables. Parameters ---------- config : str a configuration file Returns ------- no_frames : int number of frames that will be processed detector : str a string defining the first prefix in area detector, it has to match the area detector configuration """ conf = utils.get_config(config) if conf is None: print('configuration file is missing') exit(-1) try: no_frames = conf['no_frames'] except KeyError: print('no_frames parameter not configured.') return None try: aggregate_limit = conf['aggregate_limit'] except KeyError: #print ('aggregate_limit parameter not configured.') aggregate_limit = no_frames try: detector = conf['detector'] except KeyError: print('detector parameter not configured.') detector = None return int(no_frames), aggregate_limit, detector
def init(config): conf = utils.get_config(config) if conf is None: print('configuration file is missing') exit(-1) logger = utils.get_logger(__name__, conf) try: zmq_port = conf['zmq_rcv_port'] except: zmq_port = None print('configuration error: zmq_port not configured') try: detector = conf['detector'] except KeyError: print('configuration error: detector parameter not configured.') return None try: detector_basic = conf['detector_basic'] except KeyError: print('configuration error: detector_basic parameter not configured.') return None try: detector_image = conf['detector_image'] except KeyError: print('configuration error: detector_image parameter not configured.') return None try: no_frames = conf['no_frames'] except KeyError: print('no_frames parameter not configured.') return None return detector, detector_basic, detector_image, logger, zmq_port, no_frames
def init(config): """ This function initializes variables according to configuration. It gets values from the configuration file, evaluates and processes the values. If mandatory file or directory is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance data_tags : dict a dictionary od data_type/hdf tag limits : dictionary a dictionary containing limit values read from the configured 'limit' file quality_checks : dict a dictionary containing quality check functions ids extensions : list a list containing extensions of files to be monitored read from the configuration file file_type : int data file type; currently supporting FILE_TYPE_HDF and FILE_TYPE_GE report_type : int report type; currently supporting 'none', 'error', and 'full' report_dir : str a directory where report files will be located consumers : dict a dictionary containing consumer processes to run, and their parameters """ conf = utils.get_config(config) if conf is None: print('configuration file is missing') exit(0) logger = utils.get_logger(__name__, conf) try: file_type = conf['file_type'] except KeyError: print('file type is not configured') exit(0) if file_type == const.FILE_TYPE_HDF: tagsfile = utils.get_file(conf, 'data_tags', logger) if tagsfile is None: sys.exit(0) with open(tagsfile) as tags_file: data_tags = json.loads(tags_file.read()) else: data_tags = None limitsfile = utils.get_file(conf, 'limits', logger) if limitsfile is None: sys.exit(0) with open(limitsfile) as limits_file: limits = json.loads(limits_file.read()) try: extensions = conf['extensions'] except KeyError: logger.warning( 'no file extension specified. Monitoring for all files.') extensions = [''] qcfile = utils.get_file(conf, 'quality_checks', logger) if qcfile is None: sys.exit(0) with open(qcfile) as qc_file: dict = json.loads(qc_file.read()) quality_checks = utils.get_quality_checks(dict) try: report_type = conf['report_type'] except KeyError: report_type = const.REPORT_FULL try: report_dir = conf['report_dir'] if not os.path.isdir(report_dir): report_dir = None except KeyError: report_dir = None try: feedback = conf['feedback_type'] except KeyError: feedback = [] return logger, data_tags, limits, quality_checks, extensions, file_type, report_type, report_dir, feedback
def init(config): """ This function initializes variables according to configuration. It gets values from the configuration file, evaluates and processes the values. If mandatory file or directory is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance data_tags : dict a dictionary od data_type/hdf tag limits : dictionary a dictionary containing limit values read from the configured 'limit' file quality_checks : dict a dictionary containing quality check functions ids extensions : list a list containing extensions of files to be monitored read from the configuration file file_type : int data file type; currently supporting FILE_TYPE_HDF and FILE_TYPE_GE report_type : int report type; currently supporting 'none', 'error', and 'full' report_dir : str a directory where report files will be located consumers : dict a dictionary containing consumer processes to run, and their parameters """ conf = utils.get_config(config) if conf is None: print ('configuration file is missing') exit(-1) logger = utils.get_logger(__name__, conf) try: file_type = conf['file_type'] except KeyError: file_type = const.FILE_TYPE_HDF if file_type == const.FILE_TYPE_HDF: tagsfile = utils.get_file(conf, 'data_tags', logger) if tagsfile is None: sys.exit(-1) with open(tagsfile) as tags_file: data_tags = json.loads(tags_file.read()) else: data_tags = None limitsfile = utils.get_file(conf, 'limits', logger) if limitsfile is None: sys.exit(-1) with open(limitsfile) as limits_file: limits = json.loads(limits_file.read()) try: extensions = conf['extensions'] except KeyError: logger.warning('no file extension specified. Monitoring for all files.') extensions = [''] qcfile = utils.get_file(conf, 'quality_checks', logger) if qcfile is None: sys.exit(-1) with open(qcfile) as qc_file: dict = json.loads(qc_file.read()) quality_checks = utils.get_quality_checks(dict) try: report_type = conf['report_type'] except KeyError: report_type = const.REPORT_FULL try: report_dir = conf['report_dir'] if not os.path.isdir(report_dir): report_dir = None except KeyError: report_dir = None consumersfile = utils.get_file(conf, 'consumers', logger, False) if consumersfile is None: consumers = None else: with open(consumersfile) as consumers_file: consumers = json.loads(consumers_file.read()) return logger, data_tags, limits, quality_checks, extensions, file_type, report_type, report_dir, consumers
def init(self, config): """ This function initializes variables according to configuration. It gets values from the configuration file, evaluates and processes the values. If mandatory parameter is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance limits : dictionary a dictionary containing limit values read from the configured 'limit' file quality_checks : dict a dictionary containing quality check functions ids feedback : list a list of strings defining real time feedback of quality checks errors. Currently supporting 'PV', 'log', and 'console' report_type : int report type; currently supporting 'none', 'error', and 'full' consumers : dict a dictionary parsed from json file representing consumers """ conf = utils.get_config(config) if conf is None: print('configuration file is missing') exit(-1) logger = utils.get_logger(__name__, conf) feed_args = [] feed_kwargs = {} limitsfile = utils.get_file(conf, 'limits', logger) if limitsfile is None: sys.exit(-1) with open(limitsfile) as limits_file: limits = json.loads(limits_file.read()) feed_args.append(limits) qcfile = utils.get_file(conf, 'quality_checks', logger) if qcfile is None: sys.exit(-1) with open(qcfile) as qc_file: dict = json.loads(qc_file.read()) feed_args.append(dict) try: no_frames = int(conf['no_frames']) except KeyError: print('no_frames parameter not configured. Continuous mode.') no_frames = -1 feed_args.append(no_frames) try: callback_pv = conf['callback_pv'] feed_kwargs['callback_pv'] = callback_pv except KeyError: pass try: detector = conf['detector'] feed_kwargs['detector'] = detector except KeyError: print('detector parameter not configured.') sys.exit(-1) try: consumers = conf['zmq_snd_port'] feed_kwargs['consumers'] = consumers except KeyError: pass try: aggregate_limit = int(conf['aggregate_limit']) except KeyError: aggregate_limit = no_frames feed_kwargs['aggregate_limit'] = aggregate_limit try: feedback = conf['feedback_type'] if len(feedback) == 0: feedback = None except KeyError: feedback = None try: decor_conf = conf['decor'] decor_map = {} for entry in decor_conf: entry = entry.split('>') decor_map[entry[0].strip()] = entry[1].strip() if len(decor_map) == 0: decor_map = None except KeyError: decor_map = None try: report_type = conf['report_type'] except KeyError: report_type = const.REPORT_FULL return feed_args, feed_kwargs, feedback, decor_map, logger, report_type
def init(config): """ This function initializes global variables. It gets values from the configuration file, evaluates and processes the values. If mandatory file or directory is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance data_tags : dict a dictionary od data_type/hdf tag limits : dictionary a dictionary containing limit values read from the configured 'limit' file quality_checks : dict a dictionary containing quality check functions ids file_type : int data file type; currently supporting FILE_TYPE_HDF and FILE_TYPE_GE report_type : int report type; currently supporting 'none', 'error', and 'full' report_dir : str a directory where report files will be located """ conf = utils.get_config(config) logger = utils.get_logger(__name__, conf) try: file_type = conf['file_type'] except KeyError: file_type = const.FILE_TYPE_HDF if file_type == const.FILE_TYPE_HDF: tagsfile = utils.get_file(conf, 'data_tags', logger) if tagsfile is None: sys.exit(-1) with open(tagsfile) as tags_file: data_tags = json.loads(tags_file.read()) else: data_tags = None limitsfile = utils.get_file(conf, 'limits', logger) if limitsfile is None: sys.exit(-1) with open(limitsfile) as limits_file: limits = json.loads(limits_file.read()) qcfile = utils.get_file(conf, 'quality_checks', logger) if qcfile is None: sys.exit(-1) with open(qcfile) as qc_file: dict = json.loads(qc_file.read()) quality_checks = utils.get_quality_checks(dict) try: report_type = conf['report_type'] except KeyError: report_type = const.REPORT_FULL try: report_dir = conf['report_dir'] if not os.path.isdir(report_dir): report_dir = None except KeyError: report_dir = None return logger, data_tags, limits, quality_checks, file_type, report_type, report_dir
def init(config): """ This function initializes global variables. It gets values from the configuration file, evaluates and processes the values. If mandatory parameter is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance limits : dictionary a dictionary containing limit values read from the configured 'limit' file quality_checks : dict a dictionary containing quality check functions ids feedback : list a list of strings defining real time feedback of quality checks errors. Currently supporting 'PV', 'log', and 'console' feedback_pv : str a name of process variable that is used for quality feedback report_type : int report type; currently supporting 'none', 'error', and 'full' """ conf = utils.get_config(config) logger = utils.get_logger(__name__, conf) limitsfile = utils.get_file(conf, 'limits', logger) if limitsfile is None: sys.exit(-1) with open(limitsfile) as limits_file: limits = json.loads(limits_file.read()) qcfile = utils.get_file(conf, 'quality_checks', logger) if qcfile is None: sys.exit(-1) with open(qcfile) as qc_file: dict = json.loads(qc_file.read()) quality_checks = utils.get_quality_checks(dict) feedback_pv = None try: feedback = conf['feedback_type'] if 'pv' in feedback: try: feedback_pv = conf['feedback_pv'] except KeyError: feedback.remove('pv') except KeyError: feedback = None try: report_type = conf['report_type'] except KeyError: report_type = const.REPORT_FULL return logger, limits, quality_checks, feedback, feedback_pv, report_type
def init(config): """ This function initializes variables according to configuration. It gets values from the configuration file, evaluates and processes the values. If mandatory parameter is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance limits : dictionary a dictionary containing limit values read from the configured 'limit' file quality_checks : dict a dictionary containing quality check functions ids feedback : list a list of strings defining real time feedback of quality checks errors. Currently supporting 'PV', 'log', and 'console' report_type : int report type; currently supporting 'none', 'error', and 'full' consumers : dict a dictionary parsed from json file representing consumers """ conf = utils.get_config(config) if conf is None: print ('configuration file is missing') exit(-1) logger = utils.get_logger(__name__, conf) limitsfile = utils.get_file(conf, 'limits', logger) if limitsfile is None: sys.exit(-1) with open(limitsfile) as limits_file: limits = json.loads(limits_file.read()) qcfile = utils.get_file(conf, 'quality_checks', logger) if qcfile is None: sys.exit(-1) with open(qcfile) as qc_file: dict = json.loads(qc_file.read()) quality_checks = utils.get_quality_checks(dict) try: feedback = conf['feedback_type'] except KeyError: feedback = None try: report_type = conf['report_type'] except KeyError: report_type = const.REPORT_FULL consumersfile = utils.get_file(conf, 'consumers', logger, False) if consumersfile is None: consumers = None else: with open(consumersfile) as consumers_file: consumers = json.loads(consumers_file.read()) return logger, limits, quality_checks, feedback, report_type, consumers
def init(config): """ This function initializes variables according to configuration. It gets values from the configuration file, evaluates and processes the values. If mandatory file or directory is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance limits : dictionary a dictionary containing limit values read from the configured 'limit' file quality_checks : dict a dictionary containing quality check functions ids extensions : list a list containing extensions of files to be monitored read from the configuration file report_type : int report type; currently supporting 'none', 'error', and 'full' consumers : dict a dictionary containing consumer processes to run, and their parameters """ conf = utils.get_config(config) if conf is None: print('configuration file is missing') exit(-1) logger = utils.get_logger(__name__, conf) limitsfile = utils.get_file(conf, 'limits', logger) if limitsfile is None: sys.exit(-1) with open(limitsfile) as limits_file: limits = json.loads(limits_file.read()) try: extensions = conf['extensions'] except KeyError: logger.warning( 'no file extension specified. Monitoring for all files.') extensions = [''] qcfile = utils.get_file(conf, 'quality_checks', logger) if qcfile is None: sys.exit(-1) with open(qcfile) as qc_file: dict = json.loads(qc_file.read()) #quality_checks = utils.get_quality_checks(dict) quality_checks = dict try: report_type = conf['report_type'] except KeyError: report_type = const.REPORT_FULL consumersfile = utils.get_file(conf, 'consumers', logger, False) if consumersfile is None: consumers = None else: with open(consumersfile) as consumers_file: consumers = json.loads(consumers_file.read()) return logger, limits, quality_checks, extensions, report_type, consumers
def init(config): """ This function initializes variables according to configuration. It gets values from the configuration file, evaluates and processes the values. If mandatory parameter is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance limits : dictionary a dictionary containing limit values read from the configured 'limit' file quality_checks : dict a dictionary containing quality check functions ids feedback : list a list of strings defining real time feedback of quality checks errors. Currently supporting 'PV', 'log', and 'console' report_type : int report type; currently supporting 'none', 'error', and 'full' consumers : dict a dictionary parsed from json file representing consumers zmq_host : str ZeroMQ server host name zmq_rcv_port : str ZeroMQ port detector : str detector name, only needed if feedback contains pv """ conf = utils.get_config(config) if conf is None: print ('configuration file is missing') exit(-1) logger = utils.get_logger(__name__, conf) limitsfile = utils.get_file(conf, 'limits', logger) if limitsfile is None: sys.exit(-1) with open(limitsfile) as limits_file: limits = json.loads(limits_file.read()) qcfile = utils.get_file(conf, 'quality_checks', logger) if qcfile is None: sys.exit(-1) with open(qcfile) as qc_file: dict = json.loads(qc_file.read()) #quality_checks = utils.get_quality_checks(dict) quality_checks = dict try: feedback = conf['feedback_type'] except KeyError: feedback = None try: report_type = conf['report_type'] except KeyError: report_type = const.REPORT_FULL try: zmq_host = conf['zmq_host'] except: zmq_host = 'localhost' try: zmq_rcv_port = conf['zmq_rcv_port'] except: zmq_rcv_port = None print ('configuration error: zmq_rcv_port not configured') try: detector = conf['detector'] except KeyError: print ('configuration error: detector parameter not configured.') return None try: consumers = conf['zmq_snd_port'] except KeyError: consumers = None return logger, limits, quality_checks, feedback, report_type, consumers, zmq_host, zmq_rcv_port, detector
def init(config): """ This function initializes variables according to configuration. It gets values from the configuration file, evaluates and processes the values. If mandatory parameter is missing, the script logs an error and exits. Parameters ---------- config : str configuration file name, including path Returns ------- logger : Logger logger instance limits : dictionary a dictionary containing limit values read from the configured 'limit' file quality_checks : dict a dictionary containing quality check functions ids feedback : list a list of strings defining real time feedback of quality checks errors. Currently supporting 'PV', 'log', and 'console' zmq_snd_port : int a port used to send the verified data out pva_name : str pvaccess name detector : str detector name """ conf = utils.get_config(config) if conf is None: print ('configuration file is missing') exit(-1) try: pva_name = conf['pva_name'] except KeyError: print ('pva_name not configured') exit(-1) logger = utils.get_logger(__name__, conf) limitsfile = utils.get_file(conf, 'limits', logger) if limitsfile is None: sys.exit(-1) with open(limitsfile) as limits_file: limits = json.loads(limits_file.read()) qcfile = utils.get_file(conf, 'quality_checks', logger) if qcfile is None: sys.exit(-1) with open(qcfile) as qc_file: dict = json.loads(qc_file.read()) quality_checks = dict try: feedback = conf['feedback_type'] if len(feedback) == 0: feedback = None except KeyError: feedback = None try: zmq_snd_port = conf['zmq_snd_port'] except KeyError: zmq_snd_port = None try: detector = conf['detector'] except KeyError: print ('detector parameter not configured.') detector = None return logger, limits, quality_checks, feedback, zmq_snd_port, pva_name, detector
def parse_config(config): """ This function parses the configuration file. It must return the specified variables. Parameters ---------- config : str a configuration file Returns ------- no_frames : int number of frames that will be processed detector : str a string defining the first prefix in area detector, it has to match the area detector configuration detector_basic : str a string defining the second prefix in area detector, defining the basic parameters, it has to match the area detector configuration detector_image : str a string defining the second prefix in area detector, defining the image parameters, it has to match the area detector configuration """ conf = utils.get_config(config) if conf is None: print ('configuration file is missing') exit(-1) try: no_frames = conf['no_frames'] except KeyError: print ('no_frames parameter not configured.') return None try: aggregate_limit = conf['aggregate_limit'] except KeyError: #print ('aggregate_limit parameter not configured.') aggregate_limit = no_frames try: detector = conf['detector'] except KeyError: print ('configuration error: detector parameter not configured.') return None try: detector_basic = conf['detector_basic'] except KeyError: print ('configuration error: detector_basic parameter not configured.') return None try: detector_image = conf['detector_image'] except KeyError: print ('configuration error: detector_image parameter not configured.') return None return int(no_frames), aggregate_limit, detector, detector_basic, detector_image