class PimaticApp(object): def __init__(self, options): self._options = options self._pimatic = None self._args = None self._opts = None self._config = None self._logger = None self._node = {'url': None, 'username': None, 'password': None} self._parser = ArgumentParser(description='%s [PimaticAPI Rev. %s (c) %s]' % (self.app, __version__, __author__)) self._parser.add_argument('-c', '--config', type=str, help='use alternate configuration file(s)') self._parser.add_argument('-s', '--secrets', type=str, help='use alternate secrets file(s)') self._parser.add_argument('-l', '--loglevel', type=str, choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='debug log level') self._parser.add_argument('-n', '--node', type=str, help='pimatic node instance') self._parser.add_argument('-u', '--username', type=str, help='pimatic username') self._parser.add_argument('-p', '--password', type=str, help='pimatic password') def parse(self): self._args = self._parser.parse_args() self._opts = Options(self._options, self._args, '[config]', prefix='PIMATIC') self._config = self._opts.config_parser if self._config is None: LogAdapter(get_logger(), {'package': self.app}).critical("Missing configuration file!") exit(1) self._logger = LogAdapter(self._opts.logger, {'package': self.app}) # region Get node configuration and logger settings # set log level of requests module logging.getLogger('requests').setLevel(log_level(self._opts.loglevel_requests)) logging.getLogger('urllib3').setLevel(log_level(self._opts.loglevel_requests)) self._logger.debug(u'Parsing configuration file %s' % (self._opts.config_file)) if self._config.has_option('options', 'secrets'): secrets = self._config.get('options', 'secrets') path = os.path.expanduser(secrets) if os.path.isfile(path): secret = ConfigParser() secret.read(path) if self._opts.node: if self._opts.node.startswith('http'): self._node['url'] = self._opts.node else: node_section = 'node_' + self._opts.node if secret.has_section(node_section): protocol = secret.get(node_section, 'protocol') server = secret.get(node_section, 'server') port = secret.get(node_section, protocol + '_port') self._node['url']=protocol + '://' + server + ':' + port self._node['username'] = secret.get(node_section, 'username') self._node['password'] = secret.get(node_section, 'password') else: self._logger.critical(u'Invalid node tag %s' % (node_section)) exit(1) else: self._logger.critical(u'Missing pimatic node specification!') exit(1) if self._opts.username: self._node['username'] = self._opts.username if self._opts.password: self._node['password'] = self._opts.password # endregion self._logger.info(u'url: %s' % (self._node['url'])) def session(self): self._pimatic = PimaticAPI(server=self._node['url'], username=self._node['username'], password=self._node['password'], logger=self._logger) return self._pimatic @property def app(self): return self._options.get('app')
def main(): from argparse import ArgumentParser from pysync import __version__, __author__ from pysync import SyncError, SyncSessionError, SyncInitError options = { 'secrets': '~/.pysync.secrets', 'loglevel_requests': 'ERROR' # 'loglevel': 'INFO' } # region Command line arguments parser = ArgumentParser(description='PySnc Engine Rev. %s (c) %s' % (__version__, __author__)) parser.add_argument('-c', '--config', type=str, help='use alternate configuration file(s)') parser.add_argument('--relations', type=str, help='list of pysync relations to process') parser.add_argument('--rebuild', action='store_true', help='rebuild map file') parser.add_argument('--reset', type=str, help='delete entries and recreate from left/right') parser.add_argument('--update', type=str, help='force update on left/right side') parser.add_argument( '-l', '--loglevel', type=str, choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='debug log level') args = parser.parse_args() opts = Options(options, args, '[config]', prefix='PYSYNC') config = opts.config_parser if config is None: LogAdapter(get_logger(), { 'package': 'main' }).critical("Missing configuration file!") exit(1) logger = LogAdapter(opts.logger, {'package': 'main'}) # endregion # region Basic configuration and logger settings # set log level of requests module logging.getLogger('requests').setLevel(log_level(opts.loglevel_requests)) logging.getLogger('urllib3').setLevel(log_level(opts.loglevel_requests)) logger.debug(u'Parsing configuration file %s' % (opts.config_file)) if opts.relations: relations = list(opts.relations.split(',')) else: logger.critical('Missing relations!') exit(1) # endregion for relation in relations: try: left_opts, right_opts, relation_opts = parse_config( relation, config, opts.logger) except Exception as e: logger.exception( 'Error parsing configuration options. Skipping sync for [%s]' % (relation)) continue if lock(relation, relation_opts, opts.logger): # initialise web service sessions via @staticmethod session() # and initialize sync engine classes try: label = left_opts.get('label') session_lockfile = left_opts.get('session_lockfile', True) left_session = left_opts['class'].session( left_opts, opts.logger) label = right_opts.get('label') session_lockfile = right_opts.get('session_lockfile', True) right_session = right_opts['class'].session( right_opts, opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception( 'Session initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) if not session_lockfile: unlock(relation, relation_opts, opts.logger) continue # initialize sync map relation_opts['sync'] = {'map': None} if os.path.isfile(relation_opts['map']): #################### # incremental sync # #################### with codecs.open(relation_opts['map'], 'r', encoding='utf-8') as fp: relation_opts['sync'] = json.load(fp) logger.info(u'%s: starting incremental sync for %d items' % (relation, len(relation_opts['sync'].get('map')))) # merge signature from map file left_opts.update({'signature': relation_opts['sync']['left']}) right_opts.update( {'signature': relation_opts['sync']['right']}) try: engine_lockfile = left_opts.get('engine_lockfile', True) left = left_opts['class'](left_session, left_opts, logger=opts.logger) engine_lockfile = right_opts.get('engine_lockfile', True) right = right_opts['class'](right_session, right_opts, logger=opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception( 'Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) if not engine_lockfile: unlock(relation, relation_opts, opts.logger) continue if opts['update']: try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.update(opts['update']) except Exception as e: logger.exception( 'Unexpected error when processing update option! Skipping sync for [%s]' % relation) unlock(relation, relation_opts, opts.logger) continue if opts.reset: try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.reset(opts.reset) except Exception as e: logger.exception( 'Unexpected error when processing reset option!') check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) continue if opts.rebuild: relation_opts['sync'] = {'map': None} else: ################ # initial sync # ################ logger.info(u'%s: Starting initial sync' % (relation)) for opt in ['update', 'reset', 'rebuild']: if opts.get(opt): logger.warning( 'Ignoring option [%s] for initial sync' % (opt)) try: left = left_opts['class'](left_session, left_opts, logger=opts.logger) right = right_opts['class'](right_session, right_opts, logger=opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception( 'Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) # unlock(relation, relation_opts, opts.logger) continue try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.process() except Exception as e: logger.exception('Unexpected error when processing sync map!') check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) continue # check/modify sync map by backend engine relation_opts = left.commit_sync('left', relation_opts, logger) relation_opts = right.commit_sync('right', relation_opts, logger) count, errors = check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) unlock(relation, relation_opts, logger) logger.info(u'%s: %s %s' % (relation, left.label, left._changes)) logger.info(u'%s: %s %s' % (relation, right.label, right._changes)) logger.info(u'%s: finished %s sync for %d items with %d errors' % (relation, pysync.direction, count, errors)) left_opts['class'].end_session(logger) right_opts['class'].end_session(logger)
def main(): from argparse import ArgumentParser from pysync import __version__, __author__ from pysync import SyncError, SyncSessionError, SyncInitError options = { 'secrets': '~/.pysync.secrets', 'loglevel_requests': 'ERROR' # 'loglevel': 'INFO' } # region Command line arguments parser = ArgumentParser(description='PySnc Engine Rev. %s (c) %s' % (__version__, __author__)) parser.add_argument('-c', '--config', type=str, help='use alternate configuration file(s)') parser.add_argument('--relations', type=str, help='list of pysync relations to process') parser.add_argument('--rebuild', action='store_true', help='rebuild map file') parser.add_argument('--reset', type=str, help='delete entries and recreate from left/right') parser.add_argument('--update', type=str, help='force update on left/right side') parser.add_argument('-l', '--loglevel', type=str, choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='debug log level') args = parser.parse_args() opts = Options(options, args, '[config]', prefix='PYSYNC') config = opts.config_parser if config is None: LogAdapter(get_logger(), {'package': 'main'}).critical("Missing configuration file!") exit(1) logger = LogAdapter(opts.logger, {'package': 'main'}) # endregion # region Basic configuration and logger settings # set log level of requests module logging.getLogger('requests').setLevel(log_level(opts.loglevel_requests)) logging.getLogger('urllib3').setLevel(log_level(opts.loglevel_requests)) logger.debug(u'Parsing configuration file %s' % (opts.config_file)) if opts.relations: relations = list(opts.relations.split(',')) else: logger.critical('Missing relations!') exit(1) # endregion for relation in relations: try: left_opts, right_opts, relation_opts = parse_config(relation, config, opts.logger) except Exception as e: logger.exception('Error parsing configuration options. Skipping sync for [%s]' % (relation)) continue if lock(relation, relation_opts, opts.logger): # initialise web service sessions via @staticmethod session() # and initialize sync engine classes try: label = left_opts.get('label') session_lockfile = left_opts.get('session_lockfile', True) left_session = left_opts['class'].session(left_opts, opts.logger) label = right_opts.get('label') session_lockfile = right_opts.get('session_lockfile', True) right_session = right_opts['class'].session(right_opts, opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception('Session initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) if not session_lockfile: unlock(relation, relation_opts, opts.logger) continue # initialize sync map relation_opts['sync'] = {'map': None} if os.path.isfile(relation_opts['map']): #################### # incremental sync # #################### with codecs.open(relation_opts['map'], 'r', encoding='utf-8') as fp: relation_opts['sync'] = json.load(fp) logger.info(u'%s: starting incremental sync for %d items' % (relation, len(relation_opts['sync'].get('map')))) # merge signature from map file left_opts.update({'signature': relation_opts['sync']['left']}) right_opts.update({'signature': relation_opts['sync']['right']}) try: engine_lockfile = left_opts.get('engine_lockfile', True) left = left_opts['class'](left_session, left_opts, logger=opts.logger) engine_lockfile = right_opts.get('engine_lockfile', True) right = right_opts['class'](right_session, right_opts, logger=opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception('Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) if not engine_lockfile: unlock(relation, relation_opts, opts.logger) continue if opts['update']: try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.update(opts['update']) except Exception as e: logger.exception('Unexpected error when processing update option! Skipping sync for [%s]' % relation) unlock(relation, relation_opts, opts.logger) continue if opts.reset: try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.reset(opts.reset) except Exception as e: logger.exception('Unexpected error when processing reset option!') check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) continue if opts.rebuild: relation_opts['sync'] = {'map': None} else: ################ # initial sync # ################ logger.info(u'%s: Starting initial sync' % (relation)) for opt in ['update', 'reset', 'rebuild']: if opts.get(opt): logger.warning('Ignoring option [%s] for initial sync' % (opt)) try: left = left_opts['class'](left_session, left_opts, logger=opts.logger) right = right_opts['class'](right_session, right_opts, logger=opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception('Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) # unlock(relation, relation_opts, opts.logger) continue try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.process() except Exception as e: logger.exception('Unexpected error when processing sync map!') check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) continue # check/modify sync map by backend engine relation_opts = left.commit_sync('left', relation_opts, logger) relation_opts = right.commit_sync('right', relation_opts, logger) count, errors = check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) unlock(relation, relation_opts, logger) logger.info(u'%s: %s %s' % (relation, left.label, left._changes)) logger.info(u'%s: %s %s' % (relation, right.label, right._changes)) logger.info(u'%s: finished %s sync for %d items with %d errors' % (relation, pysync.direction, count, errors)) left_opts['class'].end_session(logger) right_opts['class'].end_session(logger)
def parse_config(relation, config, _logger): from ConfigParser import ConfigParser from oxsync import OxTaskSync from ensync import EnClientSync from tdsync import ToodledoSync logger = LogAdapter(_logger, {'package': 'config'}) class_map = { 'OxTaskSync': OxTaskSync, 'EnClientSync': EnClientSync, 'ToodledoSync': ToodledoSync } relation_section = 'relation' + '_' + relation if config.has_section(relation_section): rel = {} for key in config.options(relation_section): rel[key] = config.get(relation_section, key) if rel.get('map'): rel['map'] = os.path.expanduser(rel.get('map')) else: logging.critical( 'Configuration error: missing map file path for %s' % (relation)) exit(1) left = config.get(relation_section, 'left') if left: engine_section_left = 'engine' + '_' + left else: logging.critical( 'Configuraion error: missing left engine refernce') exit(1) left = None if config.has_section(engine_section_left): left = {} for key in config.options(engine_section_left): left[key] = config.get(engine_section_left, key) else: logging.critical('Configuration error: missing section [%s]' % (engine_section_left)) exit(1) right = config.get(relation_section, 'right') if right: engine_section_right = 'engine' + '_' + right else: logging.critical( 'Configuraion error: missing right engine refernce') exit(1) right = None if config.has_section(engine_section_right): right = {} for key in config.options(engine_section_right): right[key] = config.get(engine_section_right, key) else: logging.critical('Configuration error: missing section [%s]' % (engine_section_right)) exit(1) for engine in [left, right]: secrets = engine['secrets'] path = os.path.expanduser(secrets) if os.path.isfile(path): secret = ConfigParser() secret.read(path) if secret.has_section('secrets'): engine['secrets'] = {'key': path} for key in secret.options('secrets'): engine['secrets'][key] = secret.get('secrets', key) else: logger.critical( 'Configuration error: missing [secrets] in %s' % (path)) exit(1) else: if config.has_option('options', 'secrets'): secrets = config.get('options', 'secrets') path = os.path.expanduser(secrets) if os.path.isfile(path): secret = ConfigParser() secret.read(path) section = engine['secrets'][1:-1] if secret.has_section(section): engine['secrets'] = {'key': section} for key in secret.options(section): engine['secrets'][key] = secret.get( section, key) else: logger.critical( 'Configuration error: missing [%s] in %s' % (section, path)) exit(1) else: logger.critical( 'Configuration error: missing secret file %s' % (path)) exit(1) else: logger.critical( 'Configuration error: missing secrets in [options]') exit(1) else: logging.critical('Configuration error: missing section [%s]' % (relation_section)) exit(1) for options in [left, right]: if options.get('class'): cn = options['class'] if class_map.get(cn): options['class'] = class_map[cn] else: logger.critical( 'Configuration error: unknown sync engine [%s]' % (cn)) exit(1) else: logger.critical('configuration error: missing class tag') exit(1) return Options(left), Options(right), Options(rel)
def parse_config(relation, config, _logger): from ConfigParser import ConfigParser from oxsync import OxTaskSync from ensync import EnClientSync from tdsync import ToodledoSync logger = LogAdapter(_logger, {'package': 'config'}) class_map = { 'OxTaskSync': OxTaskSync, 'EnClientSync': EnClientSync, 'ToodledoSync': ToodledoSync } relation_section = 'relation' + '_' + relation if config.has_section(relation_section): rel = {} for key in config.options(relation_section): rel[key] = config.get(relation_section, key) if rel.get('map'): rel['map'] = os.path.expanduser(rel.get('map')) else: logging.critical('Configuration error: missing map file path for %s' % (relation)) exit(1) left = config.get(relation_section, 'left') if left: engine_section_left = 'engine' + '_' + left else: logging.critical('Configuraion error: missing left engine refernce') exit(1) left = None if config.has_section(engine_section_left): left = {} for key in config.options(engine_section_left): left[key] = config.get(engine_section_left, key) else: logging.critical('Configuration error: missing section [%s]' % (engine_section_left)) exit(1) right = config.get(relation_section, 'right') if right: engine_section_right = 'engine' + '_' + right else: logging.critical('Configuraion error: missing right engine refernce') exit(1) right = None if config.has_section(engine_section_right): right = {} for key in config.options(engine_section_right): right[key] = config.get(engine_section_right, key) else: logging.critical('Configuration error: missing section [%s]' % (engine_section_right)) exit(1) for engine in [left, right]: secrets = engine['secrets'] path = os.path.expanduser(secrets) if os.path.isfile(path): secret = ConfigParser() secret.read(path) if secret.has_section('secrets'): engine['secrets'] = {'key': path} for key in secret.options('secrets'): engine['secrets'][key] = secret.get('secrets', key) else: logger.critical('Configuration error: missing [secrets] in %s' % (path)) exit(1) else: if config.has_option('options', 'secrets'): secrets = config.get('options', 'secrets') path = os.path.expanduser(secrets) if os.path.isfile(path): secret = ConfigParser() secret.read(path) section = engine['secrets'][1:-1] if secret.has_section(section): engine['secrets'] = {'key': section} for key in secret.options(section): engine['secrets'][key] = secret.get(section, key) else: logger.critical('Configuration error: missing [%s] in %s' % (section, path)) exit(1) else: logger.critical('Configuration error: missing secret file %s' % (path)) exit(1) else: logger.critical('Configuration error: missing secrets in [options]') exit(1) else: logging.critical('Configuration error: missing section [%s]' % (relation_section)) exit(1) for options in [left, right]: if options.get('class'): cn = options['class'] if class_map.get(cn): options['class'] = class_map[cn] else: logger.critical('Configuration error: unknown sync engine [%s]' % (cn)) exit(1) else: logger.critical('configuration error: missing class tag') exit(1) return Options(left), Options(right), Options(rel)