def unlock(relation, opts, _logger): logger = LogAdapter(_logger, {'package': 'unlock'}) map_file = os.path.expanduser(opts['map']) lck_file = os.path.splitext(map_file)[0] + '.lck' if os.path.isfile(lck_file): logger.debug(u'Unlocking relation [%s]' % (relation)) os.remove(lck_file) return True else: logger.warning('Lockfile %s for relation [%s] not found!' % (lck_file, relation)) return False
def unlock(relation, opts, _logger): logger = LogAdapter(_logger, {'package': 'unlock'}) map_file = os.path.expanduser(opts['map']) lck_file = os.path.splitext(map_file)[0] + '.lck' if os.path.isfile(lck_file): logger.debug(u'Unlocking relation [%s]' % (relation)) os.remove(lck_file) return True else: logger.warning('Lockfile %s for relation [%s] not found!' % (lck_file, relation)) return False
def main(): from argparse import ArgumentParser from pysync import __version__, __author__ from pysync import SyncError, SyncSessionError, SyncInitError options = { 'secrets': '~/.pysync.secrets', 'loglevel_requests': 'ERROR' # 'loglevel': 'INFO' } # region Command line arguments parser = ArgumentParser(description='PySnc Engine Rev. %s (c) %s' % (__version__, __author__)) parser.add_argument('-c', '--config', type=str, help='use alternate configuration file(s)') parser.add_argument('--relations', type=str, help='list of pysync relations to process') parser.add_argument('--rebuild', action='store_true', help='rebuild map file') parser.add_argument('--reset', type=str, help='delete entries and recreate from left/right') parser.add_argument('--update', type=str, help='force update on left/right side') parser.add_argument( '-l', '--loglevel', type=str, choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='debug log level') args = parser.parse_args() opts = Options(options, args, '[config]', prefix='PYSYNC') config = opts.config_parser if config is None: LogAdapter(get_logger(), { 'package': 'main' }).critical("Missing configuration file!") exit(1) logger = LogAdapter(opts.logger, {'package': 'main'}) # endregion # region Basic configuration and logger settings # set log level of requests module logging.getLogger('requests').setLevel(log_level(opts.loglevel_requests)) logging.getLogger('urllib3').setLevel(log_level(opts.loglevel_requests)) logger.debug(u'Parsing configuration file %s' % (opts.config_file)) if opts.relations: relations = list(opts.relations.split(',')) else: logger.critical('Missing relations!') exit(1) # endregion for relation in relations: try: left_opts, right_opts, relation_opts = parse_config( relation, config, opts.logger) except Exception as e: logger.exception( 'Error parsing configuration options. Skipping sync for [%s]' % (relation)) continue if lock(relation, relation_opts, opts.logger): # initialise web service sessions via @staticmethod session() # and initialize sync engine classes try: label = left_opts.get('label') session_lockfile = left_opts.get('session_lockfile', True) left_session = left_opts['class'].session( left_opts, opts.logger) label = right_opts.get('label') session_lockfile = right_opts.get('session_lockfile', True) right_session = right_opts['class'].session( right_opts, opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception( 'Session initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) if not session_lockfile: unlock(relation, relation_opts, opts.logger) continue # initialize sync map relation_opts['sync'] = {'map': None} if os.path.isfile(relation_opts['map']): #################### # incremental sync # #################### with codecs.open(relation_opts['map'], 'r', encoding='utf-8') as fp: relation_opts['sync'] = json.load(fp) logger.info(u'%s: starting incremental sync for %d items' % (relation, len(relation_opts['sync'].get('map')))) # merge signature from map file left_opts.update({'signature': relation_opts['sync']['left']}) right_opts.update( {'signature': relation_opts['sync']['right']}) try: engine_lockfile = left_opts.get('engine_lockfile', True) left = left_opts['class'](left_session, left_opts, logger=opts.logger) engine_lockfile = right_opts.get('engine_lockfile', True) right = right_opts['class'](right_session, right_opts, logger=opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception( 'Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) if not engine_lockfile: unlock(relation, relation_opts, opts.logger) continue if opts['update']: try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.update(opts['update']) except Exception as e: logger.exception( 'Unexpected error when processing update option! Skipping sync for [%s]' % relation) unlock(relation, relation_opts, opts.logger) continue if opts.reset: try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.reset(opts.reset) except Exception as e: logger.exception( 'Unexpected error when processing reset option!') check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) continue if opts.rebuild: relation_opts['sync'] = {'map': None} else: ################ # initial sync # ################ logger.info(u'%s: Starting initial sync' % (relation)) for opt in ['update', 'reset', 'rebuild']: if opts.get(opt): logger.warning( 'Ignoring option [%s] for initial sync' % (opt)) try: left = left_opts['class'](left_session, left_opts, logger=opts.logger) right = right_opts['class'](right_session, right_opts, logger=opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception( 'Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) # unlock(relation, relation_opts, opts.logger) continue try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.process() except Exception as e: logger.exception('Unexpected error when processing sync map!') check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) continue # check/modify sync map by backend engine relation_opts = left.commit_sync('left', relation_opts, logger) relation_opts = right.commit_sync('right', relation_opts, logger) count, errors = check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) unlock(relation, relation_opts, logger) logger.info(u'%s: %s %s' % (relation, left.label, left._changes)) logger.info(u'%s: %s %s' % (relation, right.label, right._changes)) logger.info(u'%s: finished %s sync for %d items with %d errors' % (relation, pysync.direction, count, errors)) left_opts['class'].end_session(logger) right_opts['class'].end_session(logger)
def main(): from argparse import ArgumentParser from pysync import __version__, __author__ from pysync import SyncError, SyncSessionError, SyncInitError options = { 'secrets': '~/.pysync.secrets', 'loglevel_requests': 'ERROR' # 'loglevel': 'INFO' } # region Command line arguments parser = ArgumentParser(description='PySnc Engine Rev. %s (c) %s' % (__version__, __author__)) parser.add_argument('-c', '--config', type=str, help='use alternate configuration file(s)') parser.add_argument('--relations', type=str, help='list of pysync relations to process') parser.add_argument('--rebuild', action='store_true', help='rebuild map file') parser.add_argument('--reset', type=str, help='delete entries and recreate from left/right') parser.add_argument('--update', type=str, help='force update on left/right side') parser.add_argument('-l', '--loglevel', type=str, choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='debug log level') args = parser.parse_args() opts = Options(options, args, '[config]', prefix='PYSYNC') config = opts.config_parser if config is None: LogAdapter(get_logger(), {'package': 'main'}).critical("Missing configuration file!") exit(1) logger = LogAdapter(opts.logger, {'package': 'main'}) # endregion # region Basic configuration and logger settings # set log level of requests module logging.getLogger('requests').setLevel(log_level(opts.loglevel_requests)) logging.getLogger('urllib3').setLevel(log_level(opts.loglevel_requests)) logger.debug(u'Parsing configuration file %s' % (opts.config_file)) if opts.relations: relations = list(opts.relations.split(',')) else: logger.critical('Missing relations!') exit(1) # endregion for relation in relations: try: left_opts, right_opts, relation_opts = parse_config(relation, config, opts.logger) except Exception as e: logger.exception('Error parsing configuration options. Skipping sync for [%s]' % (relation)) continue if lock(relation, relation_opts, opts.logger): # initialise web service sessions via @staticmethod session() # and initialize sync engine classes try: label = left_opts.get('label') session_lockfile = left_opts.get('session_lockfile', True) left_session = left_opts['class'].session(left_opts, opts.logger) label = right_opts.get('label') session_lockfile = right_opts.get('session_lockfile', True) right_session = right_opts['class'].session(right_opts, opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception('Session initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) if not session_lockfile: unlock(relation, relation_opts, opts.logger) continue # initialize sync map relation_opts['sync'] = {'map': None} if os.path.isfile(relation_opts['map']): #################### # incremental sync # #################### with codecs.open(relation_opts['map'], 'r', encoding='utf-8') as fp: relation_opts['sync'] = json.load(fp) logger.info(u'%s: starting incremental sync for %d items' % (relation, len(relation_opts['sync'].get('map')))) # merge signature from map file left_opts.update({'signature': relation_opts['sync']['left']}) right_opts.update({'signature': relation_opts['sync']['right']}) try: engine_lockfile = left_opts.get('engine_lockfile', True) left = left_opts['class'](left_session, left_opts, logger=opts.logger) engine_lockfile = right_opts.get('engine_lockfile', True) right = right_opts['class'](right_session, right_opts, logger=opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception('Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) if not engine_lockfile: unlock(relation, relation_opts, opts.logger) continue if opts['update']: try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.update(opts['update']) except Exception as e: logger.exception('Unexpected error when processing update option! Skipping sync for [%s]' % relation) unlock(relation, relation_opts, opts.logger) continue if opts.reset: try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.reset(opts.reset) except Exception as e: logger.exception('Unexpected error when processing reset option!') check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) continue if opts.rebuild: relation_opts['sync'] = {'map': None} else: ################ # initial sync # ################ logger.info(u'%s: Starting initial sync' % (relation)) for opt in ['update', 'reset', 'rebuild']: if opts.get(opt): logger.warning('Ignoring option [%s] for initial sync' % (opt)) try: left = left_opts['class'](left_session, left_opts, logger=opts.logger) right = right_opts['class'](right_session, right_opts, logger=opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception('Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) # unlock(relation, relation_opts, opts.logger) continue try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.process() except Exception as e: logger.exception('Unexpected error when processing sync map!') check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) continue # check/modify sync map by backend engine relation_opts = left.commit_sync('left', relation_opts, logger) relation_opts = right.commit_sync('right', relation_opts, logger) count, errors = check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) unlock(relation, relation_opts, logger) logger.info(u'%s: %s %s' % (relation, left.label, left._changes)) logger.info(u'%s: %s %s' % (relation, right.label, right._changes)) logger.info(u'%s: finished %s sync for %d items with %d errors' % (relation, pysync.direction, count, errors)) left_opts['class'].end_session(logger) right_opts['class'].end_session(logger)