def unlock(relation, opts, _logger):
logger = LogAdapter(_logger, {'package': 'unlock'})
map_file = os.path.expanduser(opts['map'])
lck_file = os.path.splitext(map_file)[0] + '.lck'
if os.path.isfile(lck_file):
logger.debug(u'Unlocking relation [%s]' % (relation))
os.remove(lck_file)
return True
else:
logger.warning('Lockfile %s for relation [%s] not found!' % (lck_file, relation))
return False
def unlock(relation, opts, _logger):
logger = LogAdapter(_logger, {'package': 'unlock'})
map_file = os.path.expanduser(opts['map'])
lck_file = os.path.splitext(map_file)[0] + '.lck'
if os.path.isfile(lck_file):
logger.debug(u'Unlocking relation [%s]' % (relation))
os.remove(lck_file)
return True
else:
logger.warning('Lockfile %s for relation [%s] not found!' %
(lck_file, relation))
return False
def main():
from argparse import ArgumentParser
from pysync import __version__, __author__
from pysync import SyncError, SyncSessionError, SyncInitError
options = {
'secrets': '~/.pysync.secrets',
'loglevel_requests': 'ERROR'
# 'loglevel': 'INFO'
}
# region Command line arguments
parser = ArgumentParser(description='PySnc Engine Rev. %s (c) %s' %
(__version__, __author__))
parser.add_argument('-c',
'--config',
type=str,
help='use alternate configuration file(s)')
parser.add_argument('--relations',
type=str,
help='list of pysync relations to process')
parser.add_argument('--rebuild',
action='store_true',
help='rebuild map file')
parser.add_argument('--reset',
type=str,
help='delete entries and recreate from left/right')
parser.add_argument('--update',
type=str,
help='force update on left/right side')
parser.add_argument(
'-l',
'--loglevel',
type=str,
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help='debug log level')
args = parser.parse_args()
opts = Options(options, args, '[config]', prefix='PYSYNC')
config = opts.config_parser
if config is None:
LogAdapter(get_logger(), {
'package': 'main'
}).critical("Missing configuration file!")
exit(1)
logger = LogAdapter(opts.logger, {'package': 'main'})
# endregion
# region Basic configuration and logger settings
# set log level of requests module
logging.getLogger('requests').setLevel(log_level(opts.loglevel_requests))
logging.getLogger('urllib3').setLevel(log_level(opts.loglevel_requests))
logger.debug(u'Parsing configuration file %s' % (opts.config_file))
if opts.relations:
relations = list(opts.relations.split(','))
else:
logger.critical('Missing relations!')
exit(1)
# endregion
for relation in relations:
try:
left_opts, right_opts, relation_opts = parse_config(
relation, config, opts.logger)
except Exception as e:
logger.exception(
'Error parsing configuration options. Skipping sync for [%s]' %
(relation))
continue
if lock(relation, relation_opts, opts.logger):
# initialise web service sessions via @staticmethod session()
# and initialize sync engine classes
try:
label = left_opts.get('label')
session_lockfile = left_opts.get('session_lockfile', True)
left_session = left_opts['class'].session(
left_opts, opts.logger)
label = right_opts.get('label')
session_lockfile = right_opts.get('session_lockfile', True)
right_session = right_opts['class'].session(
right_opts, opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception(
'Session initialization for [%s] failed! Skipping sync for [%s]'
% (label, relation))
if not session_lockfile:
unlock(relation, relation_opts, opts.logger)
continue
# initialize sync map
relation_opts['sync'] = {'map': None}
if os.path.isfile(relation_opts['map']):
####################
# incremental sync #
####################
with codecs.open(relation_opts['map'], 'r',
encoding='utf-8') as fp:
relation_opts['sync'] = json.load(fp)
logger.info(u'%s: starting incremental sync for %d items' %
(relation, len(relation_opts['sync'].get('map'))))
# merge signature from map file
left_opts.update({'signature': relation_opts['sync']['left']})
right_opts.update(
{'signature': relation_opts['sync']['right']})
try:
engine_lockfile = left_opts.get('engine_lockfile', True)
left = left_opts['class'](left_session,
left_opts,
logger=opts.logger)
engine_lockfile = right_opts.get('engine_lockfile', True)
right = right_opts['class'](right_session,
right_opts,
logger=opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception(
'Engine initialization for [%s] failed! Skipping sync for [%s]'
% (label, relation))
if not engine_lockfile:
unlock(relation, relation_opts, opts.logger)
continue
if opts['update']:
try:
pysync = PySync(left, right, relation_opts,
opts.logger)
relation_opts['sync'] = pysync.update(opts['update'])
except Exception as e:
logger.exception(
'Unexpected error when processing update option! Skipping sync for [%s]'
% relation)
unlock(relation, relation_opts, opts.logger)
continue
if opts.reset:
try:
pysync = PySync(left, right, relation_opts,
opts.logger)
relation_opts['sync'] = pysync.reset(opts.reset)
except Exception as e:
logger.exception(
'Unexpected error when processing reset option!')
check_sync_map(relation, pysync.direction, left, right,
relation_opts, logger)
continue
if opts.rebuild:
relation_opts['sync'] = {'map': None}
else:
################
# initial sync #
################
logger.info(u'%s: Starting initial sync' % (relation))
for opt in ['update', 'reset', 'rebuild']:
if opts.get(opt):
logger.warning(
'Ignoring option [%s] for initial sync' % (opt))
try:
left = left_opts['class'](left_session,
left_opts,
logger=opts.logger)
right = right_opts['class'](right_session,
right_opts,
logger=opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception(
'Engine initialization for [%s] failed! Skipping sync for [%s]'
% (label, relation))
# unlock(relation, relation_opts, opts.logger)
continue
try:
pysync = PySync(left, right, relation_opts, opts.logger)
relation_opts['sync'] = pysync.process()
except Exception as e:
logger.exception('Unexpected error when processing sync map!')
check_sync_map(relation, pysync.direction, left, right,
relation_opts, logger)
continue
# check/modify sync map by backend engine
relation_opts = left.commit_sync('left', relation_opts, logger)
relation_opts = right.commit_sync('right', relation_opts, logger)
count, errors = check_sync_map(relation, pysync.direction, left,
right, relation_opts, logger)
unlock(relation, relation_opts, logger)
logger.info(u'%s: %s %s' % (relation, left.label, left._changes))
logger.info(u'%s: %s %s' % (relation, right.label, right._changes))
logger.info(u'%s: finished %s sync for %d items with %d errors' %
(relation, pysync.direction, count, errors))
left_opts['class'].end_session(logger)
right_opts['class'].end_session(logger)
def main():
from argparse import ArgumentParser
from pysync import __version__, __author__
from pysync import SyncError, SyncSessionError, SyncInitError
options = {
'secrets': '~/.pysync.secrets',
'loglevel_requests': 'ERROR'
# 'loglevel': 'INFO'
}
# region Command line arguments
parser = ArgumentParser(description='PySnc Engine Rev. %s (c) %s' % (__version__, __author__))
parser.add_argument('-c', '--config', type=str, help='use alternate configuration file(s)')
parser.add_argument('--relations', type=str, help='list of pysync relations to process')
parser.add_argument('--rebuild', action='store_true', help='rebuild map file')
parser.add_argument('--reset', type=str, help='delete entries and recreate from left/right')
parser.add_argument('--update', type=str, help='force update on left/right side')
parser.add_argument('-l', '--loglevel', type=str,
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help='debug log level')
args = parser.parse_args()
opts = Options(options, args, '[config]', prefix='PYSYNC')
config = opts.config_parser
if config is None:
LogAdapter(get_logger(), {'package': 'main'}).critical("Missing configuration file!")
exit(1)
logger = LogAdapter(opts.logger, {'package': 'main'})
# endregion
# region Basic configuration and logger settings
# set log level of requests module
logging.getLogger('requests').setLevel(log_level(opts.loglevel_requests))
logging.getLogger('urllib3').setLevel(log_level(opts.loglevel_requests))
logger.debug(u'Parsing configuration file %s' % (opts.config_file))
if opts.relations:
relations = list(opts.relations.split(','))
else:
logger.critical('Missing relations!')
exit(1)
# endregion
for relation in relations:
try:
left_opts, right_opts, relation_opts = parse_config(relation, config, opts.logger)
except Exception as e:
logger.exception('Error parsing configuration options. Skipping sync for [%s]' % (relation))
continue
if lock(relation, relation_opts, opts.logger):
# initialise web service sessions via @staticmethod session()
# and initialize sync engine classes
try:
label = left_opts.get('label')
session_lockfile = left_opts.get('session_lockfile', True)
left_session = left_opts['class'].session(left_opts, opts.logger)
label = right_opts.get('label')
session_lockfile = right_opts.get('session_lockfile', True)
right_session = right_opts['class'].session(right_opts, opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception('Session initialization for [%s] failed! Skipping sync for [%s]' % (label, relation))
if not session_lockfile:
unlock(relation, relation_opts, opts.logger)
continue
# initialize sync map
relation_opts['sync'] = {'map': None}
if os.path.isfile(relation_opts['map']):
####################
# incremental sync #
####################
with codecs.open(relation_opts['map'], 'r', encoding='utf-8') as fp:
relation_opts['sync'] = json.load(fp)
logger.info(u'%s: starting incremental sync for %d items' % (relation, len(relation_opts['sync'].get('map'))))
# merge signature from map file
left_opts.update({'signature': relation_opts['sync']['left']})
right_opts.update({'signature': relation_opts['sync']['right']})
try:
engine_lockfile = left_opts.get('engine_lockfile', True)
left = left_opts['class'](left_session, left_opts, logger=opts.logger)
engine_lockfile = right_opts.get('engine_lockfile', True)
right = right_opts['class'](right_session, right_opts, logger=opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception('Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation))
if not engine_lockfile:
unlock(relation, relation_opts, opts.logger)
continue
if opts['update']:
try:
pysync = PySync(left, right, relation_opts, opts.logger)
relation_opts['sync'] = pysync.update(opts['update'])
except Exception as e:
logger.exception('Unexpected error when processing update option! Skipping sync for [%s]' % relation)
unlock(relation, relation_opts, opts.logger)
continue
if opts.reset:
try:
pysync = PySync(left, right, relation_opts, opts.logger)
relation_opts['sync'] = pysync.reset(opts.reset)
except Exception as e:
logger.exception('Unexpected error when processing reset option!')
check_sync_map(relation, pysync.direction, left, right, relation_opts, logger)
continue
if opts.rebuild:
relation_opts['sync'] = {'map': None}
else:
################
# initial sync #
################
logger.info(u'%s: Starting initial sync' % (relation))
for opt in ['update', 'reset', 'rebuild']:
if opts.get(opt):
logger.warning('Ignoring option [%s] for initial sync' % (opt))
try:
left = left_opts['class'](left_session, left_opts, logger=opts.logger)
right = right_opts['class'](right_session, right_opts, logger=opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception('Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation))
# unlock(relation, relation_opts, opts.logger)
continue
try:
pysync = PySync(left, right, relation_opts, opts.logger)
relation_opts['sync'] = pysync.process()
except Exception as e:
logger.exception('Unexpected error when processing sync map!')
check_sync_map(relation, pysync.direction, left, right, relation_opts, logger)
continue
# check/modify sync map by backend engine
relation_opts = left.commit_sync('left', relation_opts, logger)
relation_opts = right.commit_sync('right', relation_opts, logger)
count, errors = check_sync_map(relation, pysync.direction, left, right, relation_opts, logger)
unlock(relation, relation_opts, logger)
logger.info(u'%s: %s %s' % (relation, left.label, left._changes))
logger.info(u'%s: %s %s' % (relation, right.label, right._changes))
logger.info(u'%s: finished %s sync for %d items with %d errors' % (relation, pysync.direction, count, errors))
left_opts['class'].end_session(logger)
right_opts['class'].end_session(logger)
