def main():
from ConfigParser import ConfigParser
from argparse import ArgumentParser
options = {
'option': 'OPTION',
'secrets': {'token': 'secret'},
'loglevel': 'INFO'
}
# region Command line arguments
parser = ArgumentParser(description='PySnc Engine Rev. 0.1 (c) Bernd Strebel')
parser.add_argument('-c', '--config', type=str, help='use alternate configuration file')
parser.add_argument('-l', '--loglevel', type=str,
choices=['DEBUG', 'INFO', 'WARN', 'WARNING', 'ERROR', 'CRITICAL',
'debug', 'info', 'warn', 'warning', 'error', 'critical'],
help='debug log level')
args = parser.parse_args()
opts = Options(options, args, config=True)
# _logger = get_logger('root',log_level(opts.loglevel))
# logger = LogAdapter(_logger, {'package': 'init'})
# logger.info("Console logger initilized")
# endregion
# region (I) Basic configuration and logger settings from config file
# if opts.config_file:
# if os.path.isfile(opts.config_file):
# logging.config.fileConfig(opts.config_file)
# config = ConfigParser(options)
# config.read(opts.config_file)
# else:
# logger.critical('Configuration file %s not found!' % (opts.config_file))
# exit(1)
# else:
# logger.critical("Missing configuration file!")
# exit(1)
#
# _logger = logging.getLogger()
# _logger.setLevel(log_level(opts.loglevel))
#
# logger = LogAdapter(_logger, {'package': 'main'})
# endregion
# region (II) Basic configuration and logger settings from config parser
config = opts.config_parser
logger = LogAdapter(opts.logger, {'package': 'main'})
# endregion
logger.info('Default logger configured from %s' % (opts.config_file))
print opts.option
s = opts.get('string_option', False)
t = opts['secrets']['token']
e = opts['empty']
b = opts.get('uni')
u = opts['uni']
pass
def parse_config(relation, config, _logger):
from ConfigParser import ConfigParser
from oxsync import OxTaskSync
from ensync import EnClientSync
from tdsync import ToodledoSync
logger = LogAdapter(_logger, {'package': 'config'})
class_map = {
'OxTaskSync': OxTaskSync,
'EnClientSync': EnClientSync,
'ToodledoSync': ToodledoSync
}
relation_section = 'relation' + '_' + relation
if config.has_section(relation_section):
rel = {}
for key in config.options(relation_section):
rel[key] = config.get(relation_section, key)
if rel.get('map'):
rel['map'] = os.path.expanduser(rel.get('map'))
else:
logging.critical(
'Configuration error: missing map file path for %s' %
(relation))
exit(1)
left = config.get(relation_section, 'left')
if left:
engine_section_left = 'engine' + '_' + left
else:
logging.critical(
'Configuraion error: missing left engine refernce')
exit(1)
left = None
if config.has_section(engine_section_left):
left = {}
for key in config.options(engine_section_left):
left[key] = config.get(engine_section_left, key)
else:
logging.critical('Configuration error: missing section [%s]' %
(engine_section_left))
exit(1)
right = config.get(relation_section, 'right')
if right:
engine_section_right = 'engine' + '_' + right
else:
logging.critical(
'Configuraion error: missing right engine refernce')
exit(1)
right = None
if config.has_section(engine_section_right):
right = {}
for key in config.options(engine_section_right):
right[key] = config.get(engine_section_right, key)
else:
logging.critical('Configuration error: missing section [%s]' %
(engine_section_right))
exit(1)
for engine in [left, right]:
secrets = engine['secrets']
path = os.path.expanduser(secrets)
if os.path.isfile(path):
secret = ConfigParser()
secret.read(path)
if secret.has_section('secrets'):
engine['secrets'] = {'key': path}
for key in secret.options('secrets'):
engine['secrets'][key] = secret.get('secrets', key)
else:
logger.critical(
'Configuration error: missing [secrets] in %s' %
(path))
exit(1)
else:
if config.has_option('options', 'secrets'):
secrets = config.get('options', 'secrets')
path = os.path.expanduser(secrets)
if os.path.isfile(path):
secret = ConfigParser()
secret.read(path)
section = engine['secrets'][1:-1]
if secret.has_section(section):
engine['secrets'] = {'key': section}
for key in secret.options(section):
engine['secrets'][key] = secret.get(
section, key)
else:
logger.critical(
'Configuration error: missing [%s] in %s' %
(section, path))
exit(1)
else:
logger.critical(
'Configuration error: missing secret file %s' %
(path))
exit(1)
else:
logger.critical(
'Configuration error: missing secrets in [options]')
exit(1)
else:
logging.critical('Configuration error: missing section [%s]' %
(relation_section))
exit(1)
for options in [left, right]:
if options.get('class'):
cn = options['class']
if class_map.get(cn):
options['class'] = class_map[cn]
else:
logger.critical(
'Configuration error: unknown sync engine [%s]' % (cn))
exit(1)
else:
logger.critical('configuration error: missing class tag')
exit(1)
return Options(left), Options(right), Options(rel)
def main():
from argparse import ArgumentParser
from pysync import __version__, __author__
from pysync import SyncError, SyncSessionError, SyncInitError
options = {
'secrets': '~/.pysync.secrets',
'loglevel_requests': 'ERROR'
# 'loglevel': 'INFO'
}
# region Command line arguments
parser = ArgumentParser(description='PySnc Engine Rev. %s (c) %s' %
(__version__, __author__))
parser.add_argument('-c',
'--config',
type=str,
help='use alternate configuration file(s)')
parser.add_argument('--relations',
type=str,
help='list of pysync relations to process')
parser.add_argument('--rebuild',
action='store_true',
help='rebuild map file')
parser.add_argument('--reset',
type=str,
help='delete entries and recreate from left/right')
parser.add_argument('--update',
type=str,
help='force update on left/right side')
parser.add_argument(
'-l',
'--loglevel',
type=str,
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help='debug log level')
args = parser.parse_args()
opts = Options(options, args, '[config]', prefix='PYSYNC')
config = opts.config_parser
if config is None:
LogAdapter(get_logger(), {
'package': 'main'
}).critical("Missing configuration file!")
exit(1)
logger = LogAdapter(opts.logger, {'package': 'main'})
# endregion
# region Basic configuration and logger settings
# set log level of requests module
logging.getLogger('requests').setLevel(log_level(opts.loglevel_requests))
logging.getLogger('urllib3').setLevel(log_level(opts.loglevel_requests))
logger.debug(u'Parsing configuration file %s' % (opts.config_file))
if opts.relations:
relations = list(opts.relations.split(','))
else:
logger.critical('Missing relations!')
exit(1)
# endregion
for relation in relations:
try:
left_opts, right_opts, relation_opts = parse_config(
relation, config, opts.logger)
except Exception as e:
logger.exception(
'Error parsing configuration options. Skipping sync for [%s]' %
(relation))
continue
if lock(relation, relation_opts, opts.logger):
# initialise web service sessions via @staticmethod session()
# and initialize sync engine classes
try:
label = left_opts.get('label')
session_lockfile = left_opts.get('session_lockfile', True)
left_session = left_opts['class'].session(
left_opts, opts.logger)
label = right_opts.get('label')
session_lockfile = right_opts.get('session_lockfile', True)
right_session = right_opts['class'].session(
right_opts, opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception(
'Session initialization for [%s] failed! Skipping sync for [%s]'
% (label, relation))
if not session_lockfile:
unlock(relation, relation_opts, opts.logger)
continue
# initialize sync map
relation_opts['sync'] = {'map': None}
if os.path.isfile(relation_opts['map']):
####################
# incremental sync #
####################
with codecs.open(relation_opts['map'], 'r',
encoding='utf-8') as fp:
relation_opts['sync'] = json.load(fp)
logger.info(u'%s: starting incremental sync for %d items' %
(relation, len(relation_opts['sync'].get('map'))))
# merge signature from map file
left_opts.update({'signature': relation_opts['sync']['left']})
right_opts.update(
{'signature': relation_opts['sync']['right']})
try:
engine_lockfile = left_opts.get('engine_lockfile', True)
left = left_opts['class'](left_session,
left_opts,
logger=opts.logger)
engine_lockfile = right_opts.get('engine_lockfile', True)
right = right_opts['class'](right_session,
right_opts,
logger=opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception(
'Engine initialization for [%s] failed! Skipping sync for [%s]'
% (label, relation))
if not engine_lockfile:
unlock(relation, relation_opts, opts.logger)
continue
if opts['update']:
try:
pysync = PySync(left, right, relation_opts,
opts.logger)
relation_opts['sync'] = pysync.update(opts['update'])
except Exception as e:
logger.exception(
'Unexpected error when processing update option! Skipping sync for [%s]'
% relation)
unlock(relation, relation_opts, opts.logger)
continue
if opts.reset:
try:
pysync = PySync(left, right, relation_opts,
opts.logger)
relation_opts['sync'] = pysync.reset(opts.reset)
except Exception as e:
logger.exception(
'Unexpected error when processing reset option!')
check_sync_map(relation, pysync.direction, left, right,
relation_opts, logger)
continue
if opts.rebuild:
relation_opts['sync'] = {'map': None}
else:
################
# initial sync #
################
logger.info(u'%s: Starting initial sync' % (relation))
for opt in ['update', 'reset', 'rebuild']:
if opts.get(opt):
logger.warning(
'Ignoring option [%s] for initial sync' % (opt))
try:
left = left_opts['class'](left_session,
left_opts,
logger=opts.logger)
right = right_opts['class'](right_session,
right_opts,
logger=opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception(
'Engine initialization for [%s] failed! Skipping sync for [%s]'
% (label, relation))
# unlock(relation, relation_opts, opts.logger)
continue
try:
pysync = PySync(left, right, relation_opts, opts.logger)
relation_opts['sync'] = pysync.process()
except Exception as e:
logger.exception('Unexpected error when processing sync map!')
check_sync_map(relation, pysync.direction, left, right,
relation_opts, logger)
continue
# check/modify sync map by backend engine
relation_opts = left.commit_sync('left', relation_opts, logger)
relation_opts = right.commit_sync('right', relation_opts, logger)
count, errors = check_sync_map(relation, pysync.direction, left,
right, relation_opts, logger)
unlock(relation, relation_opts, logger)
logger.info(u'%s: %s %s' % (relation, left.label, left._changes))
logger.info(u'%s: %s %s' % (relation, right.label, right._changes))
logger.info(u'%s: finished %s sync for %d items with %d errors' %
(relation, pysync.direction, count, errors))
left_opts['class'].end_session(logger)
right_opts['class'].end_session(logger)
def main():
from argparse import ArgumentParser
from pysync import __version__, __author__
from pysync import SyncError, SyncSessionError, SyncInitError
options = {
'secrets': '~/.pysync.secrets',
'loglevel_requests': 'ERROR'
# 'loglevel': 'INFO'
}
# region Command line arguments
parser = ArgumentParser(description='PySnc Engine Rev. %s (c) %s' % (__version__, __author__))
parser.add_argument('-c', '--config', type=str, help='use alternate configuration file(s)')
parser.add_argument('--relations', type=str, help='list of pysync relations to process')
parser.add_argument('--rebuild', action='store_true', help='rebuild map file')
parser.add_argument('--reset', type=str, help='delete entries and recreate from left/right')
parser.add_argument('--update', type=str, help='force update on left/right side')
parser.add_argument('-l', '--loglevel', type=str,
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help='debug log level')
args = parser.parse_args()
opts = Options(options, args, '[config]', prefix='PYSYNC')
config = opts.config_parser
if config is None:
LogAdapter(get_logger(), {'package': 'main'}).critical("Missing configuration file!")
exit(1)
logger = LogAdapter(opts.logger, {'package': 'main'})
# endregion
# region Basic configuration and logger settings
# set log level of requests module
logging.getLogger('requests').setLevel(log_level(opts.loglevel_requests))
logging.getLogger('urllib3').setLevel(log_level(opts.loglevel_requests))
logger.debug(u'Parsing configuration file %s' % (opts.config_file))
if opts.relations:
relations = list(opts.relations.split(','))
else:
logger.critical('Missing relations!')
exit(1)
# endregion
for relation in relations:
try:
left_opts, right_opts, relation_opts = parse_config(relation, config, opts.logger)
except Exception as e:
logger.exception('Error parsing configuration options. Skipping sync for [%s]' % (relation))
continue
if lock(relation, relation_opts, opts.logger):
# initialise web service sessions via @staticmethod session()
# and initialize sync engine classes
try:
label = left_opts.get('label')
session_lockfile = left_opts.get('session_lockfile', True)
left_session = left_opts['class'].session(left_opts, opts.logger)
label = right_opts.get('label')
session_lockfile = right_opts.get('session_lockfile', True)
right_session = right_opts['class'].session(right_opts, opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception('Session initialization for [%s] failed! Skipping sync for [%s]' % (label, relation))
if not session_lockfile:
unlock(relation, relation_opts, opts.logger)
continue
# initialize sync map
relation_opts['sync'] = {'map': None}
if os.path.isfile(relation_opts['map']):
####################
# incremental sync #
####################
with codecs.open(relation_opts['map'], 'r', encoding='utf-8') as fp:
relation_opts['sync'] = json.load(fp)
logger.info(u'%s: starting incremental sync for %d items' % (relation, len(relation_opts['sync'].get('map'))))
# merge signature from map file
left_opts.update({'signature': relation_opts['sync']['left']})
right_opts.update({'signature': relation_opts['sync']['right']})
try:
engine_lockfile = left_opts.get('engine_lockfile', True)
left = left_opts['class'](left_session, left_opts, logger=opts.logger)
engine_lockfile = right_opts.get('engine_lockfile', True)
right = right_opts['class'](right_session, right_opts, logger=opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception('Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation))
if not engine_lockfile:
unlock(relation, relation_opts, opts.logger)
continue
if opts['update']:
try:
pysync = PySync(left, right, relation_opts, opts.logger)
relation_opts['sync'] = pysync.update(opts['update'])
except Exception as e:
logger.exception('Unexpected error when processing update option! Skipping sync for [%s]' % relation)
unlock(relation, relation_opts, opts.logger)
continue
if opts.reset:
try:
pysync = PySync(left, right, relation_opts, opts.logger)
relation_opts['sync'] = pysync.reset(opts.reset)
except Exception as e:
logger.exception('Unexpected error when processing reset option!')
check_sync_map(relation, pysync.direction, left, right, relation_opts, logger)
continue
if opts.rebuild:
relation_opts['sync'] = {'map': None}
else:
################
# initial sync #
################
logger.info(u'%s: Starting initial sync' % (relation))
for opt in ['update', 'reset', 'rebuild']:
if opts.get(opt):
logger.warning('Ignoring option [%s] for initial sync' % (opt))
try:
left = left_opts['class'](left_session, left_opts, logger=opts.logger)
right = right_opts['class'](right_session, right_opts, logger=opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception('Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation))
# unlock(relation, relation_opts, opts.logger)
continue
try:
pysync = PySync(left, right, relation_opts, opts.logger)
relation_opts['sync'] = pysync.process()
except Exception as e:
logger.exception('Unexpected error when processing sync map!')
check_sync_map(relation, pysync.direction, left, right, relation_opts, logger)
continue
# check/modify sync map by backend engine
relation_opts = left.commit_sync('left', relation_opts, logger)
relation_opts = right.commit_sync('right', relation_opts, logger)
count, errors = check_sync_map(relation, pysync.direction, left, right, relation_opts, logger)
unlock(relation, relation_opts, logger)
logger.info(u'%s: %s %s' % (relation, left.label, left._changes))
logger.info(u'%s: %s %s' % (relation, right.label, right._changes))
logger.info(u'%s: finished %s sync for %d items with %d errors' % (relation, pysync.direction, count, errors))
left_opts['class'].end_session(logger)
right_opts['class'].end_session(logger)
