def unlock(relation, opts, _logger):
logger = LogAdapter(_logger, {'package': 'unlock'})
map_file = os.path.expanduser(opts['map'])
lck_file = os.path.splitext(map_file)[0] + '.lck'
if os.path.isfile(lck_file):
logger.debug(u'Unlocking relation [%s]' % (relation))
os.remove(lck_file)
return True
else:
logger.warning('Lockfile %s for relation [%s] not found!' % (lck_file, relation))
return False
def rollback(relation, opts, _logger):
logger = LogAdapter(_logger, {'package': 'rollback'})
map_file = os.path.expanduser(opts['map'])
lck_file = os.path.splitext(map_file)[0] + '.lck'
if os.path.isfile(lck_file):
with codecs.open(lck_file, 'r', encoding='utf-8') as fp:
content = fp.read()
logger.debug(u'Rollback relation [%s]' % (relation))
with codecs.open(map_file, 'w', encoding='utf-8') as fp:
fp.write(content)
os.remove(lck_file)
return True
return False
def rollback(relation, opts, _logger):
logger = LogAdapter(_logger, {'package': 'rollback'})
map_file = os.path.expanduser(opts['map'])
lck_file = os.path.splitext(map_file)[0] + '.lck'
if os.path.isfile(lck_file):
with codecs.open(lck_file, 'r', encoding='utf-8') as fp:
content = fp.read()
logger.debug(u'Rollback relation [%s]' % (relation))
with codecs.open(map_file, 'w', encoding='utf-8') as fp:
fp.write(content)
os.remove(lck_file)
return True
return False
def session(options, _logger):
if options:
if options.get('secrets'):
secrets = options['secrets']
if secrets.get('server'):
if secrets.get('user'):
if secrets.get('password'):
ox = OxHttpAPI.get_session(
server=secrets['server'],
user=secrets['user'],
password=secrets['password'],
logger=_logger)
if ox and ox.authenticated:
return ox
else:
raise SyncSessionError(
'Login for [%s] at [%s] failed!' %
(secrets['user'], secrets['server']))
return None
LogAdapter(_logger, {
'package': 'oxsync'
}).error('Missing credentials in Open-Xchange options')
raise SyncSessionError('Missing credentials in Open-Xchange options')
return None
def lock(relation, opts, _logger):
logger = LogAdapter(_logger, {'package': 'lock'})
map_file = os.path.expanduser(opts['map'])
lck_file = os.path.splitext(map_file)[0] + '.lck'
if os.path.isfile(lck_file):
logger.error('Relation [%s] locked. Remove %s to unlock synchronization' % (relation, lck_file))
return False
if os.path.isfile(map_file):
with codecs.open(map_file, 'r', encoding='utf-8') as fp:
content = fp.read()
logger.debug(u'Locking relation [%s]' % (relation))
with codecs.open(lck_file, 'w', encoding='utf-8') as fp:
fp.write(content)
else:
open(lck_file, 'a').close()
return True
def parse(self):
self._args = self._parser.parse_args()
self._opts = Options(self._options, self._args, '[config]', prefix='PIMATIC')
self._config = self._opts.config_parser
if self._config is None:
LogAdapter(get_logger(), {'package': self.app}).critical("Missing configuration file!")
exit(1)
self._logger = LogAdapter(self._opts.logger, {'package': self.app})
# region Get node configuration and logger settings
# set log level of requests module
logging.getLogger('requests').setLevel(log_level(self._opts.loglevel_requests))
logging.getLogger('urllib3').setLevel(log_level(self._opts.loglevel_requests))
self._logger.debug(u'Parsing configuration file %s' % (self._opts.config_file))
if self._config.has_option('options', 'secrets'):
secrets = self._config.get('options', 'secrets')
path = os.path.expanduser(secrets)
if os.path.isfile(path):
secret = ConfigParser()
secret.read(path)
if self._opts.node:
if self._opts.node.startswith('http'):
self._node['url'] = self._opts.node
else:
node_section = 'node_' + self._opts.node
if secret.has_section(node_section):
protocol = secret.get(node_section, 'protocol')
server = secret.get(node_section, 'server')
port = secret.get(node_section, protocol + '_port')
self._node['url']=protocol + '://' + server + ':' + port
self._node['username'] = secret.get(node_section, 'username')
self._node['password'] = secret.get(node_section, 'password')
else:
self._logger.critical(u'Invalid node tag %s' % (node_section))
exit(1)
else:
self._logger.critical(u'Missing pimatic node specification!')
exit(1)
if self._opts.username:
self._node['username'] = self._opts.username
if self._opts.password:
self._node['password'] = self._opts.password
# endregion
self._logger.info(u'url: %s' % (self._node['url']))
def session(options, _logger):
if options:
if options.get('secrets'):
if options['secrets'].get('token'):
return EnClient.get_client(token=options['secrets']['token'],
logger=_logger)
error = u'Missing credentials in Evernote options!'
LogAdapter(_logger, {'package': 'ensync'}).error(error)
raise SyncSessionError(error)
return None
def __init__(self, left=None, right=None, opts=None, logger=None):
if logger is None:
self._logger = get_logger('pysync', logging.DEBUG)
else:
self._logger = logger
self._adapter = LogAdapter(self._logger, {'package': 'pysync'})
self.logger.debug(u'Initalizing PySync with %s and %s ...' %
(left, right))
self._opts = opts
self._left = left
self._right = right
self._sync = opts['sync']
self._new_sync = None
def unlock(relation, opts, _logger):
logger = LogAdapter(_logger, {'package': 'unlock'})
map_file = os.path.expanduser(opts['map'])
lck_file = os.path.splitext(map_file)[0] + '.lck'
if os.path.isfile(lck_file):
logger.debug(u'Unlocking relation [%s]' % (relation))
os.remove(lck_file)
return True
else:
logger.warning('Lockfile %s for relation [%s] not found!' %
(lck_file, relation))
return False
def __init__(self, server, user=None, password=None, logger=None, verify=True):
from pyutils import LogAdapter, get_logger
self._server = server
self._user = user
self._password = password
self._verify = verify
self._session = None
self._cookies = None
self._offline = None
self._utc_offset = None
if logger is None:
self._logger = get_logger('oxapi', logging.DEBUG)
else:
self._logger = logger
self._adapter = LogAdapter(self._logger, {'package': 'oxapi', 'callback': OxHttpAPI.hide_password})
def __init__(self, options, logger=None, package='sync'):
if logger is None:
self._logger = get_logger('sync', logging.DEBUG)
else:
self._logger = logger
self._adapter = LogAdapter(self._logger, {'package': package})
self._options = options
self._key = None
self._items = {}
self._filter_expr = options.get('filter_expr')
self._filter_module = options.get('filter_module')
self._deleted = {}
self._modified = {}
self._created = {}
self._changes = {'deleted': 0, 'created': 0, 'modified': 0}
def session(options, _logger):
if options:
if options.get('cache') is not None:
if options.get('secrets'):
secrets = options['secrets']
if secrets.get('client_id'):
if secrets.get('client_secret'):
return ToodledoAPI.get_session(
cache=options.get('cache'),
client_id=secrets['client_id'],
client_secret=secrets['client_secret'],
logger=_logger)
else:
error = 'Missing secrets in Toodledo options'
else:
error = 'Missing cache specification in Toodledo options'
else:
error = 'Missing Toodledo options'
LogAdapter(_logger, {'package': 'tdsync'}).error(error)
raise SyncSessionError(error)
return None
def lock(relation, opts, _logger):
logger = LogAdapter(_logger, {'package': 'lock'})
map_file = os.path.expanduser(opts['map'])
lck_file = os.path.splitext(map_file)[0] + '.lck'
if os.path.isfile(lck_file):
logger.error(
'Relation [%s] locked. Remove %s to unlock synchronization' %
(relation, lck_file))
return False
if os.path.isfile(map_file):
with codecs.open(map_file, 'r', encoding='utf-8') as fp:
content = fp.read()
logger.debug(u'Locking relation [%s]' % (relation))
with codecs.open(lck_file, 'w', encoding='utf-8') as fp:
fp.write(content)
else:
open(lck_file, 'a').close()
return True
def __init__(self,
cache='~/.tdapi',
client_id=None,
client_secret=None,
logger=None):
# region Logging
from pyutils import get_logger, LogAdapter
if logger is None:
self._logger = get_logger('tdapi', 'DEBUG')
else:
self._logger = logger
self._adapter = LogAdapter(self._logger, {'package': 'tdapi'})
# endregion
self._cache_dir = os.path.expanduser(cache)
if not os.path.isdir(self._cache_dir):
error = 'Cache directory [%s] not found!' % (self._cache_dir)
self.logger.critical(error)
raise ToodledoApiError(error)
if not os.path.isfile(self.cache_file('session')):
error = 'Session cache file [%s] not found!' % (
self.cache_file('session'))
self.logger.critical(error)
raise ToodledoApiError(error)
self._client_id = client_id or os.environ.get('TOODLEDO_CLIENT_ID')
self._client_secret = client_secret or os.environ.get(
'TOODLEDO_CLIENT_SECRET')
self._cache = {
'session': None,
'account': None,
'lists': None,
'tasks': None
}
self._offline = None
self._account = None
self._lists = None
self._tasks = None
from tdapi import ToodledoFolders, ToodledoContexts, ToodledoGoals, ToodledoLocations
from tdapi import ToodledoFolder, ToodledoContext, ToodledoGoal, ToodledoLocation
self._class_map = {
'folders': {
'collection': ToodledoFolders,
'item': ToodledoFolder,
'auto': True
},
'contexts': {
'collection': ToodledoContexts,
'item': ToodledoContext,
'auto': True
},
'goals': {
'collection': ToodledoGoals,
'item': ToodledoGoal,
'auto': True
},
'locations': {
'collection': ToodledoLocations,
'item': ToodledoLocation,
'auto': True
}
}
# load cache files
for key in self._cache:
fn = self.cache_file(key)
if os.path.isfile(fn):
with codecs.open(fn, 'r', encoding='utf-8') as fh:
self._cache[key] = json.load(fh)
class PimaticApp(object):
def __init__(self, options):
self._options = options
self._pimatic = None
self._args = None
self._opts = None
self._config = None
self._logger = None
self._node = {'url': None, 'username': None, 'password': None}
self._parser = ArgumentParser(description='%s [PimaticAPI Rev. %s (c) %s]' % (self.app, __version__, __author__))
self._parser.add_argument('-c', '--config', type=str, help='use alternate configuration file(s)')
self._parser.add_argument('-s', '--secrets', type=str, help='use alternate secrets file(s)')
self._parser.add_argument('-l', '--loglevel', type=str, choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help='debug log level')
self._parser.add_argument('-n', '--node', type=str, help='pimatic node instance')
self._parser.add_argument('-u', '--username', type=str, help='pimatic username')
self._parser.add_argument('-p', '--password', type=str, help='pimatic password')
def parse(self):
self._args = self._parser.parse_args()
self._opts = Options(self._options, self._args, '[config]', prefix='PIMATIC')
self._config = self._opts.config_parser
if self._config is None:
LogAdapter(get_logger(), {'package': self.app}).critical("Missing configuration file!")
exit(1)
self._logger = LogAdapter(self._opts.logger, {'package': self.app})
# region Get node configuration and logger settings
# set log level of requests module
logging.getLogger('requests').setLevel(log_level(self._opts.loglevel_requests))
logging.getLogger('urllib3').setLevel(log_level(self._opts.loglevel_requests))
self._logger.debug(u'Parsing configuration file %s' % (self._opts.config_file))
if self._config.has_option('options', 'secrets'):
secrets = self._config.get('options', 'secrets')
path = os.path.expanduser(secrets)
if os.path.isfile(path):
secret = ConfigParser()
secret.read(path)
if self._opts.node:
if self._opts.node.startswith('http'):
self._node['url'] = self._opts.node
else:
node_section = 'node_' + self._opts.node
if secret.has_section(node_section):
protocol = secret.get(node_section, 'protocol')
server = secret.get(node_section, 'server')
port = secret.get(node_section, protocol + '_port')
self._node['url']=protocol + '://' + server + ':' + port
self._node['username'] = secret.get(node_section, 'username')
self._node['password'] = secret.get(node_section, 'password')
else:
self._logger.critical(u'Invalid node tag %s' % (node_section))
exit(1)
else:
self._logger.critical(u'Missing pimatic node specification!')
exit(1)
if self._opts.username:
self._node['username'] = self._opts.username
if self._opts.password:
self._node['password'] = self._opts.password
# endregion
self._logger.info(u'url: %s' % (self._node['url']))
def session(self):
self._pimatic = PimaticAPI(server=self._node['url'],
username=self._node['username'],
password=self._node['password'],
logger=self._logger)
return self._pimatic
@property
def app(self): return self._options.get('app')
def main():
from ConfigParser import ConfigParser
from argparse import ArgumentParser
options = {
'option': 'OPTION',
'secrets': {'token': 'secret'},
'loglevel': 'INFO'
}
# region Command line arguments
parser = ArgumentParser(description='PySnc Engine Rev. 0.1 (c) Bernd Strebel')
parser.add_argument('-c', '--config', type=str, help='use alternate configuration file')
parser.add_argument('-l', '--loglevel', type=str,
choices=['DEBUG', 'INFO', 'WARN', 'WARNING', 'ERROR', 'CRITICAL',
'debug', 'info', 'warn', 'warning', 'error', 'critical'],
help='debug log level')
args = parser.parse_args()
opts = Options(options, args, config=True)
# _logger = get_logger('root',log_level(opts.loglevel))
# logger = LogAdapter(_logger, {'package': 'init'})
# logger.info("Console logger initilized")
# endregion
# region (I) Basic configuration and logger settings from config file
# if opts.config_file:
# if os.path.isfile(opts.config_file):
# logging.config.fileConfig(opts.config_file)
# config = ConfigParser(options)
# config.read(opts.config_file)
# else:
# logger.critical('Configuration file %s not found!' % (opts.config_file))
# exit(1)
# else:
# logger.critical("Missing configuration file!")
# exit(1)
#
# _logger = logging.getLogger()
# _logger.setLevel(log_level(opts.loglevel))
#
# logger = LogAdapter(_logger, {'package': 'main'})
# endregion
# region (II) Basic configuration and logger settings from config parser
config = opts.config_parser
logger = LogAdapter(opts.logger, {'package': 'main'})
# endregion
logger.info('Default logger configured from %s' % (opts.config_file))
print opts.option
s = opts.get('string_option', False)
t = opts['secrets']['token']
e = opts['empty']
b = opts.get('uni')
u = opts['uni']
pass
def parse_config(relation, config, _logger):
from ConfigParser import ConfigParser
from oxsync import OxTaskSync
from ensync import EnClientSync
from tdsync import ToodledoSync
logger = LogAdapter(_logger, {'package': 'config'})
class_map = {
'OxTaskSync': OxTaskSync,
'EnClientSync': EnClientSync,
'ToodledoSync': ToodledoSync
}
relation_section = 'relation' + '_' + relation
if config.has_section(relation_section):
rel = {}
for key in config.options(relation_section):
rel[key] = config.get(relation_section, key)
if rel.get('map'):
rel['map'] = os.path.expanduser(rel.get('map'))
else:
logging.critical('Configuration error: missing map file path for %s' % (relation))
exit(1)
left = config.get(relation_section, 'left')
if left:
engine_section_left = 'engine' + '_' + left
else:
logging.critical('Configuraion error: missing left engine refernce')
exit(1)
left = None
if config.has_section(engine_section_left):
left = {}
for key in config.options(engine_section_left):
left[key] = config.get(engine_section_left, key)
else:
logging.critical('Configuration error: missing section [%s]' % (engine_section_left))
exit(1)
right = config.get(relation_section, 'right')
if right:
engine_section_right = 'engine' + '_' + right
else:
logging.critical('Configuraion error: missing right engine refernce')
exit(1)
right = None
if config.has_section(engine_section_right):
right = {}
for key in config.options(engine_section_right):
right[key] = config.get(engine_section_right, key)
else:
logging.critical('Configuration error: missing section [%s]' % (engine_section_right))
exit(1)
for engine in [left, right]:
secrets = engine['secrets']
path = os.path.expanduser(secrets)
if os.path.isfile(path):
secret = ConfigParser()
secret.read(path)
if secret.has_section('secrets'):
engine['secrets'] = {'key': path}
for key in secret.options('secrets'):
engine['secrets'][key] = secret.get('secrets', key)
else:
logger.critical('Configuration error: missing [secrets] in %s' % (path))
exit(1)
else:
if config.has_option('options', 'secrets'):
secrets = config.get('options', 'secrets')
path = os.path.expanduser(secrets)
if os.path.isfile(path):
secret = ConfigParser()
secret.read(path)
section = engine['secrets'][1:-1]
if secret.has_section(section):
engine['secrets'] = {'key': section}
for key in secret.options(section):
engine['secrets'][key] = secret.get(section, key)
else:
logger.critical('Configuration error: missing [%s] in %s' % (section, path))
exit(1)
else:
logger.critical('Configuration error: missing secret file %s' % (path))
exit(1)
else:
logger.critical('Configuration error: missing secrets in [options]')
exit(1)
else:
logging.critical('Configuration error: missing section [%s]' % (relation_section))
exit(1)
for options in [left, right]:
if options.get('class'):
cn = options['class']
if class_map.get(cn):
options['class'] = class_map[cn]
else:
logger.critical('Configuration error: unknown sync engine [%s]' % (cn))
exit(1)
else:
logger.critical('configuration error: missing class tag')
exit(1)
return Options(left), Options(right), Options(rel)
def main():
from argparse import ArgumentParser
from pysync import __version__, __author__
from pysync import SyncError, SyncSessionError, SyncInitError
options = {
'secrets': '~/.pysync.secrets',
'loglevel_requests': 'ERROR'
# 'loglevel': 'INFO'
}
# region Command line arguments
parser = ArgumentParser(description='PySnc Engine Rev. %s (c) %s' %
(__version__, __author__))
parser.add_argument('-c',
'--config',
type=str,
help='use alternate configuration file(s)')
parser.add_argument('--relations',
type=str,
help='list of pysync relations to process')
parser.add_argument('--rebuild',
action='store_true',
help='rebuild map file')
parser.add_argument('--reset',
type=str,
help='delete entries and recreate from left/right')
parser.add_argument('--update',
type=str,
help='force update on left/right side')
parser.add_argument(
'-l',
'--loglevel',
type=str,
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help='debug log level')
args = parser.parse_args()
opts = Options(options, args, '[config]', prefix='PYSYNC')
config = opts.config_parser
if config is None:
LogAdapter(get_logger(), {
'package': 'main'
}).critical("Missing configuration file!")
exit(1)
logger = LogAdapter(opts.logger, {'package': 'main'})
# endregion
# region Basic configuration and logger settings
# set log level of requests module
logging.getLogger('requests').setLevel(log_level(opts.loglevel_requests))
logging.getLogger('urllib3').setLevel(log_level(opts.loglevel_requests))
logger.debug(u'Parsing configuration file %s' % (opts.config_file))
if opts.relations:
relations = list(opts.relations.split(','))
else:
logger.critical('Missing relations!')
exit(1)
# endregion
for relation in relations:
try:
left_opts, right_opts, relation_opts = parse_config(
relation, config, opts.logger)
except Exception as e:
logger.exception(
'Error parsing configuration options. Skipping sync for [%s]' %
(relation))
continue
if lock(relation, relation_opts, opts.logger):
# initialise web service sessions via @staticmethod session()
# and initialize sync engine classes
try:
label = left_opts.get('label')
session_lockfile = left_opts.get('session_lockfile', True)
left_session = left_opts['class'].session(
left_opts, opts.logger)
label = right_opts.get('label')
session_lockfile = right_opts.get('session_lockfile', True)
right_session = right_opts['class'].session(
right_opts, opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception(
'Session initialization for [%s] failed! Skipping sync for [%s]'
% (label, relation))
if not session_lockfile:
unlock(relation, relation_opts, opts.logger)
continue
# initialize sync map
relation_opts['sync'] = {'map': None}
if os.path.isfile(relation_opts['map']):
####################
# incremental sync #
####################
with codecs.open(relation_opts['map'], 'r',
encoding='utf-8') as fp:
relation_opts['sync'] = json.load(fp)
logger.info(u'%s: starting incremental sync for %d items' %
(relation, len(relation_opts['sync'].get('map'))))
# merge signature from map file
left_opts.update({'signature': relation_opts['sync']['left']})
right_opts.update(
{'signature': relation_opts['sync']['right']})
try:
engine_lockfile = left_opts.get('engine_lockfile', True)
left = left_opts['class'](left_session,
left_opts,
logger=opts.logger)
engine_lockfile = right_opts.get('engine_lockfile', True)
right = right_opts['class'](right_session,
right_opts,
logger=opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception(
'Engine initialization for [%s] failed! Skipping sync for [%s]'
% (label, relation))
if not engine_lockfile:
unlock(relation, relation_opts, opts.logger)
continue
if opts['update']:
try:
pysync = PySync(left, right, relation_opts,
opts.logger)
relation_opts['sync'] = pysync.update(opts['update'])
except Exception as e:
logger.exception(
'Unexpected error when processing update option! Skipping sync for [%s]'
% relation)
unlock(relation, relation_opts, opts.logger)
continue
if opts.reset:
try:
pysync = PySync(left, right, relation_opts,
opts.logger)
relation_opts['sync'] = pysync.reset(opts.reset)
except Exception as e:
logger.exception(
'Unexpected error when processing reset option!')
check_sync_map(relation, pysync.direction, left, right,
relation_opts, logger)
continue
if opts.rebuild:
relation_opts['sync'] = {'map': None}
else:
################
# initial sync #
################
logger.info(u'%s: Starting initial sync' % (relation))
for opt in ['update', 'reset', 'rebuild']:
if opts.get(opt):
logger.warning(
'Ignoring option [%s] for initial sync' % (opt))
try:
left = left_opts['class'](left_session,
left_opts,
logger=opts.logger)
right = right_opts['class'](right_session,
right_opts,
logger=opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception(
'Engine initialization for [%s] failed! Skipping sync for [%s]'
% (label, relation))
# unlock(relation, relation_opts, opts.logger)
continue
try:
pysync = PySync(left, right, relation_opts, opts.logger)
relation_opts['sync'] = pysync.process()
except Exception as e:
logger.exception('Unexpected error when processing sync map!')
check_sync_map(relation, pysync.direction, left, right,
relation_opts, logger)
continue
# check/modify sync map by backend engine
relation_opts = left.commit_sync('left', relation_opts, logger)
relation_opts = right.commit_sync('right', relation_opts, logger)
count, errors = check_sync_map(relation, pysync.direction, left,
right, relation_opts, logger)
unlock(relation, relation_opts, logger)
logger.info(u'%s: %s %s' % (relation, left.label, left._changes))
logger.info(u'%s: %s %s' % (relation, right.label, right._changes))
logger.info(u'%s: finished %s sync for %d items with %d errors' %
(relation, pysync.direction, count, errors))
left_opts['class'].end_session(logger)
right_opts['class'].end_session(logger)
def main():
from argparse import ArgumentParser
from pysync import __version__, __author__
from pysync import SyncError, SyncSessionError, SyncInitError
options = {
'secrets': '~/.pysync.secrets',
'loglevel_requests': 'ERROR'
# 'loglevel': 'INFO'
}
# region Command line arguments
parser = ArgumentParser(description='PySnc Engine Rev. %s (c) %s' % (__version__, __author__))
parser.add_argument('-c', '--config', type=str, help='use alternate configuration file(s)')
parser.add_argument('--relations', type=str, help='list of pysync relations to process')
parser.add_argument('--rebuild', action='store_true', help='rebuild map file')
parser.add_argument('--reset', type=str, help='delete entries and recreate from left/right')
parser.add_argument('--update', type=str, help='force update on left/right side')
parser.add_argument('-l', '--loglevel', type=str,
choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
help='debug log level')
args = parser.parse_args()
opts = Options(options, args, '[config]', prefix='PYSYNC')
config = opts.config_parser
if config is None:
LogAdapter(get_logger(), {'package': 'main'}).critical("Missing configuration file!")
exit(1)
logger = LogAdapter(opts.logger, {'package': 'main'})
# endregion
# region Basic configuration and logger settings
# set log level of requests module
logging.getLogger('requests').setLevel(log_level(opts.loglevel_requests))
logging.getLogger('urllib3').setLevel(log_level(opts.loglevel_requests))
logger.debug(u'Parsing configuration file %s' % (opts.config_file))
if opts.relations:
relations = list(opts.relations.split(','))
else:
logger.critical('Missing relations!')
exit(1)
# endregion
for relation in relations:
try:
left_opts, right_opts, relation_opts = parse_config(relation, config, opts.logger)
except Exception as e:
logger.exception('Error parsing configuration options. Skipping sync for [%s]' % (relation))
continue
if lock(relation, relation_opts, opts.logger):
# initialise web service sessions via @staticmethod session()
# and initialize sync engine classes
try:
label = left_opts.get('label')
session_lockfile = left_opts.get('session_lockfile', True)
left_session = left_opts['class'].session(left_opts, opts.logger)
label = right_opts.get('label')
session_lockfile = right_opts.get('session_lockfile', True)
right_session = right_opts['class'].session(right_opts, opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception('Session initialization for [%s] failed! Skipping sync for [%s]' % (label, relation))
if not session_lockfile:
unlock(relation, relation_opts, opts.logger)
continue
# initialize sync map
relation_opts['sync'] = {'map': None}
if os.path.isfile(relation_opts['map']):
####################
# incremental sync #
####################
with codecs.open(relation_opts['map'], 'r', encoding='utf-8') as fp:
relation_opts['sync'] = json.load(fp)
logger.info(u'%s: starting incremental sync for %d items' % (relation, len(relation_opts['sync'].get('map'))))
# merge signature from map file
left_opts.update({'signature': relation_opts['sync']['left']})
right_opts.update({'signature': relation_opts['sync']['right']})
try:
engine_lockfile = left_opts.get('engine_lockfile', True)
left = left_opts['class'](left_session, left_opts, logger=opts.logger)
engine_lockfile = right_opts.get('engine_lockfile', True)
right = right_opts['class'](right_session, right_opts, logger=opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception('Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation))
if not engine_lockfile:
unlock(relation, relation_opts, opts.logger)
continue
if opts['update']:
try:
pysync = PySync(left, right, relation_opts, opts.logger)
relation_opts['sync'] = pysync.update(opts['update'])
except Exception as e:
logger.exception('Unexpected error when processing update option! Skipping sync for [%s]' % relation)
unlock(relation, relation_opts, opts.logger)
continue
if opts.reset:
try:
pysync = PySync(left, right, relation_opts, opts.logger)
relation_opts['sync'] = pysync.reset(opts.reset)
except Exception as e:
logger.exception('Unexpected error when processing reset option!')
check_sync_map(relation, pysync.direction, left, right, relation_opts, logger)
continue
if opts.rebuild:
relation_opts['sync'] = {'map': None}
else:
################
# initial sync #
################
logger.info(u'%s: Starting initial sync' % (relation))
for opt in ['update', 'reset', 'rebuild']:
if opts.get(opt):
logger.warning('Ignoring option [%s] for initial sync' % (opt))
try:
left = left_opts['class'](left_session, left_opts, logger=opts.logger)
right = right_opts['class'](right_session, right_opts, logger=opts.logger)
except Exception as e:
# TODO: check exception type, unlock() only in case of an temp. network error etc.
logger.exception('Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation))
# unlock(relation, relation_opts, opts.logger)
continue
try:
pysync = PySync(left, right, relation_opts, opts.logger)
relation_opts['sync'] = pysync.process()
except Exception as e:
logger.exception('Unexpected error when processing sync map!')
check_sync_map(relation, pysync.direction, left, right, relation_opts, logger)
continue
# check/modify sync map by backend engine
relation_opts = left.commit_sync('left', relation_opts, logger)
relation_opts = right.commit_sync('right', relation_opts, logger)
count, errors = check_sync_map(relation, pysync.direction, left, right, relation_opts, logger)
unlock(relation, relation_opts, logger)
logger.info(u'%s: %s %s' % (relation, left.label, left._changes))
logger.info(u'%s: %s %s' % (relation, right.label, right._changes))
logger.info(u'%s: finished %s sync for %d items with %d errors' % (relation, pysync.direction, count, errors))
left_opts['class'].end_session(logger)
right_opts['class'].end_session(logger)
def parse_config(relation, config, _logger):
from ConfigParser import ConfigParser
from oxsync import OxTaskSync
from ensync import EnClientSync
from tdsync import ToodledoSync
logger = LogAdapter(_logger, {'package': 'config'})
class_map = {
'OxTaskSync': OxTaskSync,
'EnClientSync': EnClientSync,
'ToodledoSync': ToodledoSync
}
relation_section = 'relation' + '_' + relation
if config.has_section(relation_section):
rel = {}
for key in config.options(relation_section):
rel[key] = config.get(relation_section, key)
if rel.get('map'):
rel['map'] = os.path.expanduser(rel.get('map'))
else:
logging.critical(
'Configuration error: missing map file path for %s' %
(relation))
exit(1)
left = config.get(relation_section, 'left')
if left:
engine_section_left = 'engine' + '_' + left
else:
logging.critical(
'Configuraion error: missing left engine refernce')
exit(1)
left = None
if config.has_section(engine_section_left):
left = {}
for key in config.options(engine_section_left):
left[key] = config.get(engine_section_left, key)
else:
logging.critical('Configuration error: missing section [%s]' %
(engine_section_left))
exit(1)
right = config.get(relation_section, 'right')
if right:
engine_section_right = 'engine' + '_' + right
else:
logging.critical(
'Configuraion error: missing right engine refernce')
exit(1)
right = None
if config.has_section(engine_section_right):
right = {}
for key in config.options(engine_section_right):
right[key] = config.get(engine_section_right, key)
else:
logging.critical('Configuration error: missing section [%s]' %
(engine_section_right))
exit(1)
for engine in [left, right]:
secrets = engine['secrets']
path = os.path.expanduser(secrets)
if os.path.isfile(path):
secret = ConfigParser()
secret.read(path)
if secret.has_section('secrets'):
engine['secrets'] = {'key': path}
for key in secret.options('secrets'):
engine['secrets'][key] = secret.get('secrets', key)
else:
logger.critical(
'Configuration error: missing [secrets] in %s' %
(path))
exit(1)
else:
if config.has_option('options', 'secrets'):
secrets = config.get('options', 'secrets')
path = os.path.expanduser(secrets)
if os.path.isfile(path):
secret = ConfigParser()
secret.read(path)
section = engine['secrets'][1:-1]
if secret.has_section(section):
engine['secrets'] = {'key': section}
for key in secret.options(section):
engine['secrets'][key] = secret.get(
section, key)
else:
logger.critical(
'Configuration error: missing [%s] in %s' %
(section, path))
exit(1)
else:
logger.critical(
'Configuration error: missing secret file %s' %
(path))
exit(1)
else:
logger.critical(
'Configuration error: missing secrets in [options]')
exit(1)
else:
logging.critical('Configuration error: missing section [%s]' %
(relation_section))
exit(1)
for options in [left, right]:
if options.get('class'):
cn = options['class']
if class_map.get(cn):
options['class'] = class_map[cn]
else:
logger.critical(
'Configuration error: unknown sync engine [%s]' % (cn))
exit(1)
else:
logger.critical('configuration error: missing class tag')
exit(1)
return Options(left), Options(right), Options(rel)
# logging configuration
sh = logging.StreamHandler()
sf = logging.Formatter('%(levelname)-7s %(module)s %(message)s')
sh.setFormatter(sf)
fh = logging.FileHandler('sync.log', encoding='utf-8')
ff = logging.Formatter('%(asctime)s %(levelname)-7s %(module)s %(message)s')
fh.setFormatter(ff)
# logging via LoggerAdapter
root = logging.getLogger()
root.setLevel(logging.INFO)
root.addHandler(sh)
root.addHandler(fh)
logger = LogAdapter(root, {'package': 'root'})
# set log level of requests module
requests = logging.getLogger('requests')
requests.setLevel(logging.ERROR)
#logger.setLevel(logging.DEBUG)
logger.debug("Logging initialized ...")
from private import en_dev_token, ox_server, ox_user, ox_password
ox = OxHttpAPI.get_session(server=ox_server, user=ox_user, password=ox_password, logger=root)
en = EnClient.get_client(token=en_dev_token, logger=root)
#en = {'token': os.environ.get('EVERNOTE_TOKEN'), 'sandbox': False}
right = {'class': OxTaskSync, 'session': ox, 'options': { 'folder': 'OxSync', 'key': 'title'}}
def __init__(self, engine, other, package):
self._engine = engine
self._other = other
self._update = None
self._adapter = LogAdapter(engine._logger, {'package': package})
import os, sys
# print sys.path
import logging
from pyutils import LogAdapter
# stream handler configuration
sh = logging.StreamHandler()
sf = logging.Formatter("%(levelname)-8s %(module)s %(message)s")
sh.setFormatter(sf)
# fh = logging.FileHandler('sync.log', encoding='utf-8')
# ff = logging.Formatter('%(asctime)s %(levelname)-8s %(module)s %(message)s')
# fh.setFormatter(ff)
# logging via LoggerAdapter
root = logging.getLogger()
root.setLevel(logging.DEBUG)
root.addHandler(sh)
# root.addHandler(fh)
adapter = LogAdapter(root, {"package": "pyutils"})
adapter.debug("Log message")
try:
1 / 0
except Exception as e:
adapter.exception("Catched exception!")