def unlock(relation, opts, _logger): logger = LogAdapter(_logger, {'package': 'unlock'}) map_file = os.path.expanduser(opts['map']) lck_file = os.path.splitext(map_file)[0] + '.lck' if os.path.isfile(lck_file): logger.debug(u'Unlocking relation [%s]' % (relation)) os.remove(lck_file) return True else: logger.warning('Lockfile %s for relation [%s] not found!' % (lck_file, relation)) return False
def unlock(relation, opts, _logger): logger = LogAdapter(_logger, {'package': 'unlock'}) map_file = os.path.expanduser(opts['map']) lck_file = os.path.splitext(map_file)[0] + '.lck' if os.path.isfile(lck_file): logger.debug(u'Unlocking relation [%s]' % (relation)) os.remove(lck_file) return True else: logger.warning('Lockfile %s for relation [%s] not found!' % (lck_file, relation)) return False
def rollback(relation, opts, _logger): logger = LogAdapter(_logger, {'package': 'rollback'}) map_file = os.path.expanduser(opts['map']) lck_file = os.path.splitext(map_file)[0] + '.lck' if os.path.isfile(lck_file): with codecs.open(lck_file, 'r', encoding='utf-8') as fp: content = fp.read() logger.debug(u'Rollback relation [%s]' % (relation)) with codecs.open(map_file, 'w', encoding='utf-8') as fp: fp.write(content) os.remove(lck_file) return True return False
def rollback(relation, opts, _logger): logger = LogAdapter(_logger, {'package': 'rollback'}) map_file = os.path.expanduser(opts['map']) lck_file = os.path.splitext(map_file)[0] + '.lck' if os.path.isfile(lck_file): with codecs.open(lck_file, 'r', encoding='utf-8') as fp: content = fp.read() logger.debug(u'Rollback relation [%s]' % (relation)) with codecs.open(map_file, 'w', encoding='utf-8') as fp: fp.write(content) os.remove(lck_file) return True return False
def lock(relation, opts, _logger): logger = LogAdapter(_logger, {'package': 'lock'}) map_file = os.path.expanduser(opts['map']) lck_file = os.path.splitext(map_file)[0] + '.lck' if os.path.isfile(lck_file): logger.error('Relation [%s] locked. Remove %s to unlock synchronization' % (relation, lck_file)) return False if os.path.isfile(map_file): with codecs.open(map_file, 'r', encoding='utf-8') as fp: content = fp.read() logger.debug(u'Locking relation [%s]' % (relation)) with codecs.open(lck_file, 'w', encoding='utf-8') as fp: fp.write(content) else: open(lck_file, 'a').close() return True
def lock(relation, opts, _logger): logger = LogAdapter(_logger, {'package': 'lock'}) map_file = os.path.expanduser(opts['map']) lck_file = os.path.splitext(map_file)[0] + '.lck' if os.path.isfile(lck_file): logger.error( 'Relation [%s] locked. Remove %s to unlock synchronization' % (relation, lck_file)) return False if os.path.isfile(map_file): with codecs.open(map_file, 'r', encoding='utf-8') as fp: content = fp.read() logger.debug(u'Locking relation [%s]' % (relation)) with codecs.open(lck_file, 'w', encoding='utf-8') as fp: fp.write(content) else: open(lck_file, 'a').close() return True
class PimaticApp(object): def __init__(self, options): self._options = options self._pimatic = None self._args = None self._opts = None self._config = None self._logger = None self._node = {'url': None, 'username': None, 'password': None} self._parser = ArgumentParser(description='%s [PimaticAPI Rev. %s (c) %s]' % (self.app, __version__, __author__)) self._parser.add_argument('-c', '--config', type=str, help='use alternate configuration file(s)') self._parser.add_argument('-s', '--secrets', type=str, help='use alternate secrets file(s)') self._parser.add_argument('-l', '--loglevel', type=str, choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='debug log level') self._parser.add_argument('-n', '--node', type=str, help='pimatic node instance') self._parser.add_argument('-u', '--username', type=str, help='pimatic username') self._parser.add_argument('-p', '--password', type=str, help='pimatic password') def parse(self): self._args = self._parser.parse_args() self._opts = Options(self._options, self._args, '[config]', prefix='PIMATIC') self._config = self._opts.config_parser if self._config is None: LogAdapter(get_logger(), {'package': self.app}).critical("Missing configuration file!") exit(1) self._logger = LogAdapter(self._opts.logger, {'package': self.app}) # region Get node configuration and logger settings # set log level of requests module logging.getLogger('requests').setLevel(log_level(self._opts.loglevel_requests)) logging.getLogger('urllib3').setLevel(log_level(self._opts.loglevel_requests)) self._logger.debug(u'Parsing configuration file %s' % (self._opts.config_file)) if self._config.has_option('options', 'secrets'): secrets = self._config.get('options', 'secrets') path = os.path.expanduser(secrets) if os.path.isfile(path): secret = ConfigParser() secret.read(path) if self._opts.node: if self._opts.node.startswith('http'): self._node['url'] = self._opts.node else: node_section = 'node_' + self._opts.node if secret.has_section(node_section): protocol = secret.get(node_section, 'protocol') server = secret.get(node_section, 'server') port = secret.get(node_section, protocol + '_port') self._node['url']=protocol + '://' + server + ':' + port self._node['username'] = secret.get(node_section, 'username') self._node['password'] = secret.get(node_section, 'password') else: self._logger.critical(u'Invalid node tag %s' % (node_section)) exit(1) else: self._logger.critical(u'Missing pimatic node specification!') exit(1) if self._opts.username: self._node['username'] = self._opts.username if self._opts.password: self._node['password'] = self._opts.password # endregion self._logger.info(u'url: %s' % (self._node['url'])) def session(self): self._pimatic = PimaticAPI(server=self._node['url'], username=self._node['username'], password=self._node['password'], logger=self._logger) return self._pimatic @property def app(self): return self._options.get('app')
def main(): from argparse import ArgumentParser from pysync import __version__, __author__ from pysync import SyncError, SyncSessionError, SyncInitError options = { 'secrets': '~/.pysync.secrets', 'loglevel_requests': 'ERROR' # 'loglevel': 'INFO' } # region Command line arguments parser = ArgumentParser(description='PySnc Engine Rev. %s (c) %s' % (__version__, __author__)) parser.add_argument('-c', '--config', type=str, help='use alternate configuration file(s)') parser.add_argument('--relations', type=str, help='list of pysync relations to process') parser.add_argument('--rebuild', action='store_true', help='rebuild map file') parser.add_argument('--reset', type=str, help='delete entries and recreate from left/right') parser.add_argument('--update', type=str, help='force update on left/right side') parser.add_argument( '-l', '--loglevel', type=str, choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='debug log level') args = parser.parse_args() opts = Options(options, args, '[config]', prefix='PYSYNC') config = opts.config_parser if config is None: LogAdapter(get_logger(), { 'package': 'main' }).critical("Missing configuration file!") exit(1) logger = LogAdapter(opts.logger, {'package': 'main'}) # endregion # region Basic configuration and logger settings # set log level of requests module logging.getLogger('requests').setLevel(log_level(opts.loglevel_requests)) logging.getLogger('urllib3').setLevel(log_level(opts.loglevel_requests)) logger.debug(u'Parsing configuration file %s' % (opts.config_file)) if opts.relations: relations = list(opts.relations.split(',')) else: logger.critical('Missing relations!') exit(1) # endregion for relation in relations: try: left_opts, right_opts, relation_opts = parse_config( relation, config, opts.logger) except Exception as e: logger.exception( 'Error parsing configuration options. Skipping sync for [%s]' % (relation)) continue if lock(relation, relation_opts, opts.logger): # initialise web service sessions via @staticmethod session() # and initialize sync engine classes try: label = left_opts.get('label') session_lockfile = left_opts.get('session_lockfile', True) left_session = left_opts['class'].session( left_opts, opts.logger) label = right_opts.get('label') session_lockfile = right_opts.get('session_lockfile', True) right_session = right_opts['class'].session( right_opts, opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception( 'Session initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) if not session_lockfile: unlock(relation, relation_opts, opts.logger) continue # initialize sync map relation_opts['sync'] = {'map': None} if os.path.isfile(relation_opts['map']): #################### # incremental sync # #################### with codecs.open(relation_opts['map'], 'r', encoding='utf-8') as fp: relation_opts['sync'] = json.load(fp) logger.info(u'%s: starting incremental sync for %d items' % (relation, len(relation_opts['sync'].get('map')))) # merge signature from map file left_opts.update({'signature': relation_opts['sync']['left']}) right_opts.update( {'signature': relation_opts['sync']['right']}) try: engine_lockfile = left_opts.get('engine_lockfile', True) left = left_opts['class'](left_session, left_opts, logger=opts.logger) engine_lockfile = right_opts.get('engine_lockfile', True) right = right_opts['class'](right_session, right_opts, logger=opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception( 'Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) if not engine_lockfile: unlock(relation, relation_opts, opts.logger) continue if opts['update']: try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.update(opts['update']) except Exception as e: logger.exception( 'Unexpected error when processing update option! Skipping sync for [%s]' % relation) unlock(relation, relation_opts, opts.logger) continue if opts.reset: try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.reset(opts.reset) except Exception as e: logger.exception( 'Unexpected error when processing reset option!') check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) continue if opts.rebuild: relation_opts['sync'] = {'map': None} else: ################ # initial sync # ################ logger.info(u'%s: Starting initial sync' % (relation)) for opt in ['update', 'reset', 'rebuild']: if opts.get(opt): logger.warning( 'Ignoring option [%s] for initial sync' % (opt)) try: left = left_opts['class'](left_session, left_opts, logger=opts.logger) right = right_opts['class'](right_session, right_opts, logger=opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception( 'Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) # unlock(relation, relation_opts, opts.logger) continue try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.process() except Exception as e: logger.exception('Unexpected error when processing sync map!') check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) continue # check/modify sync map by backend engine relation_opts = left.commit_sync('left', relation_opts, logger) relation_opts = right.commit_sync('right', relation_opts, logger) count, errors = check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) unlock(relation, relation_opts, logger) logger.info(u'%s: %s %s' % (relation, left.label, left._changes)) logger.info(u'%s: %s %s' % (relation, right.label, right._changes)) logger.info(u'%s: finished %s sync for %d items with %d errors' % (relation, pysync.direction, count, errors)) left_opts['class'].end_session(logger) right_opts['class'].end_session(logger)
import os, sys # print sys.path import logging from pyutils import LogAdapter # stream handler configuration sh = logging.StreamHandler() sf = logging.Formatter("%(levelname)-8s %(module)s %(message)s") sh.setFormatter(sf) # fh = logging.FileHandler('sync.log', encoding='utf-8') # ff = logging.Formatter('%(asctime)s %(levelname)-8s %(module)s %(message)s') # fh.setFormatter(ff) # logging via LoggerAdapter root = logging.getLogger() root.setLevel(logging.DEBUG) root.addHandler(sh) # root.addHandler(fh) adapter = LogAdapter(root, {"package": "pyutils"}) adapter.debug("Log message") try: 1 / 0 except Exception as e: adapter.exception("Catched exception!")
def main(): from argparse import ArgumentParser from pysync import __version__, __author__ from pysync import SyncError, SyncSessionError, SyncInitError options = { 'secrets': '~/.pysync.secrets', 'loglevel_requests': 'ERROR' # 'loglevel': 'INFO' } # region Command line arguments parser = ArgumentParser(description='PySnc Engine Rev. %s (c) %s' % (__version__, __author__)) parser.add_argument('-c', '--config', type=str, help='use alternate configuration file(s)') parser.add_argument('--relations', type=str, help='list of pysync relations to process') parser.add_argument('--rebuild', action='store_true', help='rebuild map file') parser.add_argument('--reset', type=str, help='delete entries and recreate from left/right') parser.add_argument('--update', type=str, help='force update on left/right side') parser.add_argument('-l', '--loglevel', type=str, choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help='debug log level') args = parser.parse_args() opts = Options(options, args, '[config]', prefix='PYSYNC') config = opts.config_parser if config is None: LogAdapter(get_logger(), {'package': 'main'}).critical("Missing configuration file!") exit(1) logger = LogAdapter(opts.logger, {'package': 'main'}) # endregion # region Basic configuration and logger settings # set log level of requests module logging.getLogger('requests').setLevel(log_level(opts.loglevel_requests)) logging.getLogger('urllib3').setLevel(log_level(opts.loglevel_requests)) logger.debug(u'Parsing configuration file %s' % (opts.config_file)) if opts.relations: relations = list(opts.relations.split(',')) else: logger.critical('Missing relations!') exit(1) # endregion for relation in relations: try: left_opts, right_opts, relation_opts = parse_config(relation, config, opts.logger) except Exception as e: logger.exception('Error parsing configuration options. Skipping sync for [%s]' % (relation)) continue if lock(relation, relation_opts, opts.logger): # initialise web service sessions via @staticmethod session() # and initialize sync engine classes try: label = left_opts.get('label') session_lockfile = left_opts.get('session_lockfile', True) left_session = left_opts['class'].session(left_opts, opts.logger) label = right_opts.get('label') session_lockfile = right_opts.get('session_lockfile', True) right_session = right_opts['class'].session(right_opts, opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception('Session initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) if not session_lockfile: unlock(relation, relation_opts, opts.logger) continue # initialize sync map relation_opts['sync'] = {'map': None} if os.path.isfile(relation_opts['map']): #################### # incremental sync # #################### with codecs.open(relation_opts['map'], 'r', encoding='utf-8') as fp: relation_opts['sync'] = json.load(fp) logger.info(u'%s: starting incremental sync for %d items' % (relation, len(relation_opts['sync'].get('map')))) # merge signature from map file left_opts.update({'signature': relation_opts['sync']['left']}) right_opts.update({'signature': relation_opts['sync']['right']}) try: engine_lockfile = left_opts.get('engine_lockfile', True) left = left_opts['class'](left_session, left_opts, logger=opts.logger) engine_lockfile = right_opts.get('engine_lockfile', True) right = right_opts['class'](right_session, right_opts, logger=opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception('Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) if not engine_lockfile: unlock(relation, relation_opts, opts.logger) continue if opts['update']: try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.update(opts['update']) except Exception as e: logger.exception('Unexpected error when processing update option! Skipping sync for [%s]' % relation) unlock(relation, relation_opts, opts.logger) continue if opts.reset: try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.reset(opts.reset) except Exception as e: logger.exception('Unexpected error when processing reset option!') check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) continue if opts.rebuild: relation_opts['sync'] = {'map': None} else: ################ # initial sync # ################ logger.info(u'%s: Starting initial sync' % (relation)) for opt in ['update', 'reset', 'rebuild']: if opts.get(opt): logger.warning('Ignoring option [%s] for initial sync' % (opt)) try: left = left_opts['class'](left_session, left_opts, logger=opts.logger) right = right_opts['class'](right_session, right_opts, logger=opts.logger) except Exception as e: # TODO: check exception type, unlock() only in case of an temp. network error etc. logger.exception('Engine initialization for [%s] failed! Skipping sync for [%s]' % (label, relation)) # unlock(relation, relation_opts, opts.logger) continue try: pysync = PySync(left, right, relation_opts, opts.logger) relation_opts['sync'] = pysync.process() except Exception as e: logger.exception('Unexpected error when processing sync map!') check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) continue # check/modify sync map by backend engine relation_opts = left.commit_sync('left', relation_opts, logger) relation_opts = right.commit_sync('right', relation_opts, logger) count, errors = check_sync_map(relation, pysync.direction, left, right, relation_opts, logger) unlock(relation, relation_opts, logger) logger.info(u'%s: %s %s' % (relation, left.label, left._changes)) logger.info(u'%s: %s %s' % (relation, right.label, right._changes)) logger.info(u'%s: finished %s sync for %d items with %d errors' % (relation, pysync.direction, count, errors)) left_opts['class'].end_session(logger) right_opts['class'].end_session(logger)
fh.setFormatter(ff) # logging via LoggerAdapter root = logging.getLogger() root.setLevel(logging.INFO) root.addHandler(sh) root.addHandler(fh) logger = LogAdapter(root, {'package': 'root'}) # set log level of requests module requests = logging.getLogger('requests') requests.setLevel(logging.ERROR) #logger.setLevel(logging.DEBUG) logger.debug("Logging initialized ...") from private import en_dev_token, ox_server, ox_user, ox_password ox = OxHttpAPI.get_session(server=ox_server, user=ox_user, password=ox_password, logger=root) en = EnClient.get_client(token=en_dev_token, logger=root) #en = {'token': os.environ.get('EVERNOTE_TOKEN'), 'sandbox': False} right = {'class': OxTaskSync, 'session': ox, 'options': { 'folder': 'OxSync', 'key': 'title'}} left = { 'class': EnClientSync, 'session': en, 'options': {'notebook': 'OxSync', 'key': 'title'}} sync_file = 'sync_map.json' sync_map = None if os.path.isfile('sync_map.json'): with codecs.open(sync_file, 'r', encoding='utf-8') as fp: