def run(self): """ Start the thread """ very_first = False self.start_watcher() while not self.interrupt: try: # logging.info('Starting cycle with cycles local %i and remote %is' % (self.local_seq, self.remote_seq)) self.processing_signals = {} self.init_global_progress() if very_first: self.global_progress['status_indexing'] = 1 interval = int(time.time() - self.last_run) if (self.online_status and interval < self.online_timer) or (not self.online_status and interval < self.offline_timer): time.sleep(self.event_timer) continue if not self.job_status_running: logging.debug("self.online_timer: %s" % self.online_timer) self.logger.log_state(_('Status: Paused'), "sync") self.sleep_offline() continue if self.job_config.frequency == 'time': start_time = datetime.time(int(self.job_config.start_time['h']), int(self.job_config.start_time['m'])) end_time = datetime.time(int(self.job_config.start_time['h']), int(self.job_config.start_time['m']), 59) now = datetime.datetime.now().time() if not start_time < now < end_time: self.logger.log_state(_('Status: scheduled for %s') % str(start_time), "sync") self.sleep_offline() continue else: logging.info("Now triggering synchro as expected at time " + str(start_time)) if not self.system.check_basepath(): log = _('Cannot find local folder! Did you disconnect a volume? Waiting %s seconds before retry') % self.offline_timer logging.error(log) self.logger.log_state(_('Cannot find local folder, did you disconnect a volume?'), "error") self.sleep_offline() continue # Before starting infinite loop, small check that remote folder still exists if not self.sdk.check_basepath(): # if it doesn't try to create it self.sdk.remote_folder = os.path.join("/", self.sdk.remote_folder) try: logging.info("Creating remote directory.") self.sdk.mkdir("") continue except Exception as e: logging.exception(e) if not self.sdk.check_basepath(): log = _('Cannot find remote folder, maybe it was renamed? Sync cannot start, please check the configuration.') logging.error(log) self.logger.log_state(log, 'error') self.sleep_offline() continue if self.watcher: for snap_path in self.marked_for_snapshot_pathes: logging.info('LOCAL SNAPSHOT : loading snapshot for directory %s' % snap_path) if self.interrupt or not self.job_status_running: raise InterruptException() self.watcher.check_from_snapshot(snap_path) self.marked_for_snapshot_pathes = [] writewait = .5 # To avoid reading events before they're written (db lock) wait for writing to finish while self.event_handler.locked: logging.info("Waiting for changes to be written before retrieving remote changes.") if writewait < 5: writewait += .5 time.sleep(writewait) # Load local and/or remote changes, depending on the direction self.current_store = SqliteChangeStore(self.configs_path + '/changes.sqlite', self.job_config.filters['includes'], self.job_config.filters['excludes'], self.job_config.poolsize, local_sdk=self.system, remote_sdk=self.sdk, job_config=self.job_config) self.current_store.open() try: if self.job_config.direction != 'up': logging.info( 'Loading remote changes with sequence {0:s} for job id {1:s}'.format(str(self.remote_seq), str(self.job_config.id))) if self.remote_seq == 0: self.logger.log_state(_('Gathering data from remote workspace, this can take a while...'), 'sync') very_first = True self.remote_target_seq = self.load_remote_changes_in_store(self.remote_seq, self.current_store) self.current_store.sync() else: self.remote_target_seq = 1 self.ping_remote() except RequestException as ce: logging.exception(ce) if not connection_helper.is_connected_to_internet(self.sdk.proxies): error = _('No Internet connection detected! Waiting for %s seconds to retry') % self.offline_timer else: error = _('Connection to server failed, server is probably down. Waiting %s seconds to retry') % self.offline_timer self.marked_for_snapshot_pathes = [] logging.error(error) self.logger.log_state(error, "wait") self.sleep_offline() continue except Exception as e: error = 'Error while connecting to remote server (%s), waiting for %i seconds before retempting ' % (e.message, self.offline_timer) logging.exception(e) self.logger.log_state(_('Error while connecting to remote server (%s)') % e.message, "error") self.marked_for_snapshot_pathes = [] self.sleep_offline() continue self.online_status = True if not self.job_config.server_configs: self.job_config.server_configs = self.sdk.load_server_configs() self.sdk.set_server_configs(self.job_config.server_configs) if self.job_config.direction != 'down' or (self.job_config.direction == 'down' and self.job_config.solve != 'remote'): logging.info( 'Loading local changes with sequence {0:s} for job id {1:s}'.format(str(self.local_seq), str(self.job_config.id))) self.local_target_seq = self.db_handler.get_local_changes_as_stream(self.local_seq, self.current_store.flatten_and_store) self.current_store.sync() else: self.local_target_seq = 1 if not connection_helper.internet_ok: connection_helper.is_connected_to_internet(self.sdk.proxies) changes_length = len(self.current_store) if not changes_length: self.processing = False logging.info('No changes detected in ' + self.job_config.id) self.update_min_seqs_from_store() self.exit_loop_clean(self.logger) very_first = False #logging.info("CheckSync of " + self.job_config.id) #self.db_handler.list_non_idle_nodes() if not self.watcher.isAlive() and not self.interrupt: logging.info("File watcher died, restarting...") self.watcher.stop() self.watcher = LocalWatcher(self.job_config.directory, self.configs_path, event_handler=self.event_handler) self.start_watcher() continue self.global_progress['status_indexing'] = 1 logging.info('Reducing changes for ' + self.job_config.id) self.logger.log_state(_('Merging changes between remote and local, please wait...'), 'sync') # We are updating the status to IDLE here for the nodes which has status as NEW # The reason is when we create a new sync on the existing folder, some of the files might # already be synchronized and we ignore those files while we Dedup changes and those files # remain untouched later. # So the flow of node status change will occur as follows # NEW (as soon as we create a new sync task) # | # IDLE (here, just before we reduce changes) # | # PENDING (those files/folders which remain after reducing changes and to be actually processed) # | # UP / DOWN / CONFLICT (corresponding the operation which occurs) # | # IDLE (The final state once upload/ download happens or once when the conflict is resolved) self.db_handler.update_bulk_node_status_as_idle() logging.debug('[CMERGER] Delete Copies ' + self.job_config.id) self.current_store.delete_copies() self.update_min_seqs_from_store() logging.debug('[CMERGER] Dedup changes ' + self.job_config.id) self.current_store.dedup_changes() self.update_min_seqs_from_store() if not self.storage_watcher or very_first: logging.debug('[CMERGER] Detect unnecessary changes ' + self.ws_id) self.logger.log_state(_('Detecting unecessary changes...'), 'sync') self.current_store.detect_unnecessary_changes() logging.debug('[CMERGER] Done detecting unnecessary changes') self.logger.log_state(_('Done detecting unecessary changes...'), 'sync') self.update_min_seqs_from_store() logging.debug('Clearing op and pruning folders moves ' + self.job_config.id) self.current_store.clear_operations_buffer() self.current_store.prune_folders_moves() self.update_min_seqs_from_store() logging.debug('Store conflicts ' + self.job_config.id) store_conflicts = self.current_store.clean_and_detect_conflicts(self.db_handler) if store_conflicts: if self.job_config.solve == 'both': logging.info('Marking nodes SOLVED:KEEPBOTH') for row in self.db_handler.list_conflict_nodes(): self.db_handler.update_node_status(row['node_path'], 'SOLVED:KEEPBOTH') store_conflicts = self.current_store.clean_and_detect_conflicts(self.db_handler) if store_conflicts: logging.info('Conflicts detected, cannot continue!') self.logger.log_state(_('Conflicts detected, cannot continue!'), 'error') self.current_store.close() self.sleep_offline() self.logger.log_notif(_('Conflicts detected, cannot continue!'), 'error') continue if self.job_config.direction == 'down' and self.job_config.solve != 'remote': self.current_store.remove_based_on_location('local') self.update_min_seqs_from_store() changes_length = len(self.current_store) if not changes_length: logging.info('No changes detected for ' + self.job_config.id) self.exit_loop_clean(self.logger) very_first = False continue self.current_store.update_pending_status(self.db_handler, self.local_seq) self.global_progress['status_indexing'] = 0 import change_processor self.global_progress['queue_length'] = changes_length logging.info('Processing %i changes' % changes_length) self.logger.log_state(_('Processing %i changes') % changes_length, "start") counter = [1] def processor_callback(change): try: if self.interrupt or not self.job_status_running: raise InterruptException() self.update_current_tasks() self.update_global_progress() Processor = StorageChangeProcessor if self.storage_watcher else ChangeProcessor proc = Processor(change, self.current_store, self.job_config, self.system, self.sdk, self.db_handler, self.event_logger) proc.process_change() self.update_min_seqs_from_store(success=True) self.global_progress['queue_done'] = float(counter[0]) counter[0] += 1 self.update_current_tasks() self.update_global_progress() time.sleep(0.1) if self.interrupt or not self.job_status_running: raise InterruptException() except ProcessException as pe: logging.error(pe.message) return False except InterruptException as i: raise i except PydioSdkDefaultException as p: raise p except Exception as ex: logging.exception(ex) return False return True def processor_callback2(change): try: if self.interrupt or not self.job_status_running: raise InterruptException() Processor = StorageChangeProcessor if self.storage_watcher else ChangeProcessor proc = Processor(change, self.current_store, self.job_config, self.system, self.sdk, self.db_handler, self.event_logger) proc.process_change() if self.interrupt or not self.job_status_running: raise InterruptException() except PydioSdkException as pe: if pe.message.find("Original file") > -1: pe.code = 1404 raise pe except ProcessException as pe: logging.error(pe.message) return False except PydioSdkDefaultException as p: raise p except InterruptException as i: raise i return True try: if sys.platform.startswith('win'): self.marked_for_snapshot_pathes = list(set(self.current_store.find_modified_parents()) - set(self.marked_for_snapshot_pathes)) if not self.processing: self.processing = True for i in self.current_store.process_changes_with_callback(processor_callback, processor_callback2): if self.interrupt: raise InterruptException #logging.info("Updating seqs") self.current_store.process_pending_changes() self.update_min_seqs_from_store(success=True) self.global_progress['queue_done'] = float(counter[0]) counter[0] += 1 self.update_current_tasks() self.update_global_progress() time.sleep(0.05) # Allow for changes to be noticeable in UI time.sleep(.5) self.current_store.process_pending_changes() self.update_min_seqs_from_store(success=True) self.update_current_tasks() self.update_global_progress() #logging.info("DONE WITH CHANGES") self.processing = False except InterruptException as iexc: pass self.logger.log_state(_('%i files modified') % self.global_progress['queue_done'], 'success') if self.global_progress['queue_done']: self.logger.log_notif(_('%i files modified') % self.global_progress['queue_done'], 'success') self.exit_loop_clean(self.logger) except PydioSdkDefaultException as re: logging.error(re.message) self.logger.log_state(re.message, 'error') except SSLError as rt: logging.error(rt.message) self.logger.log_state(_('An SSL error happened, please check the logs'), 'error') except ProxyError as rt: logging.error(rt.message) self.logger.log_state(_('A proxy error happened, please check the logs'), 'error') except TooManyRedirects as rt: logging.error(rt.message) self.logger.log_state(_('Connection error: too many redirects'), 'error') except ChunkedEncodingError as rt: logging.error(rt.message) self.logger.log_state(_('Chunked encoding error, please check the logs'), 'error') except ContentDecodingError as rt: logging.error(rt.message) self.logger.log_state(_('Content Decoding error, please check the logs'), 'error') except InvalidSchema as rt: logging.error(rt.message) self.logger.log_state(_('Http connection error: invalid schema.'), 'error') except InvalidURL as rt: logging.error(rt.message) self.logger.log_state(_('Http connection error: invalid URL.'), 'error') except Timeout as to: logging.error(to) self.logger.log_state(_('Connection timeout, will retry later.'), 'error') except RequestException as ree: logging.error(ree.message) self.logger.log_state(_('Cannot resolve domain!'), 'error') except Exception as e: if not (e.message.lower().count('[quota limit reached]') or e.message.lower().count('[file permissions]')): logging.exception('Unexpected Error: %s' % e.message) self.logger.log_state(_('Unexpected Error: %s') % e.message, 'error') else: logging.exception(e) self.sleep_offline() logging.debug('Finished this cycle, waiting for %i seconds' % self.online_timer) very_first = False
def run(self): """ Start the thread """ logger = EventLogger(self.configs_path) very_first = False if self.watcher: if self.watcher_first_run: def status_callback(status): logger.log_state(status, 'sync') self.init_global_progress() try: self.global_progress['status_indexing'] = 1 logger.log_state(_('Checking changes since last launch...'), "sync") very_first = True self.watcher.check_from_snapshot(state_callback=status_callback) except DBCorruptedException as e: self.stop() JobsLoader.Instance().clear_job_data(self.job_config.id) logging.error(e) return except Exception as e: self.interrupt = True logger.log_state(_('Oops, error while indexing the local folder. Pausing the task.'), 'error') logging.error(e) self.watcher_first_run = False self.watcher.start() while not self.interrupt: try: # logging.info('Starting cycle with cycles local %i and remote %is' % (self.local_seq, self.remote_seq)) self.processing_signals = {} self.init_global_progress() if very_first: self.global_progress['status_indexing'] = 1 interval = int(time.time() - self.last_run) if (self.online_status and interval < self.online_timer) or (not self.online_status and interval < self.offline_timer): time.sleep(self.event_timer) continue if not self.job_status_running: logging.debug("self.online_timer: %s" % self.online_timer) logger.log_state(_('Status: Paused'), "sync") self.sleep_offline() continue if self.job_config.frequency == 'time': start_time = datetime.time(int(self.job_config.start_time['h']), int(self.job_config.start_time['m'])) end_time = datetime.time(int(self.job_config.start_time['h']), int(self.job_config.start_time['m']), 59) now = datetime.datetime.now().time() if not start_time < now < end_time: logger.log_state(_('Status: scheduled for %s') % str(start_time), "sync") self.sleep_offline() continue else: logging.info("Now triggering synchro as expected at time " + str(start_time)) if not self.system.check_basepath(): log = _('Cannot find local folder! Did you disconnect a volume? Waiting %s seconds before retry') % self.offline_timer logging.error(log) logger.log_state(_('Cannot find local folder, did you disconnect a volume?'), "error") self.sleep_offline() continue if self.watcher: for snap_path in self.marked_for_snapshot_pathes: logging.info('LOCAL SNAPSHOT : loading snapshot for directory %s' % snap_path) if self.interrupt or not self.job_status_running: raise InterruptException() self.watcher.check_from_snapshot(snap_path) # Load local and/or remote changes, depending on the direction from pydio.job.change_stores import SqliteChangeStore self.current_store = SqliteChangeStore(self.configs_path + '/changes.sqlite', self.job_config.filters['includes'], self.job_config.filters['excludes']) self.current_store.open() try: if self.job_config.direction != 'up': logging.info('Loading remote changes with sequence %s' % str(self.remote_seq)) if self.remote_seq == 0: logger.log_state(_('Gathering data from remote workspace, this can take a while...'), 'sync') very_first = True self.remote_target_seq = self.load_remote_changes_in_store(self.remote_seq, self.current_store) self.current_store.sync() else: self.remote_target_seq = 1 self.ping_remote() except RequestException as ce: if not connection_helper.is_connected_to_internet(): error = _('No Internet connection detected! Waiting for %s seconds to retry') % self.offline_timer else: error = _('Connection to server failed, server is probably down. Waiting %s seconds to retry') % self.offline_timer self.marked_for_snapshot_pathes = [] logging.error(error) logger.log_state(error, "wait") self.sleep_offline() continue except Exception as e: error = 'Error while connecting to remote server (%s), waiting for %i seconds before retempting ' % (e.message, self.offline_timer) logging.error(error) logger.log_state(_('Error while connecting to remote server (%s)') % e.message, "error") self.marked_for_snapshot_pathes = [] self.sleep_offline() continue self.online_status = True if not self.job_config.server_configs: self.job_config.server_configs = self.sdk.load_server_configs() self.sdk.set_server_configs(self.job_config.server_configs) if self.job_config.direction != 'down': logging.info('Loading local changes with sequence ' + str(self.local_seq)) self.local_target_seq = self.db_handler.get_local_changes_as_stream(self.local_seq, self.current_store.flatten_and_store) self.current_store.sync() else: self.local_target_seq = 1 if not connection_helper.internet_ok: connection_helper.is_connected_to_internet() changes_length = len(self.current_store) if not changes_length: logging.info('No changes detected') self.update_min_seqs_from_store() self.exit_loop_clean(logger) very_first = False continue self.global_progress['status_indexing'] = 1 logging.info('Reducing changes') logger.log_state(_('Merging changes between remote and local, please wait...'), 'sync') logging.debug('Delete Copies') self.current_store.delete_copies() self.update_min_seqs_from_store() logging.debug('Dedup changes') self.current_store.dedup_changes() self.update_min_seqs_from_store() if not self.storage_watcher or very_first: logging.debug('Detect unnecessary changes') self.current_store.detect_unnecessary_changes(local_sdk=self.system, remote_sdk=self.sdk) self.update_min_seqs_from_store() logging.debug('Clearing op and pruning folders moves') self.current_store.clear_operations_buffer() self.current_store.prune_folders_moves() self.update_min_seqs_from_store() logging.debug('Store conflicts') store_conflicts = self.current_store.clean_and_detect_conflicts(self.db_handler) if store_conflicts: logging.info('Conflicts detected, cannot continue!') logger.log_state(_('Conflicts detected, cannot continue!'), 'error') self.current_store.close() self.sleep_offline() continue changes_length = len(self.current_store) if not changes_length: logging.info('No changes detected') self.exit_loop_clean(logger) very_first = False continue self.global_progress['status_indexing'] = 0 import change_processor self.global_progress['queue_length'] = changes_length logging.info('Processing %i changes' % changes_length) logger.log_state(_('Processing %i changes') % changes_length, "start") counter = [1] def processor_callback(change): try: if self.interrupt or not self.job_status_running: raise InterruptException() self.update_current_tasks() self.update_global_progress() Processor = StorageChangeProcessor if self.storage_watcher else ChangeProcessor proc = Processor(change, self.current_store, self.job_config, self.system, self.sdk, self.db_handler, self.event_logger) proc.process_change() self.update_min_seqs_from_store(success=True) self.global_progress['queue_done'] = float(counter[0]) counter[0] += 1 self.update_current_tasks() self.update_global_progress() time.sleep(0.1) if self.interrupt or not self.job_status_running: raise InterruptException() except ProcessException as pe: logging.error(pe.message) return False except InterruptException as i: raise i except PydioSdkDefaultException as p: raise p except Exception as ex: logging.exception(ex.message) return False return True try: if sys.platform.startswith('win'): self.marked_for_snapshot_pathes = list(set(self.current_store.find_modified_parents()) - set(self.marked_for_snapshot_pathes)) self.current_store.process_changes_with_callback(processor_callback) except InterruptException as iexc: pass logger.log_state(_('%i files modified') % self.global_progress['queue_done'], 'success') if self.global_progress['queue_done']: logger.log_notif(_('%i files modified') % self.global_progress['queue_done'], 'success') self.exit_loop_clean(logger) except PydioSdkDefaultException as re: logging.error(re.message) logger.log_state(re.message, 'error') except SSLError as rt: logging.error(rt.message) logger.log_state(_('An SSL error happened, please check the logs'), 'error') except ProxyError as rt: logging.error(rt.message) logger.log_state(_('A proxy error happened, please check the logs'), 'error') except TooManyRedirects as rt: logging.error(rt.message) logger.log_state(_('Connection error: too many redirects'), 'error') except ChunkedEncodingError as rt: logging.error(rt.message) logger.log_state(_('Chunked encoding error, please check the logs'), 'error') except ContentDecodingError as rt: logging.error(rt.message) logger.log_state(_('Content Decoding error, please check the logs'), 'error') except InvalidSchema as rt: logging.error(rt.message) logger.log_state(_('Http connection error: invalid schema.'), 'error') except InvalidURL as rt: logging.error(rt.message) logger.log_state(_('Http connection error: invalid URL.'), 'error') except Timeout as to: logging.error(to) logger.log_state(_('Connection timeout, will retry later.'), 'error') except RequestException as ree: logging.error(ree.message) logger.log_state(_('Cannot resolve domain!'), 'error') except Exception as e: if not (e.message.lower().count('[quota limit reached]') or e.message.lower().count('[file permissions]')): logging.exception('Unexpected Error: %s' % e.message) logger.log_state(_('Unexpected Error: %s') % e.message, 'error') logging.debug('Finished this cycle, waiting for %i seconds' % self.online_timer) very_first = False
def run(self): """ Start the thread """ logger = EventLogger(self.configs_path) very_first = False if self.watcher: if self.watcher_first_run: def status_callback(status): logger.log_state(status, 'sync') self.init_global_progress() try: self.global_progress['status_indexing'] = 1 logger.log_state( _('Checking changes since last launch...'), "sync") very_first = True self.watcher.check_from_snapshot( state_callback=status_callback) except DBCorruptedException as e: self.stop() JobsLoader.Instance().clear_job_data(self.job_config.id) logging.error(e) return except Exception as e: self.interrupt = True logger.log_state( _('Oops, error while indexing the local folder. Pausing the task.' ), 'error') logging.error(e) self.watcher_first_run = False self.watcher.start() while not self.interrupt: try: # logging.info('Starting cycle with cycles local %i and remote %is' % (self.local_seq, self.remote_seq)) self.processing_signals = {} self.init_global_progress() if very_first: self.global_progress['status_indexing'] = 1 interval = int(time.time() - self.last_run) if (self.online_status and interval < self.online_timer) or ( not self.online_status and interval < self.offline_timer): time.sleep(self.event_timer) continue if not self.job_status_running: logging.debug("self.online_timer: %s" % self.online_timer) logger.log_state(_('Status: Paused'), "sync") self.sleep_offline() continue if self.job_config.frequency == 'time': start_time = datetime.time( int(self.job_config.start_time['h']), int(self.job_config.start_time['m'])) end_time = datetime.time( int(self.job_config.start_time['h']), int(self.job_config.start_time['m']), 59) now = datetime.datetime.now().time() if not start_time < now < end_time: logger.log_state( _('Status: scheduled for %s') % str(start_time), "sync") self.sleep_offline() continue else: logging.info( "Now triggering synchro as expected at time " + str(start_time)) if not self.system.check_basepath(): log = _( 'Cannot find local folder! Did you disconnect a volume? Waiting %s seconds before retry' ) % self.offline_timer logging.error(log) logger.log_state( _('Cannot find local folder, did you disconnect a volume?' ), "error") self.sleep_offline() continue if self.watcher: for snap_path in self.marked_for_snapshot_pathes: logging.info( 'LOCAL SNAPSHOT : loading snapshot for directory %s' % snap_path) if self.interrupt or not self.job_status_running: raise InterruptException() self.watcher.check_from_snapshot(snap_path) # Load local and/or remote changes, depending on the direction from pydio.job.change_stores import SqliteChangeStore self.current_store = SqliteChangeStore( self.configs_path + '/changes.sqlite', self.job_config.filters['includes'], self.job_config.filters['excludes']) self.current_store.open() try: if self.job_config.direction != 'up': logging.info( 'Loading remote changes with sequence %s' % str(self.remote_seq)) if self.remote_seq == 0: logger.log_state( _('Gathering data from remote workspace, this can take a while...' ), 'sync') very_first = True self.remote_target_seq = self.load_remote_changes_in_store( self.remote_seq, self.current_store) self.current_store.sync() else: self.remote_target_seq = 1 self.ping_remote() except RequestException as ce: if not connection_helper.is_connected_to_internet(): error = _( 'No Internet connection detected! Waiting for %s seconds to retry' ) % self.offline_timer else: error = _( 'Connection to server failed, server is probably down. Waiting %s seconds to retry' ) % self.offline_timer self.marked_for_snapshot_pathes = [] logging.error(error) logger.log_state(error, "wait") self.sleep_offline() continue except Exception as e: error = 'Error while connecting to remote server (%s), waiting for %i seconds before retempting ' % ( e.message, self.offline_timer) logging.error(error) logger.log_state( _('Error while connecting to remote server (%s)') % e.message, "error") self.marked_for_snapshot_pathes = [] self.sleep_offline() continue self.online_status = True if not self.job_config.server_configs: self.job_config.server_configs = self.sdk.load_server_configs( ) self.sdk.set_server_configs(self.job_config.server_configs) if self.job_config.direction != 'down': logging.info('Loading local changes with sequence ' + str(self.local_seq)) self.local_target_seq = self.db_handler.get_local_changes_as_stream( self.local_seq, self.current_store.flatten_and_store) self.current_store.sync() else: self.local_target_seq = 1 if not connection_helper.internet_ok: connection_helper.is_connected_to_internet() changes_length = len(self.current_store) if not changes_length: logging.info('No changes detected') self.update_min_seqs_from_store() self.exit_loop_clean(logger) very_first = False continue self.global_progress['status_indexing'] = 1 logging.info('Reducing changes') logger.log_state( _('Merging changes between remote and local, please wait...' ), 'sync') logging.debug('Delete Copies') self.current_store.delete_copies() self.update_min_seqs_from_store() logging.debug('Dedup changes') self.current_store.dedup_changes() self.update_min_seqs_from_store() if not self.storage_watcher or very_first: logging.debug('Detect unnecessary changes') self.current_store.detect_unnecessary_changes( local_sdk=self.system, remote_sdk=self.sdk) self.update_min_seqs_from_store() logging.debug('Clearing op and pruning folders moves') self.current_store.clear_operations_buffer() self.current_store.prune_folders_moves() self.update_min_seqs_from_store() logging.debug('Store conflicts') store_conflicts = self.current_store.clean_and_detect_conflicts( self.db_handler) if store_conflicts: logging.info('Conflicts detected, cannot continue!') logger.log_state(_('Conflicts detected, cannot continue!'), 'error') self.current_store.close() self.sleep_offline() continue changes_length = len(self.current_store) if not changes_length: logging.info('No changes detected') self.exit_loop_clean(logger) very_first = False continue self.global_progress['status_indexing'] = 0 import change_processor self.global_progress['queue_length'] = changes_length logging.info('Processing %i changes' % changes_length) logger.log_state( _('Processing %i changes') % changes_length, "start") counter = [1] def processor_callback(change): try: if self.interrupt or not self.job_status_running: raise InterruptException() self.update_current_tasks() self.update_global_progress() Processor = StorageChangeProcessor if self.storage_watcher else ChangeProcessor proc = Processor(change, self.current_store, self.job_config, self.system, self.sdk, self.db_handler, self.event_logger) proc.process_change() self.update_min_seqs_from_store(success=True) self.global_progress['queue_done'] = float(counter[0]) counter[0] += 1 self.update_current_tasks() self.update_global_progress() time.sleep(0.1) if self.interrupt or not self.job_status_running: raise InterruptException() except ProcessException as pe: logging.error(pe.message) return False except InterruptException as i: raise i except PydioSdkDefaultException as p: raise p except Exception as ex: logging.exception(ex.message) return False return True try: if sys.platform.startswith('win'): self.marked_for_snapshot_pathes = list( set(self.current_store.find_modified_parents()) - set(self.marked_for_snapshot_pathes)) self.current_store.process_changes_with_callback( processor_callback) except InterruptException as iexc: pass logger.log_state( _('%i files modified') % self.global_progress['queue_done'], 'success') if self.global_progress['queue_done']: logger.log_notif( _('%i files modified') % self.global_progress['queue_done'], 'success') except PydioSdkDefaultException as re: logging.error(re.message) logger.log_state(re.message, 'error') except SSLError as rt: logging.error(rt.message) logger.log_state( _('An SSL error happened, please check the logs'), 'error') except ProxyError as rt: logging.error(rt.message) logger.log_state( _('A proxy error happened, please check the logs'), 'error') except TooManyRedirects as rt: logging.error(rt.message) logger.log_state(_('Connection error: too many redirects'), 'error') except ChunkedEncodingError as rt: logging.error(rt.message) logger.log_state( _('Chunked encoding error, please check the logs'), 'error') except ContentDecodingError as rt: logging.error(rt.message) logger.log_state( _('Content Decoding error, please check the logs'), 'error') except InvalidSchema as rt: logging.error(rt.message) logger.log_state(_('Http connection error: invalid schema.'), 'error') except InvalidURL as rt: logging.error(rt.message) logger.log_state(_('Http connection error: invalid URL.'), 'error') except Timeout as to: logging.error(to) logger.log_state(_('Connection timeout, will retry later.'), 'error') except RequestException as ree: logging.error(ree.message) logger.log_state(_('Cannot resolve domain!'), 'error') except Exception as e: if not (e.message.lower().count('[quota limit reached]') or e.message.lower().count('[file permissions]')): logging.exception('Unexpected Error: %s' % e.message) logger.log_state( _('Unexpected Error: %s') % e.message, 'error') logging.debug('Finished this cycle, waiting for %i seconds' % self.online_timer) self.exit_loop_clean(logger) very_first = False
def run(self): """ Start the thread """ very_first = False self.start_watcher() while not self.interrupt: try: # logging.info('Starting cycle with cycles local %i and remote %is' % (self.local_seq, self.remote_seq)) self.processing_signals = {} self.init_global_progress() if very_first: self.global_progress['status_indexing'] = 1 interval = int(time.time() - self.last_run) if (self.online_status and interval < self.online_timer) or ( not self.online_status and interval < self.offline_timer): time.sleep(self.event_timer) continue if not self.job_status_running: logging.debug("self.online_timer: %s" % self.online_timer) self.logger.log_state(_('Status: Paused'), "sync") self.sleep_offline() continue if self.job_config.frequency == 'time': start_time = datetime.time( int(self.job_config.start_time['h']), int(self.job_config.start_time['m'])) end_time = datetime.time( int(self.job_config.start_time['h']), int(self.job_config.start_time['m']), 59) now = datetime.datetime.now().time() if not start_time < now < end_time: self.logger.log_state( _('Status: scheduled for %s') % str(start_time), "sync") self.sleep_offline() continue else: logging.info( "Now triggering synchro as expected at time " + str(start_time)) if not self.system.check_basepath(): log = _( 'Cannot find local folder! Did you disconnect a volume? Waiting %s seconds before retry' ) % self.offline_timer logging.error(log) self.logger.log_state( _('Cannot find local folder, did you disconnect a volume?' ), "error") self.sleep_offline() continue # Before starting infinite loop, small check that remote folder still exists if not self.sdk.check_basepath(): # if it doesn't try to create it self.sdk.remote_folder = os.path.join( "/", self.sdk.remote_folder) try: logging.info("Creating remote directory.") self.sdk.mkdir("") continue except Exception as e: logging.exception(e) self.sleep_offline() if not self.sdk.check_basepath(): log = _( 'Cannot find remote folder, maybe it was renamed? Sync cannot start, please check the configuration.' ) logging.error(log) self.logger.log_state(log, 'error') self.sleep_offline() continue if self.watcher: for snap_path in self.marked_for_snapshot_pathes: logging.info( 'LOCAL SNAPSHOT : loading snapshot for directory %s' % snap_path) if self.interrupt or not self.job_status_running: raise InterruptException() self.watcher.check_from_snapshot(snap_path, use_transaction=False) self.marked_for_snapshot_pathes = [] writewait = .5 # To avoid reading events before they're written (db lock) wait for writing to finish totalwait = 0 while self.event_handler and self.event_handler.locked and totalwait < 60 * 3: logging.info( "Waiting for changes to be written before retrieving remote changes." ) if writewait < 5: writewait += .5 time.sleep(writewait) totalwait += writewait # Load local and/or remote changes, depending on the direction self.current_store = SqliteChangeStore( self.configs_path + '/changes.sqlite', self.job_config.filters['includes'], self.job_config.filters['excludes'], self.job_config.poolsize, local_sdk=self.system, remote_sdk=self.sdk, job_config=self.job_config, db_handler=self.db_handler) self.current_store.open() try: if self.job_config.direction != 'up': logging.info( 'Loading remote changes with sequence {0:s} for job id {1:s}' .format(str(self.remote_seq), str(self.job_config.id))) if self.remote_seq == 0: self.logger.log_state( _('Gathering data from remote workspace, this can take a while...' ), 'sync') very_first = True self.remote_target_seq = self.load_remote_changes_in_store( self.remote_seq, self.current_store) self.current_store.sync() else: self.remote_target_seq = 1 self.ping_remote() except RequestException as ce: logging.exception(ce) if not connection_helper.is_connected_to_internet( self.sdk.proxies): error = _( 'No Internet connection detected! Waiting for %s seconds to retry' ) % self.offline_timer else: error = _( 'Connection to server failed, server is probably down. Waiting %s seconds to retry' ) % self.offline_timer self.marked_for_snapshot_pathes = [] logging.error(error) self.logger.log_state(error, "wait") self.current_store.close() self.sleep_offline() continue except PydioSdkBasicAuthException as e: self.logger.log_state(_('Authentication Error'), 'error') self.logger.log_notif(_('Authentication Error'), 'error') self.current_store.close() self.sleep_offline() continue except Exception as e: error = 'Error while connecting to remote server (%s), waiting for %i seconds before retempting ' % ( e.message, self.offline_timer) logging.exception(e) self.logger.log_state( _('Error while connecting to remote server (%s)') % e.message, "error") self.marked_for_snapshot_pathes = [] self.current_store.close() self.sleep_offline() continue self.online_status = True if not self.job_config.server_configs: self.job_config.server_configs = self.sdk.load_server_configs( ) self.sdk.set_server_configs(self.job_config.server_configs) if self.job_config.direction != 'down' or ( self.job_config.direction == 'down' and self.job_config.solve != 'remote'): logging.info( 'Loading local changes with sequence {0:s} for job id {1:s}' .format(str(self.local_seq), str(self.job_config.id))) self.local_target_seq = self.db_handler.get_local_changes_as_stream( self.local_seq, self.current_store.flatten_and_store) self.current_store.sync() else: self.local_target_seq = 1 if not connection_helper.internet_ok: connection_helper.is_connected_to_internet( self.sdk.proxies) changes_length = len(self.current_store) if not changes_length: self.processing = False logging.info('No changes detected in ' + self.job_config.id) self.update_min_seqs_from_store() self.wait_for_changes = True self.exit_loop_clean(self.logger) very_first = False # logging.info("CheckSync of " + self.job_config.id) # self.db_handler.list_non_idle_nodes() if not self.watcher.isAlive() and not self.interrupt: logging.info("File watcher died, restarting...") self.watcher.stop() self.watcher = LocalWatcher( self.job_config.directory, self.configs_path, event_handler=self.event_handler) self.start_watcher() continue self.global_progress['status_indexing'] = 1 logging.info('Reducing changes for ' + self.job_config.id) self.logger.log_state( _('Merging changes between remote and local, please wait...' ), 'sync') logging.debug('[CMERGER] DETECTING MOVES ' + self.job_config.id) self.current_store.detect_win_moves() self.update_min_seqs_from_store() logging.debug('[CMERGER] REMOVING ECHO ' + self.job_config.id) self.current_store.clean_echoes() self.update_min_seqs_from_store() # We are updating the status to IDLE here for the nodes which has status as NEW # The reason is when we create a new sync on the existing folder, some of the files might # already be synchronized and we ignore those files while we Dedup changes and those files # remain untouched later. # So the flow of node status change will occur as follows # NEW (as soon as we create a new sync task) # | # IDLE (here, just before we reduce changes) # | # PENDING (those files/folders which remain after reducing changes and to be actually processed) # | # UP / DOWN / CONFLICT (corresponding the operation which occurs) # | # IDLE (The final state once upload/ download happens or once when the conflict is resolved) self.db_handler.update_bulk_node_status_as_idle() logging.debug('[CMERGER] Delete Copies ' + self.job_config.id) self.current_store.delete_copies() self.update_min_seqs_from_store() logging.debug('[CMERGER] Dedup changes ' + self.job_config.id) self.current_store.dedup_changes() self.update_min_seqs_from_store() if not self.storage_watcher or very_first: logging.debug('[CMERGER] Detect unnecessary changes ' + self.ws_id) self.logger.log_state( _('Detecting unnecessary changes...'), 'sync') self.current_store.detect_unnecessary_changes() logging.debug( '[CMERGER] Done detecting unnecessary changes') self.logger.log_state( _('Done detecting unnecessary changes...'), 'sync') self.update_min_seqs_from_store() logging.debug('Clearing op and pruning folders moves ' + self.job_config.id) self.current_store.clear_operations_buffer() self.current_store.prune_folders_moves() self.update_min_seqs_from_store() logging.debug('Store conflicts ' + self.job_config.id) store_conflicts = self.current_store.clean_and_detect_conflicts( self.db_handler) if store_conflicts: if self.job_config.solve == 'both': logging.info('Marking nodes SOLVED:KEEPBOTH') for row in self.db_handler.list_conflict_nodes(): self.db_handler.update_node_status( row['node_path'], 'SOLVED:KEEPBOTH') store_conflicts = self.current_store.clean_and_detect_conflicts( self.db_handler) if self.job_config.solve == 'local': logging.info('Marking nodes SOLVED:KEEPLOCAL') for row in self.db_handler.list_conflict_nodes(): self.db_handler.update_node_status( row['node_path'], 'SOLVED:KEEPLOCAL') store_conflicts = self.current_store.clean_and_detect_conflicts( self.db_handler) if self.job_config.solve == 'remote': logging.info('Marking nodes SOLVED:KEEPREMOTE') for row in self.db_handler.list_conflict_nodes(): self.db_handler.update_node_status( row['node_path'], 'SOLVED:KEEPREMOTE') store_conflicts = self.current_store.clean_and_detect_conflicts( self.db_handler) if store_conflicts: logging.info('Conflicts detected, cannot continue!') self.logger.log_state( _('Conflicts detected, cannot continue!'), 'error') self.current_store.close() self.sleep_offline() self.logger.log_notif( _('Conflicts detected, cannot continue!'), 'error') continue if self.job_config.direction == 'down' and self.job_config.solve != 'remote': self.current_store.remove_based_on_location('local') self.update_min_seqs_from_store() changes_length = len(self.current_store) if not changes_length: logging.info('No changes detected for ' + self.job_config.id) self.exit_loop_clean(self.logger) very_first = False continue self.current_store.update_pending_status( self.db_handler, self.local_seq) self.global_progress['status_indexing'] = 0 import change_processor self.global_progress['queue_length'] = changes_length logging.info('Processing %i changes' % changes_length) self.logger.log_state( _('Processing %i changes') % changes_length, "start") counter = [1] def processor_callback(change): try: if self.interrupt or not self.job_status_running: raise InterruptException() self.update_current_tasks() self.update_global_progress() Processor = StorageChangeProcessor if self.storage_watcher else ChangeProcessor proc = Processor(change, self.current_store, self.job_config, self.system, self.sdk, self.db_handler, self.event_logger) proc.process_change() self.update_min_seqs_from_store(success=True) self.global_progress['queue_done'] = float(counter[0]) counter[0] += 1 self.update_current_tasks() self.update_global_progress() time.sleep(0.01) if self.interrupt or not self.job_status_running: raise InterruptException() except ProcessException as pe: logging.error(pe.message) return False except InterruptException as i: raise i except PydioSdkDefaultException as p: raise p except Exception as ex: logging.exception(ex) return False return True try: if sys.platform.startswith('win'): self.marked_for_snapshot_pathes = list( set(self.current_store.find_modified_parents()) - set(self.marked_for_snapshot_pathes)) if not self.processing: self.processing = True self.current_store.process_changes_with_callback( processor_callback) # logging.info("Updating seqs") self.current_store.process_pending_changes() self.update_min_seqs_from_store(success=True) self.global_progress['queue_done'] = float(counter[0]) counter[0] += 1 self.update_current_tasks() self.update_global_progress() time.sleep( 0.05) # Allow for changes to be noticeable in UI # logging.info("DONE WITH CHANGES") self.processing = False except InterruptException as iexc: pass self.logger.log_state( _('%i files modified') % self.global_progress['queue_done'], 'success') if self.global_progress['queue_done']: self.logger.log_notif( _('%i files modified') % self.global_progress['queue_done'], 'success') very_first = False self.exit_loop_clean(self.logger) continue except PydioSdkDefaultException as re: logging.error(re.message) self.logger.log_state(re.message, 'error') self.sleep_online() except SSLError as rt: logging.error(rt.message) self.logger.log_state( _('An SSL error happened, please check the logs'), 'error') self.sleep_offline() except ProxyError as rt: logging.error(rt.message) self.logger.log_state( _('A proxy error happened, please check the logs'), 'error') self.sleep_offline() except TooManyRedirects as rt: logging.error(rt.message) self.logger.log_state( _('Connection error: too many redirects'), 'error') self.sleep_offline() except ChunkedEncodingError as rt: logging.error(rt.message) self.logger.log_state( _('Chunked encoding error, please check the logs'), 'error') self.sleep_offline() except ContentDecodingError as rt: logging.error(rt.message) self.logger.log_state( _('Content Decoding error, please check the logs'), 'error') self.sleep_online() except InvalidSchema as rt: logging.error(rt.message) self.logger.log_state( _('Http connection error: invalid schema.'), 'error') self.sleep_offline() except InvalidURL as rt: logging.error(rt.message) self.logger.log_state(_('Http connection error: invalid URL.'), 'error') self.sleep_offline() except Timeout as to: logging.error(to) self.logger.log_state( _('Connection timeout, will retry later.'), 'error') self.sleep_offline() except RequestException as ree: logging.error(ree.message) self.logger.log_state(_('Cannot resolve domain!'), 'error') self.sleep_offline() except PydioSdkBasicAuthException as e: self.logger.log_state(_('Authentication Error'), 'error') self.logger.log_notif(_('Authentication Error'), 'error') self.sleep_offline() except Exception as e: if not (e.message.lower().count('[quota limit reached]') or e.message.lower().count('[file permissions]')): logging.exception('Unexpected Error: %s' % e.message) self.logger.log_state( _('Unexpected Error: %s') % e.message, 'error') else: logging.exception(e) self.sleep_offline() self.current_store.close() self.processing = False logging.debug('Finished this cycle, waiting for %i seconds' % self.online_timer) very_first = False