def sync(self): hostname = socket.gethostname() try: provider = ComputerProvider(self.session) task_synced_provider = TaskSyncedProvider(self.session) computer = provider.by_name(hostname) sync_start = now() if FILE_SYNC_INTERVAL == 0: time.sleep(1) else: computers = provider.all_with_last_activtiy() computers = [ c for c in computers if (now() - c.last_activity).total_seconds() < 10 ] computers_names = {c.name for c in computers} for c, project, tasks in task_synced_provider.for_computer( computer.name): if c.name not in computers_names: self.logger.info( f'Computer = {c.name} ' f'is offline. Can not sync', ComponentType.WorkerSupervisor, hostname) continue if c.syncing_computer: continue excluded = list(map(str, yaml_load(project.ignore_folders))) folders_excluded = [[join('data', project.name), excluded], [join('models', project.name), []]] computer.syncing_computer = c.name provider.update() sync_directed(self.session, c, computer, folders_excluded) for t in tasks: task_synced_provider.add( TaskSynced(computer=computer.name, task=t.id)) time.sleep(FILE_SYNC_INTERVAL) computer.last_synced = sync_start computer.syncing_computer = None provider.update() except Exception as e: if Session.sqlalchemy_error(e): Session.cleanup('FileSync') self.session = Session.create_session(key='FileSync') self.logger = create_logger(self.session, 'FileSync') self.logger.error(traceback.format_exc(), ComponentType.WorkerSupervisor, hostname)
def sync(project: str, computer: str, only_from: bool, only_to: bool, online: bool): """ Syncs specified project on this computer with other computers """ check_statuses() _create_computer() _create_docker() computer = computer or socket.gethostname() provider = ComputerProvider(_session) project_provider = ProjectProvider(_session) computer = provider.by_name(computer) computers = provider.all_with_last_activtiy() p = project_provider.by_name(project) assert p, f'Project={project} is not found' sync_folders = yaml_load(p.sync_folders) ignore_folders = yaml_load(p.ignore_folders) sync_folders = correct_folders(sync_folders, p.name) ignore_folders = correct_folders(ignore_folders, p.name) if not isinstance(sync_folders, list): sync_folders = [] if not isinstance(ignore_folders, list): ignore_folders = [] folders = [[s, ignore_folders] for s in sync_folders] for c in computers: if c.name != computer.name: if online and (now() - c.last_activity).total_seconds() > 100: continue if not only_from: sync_directed(_session, computer, c, folders) if not only_to: sync_directed(_session, c, computer, folders)
def sync(self): hostname = socket.gethostname() try: provider = ComputerProvider(self.session) task_synced_provider = TaskSyncedProvider(self.session) computer = provider.by_name(hostname) sync_start = now() if FILE_SYNC_INTERVAL == 0: time.sleep(1) else: self.sync_manual(computer, provider) computers = provider.all_with_last_activtiy() computers = [ c for c in computers if (now() - c.last_activity).total_seconds() < 10 ] computers_names = {c.name for c in computers} for c, project, tasks in task_synced_provider.for_computer( computer.name): if c.sync_with_this_computer: if c.name not in computers_names: self.logger.info(f'Computer = {c.name} ' f'is offline. Can not sync', ComponentType.WorkerSupervisor, hostname) continue if c.syncing_computer: continue sync_folders = yaml_load(project.sync_folders) ignore_folders = yaml_load(project.ignore_folders) sync_folders = correct_folders(sync_folders, project.name) ignore_folders = correct_folders(ignore_folders, project.name) if not isinstance(sync_folders, list): sync_folders = [] if not isinstance(ignore_folders, list): ignore_folders = [] folders = [[s, ignore_folders] for s in sync_folders] computer.syncing_computer = c.name provider.update() sync_directed(self.session, c, computer, folders) for t in tasks: task_synced_provider.add( TaskSynced(computer=computer.name, task=t.id) ) time.sleep(FILE_SYNC_INTERVAL) computer.last_synced = sync_start computer.syncing_computer = None provider.update() except Exception as e: self.process_error(e)