def sync(self):
hostname = socket.gethostname()
try:
provider = ComputerProvider(self.session)
task_synced_provider = TaskSyncedProvider(self.session)
computer = provider.by_name(hostname)
sync_start = now()
if FILE_SYNC_INTERVAL == 0:
time.sleep(1)
else:
computers = provider.all_with_last_activtiy()
computers = [
c for c in computers
if (now() - c.last_activity).total_seconds() < 10
]
computers_names = {c.name for c in computers}
for c, project, tasks in task_synced_provider.for_computer(
computer.name):
if c.name not in computers_names:
self.logger.info(
f'Computer = {c.name} '
f'is offline. Can not sync',
ComponentType.WorkerSupervisor, hostname)
continue
if c.syncing_computer:
continue
excluded = list(map(str,
yaml_load(project.ignore_folders)))
folders_excluded = [[join('data', project.name), excluded],
[join('models', project.name), []]]
computer.syncing_computer = c.name
provider.update()
sync_directed(self.session, c, computer, folders_excluded)
for t in tasks:
task_synced_provider.add(
TaskSynced(computer=computer.name, task=t.id))
time.sleep(FILE_SYNC_INTERVAL)
computer.last_synced = sync_start
computer.syncing_computer = None
provider.update()
except Exception as e:
if Session.sqlalchemy_error(e):
Session.cleanup('FileSync')
self.session = Session.create_session(key='FileSync')
self.logger = create_logger(self.session, 'FileSync')
self.logger.error(traceback.format_exc(),
ComponentType.WorkerSupervisor, hostname)
def sync(project: str, computer: str, only_from: bool, only_to: bool,
online: bool):
"""
Syncs specified project on this computer with other computers
"""
check_statuses()
_create_computer()
_create_docker()
computer = computer or socket.gethostname()
provider = ComputerProvider(_session)
project_provider = ProjectProvider(_session)
computer = provider.by_name(computer)
computers = provider.all_with_last_activtiy()
p = project_provider.by_name(project)
assert p, f'Project={project} is not found'
sync_folders = yaml_load(p.sync_folders)
ignore_folders = yaml_load(p.ignore_folders)
sync_folders = correct_folders(sync_folders, p.name)
ignore_folders = correct_folders(ignore_folders, p.name)
if not isinstance(sync_folders, list):
sync_folders = []
if not isinstance(ignore_folders, list):
ignore_folders = []
folders = [[s, ignore_folders] for s in sync_folders]
for c in computers:
if c.name != computer.name:
if online and (now() - c.last_activity).total_seconds() > 100:
continue
if not only_from:
sync_directed(_session, computer, c, folders)
if not only_to:
sync_directed(_session, c, computer, folders)
def sync(self):
hostname = socket.gethostname()
try:
provider = ComputerProvider(self.session)
task_synced_provider = TaskSyncedProvider(self.session)
computer = provider.by_name(hostname)
sync_start = now()
if FILE_SYNC_INTERVAL == 0:
time.sleep(1)
else:
self.sync_manual(computer, provider)
computers = provider.all_with_last_activtiy()
computers = [
c for c in computers
if (now() - c.last_activity).total_seconds() < 10
]
computers_names = {c.name for c in computers}
for c, project, tasks in task_synced_provider.for_computer(
computer.name):
if c.sync_with_this_computer:
if c.name not in computers_names:
self.logger.info(f'Computer = {c.name} '
f'is offline. Can not sync',
ComponentType.WorkerSupervisor,
hostname)
continue
if c.syncing_computer:
continue
sync_folders = yaml_load(project.sync_folders)
ignore_folders = yaml_load(project.ignore_folders)
sync_folders = correct_folders(sync_folders,
project.name)
ignore_folders = correct_folders(ignore_folders,
project.name)
if not isinstance(sync_folders, list):
sync_folders = []
if not isinstance(ignore_folders, list):
ignore_folders = []
folders = [[s, ignore_folders] for s in sync_folders]
computer.syncing_computer = c.name
provider.update()
sync_directed(self.session, c, computer, folders)
for t in tasks:
task_synced_provider.add(
TaskSynced(computer=computer.name, task=t.id)
)
time.sleep(FILE_SYNC_INTERVAL)
computer.last_synced = sync_start
computer.syncing_computer = None
provider.update()
except Exception as e:
self.process_error(e)