def start_new_folder_sync_engines(self, folders=set()):
new_folders = [f for f in self.prepare_sync() if f not in folders]
for folder_name, folder_id in new_folders:
log.info('Folder sync engine started',
account_id=self.account_id,
folder_id=folder_id,
folder_name=folder_name)
thread = self.sync_engine_class(self.account_id,
folder_name,
folder_id,
self.email_address,
self.provider_name,
self.poll_frequency,
self.syncmanager_lock,
self.refresh_flags_max,
self.retry_fail_classes)
self.folder_monitors.start(thread)
while not thread_polling(thread) and \
not thread_finished(thread) and \
not thread.ready():
sleep(self.heartbeat)
# allow individual folder sync monitors to shut themselves down
# after completing the initial sync
if thread_finished(thread) or thread.ready():
log.info('Folder sync engine finished/killed',
account_id=self.account_id,
folder_id=folder_id,
folder_name=folder_name)
# note: thread is automatically removed from
# self.folder_monitors
else:
folders.add((folder_name, folder_id))
def start_new_folder_sync_engines(self):
running_monitors = {monitor.folder_name: monitor for monitor in
self.folder_monitors}
for folder_name in self.prepare_sync():
if folder_name in running_monitors:
thread = running_monitors[folder_name]
else:
log.info('Folder sync engine started',
account_id=self.account_id,
folder_name=folder_name)
thread = self.sync_engine_class(self.account_id,
self.namespace_id,
folder_name,
self.email_address,
self.provider_name,
self.syncmanager_lock)
self.folder_monitors.start(thread)
while not thread_polling(thread) and not thread.ready():
sleep(self.heartbeat)
if thread.ready():
log.info('Folder sync engine exited',
account_id=self.account_id,
folder_name=folder_name,
error=thread.exception)
def sync(self):
""" Start per-folder syncs. Only have one per-folder sync in the
'initial' state at a time.
"""
sync_folder_names_ids = self.prepare_sync()
for folder_name, folder_id in sync_folder_names_ids:
log.info('initializing folder sync')
thread = self.sync_engine_class(self.account_id,
folder_name,
folder_id,
self.email_address,
self.provider_name,
self.poll_frequency,
self.syncmanager_lock,
self.refresh_flags_max,
self.retry_fail_classes,
self.sync_status_queue)
thread.start()
self.folder_monitors.add(thread)
while not thread_polling(thread) and \
not thread_finished(thread) and \
not thread.ready():
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if thread_finished(thread) or thread.ready():
log.info('folder sync finished/killed',
folder_name=thread.folder_name)
# NOTE: Greenlet is automatically removed from the group.
self.folder_monitors.join()
def start_new_folder_sync_engines(self):
running_monitors = {
monitor.folder_id: monitor
for monitor in self.folder_monitors
}
for folder_name, folder_id in self.prepare_sync():
if folder_id in running_monitors:
thread = running_monitors[folder_id]
else:
log.info('Folder sync engine started',
account_id=self.account_id,
folder_id=folder_id,
folder_name=folder_name)
thread = self.sync_engine_class(self.account_id,
self.namespace_id, folder_name,
folder_id, self.email_address,
self.provider_name,
self.syncmanager_lock)
self.folder_monitors.start(thread)
while not thread_polling(thread) and not thread.ready():
sleep(self.heartbeat)
if thread.ready():
log.info('Folder sync engine exited',
account_id=self.account_id,
folder_id=folder_id,
folder_name=folder_name,
error=thread.exception)
def sync(self):
sync_folder_names_ids = self.prepare_sync()
thread_download_lock = BoundedSemaphore(1)
for folder_name, folder_id in sync_folder_names_ids:
log.info('initializing folder sync')
thread = GmailFolderSyncEngine(thread_download_lock,
self.account_id, folder_name,
folder_id,
self.email_address,
self.provider_name,
self.poll_frequency,
self.syncmanager_lock,
self.refresh_flags_max,
self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
if thread.should_block:
while not thread_polling(thread) and \
not thread_finished(thread) and \
not thread.ready():
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if thread_finished(thread) or thread.ready():
log.info('folder sync finished/killed',
folder_name=thread.folder_name)
# NOTE: Greenlet is automatically removed from the group.
self.folder_monitors.join()
def start_new_folder_sync_engines(self, folders=set()):
new_folders = [f for f in self.prepare_sync() if f not in folders]
for folder_name, folder_id in new_folders:
log.info('Folder sync engine started',
account_id=self.account_id,
folder_id=folder_id,
folder_name=folder_name)
thread = self.sync_engine_class(
self.account_id, folder_name, folder_id, self.email_address,
self.provider_name, self.poll_frequency, self.syncmanager_lock,
self.refresh_flags_max, self.retry_fail_classes)
self.folder_monitors.start(thread)
while not thread_polling(thread) and \
not thread_finished(thread) and \
not thread.ready():
sleep(self.heartbeat)
# allow individual folder sync monitors to shut themselves down
# after completing the initial sync
if thread_finished(thread) or thread.ready():
log.info('Folder sync engine finished/killed',
account_id=self.account_id,
folder_id=folder_id,
folder_name=folder_name)
# note: thread is automatically removed from
# self.folder_monitors
else:
folders.add((folder_name, folder_id))
def start_new_folder_sync_engines(self, folders=set()):
new_folders = [f for f in self.prepare_sync() if f not in folders]
for folder_name, folder_id in new_folders:
log.info(
"Folder sync engine started", account_id=self.account_id, folder_id=folder_id, folder_name=folder_name
)
thread = self.sync_engine_class(
self.account_id,
folder_name,
folder_id,
self.email_address,
self.provider_name,
self.poll_frequency,
self.syncmanager_lock,
self.refresh_flags_max,
self.retry_fail_classes,
)
self.folder_monitors.start(thread)
while not thread_polling(thread) and not thread_finished(thread) and not thread.ready():
sleep(self.heartbeat)
# allow individual folder sync monitors to shut themselves down
# after completing the initial sync
if thread_finished(thread) or thread.ready():
if thread.exception:
# Exceptions causing the folder sync to exit should not
# clear the heartbeat.
log.info(
"Folder sync engine exited with error",
account_id=self.account_id,
folder_id=folder_id,
folder_name=folder_name,
error=thread.exception,
)
else:
log.info(
"Folder sync engine finished",
account_id=self.account_id,
folder_id=folder_id,
folder_name=folder_name,
)
# clear the heartbeat for this folder-thread since it
# exited cleanly.
clear_heartbeat_status(self.account_id, folder_id)
# note: thread is automatically removed from
# self.folder_monitors
else:
folders.add((folder_name, folder_id))
def start_new_folder_sync_engines(self, folders=set()):
new_folders = [f for f in self.prepare_sync() if f not in folders]
for folder_name, folder_id in new_folders:
log.info('Folder sync engine started',
account_id=self.account_id,
folder_id=folder_id,
folder_name=folder_name)
thread = self.sync_engine_class(
self.account_id, folder_name, folder_id, self.email_address,
self.provider_name, self.poll_frequency, self.syncmanager_lock,
self.refresh_flags_max, self.retry_fail_classes)
self.folder_monitors.start(thread)
while not thread_polling(thread) and \
not thread_finished(thread) and \
not thread.ready():
sleep(self.heartbeat)
# allow individual folder sync monitors to shut themselves down
# after completing the initial sync
if thread_finished(thread) or thread.ready():
if thread.exception:
# Exceptions causing the folder sync to exit should not
# clear the heartbeat.
log.info('Folder sync engine exited with error',
account_id=self.account_id,
folder_id=folder_id,
folder_name=folder_name,
error=thread.exception)
else:
log.info('Folder sync engine finished',
account_id=self.account_id,
folder_id=folder_id,
folder_name=folder_name)
# clear the heartbeat for this folder-thread since it
# exited cleanly.
clear_heartbeat_status(self.account_id, folder_id)
# note: thread is automatically removed from
# self.folder_monitors
else:
folders.add((folder_name, folder_id))