def __init__(
self,
root_dir,
shares_dir,
data_dir,
partials_dir,
host="fs-1.one.ubuntu.com",
port=443,
dns_srv=None,
ssl=True,
disable_ssl_verify=False,
mark_interval=120,
broadcast_events=False,
handshake_timeout=30,
shares_symlink_name="Shared With Me",
read_limit=None,
write_limit=None,
throttling_enabled=False,
ignore_files=None,
auth_credentials=None,
monitor_class=None,
):
self.root_dir = root_dir
self.shares_dir = shares_dir
self.shares_dir_link = os.path.join(self.root_dir, shares_symlink_name)
self.data_dir = data_dir
self.partials_dir = partials_dir
tritcask_dir = os.path.join(self.data_dir, "tritcask")
self.logger = logging.getLogger("ubuntuone.SyncDaemon.Main")
user_config = config.get_user_config()
if read_limit is None:
read_limit = user_config.get_throttling_read_limit()
if write_limit is None:
write_limit = user_config.get_throttling_write_limit()
if not throttling_enabled:
throttling_enabled = user_config.get_throttling()
self.logger.info("Starting %s client version %s", clientdefs.NAME, clientdefs.VERSION)
self.logger.info("Using %r as root dir", self.root_dir)
self.logger.info("Using %r as data dir", self.data_dir)
self.logger.info("Using %r as shares root dir", self.shares_dir)
self.db = tritcask.Tritcask(tritcask_dir)
self.vm = volume_manager.VolumeManager(self)
self.fs = filesystem_manager.FileSystemManager(data_dir, partials_dir, self.vm, self.db)
self.event_q = event_queue.EventQueue(self.fs, ignore_files, monitor_class=monitor_class)
self.fs.register_eq(self.event_q)
# subscribe VM to EQ, to be unsubscribed in shutdown
self.event_q.subscribe(self.vm)
self.vm.init_root()
# we don't have the auth tokens yet, we 'll get them later
self.action_q = action_queue.ActionQueue(
self.event_q,
self,
host,
port,
dns_srv,
ssl,
disable_ssl_verify,
read_limit,
write_limit,
throttling_enabled,
)
self.hash_q = hash_queue.HashQueue(self.event_q)
events_nanny.DownloadFinishedNanny(self.fs, self.event_q, self.hash_q)
# call StateManager after having AQ
self.state_manager = StateManager(self, handshake_timeout)
self.sync = sync.Sync(self)
self.lr = local_rescan.LocalRescan(self.vm, self.fs, self.event_q, self.action_q)
self.external = SyncdaemonService(main=self, send_events=broadcast_events)
self.external.auth_credentials = auth_credentials
if user_config.get_autoconnect():
self.external.connect(autoconnecting=True)
self.mark = task.LoopingCall(self.log_mark)
self.mark.start(mark_interval)
def __init__(self, root, main, volume_manager, action_queue):
""" Creates the instance."""
super(SyncDaemon, self).__init__()
self.service = SyncdaemonService(root, main, volume_manager,
action_queue)
self.clients = []
class Main(object):
"""The one who executes the syncdaemon."""
def __init__(
self,
root_dir,
shares_dir,
data_dir,
partials_dir,
host="fs-1.one.ubuntu.com",
port=443,
dns_srv=None,
ssl=True,
disable_ssl_verify=False,
mark_interval=120,
broadcast_events=False,
handshake_timeout=30,
shares_symlink_name="Shared With Me",
read_limit=None,
write_limit=None,
throttling_enabled=False,
ignore_files=None,
auth_credentials=None,
monitor_class=None,
):
self.root_dir = root_dir
self.shares_dir = shares_dir
self.shares_dir_link = os.path.join(self.root_dir, shares_symlink_name)
self.data_dir = data_dir
self.partials_dir = partials_dir
tritcask_dir = os.path.join(self.data_dir, "tritcask")
self.logger = logging.getLogger("ubuntuone.SyncDaemon.Main")
user_config = config.get_user_config()
if read_limit is None:
read_limit = user_config.get_throttling_read_limit()
if write_limit is None:
write_limit = user_config.get_throttling_write_limit()
if not throttling_enabled:
throttling_enabled = user_config.get_throttling()
self.logger.info("Starting %s client version %s", clientdefs.NAME, clientdefs.VERSION)
self.logger.info("Using %r as root dir", self.root_dir)
self.logger.info("Using %r as data dir", self.data_dir)
self.logger.info("Using %r as shares root dir", self.shares_dir)
self.db = tritcask.Tritcask(tritcask_dir)
self.vm = volume_manager.VolumeManager(self)
self.fs = filesystem_manager.FileSystemManager(data_dir, partials_dir, self.vm, self.db)
self.event_q = event_queue.EventQueue(self.fs, ignore_files, monitor_class=monitor_class)
self.fs.register_eq(self.event_q)
# subscribe VM to EQ, to be unsubscribed in shutdown
self.event_q.subscribe(self.vm)
self.vm.init_root()
# we don't have the auth tokens yet, we 'll get them later
self.action_q = action_queue.ActionQueue(
self.event_q,
self,
host,
port,
dns_srv,
ssl,
disable_ssl_verify,
read_limit,
write_limit,
throttling_enabled,
)
self.hash_q = hash_queue.HashQueue(self.event_q)
events_nanny.DownloadFinishedNanny(self.fs, self.event_q, self.hash_q)
# call StateManager after having AQ
self.state_manager = StateManager(self, handshake_timeout)
self.sync = sync.Sync(self)
self.lr = local_rescan.LocalRescan(self.vm, self.fs, self.event_q, self.action_q)
self.external = SyncdaemonService(main=self, send_events=broadcast_events)
self.external.auth_credentials = auth_credentials
if user_config.get_autoconnect():
self.external.connect(autoconnecting=True)
self.mark = task.LoopingCall(self.log_mark)
self.mark.start(mark_interval)
def log_mark(self):
"""Log a "mark" that includes the current AQ state and queue size."""
self.logger.note(
"---- MARK (state: %s; queue: %d; offloaded: %d; " "hash: %d) ----",
self.state_manager,
len(self.action_q.queue),
len(self.action_q.disk_queue),
len(self.hash_q),
)
def wait_for_nirvana(self, last_event_interval=0.5):
"""Get a deferred that will fire on Nirvana.
That is, when there are no more events or transfers.
"""
self.logger.debug("wait_for_nirvana(%s)" % last_event_interval)
d = defer.Deferred()
def start():
"""Request the event empty notification."""
self.logger.debug("starting wait_for_nirvana")
self.event_q.add_empty_event_queue_callback(callback)
def callback():
"""Event queue is empty."""
state = self.state_manager.state
if not (
self.state_manager.queues.state == QueueManager.IDLE
and state == StateManager.QUEUE_MANAGER
and not self.action_q.queue
and self.hash_q.empty()
):
self.logger.debug(
"I can't reach Nirvana yet [state: %s queue: %d hash: %d]",
self.state_manager,
len(self.action_q.queue),
len(self.hash_q),
)
return
self.logger.debug("Nirvana reached!! I'm a Buddha")
self.event_q.remove_empty_event_queue_callback(callback)
d.callback(True)
reactor.callLater(last_event_interval, start)
return d
def wait_for(self, *waiting_events, **waiting_kwargs):
"""defer until event appears"""
return WaitingHelpingHandler(self.event_q, waiting_events, waiting_kwargs).deferred
def start(self):
"""Setup the daemon to be ready to run."""
# hook the event queue to the root dir
self.event_q.push("SYS_INIT_DONE")
return self.external.start()
@defer.inlineCallbacks
def shutdown(self, with_restart=False):
"""Shutdown the daemon; optionally restart it."""
self.event_q.push("SYS_USER_DISCONNECT")
self.event_q.push("SYS_QUIT")
self.event_q.unsubscribe(self.vm)
# shutdown these before event_q so the listeners are unsubscribed
self.state_manager.shutdown()
self.hash_q.shutdown()
self.external.shutdown(with_restart)
yield self.event_q.shutdown()
self.db.shutdown()
self.mark.stop()
def restart(self):
"""Restart the daemon.
This ultimately shuts down the daemon and asks external interface
to start one again.
"""
self.quit(exit_value=42, with_restart=True)
def check_version(self):
"""Check the client protocol version matches that of the server."""
self.action_q.check_version()
def authenticate(self):
"""Do the authentication dance."""
self.action_q.authenticate()
def set_capabilities(self):
"""Set the capabilities with the server"""
self.logger.debug("capabilities query: %r", syncdaemon.REQUIRED_CAPS)
self.action_q.set_capabilities(syncdaemon.REQUIRED_CAPS)
def server_rescan(self):
"""Do the server rescan."""
return self.vm.server_rescan()
def get_homedir(self):
"""Return the home dir point."""
return user_home
def get_rootdir(self):
"""Return the base dir/mount point."""
return self.root_dir
def get_sharesdir(self):
"""Return the dir/mount point for shares."""
return self.shares_dir
def get_sharesdir_link(self):
"""Return the dir link for shares."""
return self.shares_dir_link
@defer.inlineCallbacks
def quit(self, exit_value=0, with_restart=False):
"""Shutdown and stop the reactor."""
yield self.shutdown(with_restart)
if reactor.running:
reactor.stop()
else:
sys.exit(exit_value)
def local_rescan(self):
"""Do the local rescan."""
self.logger.note("Local rescan starting...")
d = self.lr.start()
def _wait_for_hashq():
"""Keep on calling this until the hash_q finishes."""
if len(self.hash_q):
self.logger.info("hash queue pending. Waiting for it...")
reactor.callLater(0.1, _wait_for_hashq)
else:
self.logger.info("hash queue empty. We are ready!")
# nudge the action queue into action
self.event_q.push("SYS_LOCAL_RESCAN_DONE")
def local_rescan_done(_):
"""After local rescan finished."""
self.logger.note("Local rescan finished!")
_wait_for_hashq()
def stop_the_press(failure):
"""Something went wrong in LR, can't continue."""
self.logger.error("Local rescan finished with error: %s", failure.getBriefTraceback())
self.event_q.push("SYS_UNKNOWN_ERROR")
d.addCallbacks(local_rescan_done, stop_the_press)
class SyncDaemon(Referenceable, SignalBroadcaster):
""" The Daemon ipc interface. """
__metaclass__ = RemoteMeta
# calls that will be accessible remotely
remote_calls = [
'connect',
'disconnect',
'get_rootdir',
'get_sharesdir',
'get_sharesdir_link',
'wait_for_nirvana',
'quit',
'rescan_from_scratch',
]
def __init__(self, root, main, volume_manager, action_queue):
""" Creates the instance."""
super(SyncDaemon, self).__init__()
self.service = SyncdaemonService(root, main, volume_manager,
action_queue)
self.clients = []
def connect(self):
""" Connect to the server. """
logger.debug('connect requested')
self.service.connect()
def disconnect(self):
""" Disconnect from the server. """
logger.debug('disconnect requested')
self.service.disconnect()
def get_rootdir(self):
""" Returns the root dir/mount point. """
logger.debug('called get_rootdir')
return self.service.get_rootdir()
def get_sharesdir(self):
""" Returns the shares dir/mount point. """
logger.debug('called get_sharesdir')
return self.service.get_sharesdir()
def get_sharesdir_link(self):
""" Returns the shares dir/mount point. """
logger.debug('called get_sharesdir_link')
return self.service.get_sharesdir_link()
def wait_for_nirvana(self, last_event_interval,
reply_handler=None, error_handler=None):
""" call the reply handler when there are no more
events or transfers.
"""
logger.debug('called wait_for_nirvana')
return self.service.wait_for_nirvana(last_event_interval,
remote_handler(reply_handler), remote_handler(error_handler))
def quit(self, reply_handler=None, error_handler=None):
""" shutdown the syncdaemon. """
logger.debug('Quit requested')
self.service.quit(remote_handler(reply_handler),
remote_handler(error_handler))
def rescan_from_scratch(self, volume_id):
"""Request a rescan from scratch of the volume with volume_id."""
self.service.rescan_from_scratch(volume_id)
def emit_root_mismatch(self, root_id, new_root_id):
"""Emit RootMismatch signal."""
self.emit_signal('on_root_mismatch', root_id, new_root_id)
def emit_quota_exceeded(self, volume_dict):
"""Emit QuotaExceeded signal."""
self.emit_signal('on_quota_exceeded', volume_dict)
