def config_to_tree(filter_config):
"""
build a tree as helper to filter
"""
filter_tree = Node(name=None)
for filter_obj in filter_config:
filter_tree.get_node_safe(filter_obj['path']).props.update(
{'children': filter_obj['children']})
return filter_tree
def test_get_node_safe():
""" Tests get_node_safe
"""
root = Node(None)
path = ['a', 'b', 'c', 'd']
node = root.get_node_safe(path)
assert path == node.path
path = ['a', 'b', 'f']
node = root.get_node_safe(path)
assert path == node.path
path = ['bb']
node = root.get_node_safe(path)
assert root.get_node(['bb']) is node
assert path == node.path
def init_with_files(self, files, equivalent=True, file_size=MBYTE):
"""Init sync engine with a set of files
:param files: a list of paths(lists) to populate it with
"""
tree = Node(name=None)
for file in files:
tree.get_node_safe(file).props.update({
VERSION_ID: 1,
SIZE: file_size
})
for node in tree:
if node.parent is not None and VERSION_ID not in node.props:
# all nodes without a version id are auto generated by get_node_safe and are dirs
# (implicitly created directories)
node.props[IS_DIR] = True
node.props[VERSION_ID] = IS_DIR
else:
node.props[IS_DIR] = False
self.sync_engine.merge_storage_to_sync_model(tree, FILESYSTEM_ID)
self.sync_engine.merge_storage_to_sync_model(tree, CSP_1.storage_id)
if equivalent:
for node in tree:
if node.parent is None:
continue
self.sync_engine.root_node.get_node(node.path).props['equivalents'] = \
{'new': {FILESYSTEM_ID: node.props[VERSION_ID],
CSP_1.storage_id: node.props[VERSION_ID]}}
self.sync_engine._sync_state()
self.sync_engine.state = SyncEngineState.RUNNING
return tree
class SyncEngine(ThreadingActorPriorityMailbox):
"""Responsible to keep two StorageProviders in Sync.
All functions started with 'storage_' are called by the storages, that is from the jars module.
"""
# pylint: disable=too-many-arguments, too-many-public-methods
def __init__(self, storage_metrics, task_sink, model=None):
super().__init__(self)
if model is not None:
self.root_node = model
else:
self.root_node = Node(name=None)
self.state = SyncEngineState.STOPPED
self.task_sink = task_sink
#: a iterable of StorageMetrics
self.storage_metrics = storage_metrics
# this is used for the local state transition between STATE_SYNC->RUNNING
self.local_tree_fetched = False
self.remote_tree_fetched = False
#: :class:`blinker.Signal` is called if the props of the node change
#: IMPORTANT: the signal handlers run in the same context as the
#: sync_engine. Be aware of blocking calls etc.
self.on_node_props_change = Signal()
def _handle_failure(self, exception_type, exception_value, traceback):
logger.error("In the syncengine. NOT shutting down",
exc_info=(exception_type, exception_value, traceback))
def _handle_receive(self, message):
"""Measure time, when executing something via the pykka actor"""
start_time = time.time()
return_val = super()._handle_receive(message)
took = time.time() - start_time
if took > 0.08:
logger.info('execution of %s took %.4f, that might be problematic',
message['attr_path'], took)
return return_val
@priority(100)
def get_model_copy(self):
"""Returns a copy of the model"""
return copy.deepcopy(self.root_node)
@priority(1)
def init(self):
"""Initialize sync engine by executing state sync"""
if self.state == SyncEngineState.STOPPED or self.state == SyncEngineState.OFFLINE:
self.issue_sync_task(
FetchFileTreeTask(self.storage_metrics.storage_id))
self.state = SyncEngineState.STATE_SYNC
else:
raise ValueError('Wrong state')
@priority(10)
def query(self, path):
"""Query an element
:param path: the elements path
:return: a deepcopy of the properties
"""
path = normalize_path(path)
node = self.root_node.get_node(path)
return copy.deepcopy(node.props)
@priority(10)
def query_storage_path(self, path):
"""Get the real path on the storages
:param path: the path
:return: a dict of normalized paths with the storage id as key and the normalized
path on the storage as value
"""
path = normalize_path(path)
node = self.root_node.get_node(path)
storage_paths = {}
for csp_id in node.props[STORAGE]:
if csp_id != FILESYSTEM_ID:
storage_paths[csp_id] = get_storage_path(node, csp_id)
return storage_paths
@priority(9)
def query_shared_state(self, path):
""" Queries the key subjects for the item in path
:param path: a not necessarily normalized path to an item
:return a tuple with storage_id, public_shared, shared_id
This method does a lookup if any of the parents of the item might have information about
the shared state.
"""
path = normalize_path(path)
storage_id = self.storage_metrics.storage_id
logger.debug('Querying sync state for path %s and storage_id %s', path,
storage_id)
share_id = None
public_shared = False
for node in self.root_node.iter_up_existing(path):
if node.props.get(STORAGE, {}).get(storage_id,
{}).get(PUBLIC_SHARE, False):
public_shared = True
break
for node in self.root_node.iter_up_existing(path):
share_id = node.props.get(STORAGE, {}).get(storage_id,
{}).get(SHARE_ID)
if share_id:
break
logger.debug('Found share_id:%s in path %s', share_id, node.path)
return SharedState(storage_id=storage_id,
share_id=share_id,
public_shared=public_shared)
def get_default_fsm(self, path):
"""Get node fsm for node, if not exists it will create the default fsm
:param path: the path
:return: the fsm
:raise: ValueError for the root node []
"""
if path == []:
raise ValueError('There should be no state machine for the root')
node = self.root_node.get_node_safe(path=path)
return node.props.setdefault(SE_FSM, fysom.Fysom(FSM_NODE_CONFIG))
def storage_delete(self, storage_id, path):
"""
Handler for a Delete Event on a storage
:param storage_id: the storage
:param path: the path for the event
:return:
"""
normalized_path = normalize_path(path)
logger.debug("Deletion for %s(%s) from %s", path, normalized_path,
storage_id)
# Check if the node exists
try:
del_node = self.root_node.get_node(normalized_path)
except KeyError:
logger.info("Node does not exist -> nothing to delete")
return
logger.debug("Triggering a delete event on %s and all it's children",
normalized_path)
for node in del_node:
logger.debug("triggering e_deleted for %s", node.path)
fsm = self.get_default_fsm(path=node.path)
update_storage_delete(storage_id, node)
if self.state == SyncEngineState.RUNNING:
try:
node.props[STORAGE][storage_id][EVENT_RECEIVED] = True
fsm.e_deleted(node=node,
storage_id=storage_id,
task_sink=self.issue_sync_task,
event_props={},
csps=[self.storage_metrics])
except KeyError:
# This is ok since it could be that the node was already deleted
# or at least manipulated
# If we don't catch exceptions here, the remaining children and the parent
# won't be deleted properly (see client#933 for more information).
logger.debug(
"Tried to delete %s from %s which might have already been deleted.",
path, storage_id)
def storage_create(self, storage_id, path, event_props):
"""
Handler for a Create Event on a storage
:param event_props: dictionary for the properties of an event with the
keys [modified_date, is_dir]
:param storage_id: the storage
:param path: the path
:return
"""
name = path[-1]
normed_path = normalize_path(path)
logger.debug("Got create message for %s(%s)", path, normed_path)
# Ensure full path structure has entries for the current storage
# tree nodes need normed path, display name gets unnormalized version.
curr_path = []
for elem in path[:-1]:
curr_path.append(normalize_path_element(elem))
node = self.root_node.get_node_safe(curr_path)
storage = node.props.setdefault(STORAGE,
{}).setdefault(storage_id, {})
storage[DISPLAY_NAME] = elem
event_props[DISPLAY_NAME] = name
fsm = self.get_default_fsm(path=normed_path)
node = self.root_node.get_node_safe(normed_path)
# update storage props and send blinker signal
old_props = copy.deepcopy(node.props)
update_storage_props(storage_id, node, event_props)
if old_props != node.props:
# props changed -> send signal
self.on_node_props_change.send(self,
storage_id=storage_id,
old_props=old_props,
node=node)
if self.state == SyncEngineState.RUNNING:
node.props[STORAGE][storage_id][EVENT_RECEIVED] = True
fsm.e_created(csps=[self.storage_metrics],
node=node,
task_sink=self.issue_sync_task,
event_props=event_props,
storage_id=storage_id)
def storage_move(self, storage_id, source_path, target_path, event_props):
"""Handler for a Move Event on a storage.
Currently this is just doing a delete on the old node and a create on the new node
:param storage_id: the storage
:param source_path: the old path
:param target_path: the new path
:param event_props event properties of the move
:return:
"""
self.storage_create(storage_id, target_path, event_props)
for node in self.root_node.get_node(
normalize_path(source_path)).children:
child_target_path = copy.deepcopy(target_path)
node_path = node.path[-1]
child_target_path.append(node_path)
self.storage_move(storage_id=storage_id,
source_path=node.path,
target_path=child_target_path,
event_props=node.props[STORAGE][storage_id])
self.storage_delete(storage_id, source_path)
def storage_modify(self, storage_id, path, event_props):
"""
Handler for a Modify Event on a storage
:param storage_id: the storage
:param path: the path
:param event_props event properties of the modify
"""
name = path[-1]
normed_path = normalize_path(path)
logger.debug("Got modify message for %s(%s)", path, normed_path)
event_props[DISPLAY_NAME] = name
fsm = self.get_default_fsm(path=normed_path)
node = self.root_node.get_node(normed_path)
# update storage props and send blinker signal
old_props = copy.deepcopy(node.props)
update_storage_props(storage_id, node, event_props)
if old_props != node.props:
# props changed -> send signal
self.on_node_props_change.send(self,
storage_id=storage_id,
old_props=old_props,
node=node)
if self.state == SyncEngineState.RUNNING:
if SE_FSM in node.props:
logging.debug('Triggering e_modified(current state: %s)',
node.props[SE_FSM].current)
node.props[STORAGE][storage_id][EVENT_RECEIVED] = True
fsm.e_modified(node=node,
csps=[self.storage_metrics],
task_sink=self.issue_sync_task,
event_props=event_props,
storage_id=storage_id)
if SE_FSM in node.props:
logging.debug('now in state: %s', node.props[SE_FSM].current)
def ack_task(self, task):
"""
Acknowledge a task
:param task: the task
:return: None
"""
logger.debug('acknowledging %s', task)
if isinstance(task, FetchFileTreeTask):
self._ack_fetch_file_tree_task(task)
else:
path = task.path
fsm = self.get_default_fsm(path=path)
# TODO: this creates a node for delete events as well, good idea?
node = self.root_node.get_node(path)
if task.state == SyncTask.INVALID_OPERATION:
# Fatal error -> mark node
node.props['invalid_op'] = True
if isinstance(task, UploadSyncTask) or isinstance(
task, DownloadSyncTask):
self._ack_updownload_task(fsm, node, task)
elif isinstance(task, DeleteSyncTask):
self._ack_delete_task(fsm, node, task)
elif isinstance(task, CreateDirSyncTask):
self._ack_updownload_task(fsm, node, task)
elif isinstance(task, MoveSyncTask):
self._ack_move_task(fsm, node, task)
elif isinstance(task, CancelSyncTask):
self._ack_cancel_task(fsm, node, task)
elif isinstance(task, CompareSyncTask):
self._ack_compare_task(fsm, node, task)
else:
logger.critical("Ack of task %s not handled", task)
def _ack_compare_task(self, fsm, node, task):
if task.state == SyncTask.SUCCESSFUL:
result = task.equivalents
if len(result) == 1:
# all elements are equal -> add it to the equivalents list
equivalents = node.props.setdefault('equivalents', {})
new_equivalents = equivalents.setdefault('new', {})
for storage in node.props.setdefault(STORAGE, {}):
new_equivalents[storage] = node.props[STORAGE][storage][
'version_id']
else:
logger.debug("Not implemented")
node.props['comp_task_received'] = True
if self.state == SyncEngineState.RUNNING:
fsm.e_comp_ack(task=task,
node=node,
task_sink=self.issue_sync_task,
csps=[self.storage_metrics])
def _ack_delete_task(self, fsm, node, task):
"""
Acknowledge a delete task
:param fsm: the state machine for this node
:param node: the node
:param task: the task
"""
if STORAGE not in node.props:
logger.info('No storage entries in this node!')
return
if task.target_storage_id not in node.props[STORAGE]:
logger.info('No storage entry for %s in this node!',
task.target_storage_id)
return
if task.state == SyncTask.INVALID_AUTHENTICATION:
self._handle_invalid_authentication(task.target_storage_id)
return
node.props[STORAGE][task.target_storage_id][SYNC_TASK_RUNNING] = False
node.props[STORAGE][task.target_storage_id][SYNC_TASK_FAILED] = \
not task.state == SyncTask.SUCCESSFUL
if task.state == SyncTask.SUCCESSFUL:
# update metrics
file_size = node.props[STORAGE][task.target_storage_id][SIZE]
self._update_storage_metrics(task.target_storage_id, file_size)
# update node props
node.props.setdefault('equivalents', {}).setdefault('new', {}).pop(
task.target_storage_id, None)
if len(node.props['equivalents']['new']) == 1:
node.props['equivalents']['new'] = {}
else:
logger.info("Synctask failed %s", task)
if self.state == SyncEngineState.RUNNING:
fsm.e_st_ack(task=task,
node=node,
task_sink=self.issue_sync_task,
csps=[self.storage_metrics])
def _ack_updownload_task(self, fsm, node, task):
logger.debug('task:%s path:%s', task, node.path)
# checking if account is invalid
if task.state == SyncTask.INVALID_AUTHENTICATION:
self._handle_invalid_authentication(task.target_storage_id)
node.props[STORAGE][task.target_storage_id][SYNC_TASK_RUNNING] = False
node.props[STORAGE][task.target_storage_id][SYNC_TASK_FAILED] = \
not task.state == SyncTask.SUCCESSFUL
if task.state == SyncTask.SUCCESSFUL:
equivalents = node.props.setdefault('equivalents', {})
new_equivalents = equivalents.setdefault('new', {})
source = (task.source_storage_id, task.source_version_id)
target = (task.target_storage_id, task.target_version_id)
if source in new_equivalents.items(
) or target in new_equivalents.items():
new_equivalents[
task.source_storage_id] = task.source_version_id
new_equivalents[
task.target_storage_id] = task.target_version_id
else:
equivalents['old'] = new_equivalents
equivalents['new'] = {
task.source_storage_id: task.source_version_id,
task.target_storage_id: task.target_version_id
}
else:
# if upload -> revert metrics updates
if task.target_storage_id != FILESYSTEM_ID:
file_size = node.props[STORAGE][FILESYSTEM_ID][SIZE]
self._update_storage_metrics(task.target_storage_id, file_size)
# Remove this storage id from desired storages -> new value will be determined
# in on sync
# logger.debug('removing storage %s from desired storages', task.target_storage_id)
# if task.target_storage_id in node.props.get('desired_storages', []):
# node.props['desired_storages'].remove(task.target_storage_id)
if self.state == SyncEngineState.RUNNING:
if SE_FSM in node.props:
logger.debug(
'_ack_updownload_task finished for node %s, state:%s',
node.path, node.props[SE_FSM].current)
fsm.e_st_ack(task=task,
node=node,
task_sink=self.issue_sync_task,
csps=[self.storage_metrics])
if SE_FSM in node.props:
logger.debug('state:%s afterwards', node.props[SE_FSM].current)
def _handle_invalid_authentication(self, storage_id):
# pylint: disable=no-self-use
logger.info('detected invalid account with id %s', storage_id)
# storage is not authenticated anymore
# for storage in self.csps:
# if storage.storage_id == storage_id and storage.valid_auth:
# storage.valid_auth = False
# logger.info('trigger handling of invalid account with id %s', storage_id)
# message = 'handling invalid account with id {}'.format(storage_id)
# thread = Thread(target=self.restart_client_callback, args=(message,))
# thread.start()
def _ack_move_task(self, fsm, node, task):
"""Acknowledge a move task."""
if task.state == SyncTask.SUCCESSFUL:
node.props['storage'].setdefault(task.source_storage_id,
{})[SYNC_TASK_STATE] = MOVED
if self.state == SyncEngineState.RUNNING:
fsm.e_st_move_success(task=task,
node=node,
csps=[self.storage_metrics],
task_sink=self.issue_sync_task)
else:
if self.state == SyncEngineState.RUNNING:
fsm.e_st_move_failed(task=task,
node=node,
task_sink=self.issue_sync_task,
csps=[self.storage_metrics])
def _ack_fetch_file_tree_task(self, task):
"""
Adds the fetched tree model to the local model
:param task:
"""
if task.state == SyncTask.SUCCESSFUL:
self.merge_storage_to_sync_model(storage_model=task.file_tree,
storage_id=task.storage_id)
# wait until both trees have been returned
# pylint: disable=simplifiable-if-statement
if task.storage_id == FILESYSTEM_ID:
self.local_tree_fetched = True
else:
self.remote_tree_fetched = True
self.storage_metrics = task.file_tree.props['metrics']
self.issue_sync_task(FetchFileTreeTask(FILESYSTEM_ID))
if self.remote_tree_fetched and self.local_tree_fetched:
self._sync_state()
self.state = SyncEngineState.RUNNING
# TODO: yuk! (we need to use more signals)
cc.ipc_gui.accountDeleted('')
self.local_tree_fetched = False
self.remote_tree_fetched = False
else:
# in case of failure
self.state = SyncEngineState.STOPPED # pylint: disable=redefined-variable-type
def _ack_cancel_task(self, fsm, node, task):
node.props['tasks_cancelled'] = True
if self.state == SyncEngineState.RUNNING:
fsm.e_st_ack(node=node,
task=task,
task_sink=self.issue_sync_task,
csps=[self.storage_metrics])
def storage_offline(self, **kwargs):
"""Turn the syncengine for this storage off.
Note: kwargs are required to stay compatible with older versions which pass storage_id.
"""
# pylint: disable=unused-argument
storage_id = self.root_node.props.get(STORAGE, {})
logger.info("Storage %s went offline.", storage_id)
self.state = SyncEngineState.OFFLINE
def storage_online(self, **kwargs):
"""Turn the syncengine for this storage on.
Note: kwargs are required to stay compatible with older versions which pass storage_id.
"""
# pylint: disable=unused-argument
storage_id = self.root_node.props.get(STORAGE, {})
logger.info("Storage %s is back online.", storage_id)
# if we have not been initialized before, we were probably totally offline
cur_state = self.state
if cur_state is SyncEngineState.OFFLINE:
self.init()
def pause(self):
""" Pause sync engine.
That means, it cancel all running actions an stops executing things within the stop
"""
if self.state == SyncEngineState.RUNNING:
# Cancel all tasks
self.cancel_all_tasks()
self.state = SyncEngineState.STOPPED
def resume(self):
"""
Reenables that events are passed to the FSM and triggers state sync
"""
if self.state == SyncEngineState.STOPPED:
self._sync_state()
self.state = SyncEngineState.RUNNING
##################
# Helper Methods #
##################
def print_state(self):
"""
prints all data stored in tree
"""
all_items = copy.deepcopy(self.root_node)
print_sync_model(all_items)
def _update_storage_metrics(self, storage_id, usage_diff):
""" Metrics Update
Updates the metrics of a specified csps
:param storage_id: the csp_id
:param usage_diff: positive or negative change
:return: None
"""
if self.storage_metrics.storage_id == storage_id:
self.storage_metrics.free_space += usage_diff
logger.debug(
'updating metrics of %s - free space %2.2fMB (diff %2.2fMB)',
storage_id, self.storage_metrics.free_space / (1024 * 1024),
usage_diff / (1024 * 1024))
return
def issue_sync_task(self, task):
"""
Issues a sync tasks and calculates new metrics
:param task: the synctask
"""
if isinstance(task, UploadSyncTask):
# the metrics only change if an upload is issued
logger.debug("Upload Task issued. Updating Metrics.")
node = self.root_node.get_node(task.path)
file_size = node.props[STORAGE][FILESYSTEM_ID][SIZE]
self._update_storage_metrics(task.target_storage_id,
(file_size * (-1)))
logger.debug("Issuing task '%s' on '%s' sink.", task, self.task_sink)
self.task_sink(task)
def merge_storage_to_sync_model(self, storage_model, storage_id):
"""
Merges storage model into existing sync model with necessary attributes.
"""
# create and add every node to sync model
for storage_node in storage_model:
sync_node = self.root_node.get_node_safe(
normalize_path(storage_node.path))
sync_node.props.setdefault(STORAGE, {})[storage_id] = {}
if storage_node.parent is None:
# the rest from here on is not needed for the root, but the other values are
# used to mark if the tree has been retrieved
continue
if DISPLAY_NAME not in storage_node.props:
storage_node.props[DISPLAY_NAME] = storage_node.name
update_storage_props(storage_id=storage_id,
node=sync_node,
props=storage_node.props)
if storage_node.parent is not None:
# normalize storage model, to normalize the model, so the deletion operation below
# is working
storage_node.name = normalize_path_element(storage_node.name)
for node in set(self.root_node) - set(storage_model):
storages = node.props.get(STORAGE, {})
if storage_id in storages:
del storages[storage_id]
def _sync_state(self):
"""Force an (re-)evaluation of all nodes in the model by triggering the e_check method."""
logger.debug('Starting state sync')
for node in self.root_node:
if node.parent is None:
# ignore the root
continue
fsm = self.get_default_fsm(node.path)
fsm.current = S_SYNCED
try:
fsm.e_check(csps=[self.storage_metrics],
task_sink=self.task_sink,
node=node)
except BaseException:
logger.exception('State Sync for node %s failed',
node.path,
extra={
'path': node.path,
'node props': node.props
})
logger.debug('Done with state sync')
def cancel_all_tasks(self):
"""Try to cancel all SyncTasks for each Node"""
for node in self.root_node:
if node.parent is None:
continue
fsm = node.props.get(SE_FSM)
if fsm is not None:
try:
fsm.e_cancel_all(csps=[self.storage_metrics],
task_sink=self.task_sink,
node=node)
except fysom.FysomError:
logger.info('did not cancel task since in state "%s"',
fsm.current)
logger.debug(node.props)