def __init__(self, global_conf=CoreConfig()):
"""
Initializes a DataStore object
"""
# dictionary {backend_name_string: Backend instance}
self.backends = {}
self.treefactory = TreeFactory()
self._tasks = self.treefactory.get_tasks_tree()
self.requester = requester.Requester(self, global_conf)
self.tagfile_loaded = False
self._tagstore = self.treefactory.get_tags_tree(self.requester)
self._backend_signals = BackendSignals()
self.conf = global_conf
self.tag_idmap = {}
# Flag when turned to true, all pending operation should be
# completed and then GTG should quit
self.please_quit = False
# The default backend must be loaded first. This flag turns to True
# when the default backend loading has finished.
self.is_default_backend_loaded = False
self._backend_signals.connect('default-backend-loaded',
self._activate_non_default_backends)
self._backend_mutex = threading.Lock()
def __init__(self, parameters):
"""
Instantiates a new backend. Please note that this is called also
for disabled backends. Those are not initialized, so you might
want to check out the initialize() function.
"""
if self.KEY_DEFAULT_BACKEND not in parameters:
# if it's not specified, then this is the default backend
# (for retro-compatibility with the GTG 0.2 series)
parameters[self.KEY_DEFAULT_BACKEND] = True
# default backends should get all the tasks
no_attached_tags = self.KEY_ATTACHED_TAGS not in parameters and \
self._general_description[self.BACKEND_TYPE] == self.TYPE_READWRITE
if parameters[self.KEY_DEFAULT_BACKEND] or no_attached_tags:
parameters[self.KEY_ATTACHED_TAGS] = [ALLTASKS_TAG]
self._parameters = parameters
self._signal_manager = BackendSignals()
self._is_initialized = False
# if debugging mode is enabled, tasks should be saved as soon as
# they're marked as modified. If in normal mode, we prefer speed over
# easier debugging.
if log.isEnabledFor(logging.DEBUG):
self.timer_timestep = 5
else:
self.timer_timestep = 1
self.to_set_timer = None
self.please_quit = False
self.cancellation_point = lambda: _cancellation_point(lambda: self.
please_quit)
self.to_set = deque()
self.to_remove = deque()
def _start_get_tasks(self):
"""
This function executes an imports and schedules the next
"""
self.cancellation_point()
BackendSignals().backend_sync_started(self.get_id())
self.do_periodic_import()
BackendSignals().backend_sync_ended(self.get_id())
def _connect_signals(self):
""" Connects the backends generated signals """
_signals = BackendSignals()
_signals.connect(_signals.BACKEND_RENAMED, self.refresh_title)
_signals.connect(_signals.BACKEND_STATE_TOGGLED,
self.refresh_sync_status)
_signals.connect(_signals.BACKEND_SYNC_STARTED, self.on_sync_started)
_signals.connect(_signals.BACKEND_SYNC_ENDED, self.on_sync_ended)
def notify_user_about_backup(self):
""" This function causes the inforbar to show up with the message
about file recovery.
"""
message = _("Oops, something unexpected happened! "
"GTG tried to recover your tasks from backups. \n"
) + self.backup_file_info()
BackendSignals().interaction_requested(
self.get_id(), message,
BackendSignals().INTERACTION_INFORM, "on_continue_clicked")
def _ask_user_to_confirm_authentication(self):
"""
Calls for a user interaction during authentication
"""
self.login_event.clear()
BackendSignals().interaction_requested(
self.get_id(),
"You need to authenticate to Remember The Milk. A browser "
"is opening with a login page.\n When you have logged in "
"and given GTG the requested permissions,\n "
"press the 'Confirm' button",
BackendSignals().INTERACTION_CONFIRM, "on_login")
self.login_event.wait()
def _refresh_calendar_list(self):
"""Will browse calendar list available after principal call and cache
them"""
try:
principal = self._dav_client.principal()
except caldav.lib.error.AuthorizationError as error:
message = _("You need a correct login to CalDAV\n"
"Configure CalDAV with login information.\n Error:")
BackendSignals().interaction_requested(
self.get_id(), f"{message} {error!r}",
BackendSignals().INTERACTION_INFORM, "on_continue_clicked")
raise error
for calendar in principal.calendars():
self._cache.set_calendar(calendar)
def _authenticate(self):
"""
authentication main function
"""
self.authenticated.clear()
while not self.authenticated.isSet():
if not self.token:
self.rtm = createRTM(self.PUBLIC_KEY, self.PRIVATE_KEY,
self.token)
Gio.AppInfo.launch_default_for_uri(self.rtm.getAuthURL(), None)
self.auth_confirm()
try:
time.sleep(1)
self.token = self.rtm.getToken()
except Exception:
# something went wrong.
self.token = None
continue
try:
if self._login():
self.authenticated.set()
self.on_auth()
except IOError:
BackendSignals().backend_failed(self.get_id(),
BackendSignals.ERRNO_NETWORK)
def start_get_tasks(self):
""" Loads all task from the backend and connects its signals
afterwards. """
self.backend.start_get_tasks()
self._connect_signals()
if self.backend.is_default():
BackendSignals().default_backend_loaded()
def _on_interaction_response(self, widget, event):
"""
Signal callback executed when the user gives the feedback for a
requested interaction
@param widget: not used, here for compatibility with signals callbacks
@param event: the code of the gtk response
"""
if event == Gtk.ResponseType.ACCEPT:
if self.interaction_type == BackendSignals().INTERACTION_TEXT:
self._prepare_textual_interaction()
elif self.interaction_type == BackendSignals().INTERACTION_CONFIRM:
self.hide()
threading.Thread(
target=getattr(self.backend, self.callback)).start()
else:
self.hide()
def tomboy_failed(self):
""" Handle failed tomboy connection.
Disable backend and show error in notification bar """
self.tomboy_connection_is_ok = False
BackendSignals().backend_failed(self.backend.get_id(),
BackendSignals.ERRNO_DBUS)
self.backend.quit(disable=True)
def _when_taking_too_long(self):
"""
Function that is executed when the Dbus connection seems to be
hanging. It disables the backend and signals the error to the user.
"""
log.error("Dbus connection is taking too long for the Tomboy/Gnote"
"backend!")
BackendSignals().backend_failed(self.backend.get_id(),
BackendSignals.ERRNO_DBUS)
self.backend.quit(disable=True)
def _init_signals(self):
"""Initializes the backends and gtk signals """
self.connect("cursor-changed", self.on_select_row)
_signals = BackendSignals()
_signals.connect(_signals.BACKEND_ADDED, self.on_backend_added)
_signals.connect(_signals.BACKEND_STATE_TOGGLED,
self.on_backend_state_changed)
def set_interaction_request(self, description, interaction_type, callback):
"""
Sets this infobar to request an interaction from the user
@param description: a string describing the interaction needed
@param interaction_type: a string describing the type of interaction
(yes/no, only confirm, ok/cancel...)
@param callback: the function to call when the user provides the
feedback
"""
self._populate()
self.callback = callback
self.set_message_type(Gtk.MessageType.INFO)
self.label.set_markup(description)
self.connect("response", self._on_interaction_response)
self.interaction_type = interaction_type
if interaction_type == BackendSignals().INTERACTION_CONFIRM:
self.add_button(_('Confirm'), Gtk.ResponseType.ACCEPT)
elif interaction_type == BackendSignals().INTERACTION_TEXT:
self.add_button(_('Continue'), Gtk.ResponseType.ACCEPT)
elif interaction_type == BackendSignals().INTERACTION_INFORM:
self.add_button(_('OK'), Gtk.ResponseType.ACCEPT)
self.show_all()
def do_periodic_import(self):
# Establishing connection
try:
self.cancellation_point()
client = Client('%s/api/soap/mantisconnect.php?wsdl' %
self._parameters['service-url'])
except KeyError:
self.quit(disable=True)
BackendSignals().backend_failed(
self.get_id(), BackendSignals.ERRNO_AUTHENTICATION)
return
projects = client.service.mc_projects_get_user_accessible(
self._parameters['username'], self._parameters['password'])
filters = client.service.mc_filter_get(self._parameters['username'],
self._parameters['password'], 0)
# Fetching the issues
self.cancellation_point()
my_issues = []
for filt in filters:
if filt['name'] == 'gtg':
for project in projects:
my_issues = client.service.mc_filter_get_issues(
self._parameters['username'],
self._parameters['password'], project['id'],
filt['id'], 0, 100)
for issue in my_issues:
self.cancellation_point()
self._process_mantis_issue(issue)
last_issue_list = self.sync_engine.get_all_remote()
new_issue_list = [str(issue['id']) for issue in my_issues]
for issue_link in set(last_issue_list).difference(set(new_issue_list)):
self.cancellation_point()
# we make sure that the other backends are not modifying the task
# set
with self.datastore.get_backend_mutex():
tid = self.sync_engine.get_local_id(issue_link)
self.datastore.request_task_deletion(tid)
try:
self.sync_engine.break_relationship(remote_id=issue_link)
except KeyError:
pass
return
class GenericBackend():
"""
Base class for every backend.
It defines the interface a backend must have and takes care of all the
operations common to all backends.
A particular backend should redefine all the methods marked as such.
"""
###########################################################################
# BACKEND INTERFACE #######################################################
###########################################################################
# General description of the backend: these parameters are used
# to show a description of the backend to the user when s/he is
# considering adding it.
# For an example, see the GTG/backends/backend_localfile.py file
# _general_description has this format:
# _general_description = {
# GenericBackend.BACKEND_NAME: "backend_unique_identifier", \
# GenericBackend.BACKEND_HUMAN_NAME: _("Human friendly name"), \
# GenericBackend.BACKEND_AUTHORS: ["First author", \
# "Chuck Norris"], \
# GenericBackend.BACKEND_TYPE: GenericBackend.TYPE_READWRITE, \
# GenericBackend.BACKEND_DESCRIPTION: \
# _("Short description of the backend"),\
# }
# The complete list of constants and their meaning is given below.
_general_description = {}
# These are the parameters to configure a new backend of this type. A
# parameter has a name, a type and a default value.
# For an example, see the GTG/backends/backend_localfile.py file
# _static_parameters has this format:
# _static_parameters = { \
# "param1_name": { \
# GenericBackend.PARAM_TYPE: GenericBackend.TYPE_STRING,
# GenericBackend.PARAM_DEFAULT_VALUE: "my default value",
# },
# "param2_name": {
# GenericBackend.PARAM_TYPE: GenericBackend.TYPE_INT,
# GenericBackend.PARAM_DEFAULT_VALUE: 42,
# }}
# The complete list of constants and their meaning is given below.
_static_parameters = {}
def initialize(self):
"""
Called each time it is enabled (including on backend creation).
Please note that a class instance for each disabled backend *is*
created, but it's not initialized.
Optional.
NOTE: make sure to call super().initialize()
"""
self._parameters[self.KEY_ENABLED] = True
self._is_initialized = True
# we signal that the backend has been enabled
self._signal_manager.backend_state_changed(self.get_id())
def start_get_tasks(self):
"""
This function starts submitting the tasks from the backend into GTG
core.
It's run as a separate thread.
@return: start_get_tasks() might not return or finish
"""
return
def set_task(self, task):
"""
This function is called from GTG core whenever a task should be
saved, either because it's a new one or it has been modified.
If the task id is new for the backend, then a new task must be
created. No special notification that the task is a new one is given.
@param task: the task object to save
"""
pass
def remove_task(self, tid):
""" This function is called from GTG core whenever a task must be
removed from the backend. Note that the task could be not present here.
@param tid: the id of the task to delete
"""
pass
def this_is_the_first_run(self, xml):
"""
Optional, and almost surely not needed.
Called upon the very first GTG startup.
This function is needed only in the default backend (XML localfile,
currently).
The xml parameter is an object containing GTG default tasks.
@param xml: an xml object containing the default tasks.
"""
pass
def quit(self, disable=False):
"""
Called when GTG quits or the user wants to disable the backend.
@param disable: If disable is True, the backend won't
be automatically loaded when GTG starts
"""
if self._parameters[self.KEY_ENABLED]:
self._is_initialized = False
if disable:
self._parameters[self.KEY_ENABLED] = False
# we signal that we have been disabled
self._signal_manager.backend_state_changed(self.get_id())
self._signal_manager.backend_sync_ended(self.get_id())
threading.Thread(target=self.sync).run()
def save_state(self):
"""
It's the last function executed on a quitting backend, after the
pending actions have been done.
Useful to ensure that the state is saved in a consistent manner
"""
pass
###############################################################################
# You don't need to reimplement the functions below this line #################
###############################################################################
###########################################################################
# CONSTANTS ###############################################################
###########################################################################
# BACKEND TYPE DESCRIPTION
# Each backend must have a "_general_description" attribute, which
# is a dictionary that holds the values for the following keys.
# the backend gtg internal name
# (doesn't change in translations, *must be unique*)
BACKEND_NAME = "name"
BACKEND_HUMAN_NAME = "human-friendly-name" # The name shown to the user
BACKEND_DESCRIPTION = "description" # A short description of the backend
BACKEND_AUTHORS = "authors" # a list of strings
BACKEND_TYPE = "type"
# BACKEND_TYPE is one of:
TYPE_READWRITE = "readwrite"
TYPE_READONLY = "readonly"
TYPE_IMPORT = "import"
TYPE_EXPORT = "export"
# "static_parameters" is a dictionary of dictionaries, each of which
# are a description of a parameter needed to configure the backend and
# is identified in the outer dictionary by a key which is the name of the
# parameter.
# For an example, see the GTG/backends/backend_localfile.py file
# Each dictionary contains the keys:
PARAM_DEFAULT_VALUE = "default_value" # its default value
PARAM_TYPE = "type"
# PARAM_TYPE is one of the following (changing this changes the way
# the user can configure the parameter)
# the real password is stored in the GNOME keyring
# This is just a key to find it there
TYPE_PASSWORD = "******"
TYPE_STRING = "string" # generic string, nothing fancy is done
TYPE_INT = "int" # edit box can contain only integers
TYPE_BOOL = "bool" # checkbox is shown
# list of strings. the "," character is prohibited in strings
TYPE_LIST_OF_STRINGS = "liststring"
# These parameters are common to all backends and necessary.
# They will be added automatically to your _static_parameters list
# NOTE: for now I'm disabling changing the default backend. Once it's all
# set up, we will see about that (invernizzi)
KEY_DEFAULT_BACKEND = "Default"
KEY_ENABLED = "enabled"
KEY_HUMAN_NAME = BACKEND_HUMAN_NAME
KEY_ATTACHED_TAGS = "attached-tags"
KEY_USER = "******"
KEY_PID = "pid"
_static_parameters_obligatory = {
KEY_DEFAULT_BACKEND: {
PARAM_TYPE: TYPE_BOOL,
PARAM_DEFAULT_VALUE: False,
},
KEY_HUMAN_NAME: {
PARAM_TYPE: TYPE_STRING,
PARAM_DEFAULT_VALUE: "",
},
KEY_USER: {
PARAM_TYPE: TYPE_STRING,
PARAM_DEFAULT_VALUE: "",
},
KEY_PID: {
PARAM_TYPE: TYPE_STRING,
PARAM_DEFAULT_VALUE: "",
},
KEY_ENABLED: {
PARAM_TYPE: TYPE_BOOL,
PARAM_DEFAULT_VALUE: False,
}
}
_static_parameters_obligatory_for_rw = {
KEY_ATTACHED_TAGS: {
PARAM_TYPE: TYPE_LIST_OF_STRINGS,
PARAM_DEFAULT_VALUE: [ALLTASKS_TAG],
}
}
# Handy dictionary used in type conversion (from string to type)
_type_converter = {
TYPE_STRING: str,
TYPE_INT: int,
}
@classmethod
def _get_static_parameters(cls):
"""
Helper method, used to obtain the full list of the static_parameters
(user configured and default ones)
@returns dict: the dict containing all the static parameters
"""
temp_dic = cls._static_parameters_obligatory.copy()
if cls._general_description[cls.BACKEND_TYPE] == \
cls.TYPE_READWRITE:
for key, value in \
cls._static_parameters_obligatory_for_rw.items():
temp_dic[key] = value
for key, value in cls._static_parameters.items():
temp_dic[key] = value
return temp_dic
def __init__(self, parameters):
"""
Instantiates a new backend. Please note that this is called also
for disabled backends. Those are not initialized, so you might
want to check out the initialize() function.
"""
if self.KEY_DEFAULT_BACKEND not in parameters:
# if it's not specified, then this is the default backend
# (for retro-compatibility with the GTG 0.2 series)
parameters[self.KEY_DEFAULT_BACKEND] = True
# default backends should get all the tasks
no_attached_tags = self.KEY_ATTACHED_TAGS not in parameters and \
self._general_description[self.BACKEND_TYPE] == self.TYPE_READWRITE
if parameters[self.KEY_DEFAULT_BACKEND] or no_attached_tags:
parameters[self.KEY_ATTACHED_TAGS] = [ALLTASKS_TAG]
self._parameters = parameters
self._signal_manager = BackendSignals()
self._is_initialized = False
# if debugging mode is enabled, tasks should be saved as soon as
# they're marked as modified. If in normal mode, we prefer speed over
# easier debugging.
if log.isEnabledFor(logging.DEBUG):
self.timer_timestep = 5
else:
self.timer_timestep = 1
self.to_set_timer = None
self.please_quit = False
self.cancellation_point = lambda: _cancellation_point(lambda: self.
please_quit)
self.to_set = deque()
self.to_remove = deque()
def get_attached_tags(self):
"""
Returns the list of tags which are handled by this backend
"""
if hasattr(self._parameters, self.KEY_DEFAULT_BACKEND) and \
self._parameters[self.KEY_DEFAULT_BACKEND]:
# default backends should get all the tasks
# NOTE: this shouldn't be needed, but it doesn't cost anything and
# it could avoid potential tasks losses.
return [ALLTASKS_TAG]
try:
return self._parameters[self.KEY_ATTACHED_TAGS]
except Exception:
return []
def set_attached_tags(self, tags):
"""
Changes the set of attached tags
@param tags: the new attached_tags set
"""
self._parameters[self.KEY_ATTACHED_TAGS] = tags
@classmethod
def get_static_parameters(cls):
"""
Returns a dictionary of parameters necessary to create a backend.
"""
return cls._get_static_parameters()
def get_parameters(self):
"""
Returns a dictionary of the current parameters.
"""
return self._parameters
def set_parameter(self, parameter, value):
"""
Change a parameter for this backend
@param parameter: the parameter name
@param value: the new value
"""
self._parameters[parameter] = value
@classmethod
def get_name(cls):
"""
Returns the name of the backend as it should be displayed in the UI
"""
return cls._get_from_general_description(cls.BACKEND_NAME)
@classmethod
def get_description(cls):
"""Returns a description of the backend"""
return cls._get_from_general_description(cls.BACKEND_DESCRIPTION)
@classmethod
def get_type(cls):
"""Returns the backend type(readonly, r/w, import, export) """
return cls._get_from_general_description(cls.BACKEND_TYPE)
@classmethod
def get_authors(cls):
"""
returns the backend author(s)
"""
return cls._get_from_general_description(cls.BACKEND_AUTHORS)
@classmethod
def _get_from_general_description(cls, key):
"""
Helper method to extract values from cls._general_description.
@param key: the key to extract
"""
return cls._general_description[key]
@classmethod
def cast_param_type_from_string(cls, param_value, param_type):
"""
Parameters are saved in a text format, so we have to cast them to the
appropriate type on loading. This function does exactly that.
@param param_value: the actual value of the parameter, in a string
format
@param param_type: the wanted type
@returns something: the casted param_value
"""
if param_type in cls._type_converter:
return cls._type_converter[param_type](param_value)
elif param_type == cls.TYPE_BOOL:
if param_value == "True":
return True
elif param_value == "False":
return False
else:
raise Exception(f"Unrecognized bool value '{param_type}'")
elif param_type == cls.TYPE_PASSWORD:
if param_value == -1:
return None
return Keyring().get_password(int(param_value))
elif param_type == cls.TYPE_LIST_OF_STRINGS:
the_list = param_value.split(",")
if not isinstance(the_list, list):
the_list = [the_list]
return the_list
else:
raise NotImplemented(f"I don't know what type is '{param_type}'")
def cast_param_type_to_string(self, param_type, param_value):
"""
Inverse of cast_param_type_from_string
@param param_value: the actual value of the parameter
@param param_type: the type of the parameter (password...)
@returns something: param_value casted to string
"""
if param_type == GenericBackend.TYPE_PASSWORD:
if param_value is None:
return str(-1)
else:
return str(Keyring().set_password(
"GTG stored password -" + self.get_id(), param_value))
elif param_type == GenericBackend.TYPE_LIST_OF_STRINGS:
if param_value == []:
return ""
return reduce(lambda a, b: a + "," + b, param_value)
else:
return str(param_value)
def get_id(self):
"""
returns the backends id, used in the datastore for indexing backends
@returns string: the backend id
"""
return self.get_name() + "@" + self._parameters["pid"]
@classmethod
def get_human_default_name(cls):
"""
returns the user friendly default backend name, without eventual user
modifications.
@returns string: the default "human name"
"""
return cls._general_description[cls.BACKEND_HUMAN_NAME]
def get_human_name(self):
"""
returns the user customized backend name. If the user hasn't
customized it, returns the default one.
@returns string: the "human name" of this backend
"""
if self.KEY_HUMAN_NAME in self._parameters and \
self._parameters[self.KEY_HUMAN_NAME] != "":
return self._parameters[self.KEY_HUMAN_NAME]
else:
return self.get_human_default_name()
def set_human_name(self, name):
"""
sets a custom name for the backend
@param name: the new name
"""
self._parameters[self.KEY_HUMAN_NAME] = name
# we signal the change
self._signal_manager.backend_renamed(self.get_id())
def is_enabled(self):
"""
Returns if the backend is enabled
@returns: bool
"""
return self.get_parameters()[GenericBackend.KEY_ENABLED] or \
self.is_default()
def is_default(self):
"""
Returns if the backend is default
@returns: bool
"""
return self.get_parameters()[GenericBackend.KEY_DEFAULT_BACKEND]
def is_initialized(self):
"""
Returns if the backend is up and running
@returns: is_initialized
"""
return self._is_initialized
def get_parameter_type(self, param_name):
"""
Given the name of a parameter, returns its type. If the parameter is
one of the default ones, it does not have a type: in that case, it
returns None
@param param_name: the name of the parameter
@returns string: the type, or None
"""
try:
return self.get_static_parameters()[param_name][self.PARAM_TYPE]
except Exception:
return None
def register_datastore(self, datastore):
"""
Setter function to inform the backend about the datastore that's
loading it.
@param datastore: a Datastore
"""
self.datastore = datastore
###############################################################################
# HELPER FUNCTIONS ############################################################
###############################################################################
def _store_pickled_file(self, path, data):
"""
A helper function to save some object in a file.
@param path: a relative path. A good choice is
"backend_name/object_name"
@param data: the object
"""
path = os.path.join(SYNC_DATA_DIR, path)
# mkdir -p
try:
os.makedirs(os.path.dirname(path))
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
# Shift backups
for i in range(PICKLE_BACKUP_NBR, 1, -1):
destination = f"{path}.bak.{i:d}"
source = f"{path}.bak.{i - 1}"
if os.path.exists(destination):
os.unlink(destination)
if os.path.exists(source):
os.rename(source, destination)
# Backup main file
if PICKLE_BACKUP_NBR > 0:
destination = f"{path}.bak.1"
if os.path.exists(path):
os.rename(path, destination)
# saving
with open(path, 'wb') as file:
pickle.dump(data, file)
def _load_pickled_file(self, path, default_value=None):
"""
A helper function to load some object from a file.
@param path: the relative path of the file
@param default_value: the value to return if the file is missing or
corrupt
@returns object: the needed object, or default_value
"""
path = os.path.join(SYNC_DATA_DIR, path)
if not os.path.exists(path):
return default_value
with open(path, 'rb') as file:
try:
return pickle.load(file)
except Exception:
log.error("Pickle file for backend '%s' is damaged",
self.get_name())
# Loading file failed, trying backups
for i in range(1, PICKLE_BACKUP_NBR + 1):
backup_file = f"{path}.bak.{i:d}"
if os.path.exists(backup_file):
with open(backup_file, 'rb') as file:
try:
data = pickle.load(file)
log.info("Succesfully restored backup #%d for %r", i,
self.get_name())
return data
except Exception:
log.error("Backup #%d for %r is damaged as well", i,
self.get_name())
# Data could not be loaded, degrade to default data
log.error("There is no suitable backup for %r, loading default data",
self.get_name())
return default_value
def _gtg_task_is_syncable_per_attached_tags(self, task):
"""
Helper function which checks if the given task satisfies the filtering
imposed by the tags attached to the backend.
That means, if a user wants a backend to sync only tasks tagged @works,
this function should be used to check if that is verified.
@returns bool: True if the task should be synced
"""
attached_tags = self.get_attached_tags()
if ALLTASKS_TAG in attached_tags:
return True
for tag in task.get_tags_name():
if "@" + tag in attached_tags:
return True
return False
###############################################################################
# THREADING ###################################################################
###############################################################################
def __try_launch_setting_thread(self):
"""
Helper function to launch the setting thread, if it's not running.
"""
if self.to_set_timer is None and self.is_enabled():
self.to_set_timer = threading.Timer(self.timer_timestep,
self.launch_setting_thread)
self.to_set_timer.start()
def launch_setting_thread(self, bypass_quit_request=False):
"""
This function is launched as a separate thread. Its job is to perform
the changes that have been issued from GTG core.
In particular, for each task in the self.to_set queue, a task
has to be modified or to be created (if the tid is new), and for
each task in the self.to_remove queue, a task has to be deleted
@param bypass_quit_request: if True, the thread should not be stopped
even if asked by self.please_quit = True.
It's used when the backend quits, to finish
syncing all pending tasks
"""
while not self.please_quit or bypass_quit_request:
try:
task = self.to_set.pop()
except IndexError:
break
tid = task.get_id()
if tid not in self.to_remove:
self.set_task(task)
while not self.please_quit or bypass_quit_request:
try:
tid = self.to_remove.pop()
except IndexError:
break
self.remove_task(tid)
# we release the weak lock
self.to_set_timer = None
def queue_set_task(self, task):
""" Save the task in the backend. In particular, it just enqueues the
task in the self.to_set queue. A thread will shortly run to apply the
requested changes.
@param task: the task that should be saved
"""
tid = task.get_id()
if task not in self.to_set and tid not in self.to_remove:
self.to_set.appendleft(task)
self.__try_launch_setting_thread()
def queue_remove_task(self, tid):
"""
Queues task to be removed. In particular, it just enqueues the
task in the self.to_remove queue. A thread will shortly run to apply
the requested changes.
@param tid: The Task ID of the task to be removed
"""
if tid not in self.to_remove:
self.to_remove.appendleft(tid)
self.__try_launch_setting_thread()
return None
def sync(self):
"""
Helper method. Forces the backend to perform all the pending changes.
It is usually called upon quitting the backend.
"""
if self.to_set_timer is not None:
self.please_quit = True
try:
self.to_set_timer.cancel()
except Exception:
pass
try:
self.to_set_timer.join()
except Exception:
pass
self.launch_setting_thread(bypass_quit_request=True)
self.save_state()
def do_periodic_import(self):
"""
See GenericBackend for an explanation of this function.
Connect to launchpad and updates the state of GTG tasks to reflect the
bugs on launchpad.
"""
# IMPORTANT NOTE!
# Bugs can be splitted in bug tasks (such as, you can assign a single
# bug to multiple projects: you have one bug and several bug tasks).
# At least, one bug contains a bug task (if it's referring to a single
# project).
# Here, we process bug tasks, since those are the ones that get
# assigned to someone.
# To avoid having multiple GTG Tasks for the same bug (because we use
# bug tasks, this may happen if somebody is working at the same bug for
# different projects), we use the bug self_link for indexing the tasks.
# Connecting to Launchpad
CACHE_DIR = os.path.join(SYNC_CACHE_DIR, self.get_id())
try:
self.cancellation_point()
self.launchpad = Launchpad.login_anonymously(
GTG_NAME, EDGE_SERVICE_ROOT, CACHE_DIR)
except Exception:
# The connection is not working (the exception type can be
# anything)
BackendSignals().backend_failed(self.get_id(),
BackendSignals.ERRNO_NETWORK)
return
# Getting the user data
try:
self.cancellation_point()
me = self.launchpad.people[self._parameters["username"]]
except KeyError:
self.quit(disable=True)
BackendSignals().backend_failed(
self.get_id(), BackendSignals.ERRNO_AUTHENTICATION)
return
# Fetching the bugs
self.cancellation_point()
my_bugs_tasks = me.searchTasks(assignee=me,
status=[
"New", "Incomplete", "Confirmed",
"Triaged", "In Progress",
"Fix Committed"
])
# Adding and updating
for bug_task in my_bugs_tasks:
self.cancellation_point()
self._process_launchpad_bug(bug_task)
# removing the old ones
last_bug_list = self.sync_engine.get_all_remote()
new_bug_list = [bug.self_link for bug in my_bugs_tasks]
for bug_link in set(last_bug_list).difference(set(new_bug_list)):
self.cancellation_point()
# we make sure that the other backends are not modifying the task
# set
with self.datastore.get_backend_mutex():
tid = self.sync_engine.get_local_id(bug_link)
self.datastore.request_task_deletion(tid)
try:
self.sync_engine.break_relationship(remote_id=bug_link)
except KeyError:
pass
class DataStore():
"""
A wrapper around all backends that is responsible for keeping the backend
instances. It can enable, disable, register and destroy backends, and acts
as interface between the backends and GTG core.
You should not interface yourself directly with the DataStore: use the
Requester instead (which also sends signals as you issue commands).
"""
def __init__(self, global_conf=CoreConfig()):
"""
Initializes a DataStore object
"""
# dictionary {backend_name_string: Backend instance}
self.backends = {}
self.treefactory = TreeFactory()
self._tasks = self.treefactory.get_tasks_tree()
self.requester = requester.Requester(self, global_conf)
self.tagfile_loaded = False
self._tagstore = self.treefactory.get_tags_tree(self.requester)
self._backend_signals = BackendSignals()
self.conf = global_conf
self.tag_idmap = {}
# Flag when turned to true, all pending operation should be
# completed and then GTG should quit
self.please_quit = False
# The default backend must be loaded first. This flag turns to True
# when the default backend loading has finished.
self.is_default_backend_loaded = False
self._backend_signals.connect('default-backend-loaded',
self._activate_non_default_backends)
self._backend_mutex = threading.Lock()
# Accessor to embedded objects in DataStore ##############################
def get_tagstore(self):
"""
Return the Tagstore associated with this DataStore
@return GTG.core.tagstore.TagStore: the tagstore object
"""
return self._tagstore
def get_requester(self):
"""
Return the Requester associate with this DataStore
@returns GTG.core.requester.Requester: the requester associated with
this datastore
"""
return self.requester
def get_tasks_tree(self):
"""
Return the Tree with all the tasks contained in this Datastore
@returns GTG.core.tree.Tree: a task tree (the main one)
"""
return self._tasks
# Tags functions ##########################################################
def _add_new_tag(self, name, tag, filter_func, parameters, parent_id=None):
""" Add tag into a tree """
if self._tagstore.has_node(name):
raise IndexError(f'tag {name} was already in the datastore')
self._tasks.add_filter(name, filter_func, parameters=parameters)
self._tagstore.add_node(tag, parent_id=parent_id)
tag.set_save_callback(self.save)
def new_tag(self, name, attributes={}, tid=None):
"""
Create a new tag
@returns GTG.core.tag.Tag: the new tag
"""
parameters = {'tag': name}
tag = Tag(name, req=self.requester, attributes=attributes, tid=tid)
self._add_new_tag(name, tag, self.treefactory.tag_filter, parameters)
return tag
def new_search_tag(self, name, query, attributes={}, tid=None, save=True):
"""
Create a new search tag
@returns GTG.core.tag.Tag: the new search tag/None for a invalid query
"""
try:
parameters = parse_search_query(query)
except InvalidQuery as error:
log.warning("Problem with parsing query %r (skipping): %s", query,
error.message)
return None
# Create own copy of attributes and add special attributes label, query
init_attr = dict(attributes)
init_attr["label"] = name
init_attr["query"] = query
tag = Tag(name, req=self.requester, attributes=init_attr, tid=tid)
self._add_new_tag(name,
tag,
search_filter,
parameters,
parent_id=SEARCH_TAG)
if save:
self.save_tagtree()
return tag
def remove_tag(self, name):
""" Removes a tag from the tagtree """
if self._tagstore.has_node(name):
self._tagstore.del_node(name)
self.save_tagtree()
else:
raise IndexError(f"There is no tag {name}")
def rename_tag(self, oldname, newname):
""" Give a tag a new name
This function is quite high-level method. Right now,
only renaming search bookmarks are implemented by removing
the old one and creating almost identical one with the new name.
NOTE: Implementation for regular tasks must be much more robust.
You have to replace all occurences of tag name in tasks descriptions,
their parameters and backend settings (synchronize only certain tags).
Have a fun with implementing it!
"""
tag = self.get_tag(oldname)
if not tag.is_search_tag():
for task_id in tag.get_related_tasks():
# Store old tag attributes
color = tag.get_attribute("color")
icon = tag.get_attribute("icon")
tid = tag.tid
my_task = self.get_task(task_id)
my_task.rename_tag(oldname, newname)
# Restore attributes on tag
new_tag = self.get_tag(newname)
new_tag.tid = tid
if color:
new_tag.set_attribute("color", color)
if icon:
new_tag.set_attribute("icon", icon)
self.remove_tag(oldname)
self.save_tagtree()
return None
query = tag.get_attribute("query")
self.remove_tag(oldname)
# Make sure the name is unique
if newname.startswith('!'):
newname = '_' + newname
label, num = newname, 1
while self._tagstore.has_node(label):
num += 1
label = newname + " " + str(num)
self.new_search_tag(label, query, {}, tag.tid)
def get_tag(self, tagname):
"""
Returns tag object
@return GTG.core.tag.Tag
"""
if self._tagstore.has_node(tagname):
return self._tagstore.get_node(tagname)
else:
return None
def load_tag_tree(self, tag_tree):
"""
Loads the tag tree from a xml file
"""
for element in tag_tree.iter('tag'):
tid = element.get('id')
name = element.get('name')
color = element.get('color')
icon = element.get('icon')
parent = element.get('parent')
nonactionable = element.get('nonactionable')
tag_attrs = {}
if color:
tag_attrs['color'] = '#' + color
if icon:
tag_attrs['icon'] = icon
if nonactionable:
tag_attrs['nonactionable'] = nonactionable
tag = self.new_tag(name, tag_attrs, tid)
if parent:
tag.set_parent(parent)
# Add to idmap for quick lookup based on ID
self.tag_idmap[tid] = tag
self.tagfile_loaded = True
def load_search_tree(self, search_tree):
"""Load saved searches tree."""
for element in search_tree.iter('savedSearch'):
tid = element.get('id')
name = element.get('name')
color = element.get('color')
icon = element.get('icon')
query = element.get('query')
tag_attrs = {}
if color:
tag_attrs['color'] = color
if icon:
tag_attrs['icon'] = icon
self.new_search_tag(name, query, tag_attrs, tid, False)
def get_tag_by_id(self, tid):
"""Get a tag by its ID"""
try:
return self.tag_idmap[tid]
except KeyError:
return
def save_tagtree(self):
""" Saves the tag tree to an XML file """
if not self.tagfile_loaded:
return
tags = self._tagstore.get_main_view().get_all_nodes()
for backend in self.backends.values():
if backend.get_name() == 'backend_localfile':
backend.save_tags(tags, self._tagstore)
# Tasks functions #########################################################
def get_all_tasks(self):
"""
Returns list of all keys of active tasks
@return a list of strings: a list of task ids
"""
return self._tasks.get_main_view().get_all_nodes()
def has_task(self, tid):
"""
Returns true if the tid is among the active or closed tasks for
this DataStore, False otherwise.
@param tid: Task ID to search for
@return bool: True if the task is present
"""
return self._tasks.has_node(tid)
def get_task(self, tid):
"""
Returns the internal task object for the given tid, or None if the
tid is not present in this DataStore.
@param tid: Task ID to retrieve
@returns GTG.core.task.Task or None: whether the Task is present
or not
"""
if self.has_task(tid):
return self._tasks.get_node(tid)
else:
# log.error("requested non-existent task %s", tid)
# This is not an error: it is normal to request a task which
# might not exist yet.
return None
def task_factory(self, tid, newtask=False):
"""
Instantiates the given task id as a Task object.
@param tid: a task id. Must be unique
@param newtask: True if the task has never been seen before
@return Task: a Task instance
"""
return Task(tid, self.requester, newtask)
def new_task(self):
"""
Creates a blank new task in this DataStore.
New task is created in all the backends that collect all tasks (among
them, the default backend). The default backend uses the same task id
in its own internal representation.
@return: The task object that was created.
"""
task = self.task_factory(str(uuid.uuid4()), True)
self._tasks.add_node(task)
return task
def push_task(self, task):
"""
Adds the given task object to the task tree. In other words, registers
the given task in the GTG task set.
This function is used in mutual exclusion: only a backend at a time is
allowed to push tasks.
@param task: A valid task object (a GTG.core.task.Task)
@return bool: True if the task has been accepted
"""
def adding(task):
self._tasks.add_node(task)
task.set_loaded()
if self.is_default_backend_loaded:
task.sync()
if self.has_task(task.get_id()):
return False
else:
# Thread protection
adding(task)
return True
##########################################################################
# Backends functions
##########################################################################
def get_all_backends(self, disabled=False):
"""
returns list of all registered backends for this DataStore.
@param disabled: If disabled is True, attaches also the list of
disabled backends
@return list: a list of TaskSource objects
"""
result = []
for backend in self.backends.values():
if backend.is_enabled() or disabled:
result.append(backend)
return result
def get_backend(self, backend_id):
"""
Returns a backend given its id.
@param backend_id: a backend id
@returns GTG.core.datastore.TaskSource or None: the requested backend,
or None
"""
if backend_id in self.backends:
return self.backends[backend_id]
else:
return None
def register_backend(self, backend_dic):
"""
Registers a TaskSource as a backend for this DataStore
@param backend_dic: Dictionary object containing all the
parameters to initialize the backend
(filename...). It should also contain the
backend class (under "backend"), and its
unique id (under "pid")
"""
if "backend" in backend_dic:
if "pid" not in backend_dic:
log.error("registering a backend without pid.")
return None
backend = backend_dic["backend"]
first_run = backend_dic["first_run"]
# Checking that is a new backend
if backend.get_id() in self.backends:
log.error("registering already registered backend")
return None
# creating the TaskSource which will wrap the backend,
# filtering the tasks that should hit the backend.
source = TaskSource(requester=self.requester,
backend=backend,
datastore=self)
if first_run:
backend.this_is_the_first_run(None)
self.backends[backend.get_id()] = source
# we notify that a new backend is present
self._backend_signals.backend_added(backend.get_id())
# saving the backend in the correct dictionary (backends for
# enabled backends, disabled_backends for the disabled ones)
# this is useful for retro-compatibility
if GenericBackend.KEY_ENABLED not in backend_dic:
source.set_parameter(GenericBackend.KEY_ENABLED, True)
if GenericBackend.KEY_DEFAULT_BACKEND not in backend_dic:
source.set_parameter(GenericBackend.KEY_DEFAULT_BACKEND, True)
# if it's enabled, we initialize it
if source.is_enabled() and \
(self.is_default_backend_loaded or source.is_default()):
source.initialize(connect_signals=False)
# Filling the backend
# Doing this at start is more efficient than
# after the GUI is launched
source.start_get_tasks()
return source
else:
log.error("Tried to register a backend without a pid")
def _activate_non_default_backends(self, sender=None):
"""
Non-default backends have to wait until the default loads before
being activated. This function is called after the first default
backend has loaded all its tasks.
@param sender: not used, just here for signal compatibility
"""
if self.is_default_backend_loaded:
log.debug("spurious call")
return
self.is_default_backend_loaded = True
for backend in self.backends.values():
if backend.is_enabled() and not backend.is_default():
self._backend_startup(backend)
def _backend_startup(self, backend):
"""
Helper function to launch a thread that starts a backend.
@param backend: the backend object
"""
def __backend_startup(self, backend):
"""
Helper function to start a backend
@param backend: the backend object
"""
backend.initialize()
backend.start_get_tasks()
self.flush_all_tasks(backend.get_id())
thread = threading.Thread(target=__backend_startup,
args=(self, backend))
thread.setDaemon(True)
thread.start()
def set_backend_enabled(self, backend_id, state):
"""
The backend corresponding to backend_id is enabled or disabled
according to "state".
Disable:
Quits a backend and disables it (which means it won't be
automatically loaded next time GTG is started)
Enable:
Reloads a disabled backend. Backend must be already known by the
Datastore
@param backend_id: a backend id
@param state: True to enable, False to disable
"""
if backend_id in self.backends:
backend = self.backends[backend_id]
current_state = backend.is_enabled()
if current_state is True and state is False:
# we disable the backend
# FIXME!!!
threading.Thread(target=backend.quit, kwargs={
'disable': True
}).start()
elif current_state is False and state is True:
if self.is_default_backend_loaded is True:
self._backend_startup(backend)
else:
# will be activated afterwards
backend.set_parameter(GenericBackend.KEY_ENABLED, True)
def remove_backend(self, backend_id):
"""
Removes a backend, and forgets it ever existed.
@param backend_id: a backend id
"""
if backend_id in self.backends:
backend = self.backends[backend_id]
if backend.is_enabled():
self.set_backend_enabled(backend_id, False)
# FIXME: to keep things simple, backends are not notified that they
# are completely removed (they think they're just
# deactivated). We should add a "purge" call to backend to
# let them know that they're removed, so that they can
# remove all the various files they've created. (invernizzi)
# we notify that the backend has been deleted
self._backend_signals.backend_removed(backend.get_id())
del self.backends[backend_id]
def backend_change_attached_tags(self, backend_id, tag_names):
"""
Changes the tags for which a backend should store a task
@param backend_id: a backend_id
@param tag_names: the new set of tags. This should not be a tag object,
just the tag name.
"""
backend = self.backends[backend_id]
backend.set_attached_tags(tag_names)
def flush_all_tasks(self, backend_id):
"""
This function will cause all tasks to be checked against the backend
identified with backend_id. If tasks need to be added or removed, it
will be done here.
It has to be run after the creation of a new backend (or an alteration
of its "attached tags"), so that the tasks which are already loaded in
the Tree will be saved in the proper backends
@param backend_id: a backend id
"""
def _internal_flush_all_tasks():
backend = self.backends[backend_id]
for task_id in self.get_all_tasks():
if self.please_quit:
break
backend.queue_set_task(task_id)
t = threading.Thread(target=_internal_flush_all_tasks)
t.start()
self.backends[backend_id].start_get_tasks()
def save(self, quit=False):
"""
Saves the backends parameters.
@param quit: If quit is true, backends are shut down
"""
try:
self.start_get_tasks_thread.join()
except Exception:
pass
# we ask all the backends to quit first.
if quit:
# we quit backends in parallel
threads_dic = {}
for b in self.get_all_backends():
thread = threading.Thread(target=b.quit)
threads_dic[b.get_id()] = thread
thread.start()
for backend_id, thread in threads_dic.items():
# after 20 seconds, we give up
thread.join(20)
alive = thread.is_alive()
if alive:
log.error("The %s backend stalled while quitting",
backend_id)
# we save the parameters
for b in self.get_all_backends(disabled=True):
config = self.conf.get_backend_config(b.get_name())
for key, value in b.get_parameters().items():
if key in ["backend", "xmlobject"]:
# We don't want parameters, backend, xmlobject:
# we'll create them at next startup
continue
param_type = b.get_parameter_type(key)
value = b.cast_param_type_to_string(param_type, value)
config.set(str(key), value)
config.save()
# Saving the tagstore
self.save_tagtree()
def request_task_deletion(self, tid):
"""
This is a proxy function to request a task deletion from a backend
@param tid: the tid of the task to remove
"""
self.requester.delete_task(tid)
def get_backend_mutex(self):
"""
Returns the mutex object used by backends to avoid modifying a task
at the same time.
@returns: threading.Lock
"""
return self._backend_mutex