def push_to_zeitgeist(commits): ZG = ZeitgeistClient() commits = commits def error_handler(error): print "===> ERROR:", error def ids_reply_handler(ids): global commits print len(commits) if len(commits) > 0: events = format_events(commits[0:100]) commits = commits[100:] time.sleep(0.5) ZG.insert_events(events, ids_reply_handler, error_handler) events = format_events(commits[0:100]) commits = commits[100:] ZG.insert_events(events, ids_reply_handler, error_handler)
class RemoteTestCase(unittest.TestCase): """ Helper class to implement unit tests against a remote Zeitgeist process """ @staticmethod def _get_pid(matching_string): p1 = Popen(["pgrep", "-x", "zeitgeist-daemo"], stdout=PIPE, stderr=PIPE) out = p1.communicate()[0] pid = out.decode().split('\n')[0] p2 = Popen(["ps", "--no-headers", "-fp", pid], stderr=PIPE, stdout=PIPE) pid_line = p2.communicate()[0].decode() return pid_line @staticmethod def _safe_start_subprocess(cmd, env, timeout=1, error_callback=None): """ starts `cmd` in a subprocess and check after `timeout` if everything goes well""" args = {'env': env} if not '--verbose-subprocess' in sys.argv: args['stderr'] = PIPE args['stdout'] = PIPE process = Popen(cmd, **args) # give the process some time to wake up time.sleep(timeout) error = process.poll() if error: cmd = " ".join(cmd) error = "'%s' exits with error %i." % (cmd, error) if error_callback: error += " *** %s" % error_callback(*process.communicate()) raise RuntimeError(error) return process @staticmethod def _safe_start_daemon(env=None, timeout=1): if env is None: env = os.environ.copy() def error_callback(stdout, stderr): stderr = stderr.decode() if "--replace" in stderr: return "%r | %s" % ( stderr, RemoteTestCase._get_pid("./src/zeitgeist-daemon").replace( "\n", "|")) else: return stderr return RemoteTestCase._safe_start_subprocess( ("./src/zeitgeist-daemon", "--no-datahub", "--log-level=DEBUG"), env, timeout, error_callback) def __init__(self, methodName): super(RemoteTestCase, self).__init__(methodName) self.daemon = None self.client = None def spawn_daemon(self): self.daemon = self._safe_start_daemon(env=self.env) def kill_daemon(self, kill_signal=signal.SIGKILL): os.kill(self.daemon.pid, kill_signal) return self.daemon.wait() def setUp(self, database_path=None): assert self.daemon is None assert self.client is None self.env = os.environ.copy() self.datapath = tempfile.mkdtemp(prefix="zeitgeist.datapath.") self.env.update({ "ZEITGEIST_DATABASE_PATH": database_path or ":memory:", "ZEITGEIST_DATA_PATH": self.datapath, "XDG_CACHE_HOME": os.path.join(self.datapath, "cache"), }) self.spawn_daemon() # hack to clear the state of the interface ZeitgeistDBusInterface._ZeitgeistDBusInterface__shared_state = {} # Replace the bus connection with a private one for each test case, # so that they don't share signals or other state _set_bus(dbus.SessionBus(private=True)) get_bus().set_exit_on_disconnect(False) self.client = ZeitgeistClient() def tearDown(self): assert self.daemon is not None assert self.client is not None get_bus().close() self.kill_daemon() if 'ZEITGEIST_TESTS_KEEP_TMP' in os.environ: print('\n\nAll temporary files have been preserved in %s\n' \ % self.datapath) else: shutil.rmtree(self.datapath) def insertEventsAndWait(self, events): """ Insert a set of events and spin a mainloop until the async reply is back and return the result - which should be a list of ids. This method is basically just a hack to invoke an async method in a blocking manner. """ mainloop = self.create_mainloop() result = [] def collect_ids_and_quit(ids): result.extend(ids) mainloop.quit() self.client.insert_events(events, ids_reply_handler=collect_ids_and_quit) mainloop.run() return result def findEventIdsAndWait(self, event_templates, **kwargs): """ Do search based on event_templates and spin a mainloop until the async reply is back and return the result - which should be a list of ids. This method is basically just a hack to invoke an async method in a blocking manner. """ mainloop = self.create_mainloop() result = [] def collect_ids_and_quit(ids): result.extend(ids) mainloop.quit() self.client.find_event_ids_for_templates(event_templates, collect_ids_and_quit, **kwargs) mainloop.run() return result def getEventsAndWait(self, event_ids): """ Request a set of full events and spin a mainloop until the async reply is back and return the result - which should be a list of Events. This method is basically just a hack to invoke an async method in a blocking manner. """ mainloop = self.create_mainloop() result = [] def collect_events_and_quit(events): for event in events: if event: event[0][0] = int(event.id) result.extend(events) mainloop.quit() self.client.get_events(event_ids, collect_events_and_quit) mainloop.run() return result def findEventsForTemplatesAndWait(self, event_templates, **kwargs): """ Execute ZeitgeistClient.find_events_for_templates in a blocking manner. """ mainloop = self.create_mainloop() result = [] def collect_events_and_quit(events): result.extend(events) mainloop.quit() self.client.find_events_for_templates(event_templates, collect_events_and_quit, **kwargs) mainloop.run() return result def findEventsForValuesAndWait(self, *args, **kwargs): """ Execute ZeitgeistClient.find_events_for_value in a blocking manner. """ mainloop = self.create_mainloop() result = [] def collect_events_and_quit(events): result.extend(events) mainloop.quit() self.client.find_events_for_values(collect_events_and_quit, *args, **kwargs) mainloop.run() return result def deleteEventsAndWait(self, event_ids): """ Delete events given by their id and run a loop until the result containing a timetuple describing the interval of changes is returned. This method is basically just a hack to invoke an async method in a blocking manner. """ mainloop = self.create_mainloop() result = [] def collect_timestamp_and_quit(timestamps): result.append(timestamps) mainloop.quit() self.client.delete_events(event_ids, collect_timestamp_and_quit) mainloop.run() return result[0] def findRelatedAndWait(self, subject_uris, num_events, result_type): """ Find related subject uris to given uris and return them. This method is basically just a hack to invoke an async method in a blocking manner. """ mainloop = self.create_mainloop() result = [] def callback(uri_list): result.extend(uri_list) mainloop.quit() self.client.find_related_uris_for_uris(subject_uris, callback, num_events=num_events, result_type=result_type) mainloop.run() return result @staticmethod def create_mainloop(timeout=5): class MainLoopWithFailure(object): """ Remember to wrap callbacks using the asyncTestMethod decorator. """ def __init__(self): self._mainloop = GLib.MainLoop() self.failed = False def __getattr__(self, name): return getattr(self._mainloop, name) def fail(self, message): self.failed = True self.failure_message = message mainloop.quit() def run(self): assert self.failed is False self._mainloop.run() if self.failed: raise AssertionError(self.failure_message) mainloop = MainLoopWithFailure() if timeout is not None: def cb_timeout(): mainloop.fail("Timed out -- " "operations not completed in reasonable time.") return False # stop timeout from being called again # Add an arbitrary timeout so this test won't block if it fails GLib.timeout_add_seconds(timeout, cb_timeout) return mainloop @staticmethod def get_plain_event(ev): """ Ensure that an Event instance is a Plain Old Python Object (popo), without DBus wrappings, etc. """ if not ev: return NULL_EVENT for subject in ev.subjects: if not subject.current_uri: subject.current_uri = subject.uri if not subject.current_origin: subject.current_origin = subject.origin popo = [] popo.append(list(map(str, ev[0]))) popo.append([list(map(str, subj)) for subj in ev[1]]) # We need the check here so that if D-Bus gives us an empty # byte array we don't serialize the text "dbus.Array(...)". popo.append(str(ev[2]) if ev[2] else '') return popo def assertEventsEqual(self, ev1, ev2): ev1 = self.get_plain_event(Event(ev1)) ev2 = self.get_plain_event(Event(ev2)) if ev1 is not NULL_EVENT and ev2 is not NULL_EVENT: if (ev1[0][0] and not ev2[0][0]) or (ev2[0][0] and not ev1[0][0]): ev1[0][0] = ev2[0][0] = "" # delete IDs self.assertEqual(ev1, ev2)
class ZeitgeistIntegration(plugin.Plugin): ''' Plugin that enables integration with the Zeitgeist Framework. ''' def initialize(self): ''' Inicializes the Zeitgeist client and registers itself as a Zeitgeist data source. ''' self.logger.info('Initialiazing zeitgeist plugin') editor = self.locator.get_service('editor') editor.fileOpened.connect(self._zeitgeist_log_file_open) editor.fileSaved.connect(self._zeitgeist_log_file_modified) #initialize zeitgeist client self.zeitgeist = ZeitgeistClient() self._register_data_source() def finish(self): ''' Deletes the reference to the Zeitgeist client. ''' # remove zeitgeist client del self.zeitgeist self.logger.info('Shutting down zeitgeist plugin') def _register_data_source(self): ''' Registers the plugin as a Zeitgeist data source. ''' unique_id = 'org.ninja.ide' name = 'Ninja IDE' description = 'Very versatile Python IDE' # Describe what sort of events will be inserted templates = [] for interp in (Interpretation.EVENT_INTERPRETATION.ACCESS_EVENT, Interpretation.EVENT_INTERPRETATION.MODIFY_EVENT, Interpretation.EVENT_INTERPRETATION.LEAVE_EVENT, Interpretation.EVENT_INTERPRETATION.CREATE_EVENT): event_template = Event() event_template.interpretation = interp event_template.manifestation = Manifestation.USER_ACTIVITY for subj_interp in (Interpretation.DOCUMENT.TEXT_DOCUMENT. PLAIN_TEXT_DOCUMENT.SOURCE_CODE, Interpretation.DOCUMENT.TEXT_DOCUMENT.PLAIN_TEXT_DOCUMENT): subject_template = Subject() subject_template.interpretation = subj_interp subject_template.manifestation = Manifestation.FILE_DATA_OBJECT event_template.append_subject(subject_template) templates.append(event_template) self.zeitgeist.register_data_source(unique_id, name, description, templates) def _zeitgeist_log_event(self, fileName, interpretation): ''' Registers an event over the file with a given interpretation. ''' fileName = unicode(fileName) self.logger.info('Inserting event for %s' % fileName) subject = Subject.new_for_values( uri='file://%s' % fileName, origin='file://%s' % path.dirname(fileName), text=path.basename(fileName)) event = Event.new_for_values( timestamp=int(time.time() * 1000), interpretation=interpretation, manifestation=Manifestation.USER_ACTIVITY, actor='application://ninja-ide.desktop', subjects=[subject]) def on_id_received(event_ids): self.logger.info( 'Logged %r with event id %d.' % (fileName, event_ids[0])) self.zeitgeist.insert_events([event], on_id_received) def _zeitgeist_log_file_open(self, fileName): ''' Registers an event everytime Ninja IDE opens a file. ''' self._zeitgeist_log_event(unicode(fileName), Interpretation.EVENT_INTERPRETATION.ACCESS_EVENT) def _zeitgeist_log_file_modified(self, fileName): ''' Registers an event everytime Ninja IDE modifies a file. ''' self._zeitgeist_log_event(unicode(fileName), Interpretation.EVENT_INTERPRETATION.MODIFY_EVENT)
class RemoteTestCase (unittest.TestCase): """ Helper class to implement unit tests against a remote Zeitgeist process """ @staticmethod def _get_pid(matching_string): p1 = Popen(["ps", "aux"], stdout=PIPE, stderr=PIPE) p2 = Popen(["grep", matching_string], stdin=p1.stdout, stderr=PIPE, stdout=PIPE) return p2.communicate()[0] @staticmethod def _safe_start_subprocess(cmd, env, timeout=1, error_callback=None): """ starts `cmd` in a subprocess and check after `timeout` if everything goes well""" args = { 'env': env } if not '--verbose-subprocess' in sys.argv: args['stderr'] = PIPE args['stdout'] = PIPE process = Popen(cmd, **args) # give the process some time to wake up time.sleep(timeout) error = process.poll() if error: cmd = " ".join(cmd) error = "'%s' exits with error %i." %(cmd, error) if error_callback: error += " *** %s" %error_callback(*process.communicate()) raise RuntimeError(error) return process @staticmethod def _safe_start_daemon(env=None, timeout=1): if env is None: env = os.environ.copy() def error_callback(stdout, stderr): if "--replace" in stderr: return "%r | %s" %(stderr, RemoteTestCase._get_pid( "./src/zeitgeist-daemon").replace("\n", "|")) else: return stderr return RemoteTestCase._safe_start_subprocess( ("./src/zeitgeist-daemon", "--no-datahub", "--log-level=DEBUG"), env, timeout, error_callback) def __init__(self, methodName): super(RemoteTestCase, self).__init__(methodName) self.daemon = None self.client = None def spawn_daemon(self): self.daemon = self._safe_start_daemon(env=self.env) def kill_daemon(self, kill_signal=signal.SIGKILL): os.kill(self.daemon.pid, kill_signal) return self.daemon.wait() def setUp(self, database_path=None): assert self.daemon is None assert self.client is None self.env = os.environ.copy() self.datapath = tempfile.mkdtemp(prefix="zeitgeist.datapath.") self.env.update({ "ZEITGEIST_DATABASE_PATH": database_path or ":memory:", "ZEITGEIST_DATA_PATH": self.datapath, "XDG_CACHE_HOME": os.path.join(self.datapath, "cache"), }) self.spawn_daemon() # hack to clear the state of the interface ZeitgeistDBusInterface._ZeitgeistDBusInterface__shared_state = {} # Replace the bus connection with a private one for each test case, # so that they don't share signals or other state _set_bus(dbus.SessionBus(private=True)) get_bus().set_exit_on_disconnect(False) self.client = ZeitgeistClient() def tearDown(self): assert self.daemon is not None assert self.client is not None get_bus().close() self.kill_daemon() if 'ZEITGEIST_TESTS_KEEP_TMP' in os.environ: print '\n\nAll temporary files have been preserved in %s\n' \ % self.datapath else: shutil.rmtree(self.datapath) def insertEventsAndWait(self, events): """ Insert a set of events and spin a mainloop until the async reply is back and return the result - which should be a list of ids. This method is basically just a hack to invoke an async method in a blocking manner. """ mainloop = self.create_mainloop() result = [] def collect_ids_and_quit(ids): result.extend(ids) mainloop.quit() self.client.insert_events(events, ids_reply_handler=collect_ids_and_quit) mainloop.run() return result def findEventIdsAndWait(self, event_templates, **kwargs): """ Do search based on event_templates and spin a mainloop until the async reply is back and return the result - which should be a list of ids. This method is basically just a hack to invoke an async method in a blocking manner. """ mainloop = self.create_mainloop() result = [] def collect_ids_and_quit(ids): result.extend(ids) mainloop.quit() self.client.find_event_ids_for_templates(event_templates, collect_ids_and_quit, **kwargs) mainloop.run() return result def getEventsAndWait(self, event_ids): """ Request a set of full events and spin a mainloop until the async reply is back and return the result - which should be a list of Events. This method is basically just a hack to invoke an async method in a blocking manner. """ mainloop = self.create_mainloop() result = [] def collect_events_and_quit(events): for event in events: if event: event[0][0] = int(event.id) result.extend(events) mainloop.quit() self.client.get_events(event_ids, collect_events_and_quit) mainloop.run() return result def findEventsForTemplatesAndWait(self, event_templates, **kwargs): """ Execute ZeitgeistClient.find_events_for_templates in a blocking manner. """ mainloop = self.create_mainloop() result = [] def collect_events_and_quit(events): result.extend(events) mainloop.quit() self.client.find_events_for_templates( event_templates, collect_events_and_quit, **kwargs) mainloop.run() return result def findEventsForValuesAndWait(self, *args, **kwargs): """ Execute ZeitgeistClient.find_events_for_value in a blocking manner. """ mainloop = self.create_mainloop() result = [] def collect_events_and_quit(events): result.extend(events) mainloop.quit() self.client.find_events_for_values( collect_events_and_quit, *args, **kwargs) mainloop.run() return result def deleteEventsAndWait(self, event_ids): """ Delete events given by their id and run a loop until the result containing a timetuple describing the interval of changes is returned. This method is basically just a hack to invoke an async method in a blocking manner. """ mainloop = self.create_mainloop() result = [] def collect_timestamp_and_quit(timestamps): result.append(timestamps) mainloop.quit() self.client.delete_events(event_ids, collect_timestamp_and_quit) mainloop.run() return result[0] def findRelatedAndWait(self, subject_uris, num_events, result_type): """ Find related subject uris to given uris and return them. This method is basically just a hack to invoke an async method in a blocking manner. """ mainloop = self.create_mainloop() result = [] def callback(uri_list): result.extend(uri_list) mainloop.quit() self.client.find_related_uris_for_uris(subject_uris, callback, num_events=num_events, result_type=result_type) mainloop.run() return result @staticmethod def create_mainloop(timeout=5): class MainLoopWithFailure(object): """ Remember to wrap callbacks using the asyncTestMethod decorator. """ def __init__(self): self._mainloop = gobject.MainLoop() self.failed = False def __getattr__(self, name): return getattr(self._mainloop, name) def fail(self, message): self.failed = True self.failure_message = message mainloop.quit() def run(self): assert self.failed is False self._mainloop.run() if self.failed: raise AssertionError, self.failure_message mainloop = MainLoopWithFailure() if timeout is not None: def cb_timeout(): mainloop.fail("Timed out -- " "operations not completed in reasonable time.") return False # stop timeout from being called again # Add an arbitrary timeout so this test won't block if it fails gobject.timeout_add_seconds(timeout, cb_timeout) return mainloop @staticmethod def get_plain_event(ev): """ Ensure that an Event instance is a Plain Old Python Object (popo), without DBus wrappings, etc. """ if not ev: return NULL_EVENT for subject in ev.subjects: if not subject.current_uri: subject.current_uri = subject.uri if not subject.current_origin: subject.current_origin = subject.origin popo = [] popo.append(map(unicode, ev[0])) popo.append([map(unicode, subj) for subj in ev[1]]) # We need the check here so that if D-Bus gives us an empty # byte array we don't serialize the text "dbus.Array(...)". popo.append(str(ev[2]) if ev[2] else u'') return popo def assertEventsEqual(self, ev1, ev2): ev1 = self.get_plain_event(Event(ev1)) ev2 = self.get_plain_event(Event(ev2)) if ev1 is not NULL_EVENT and ev2 is not NULL_EVENT: if (ev1[0][0] and not ev2[0][0]) or (ev2[0][0] and not ev1[0][0]): ev1[0][0] = ev2[0][0] = "" # delete IDs self.assertEqual(ev1, ev2)