def setUp(self):
self.workdir = cfg.create_monitored_test_dir()
index_dirs = [self.workdir]
CONF_OPTIONS = {
cfg.DCONF_MINER_SCHEMA: {
'index-recursive-directories':
GLib.Variant.new_strv(index_dirs),
'index-single-directories': GLib.Variant.new_strv([]),
'index-optical-discs': GLib.Variant.new_boolean(False),
'index-removable-devices': GLib.Variant.new_boolean(False),
}
}
# Use local directory if available. Installation otherwise.
if os.path.exists(os.path.join(os.getcwd(), "test-apps-data")):
self.datadir = os.path.join(os.getcwd(), "test-apps-data")
else:
self.datadir = os.path.join(cfg.DATADIR, "tracker-tests",
"test-apps-data")
self.system = TrackerSystemAbstraction()
self.system.tracker_all_testing_start(CONF_OPTIONS)
self.tracker = self.system.store
def setUp(self):
if not os.path.exists(MINER_TMP_DIR):
os.makedirs(MINER_TMP_DIR)
assert os.path.isdir(MINER_TMP_DIR)
self.system = TrackerSystemAbstraction(CONF_OPTIONS)
self.system.tracker_miner_fs_testing_start()
class ExtractorDecoratorTest(ut.TestCase):
def setUp(self):
ensure_dir_exists(cfg.TEST_MONITORED_TMP_DIR)
# It's important that this directory is NOT inside /tmp, because
# monitoring files in /tmp usually doesn't work.
self.datadir = tempfile.mkdtemp(dir=cfg.TEST_MONITORED_TMP_DIR)
config = {
cfg.DCONF_MINER_SCHEMA: {
'index-recursive-directories': GLib.Variant.new_strv([]),
'index-single-directories':
GLib.Variant.new_strv([self.datadir]),
'index-optical-discs': GLib.Variant.new_boolean(False),
'index-removable-devices': GLib.Variant.new_boolean(False),
},
'org.freedesktop.Tracker.Store': {
'graphupdated-delay': GLib.Variant('i', 100)
}
}
self.system = TrackerSystemAbstraction(config)
self.system.tracker_miner_fs_testing_start()
def tearDown(self):
self.system.tracker_miner_fs_testing_stop()
shutil.rmtree(self.datadir)
def test_reextraction(self):
"""Tests whether known files are still re-extracted on user request."""
miner_fs = self.system.miner_fs
store = self.system.store
# Insert a valid file and wait extraction of its metadata.
file_path = os.path.join(self.datadir, os.path.basename(VALID_FILE))
shutil.copy(VALID_FILE, file_path)
file_id, file_urn = store.await_resource_inserted(
VALID_FILE_CLASS, title=VALID_FILE_TITLE)
# Remove a key piece of metadata.
store.update('DELETE { <%s> nie:title ?title }'
' WHERE { <%s> nie:title ?title }' % (file_urn, file_urn))
store.await_property_changed(VALID_FILE_CLASS, file_id, 'nie:title')
assert not store.ask('ASK { <%s> nie:title ?title }' % file_urn)
log("Sending re-index request")
# Request re-indexing (same as `tracker index --file ...`)
miner_fs.index_file('file://' + os.path.join(self.datadir, file_path))
# The extractor should reindex the file and re-add the metadata that we
# deleted, so we should see the nie:title property change.
store.await_property_changed(VALID_FILE_CLASS, file_id, 'nie:title')
title_result = store.query('SELECT ?title { <%s> nie:title ?title }' %
file_urn)
assert len(title_result) == 1
self.assertEqual(title_result[0][0], VALID_FILE_TITLE)
class CommonTrackerWritebackTest (ut.TestCase):
"""
Superclass to share methods. Shouldn't be run by itself.
Start all processes including writeback, miner pointing to HOME/test-writeback-monitored
"""
@classmethod
def __prepare_directories (self):
#
# ~/test-writeback-monitored/
#
for d in ["test-writeback-monitored"]:
directory = os.path.join (WRITEBACK_TMP_DIR, d)
if (os.path.exists (directory)):
shutil.rmtree (directory)
os.makedirs (directory)
if (os.path.exists (os.getcwd() + "/test-writeback-data")):
# Use local directory if available
datadir = os.getcwd() + "/test-writeback-data"
else:
datadir = os.path.join (cfg.DATADIR, "tracker-tests",
"test-writeback-data")
for testfile in [TEST_FILE_JPEG, TEST_FILE_PNG,TEST_FILE_TIFF]:
origin = os.path.join (datadir, testfile)
log ("Copying %s -> %s" % (origin, WRITEBACK_TMP_DIR))
shutil.copy (origin, WRITEBACK_TMP_DIR)
time.sleep (2)
@classmethod
def setUpClass (self):
#print "Starting the daemon in test mode"
self.__prepare_directories ()
self.system = TrackerSystemAbstraction ()
self.system.tracker_writeback_testing_start (CONF_OPTIONS)
# Returns when ready
log ("Ready to go!")
@classmethod
def tearDownClass (self):
#print "Stopping the daemon in test mode (Doing nothing now)"
self.system.tracker_writeback_testing_stop ()
def get_test_filename_jpeg (self):
return uri (TEST_FILE_JPEG)
def get_test_filename_tiff (self):
return uri (TEST_FILE_TIFF)
def get_test_filename_png (self):
return uri (TEST_FILE_PNG)
def setUpClass(self):
#print "Starting the daemon in test mode"
self.__prepare_directories()
self.system = TrackerSystemAbstraction()
self.system.tracker_writeback_testing_start(CONF_OPTIONS)
# Returns when ready
log("Ready to go!")
class CommonTrackerWritebackTest(ut.TestCase):
"""
Superclass to share methods. Shouldn't be run by itself.
Start all processes including writeback, miner pointing to HOME/test-writeback-monitored
"""
@classmethod
def __prepare_directories(self):
#
# ~/test-writeback-monitored/
#
for d in ["test-writeback-monitored"]:
directory = os.path.join(WRITEBACK_TMP_DIR, d)
if (os.path.exists(directory)):
shutil.rmtree(directory)
os.makedirs(directory)
if (os.path.exists(os.getcwd() + "/test-writeback-data")):
# Use local directory if available
datadir = os.getcwd() + "/test-writeback-data"
else:
datadir = os.path.join(cfg.DATADIR, "tracker-tests",
"test-writeback-data")
for testfile in [TEST_FILE_JPEG, TEST_FILE_PNG, TEST_FILE_TIFF]:
origin = os.path.join(datadir, testfile)
log("Copying %s -> %s" % (origin, WRITEBACK_TMP_DIR))
shutil.copy(origin, WRITEBACK_TMP_DIR)
time.sleep(2)
@classmethod
def setUpClass(self):
#print "Starting the daemon in test mode"
self.__prepare_directories()
self.system = TrackerSystemAbstraction()
self.system.tracker_writeback_testing_start(CONF_OPTIONS)
# Returns when ready
log("Ready to go!")
@classmethod
def tearDownClass(self):
#print "Stopping the daemon in test mode (Doing nothing now)"
self.system.tracker_writeback_testing_stop()
def get_test_filename_jpeg(self):
return uri(TEST_FILE_JPEG)
def get_test_filename_tiff(self):
return uri(TEST_FILE_TIFF)
def get_test_filename_png(self):
return uri(TEST_FILE_PNG)
class CommonTrackerStoreTest(ut.TestCase):
"""
Common superclass for tests that just require a fresh store running
"""
@classmethod
def setUpClass(self):
self.system = TrackerSystemAbstraction()
self.system.tracker_store_testing_start()
self.tracker = self.system.store
@classmethod
def tearDownClass(self):
self.system.finish()
def setUp(self):
ensure_dir_exists(cfg.TEST_MONITORED_TMP_DIR)
# It's important that this directory is NOT inside /tmp, because
# monitoring files in /tmp usually doesn't work.
self.datadir = tempfile.mkdtemp(dir=cfg.TEST_MONITORED_TMP_DIR)
self.indexed_dir = os.path.join(self.datadir, 'test-monitored')
# It's important that this directory exists BEFORE we start Tracker:
# it won't monitor an indexing root for changes if it doesn't exist,
# it'll silently ignore it instead. See the tracker_crawler_start()
# function.
ensure_dir_exists(self.indexed_dir)
self.system = TrackerSystemAbstraction(
settings={
'org.freedesktop.Tracker.Store': {
'graphupdated-delay': GLib.Variant('i', 100)
}
})
config = {
cfg.DCONF_MINER_SCHEMA: {
'index-recursive-directories':
GLib.Variant.new_strv([self.indexed_dir]),
'index-single-directories':
GLib.Variant.new_strv([]),
'index-optical-discs':
GLib.Variant.new_boolean(False),
'index-removable-devices':
GLib.Variant.new_boolean(False),
'throttle':
GLib.Variant.new_int32(5),
}
}
try:
self.system.tracker_miner_fs_testing_start(config)
except RuntimeError as e:
self.fail(e)
self.tracker = self.system.store
try:
self.create_test_data()
self.tracker.reset_graph_updates_tracking()
except Exception as e:
self.tearDown()
raise
class CommonTrackerStoreTest (ut.TestCase):
"""
Common superclass for tests that just require a fresh store running
"""
@classmethod
def setUpClass (self):
#print "Starting the daemon in test mode"
self.system = TrackerSystemAbstraction ()
self.system.tracker_store_testing_start ()
self.tracker = self.system.store
@classmethod
def tearDownClass (self):
#print "Stopping the daemon in test mode (Doing nothing now)"
self.system.tracker_store_testing_stop ()
class CommonTrackerStoreTest(ut.TestCase):
"""
Common superclass for tests that just require a fresh store running
"""
@classmethod
def setUpClass(self):
#print "Starting the daemon in test mode"
self.system = TrackerSystemAbstraction()
self.system.tracker_store_testing_start()
self.tracker = self.system.store
@classmethod
def tearDownClass(self):
#print "Stopping the daemon in test mode (Doing nothing now)"
self.system.tracker_store_testing_stop()
def setUp(self):
if not os.path.exists(MINER_TMP_DIR):
os.makedirs(MINER_TMP_DIR)
assert os.path.isdir(MINER_TMP_DIR)
self.system = TrackerSystemAbstraction(CONF_OPTIONS)
self.system.tracker_miner_fs_testing_start()
def setUpClass(self):
#print "Starting the daemon in test mode"
self.__prepare_directories()
self.system = TrackerSystemAbstraction()
if (os.path.exists(os.getcwd() +
"/test-configurations/miner-basic-ops")):
# Use local directory if available
confdir = os.getcwd() + "/test-configurations/miner-basic-ops"
else:
confdir = os.path.join(cfg.DATADIR, "tracker-tests",
"test-configurations", "miner-basic-ops")
self.system.tracker_miner_fs_testing_start(CONF_OPTIONS)
self.system.tracker_miner_fs_wait_for_idle()
self.tracker = self.system.store
def setUp(self):
for d in ['test-monitored', 'test-no-monitored']:
dirname = path(d)
if os.path.exists(dirname):
shutil.rmtree(dirname)
os.makedirs(dirname)
self.system = TrackerSystemAbstraction()
self.system.tracker_miner_fs_testing_start(CONF_OPTIONS)
self.tracker = self.system.store
try:
self.prepare_directories()
self.tracker.reset_graph_updates_tracking()
except Exception as e:
self.tearDown()
raise
def setUpClass (self):
#print "Starting the daemon in test mode"
self.__prepare_directories ()
self.system = TrackerSystemAbstraction ()
self.system.tracker_writeback_testing_start (CONF_OPTIONS)
# Returns when ready
log ("Ready to go!")
class ExtractorDecoratorTest(ut.TestCase):
def setUp(self):
if not os.path.exists(MINER_TMP_DIR):
os.makedirs(MINER_TMP_DIR)
assert os.path.isdir(MINER_TMP_DIR)
self.system = TrackerSystemAbstraction(CONF_OPTIONS)
self.system.tracker_miner_fs_testing_start()
def tearDown(self):
self.system.tracker_miner_fs_testing_stop()
shutil.rmtree(MINER_TMP_DIR)
def test_reextraction(self):
"""Tests whether known files are still re-extracted on user request."""
miner_fs = self.system.miner_fs
store = self.system.store
# Insert a valid file and wait extraction of its metadata.
file_path = os.path.join(MINER_TMP_DIR, os.path.basename(VALID_FILE))
shutil.copy(VALID_FILE, file_path)
file_id, file_urn = store.await_resource_inserted(
VALID_FILE_CLASS, title=VALID_FILE_TITLE)
# Remove a key piece of metadata.
store.update('DELETE { <%s> nie:title ?title }'
' WHERE { <%s> nie:title ?title }' % (file_urn, file_urn))
store.await_property_changed(file_id, 'nie:title')
assert not store.ask('ASK { <%s> nie:title ?title }' % file_urn)
log("Sending re-index request")
# Request re-indexing (same as `tracker index --file ...`)
miner_fs.index_file(uri(file_path))
# The extractor should reindex the file and re-add the metadata that we
# deleted, so we should see the nie:title property change.
store.await_property_changed(file_id, 'nie:title')
title_result = store.query('SELECT ?title { <%s> nie:title ?title }' %
file_urn)
assert len(title_result) == 1
self.assertEqual(title_result[0][0], VALID_FILE_TITLE)
def setUp(self):
self.datadir = cfg.create_monitored_test_dir()
config = {
cfg.DCONF_MINER_SCHEMA: {
'enable-writeback': GLib.Variant.new_boolean(False),
'index-recursive-directories': GLib.Variant.new_strv([]),
'index-single-directories':
GLib.Variant.new_strv([self.datadir]),
'index-optical-discs': GLib.Variant.new_boolean(False),
'index-removable-devices': GLib.Variant.new_boolean(False),
},
'org.freedesktop.Tracker.Store': {
'graphupdated-delay': GLib.Variant('i', 100)
}
}
self.system = TrackerSystemAbstraction(config)
self.system.tracker_miner_fs_testing_start()
def setUpClass(self):
log("Using %s as temp dir\n" % MINER_TMP_DIR)
if os.path.exists(MINER_TMP_DIR):
shutil.rmtree(MINER_TMP_DIR)
os.makedirs(MINER_TMP_DIR)
self.system = TrackerSystemAbstraction()
self.system.set_up_environment(CONF_OPTIONS, None)
self.store = StoreHelper()
self.store.start()
# GraphUpdated seems to not be emitted if the extractor isn't running
# even though the file resource still gets inserted - maybe because
# INSERT SILENT is used in the FS miner?
self.extractor = ExtractorHelper()
self.extractor.start()
self.miner_fs = MinerFsHelper()
self.miner_fs.start()
class ExtractorDecoratorTest(ut.TestCase):
def setUp(self):
if not os.path.exists(MINER_TMP_DIR):
os.makedirs(MINER_TMP_DIR)
assert os.path.isdir(MINER_TMP_DIR)
self.system = TrackerSystemAbstraction(CONF_OPTIONS)
self.system.tracker_miner_fs_testing_start()
def tearDown(self):
self.system.tracker_miner_fs_testing_stop()
shutil.rmtree(MINER_TMP_DIR)
def test_reextraction(self):
"""Tests whether known files are still re-extracted on user request."""
miner_fs = self.system.miner_fs
store = self.system.store
# Insert a valid file and wait extraction of its metadata.
file_path = os.path.join(MINER_TMP_DIR, os.path.basename(VALID_FILE))
shutil.copy(VALID_FILE, file_path)
file_id, file_urn = store.await_resource_inserted(
VALID_FILE_CLASS, title=VALID_FILE_TITLE)
# Remove a key piece of metadata.
store.update(
'DELETE { <%s> nie:title ?title }'
' WHERE { <%s> nie:title ?title }' % (file_urn, file_urn))
store.await_property_changed(file_id, 'nie:title')
assert not store.ask('ASK { <%s> nie:title ?title }' % file_urn)
log("Sending re-index request")
# Request re-indexing (same as `tracker index --file ...`)
miner_fs.index_file(uri(file_path))
# The extractor should reindex the file and re-add the metadata that we
# deleted, so we should see the nie:title property change.
store.await_property_changed(file_id, 'nie:title')
title_result = store.query('SELECT ?title { <%s> nie:title ?title }' % file_urn)
assert len(title_result) == 1
self.assertEqual(title_result[0][0], VALID_FILE_TITLE)
def setUp(self):
# Create temp directory to monitor
if (os.path.exists(APPLICATIONS_TMP_DIR)):
shutil.rmtree(APPLICATIONS_TMP_DIR)
os.makedirs(APPLICATIONS_TMP_DIR)
# Use local directory if available. Installation otherwise.
if os.path.exists(os.path.join(os.getcwd(), "test-apps-data")):
self.datadir = os.path.join(os.getcwd(), "test-apps-data")
else:
self.datadir = os.path.join(cfg.DATADIR, "tracker-tests",
"test-apps-data")
self.system = TrackerSystemAbstraction()
self.system.tracker_all_testing_start(CONF_OPTIONS)
# Returns when ready
self.tracker = self.system.store
log("Ready to go!")
def setUp(self):
ensure_dir_exists(cfg.TEST_MONITORED_TMP_DIR)
# It's important that this directory is NOT inside /tmp, because
# monitoring files in /tmp usually doesn't work.
self.datadir = tempfile.mkdtemp(dir=cfg.TEST_MONITORED_TMP_DIR)
config = {
cfg.DCONF_MINER_SCHEMA: {
'index-recursive-directories': GLib.Variant.new_strv([]),
'index-single-directories':
GLib.Variant.new_strv([self.datadir]),
'index-optical-discs': GLib.Variant.new_boolean(False),
'index-removable-devices': GLib.Variant.new_boolean(False),
},
'org.freedesktop.Tracker.Store': {
'graphupdated-delay': GLib.Variant('i', 100)
}
}
self.system = TrackerSystemAbstraction(config)
self.system.tracker_miner_fs_testing_start()
class CommonTrackerMinerTest (ut.TestCase):
@classmethod
def __prepare_directories (self):
#
# ~/test-monitored/
# /file1.txt
# /dir1/
# /file2.txt
# /dir2/
# /file3.txt
#
#
# ~/test-no-monitored/
# /file0.txt
#
for d in ["test-monitored",
"test-monitored/dir1",
"test-monitored/dir1/dir2",
"test-no-monitored"]:
directory = os.path.join (MINER_TMP_DIR, d)
if (os.path.exists (directory)):
shutil.rmtree (directory)
os.makedirs (directory)
for tf in ["test-monitored/file1.txt",
"test-monitored/dir1/file2.txt",
"test-monitored/dir1/dir2/file3.txt",
"test-no-monitored/file0.txt"]:
testfile = os.path.join (MINER_TMP_DIR, tf)
if (os.path.exists (testfile)):
os.remove (testfile)
f = open (testfile, 'w')
f.write (DEFAULT_TEXT)
f.close ()
@classmethod
def setUpClass (self):
#print "Starting the daemon in test mode"
self.__prepare_directories ()
self.system = TrackerSystemAbstraction ()
if (os.path.exists (os.getcwd() + "/test-configurations/miner-basic-ops")):
# Use local directory if available
confdir = os.getcwd() + "/test-configurations/miner-basic-ops"
else:
confdir = os.path.join (cfg.DATADIR, "tracker-tests",
"test-configurations", "miner-basic-ops")
self.system.tracker_miner_fs_testing_start (CONF_OPTIONS)
self.system.tracker_miner_fs_wait_for_idle ()
self.tracker = self.system.store
@classmethod
def tearDownClass (self):
#print "Stopping the daemon in test mode (Doing nothing now)"
self.system.tracker_miner_fs_testing_stop ()
def setUpClass(self):
#print "Starting the daemon in test mode"
self.__prepare_directories()
self.system = TrackerSystemAbstraction()
self.system.tracker_writeback_testing_start(CONF_OPTIONS)
def await_resource_extraction(url):
# Make sure a resource has been crawled by the FS miner and by
# tracker-extract. The extractor adds nie:contentCreated for
# image resources, so know once this property is set the
# extraction is complete.
self.system.store.await_resource_inserted(
NFO_IMAGE, url=url, required_property='nfo:width')
await_resource_extraction(self.get_test_filename_jpeg())
await_resource_extraction(self.get_test_filename_tiff())
await_resource_extraction(self.get_test_filename_png())
# Returns when ready
log("Ready to go!")
class CommonTrackerMinerTest(ut.TestCase):
@classmethod
def __prepare_directories(self):
#
# ~/test-monitored/
# /file1.txt
# /dir1/
# /file2.txt
# /dir2/
# /file3.txt
#
#
# ~/test-no-monitored/
# /file0.txt
#
for d in [
"test-monitored", "test-monitored/dir1",
"test-monitored/dir1/dir2", "test-no-monitored"
]:
directory = os.path.join(MINER_TMP_DIR, d)
if (os.path.exists(directory)):
shutil.rmtree(directory)
os.makedirs(directory)
for tf in [
"test-monitored/file1.txt", "test-monitored/dir1/file2.txt",
"test-monitored/dir1/dir2/file3.txt",
"test-no-monitored/file0.txt"
]:
testfile = os.path.join(MINER_TMP_DIR, tf)
if (os.path.exists(testfile)):
os.remove(testfile)
f = open(testfile, 'w')
f.write(DEFAULT_TEXT)
f.close()
@classmethod
def setUpClass(self):
#print "Starting the daemon in test mode"
self.__prepare_directories()
self.system = TrackerSystemAbstraction()
if (os.path.exists(os.getcwd() +
"/test-configurations/miner-basic-ops")):
# Use local directory if available
confdir = os.getcwd() + "/test-configurations/miner-basic-ops"
else:
confdir = os.path.join(cfg.DATADIR, "tracker-tests",
"test-configurations", "miner-basic-ops")
self.system.tracker_miner_fs_testing_start(CONF_OPTIONS)
self.system.tracker_miner_fs_wait_for_idle()
self.tracker = self.system.store
@classmethod
def tearDownClass(self):
#print "Stopping the daemon in test mode (Doing nothing now)"
self.system.tracker_miner_fs_testing_stop()
def setUp(self):
self.workdir = cfg.create_monitored_test_dir()
index_dirs = [self.workdir]
CONF_OPTIONS = {
cfg.DCONF_MINER_SCHEMA: {
'index-recursive-directories':
GLib.Variant.new_strv(index_dirs),
'index-single-directories': GLib.Variant.new_strv([]),
'index-optical-discs': GLib.Variant.new_boolean(False),
'index-removable-devices': GLib.Variant.new_boolean(False),
},
'org.freedesktop.Tracker.Store': {
'graphupdated-delay': GLib.Variant.new_int32(100)
}
}
self.system = TrackerSystemAbstraction()
self.system.tracker_writeback_testing_start(CONF_OPTIONS)
self.tracker = self.system.store
self.extractor = self.system.extractor
def setUpClass (self):
#print "Starting the daemon in test mode"
self.__prepare_directories ()
self.system = TrackerSystemAbstraction ()
if (os.path.exists (os.getcwd() + "/test-configurations/miner-basic-ops")):
# Use local directory if available
confdir = os.getcwd() + "/test-configurations/miner-basic-ops"
else:
confdir = os.path.join (cfg.DATADIR, "tracker-tests",
"test-configurations", "miner-basic-ops")
self.system.tracker_miner_fs_testing_start (CONF_OPTIONS)
self.system.tracker_miner_fs_wait_for_idle ()
self.tracker = self.system.store
def setUp (self):
for d in ['test-monitored', 'test-no-monitored']:
dirname = path(d)
if os.path.exists (dirname):
shutil.rmtree(dirname)
os.makedirs(dirname)
self.system = TrackerSystemAbstraction ()
self.system.tracker_miner_fs_testing_start (CONF_OPTIONS)
self.tracker = self.system.store
try:
self.prepare_directories ()
self.tracker.reset_graph_updates_tracking ()
except Exception as e:
self.tearDown ()
raise
class OntologyChangeTestTemplate (ut.TestCase):
"""
Template class for the ontology changes tests. The tests are subclasses
of this, implementing these methods:
* set_ontology_dirs
* insert_data
* validate_status
and adding a method 'test_x_y_z' to be invoked by unittest.
Check doc in those methods for the specific details.
"""
def get_ontology_dir (self, param):
local = os.path.join (os.getcwd (), "test-ontologies", param)
if (os.path.exists (local)):
# Use local directory if available
return local
else:
return os.path.join (cfg.DATADIR, "tracker-tests",
"test-ontologies", param)
def setUp (self):
self.system = TrackerSystemAbstraction ()
def tearDown (self):
self.system.tracker_store_testing_stop ()
def template_test_ontology_change (self):
self.set_ontology_dirs ()
basic_ontologies = self.get_ontology_dir (self.FIRST_ONTOLOGY_DIR)
modified_ontologies = self.get_ontology_dir (self.SECOND_ONTOLOGY_DIR)
self.__assert_ontology_dates (basic_ontologies, modified_ontologies)
self.system.tracker_store_testing_start (ontodir=basic_ontologies)
self.tracker = self.system.store
self.insert_data ()
try:
# Boot the second set of ontologies
self.system.tracker_store_restart_with_new_ontologies (modified_ontologies)
except UnableToBootException, e:
self.fail (str(self.__class__) + " " + str(e))
self.validate_status ()
class OntologyChangeTestTemplate (ut.TestCase):
"""
Template class for the ontology changes tests. The tests are subclasses
of this, implementing these methods:
* set_ontology_dirs
* insert_data
* validate_status
and adding a method 'test_x_y_z' to be invoked by unittest.
Check doc in those methods for the specific details.
"""
def get_ontology_dir (self, param):
local = os.path.join (os.getcwd (), "test-ontologies", param)
if (os.path.exists (local)):
# Use local directory if available
return local
else:
return os.path.join (cfg.DATADIR, "tracker-tests",
"test-ontologies", param)
def setUp (self):
self.system = TrackerSystemAbstraction ()
def tearDown (self):
self.system.tracker_store_testing_stop ()
def template_test_ontology_change (self):
self.set_ontology_dirs ()
basic_ontologies = self.get_ontology_dir (self.FIRST_ONTOLOGY_DIR)
modified_ontologies = self.get_ontology_dir (self.SECOND_ONTOLOGY_DIR)
self.__assert_ontology_dates (basic_ontologies, modified_ontologies)
self.system.tracker_store_testing_start (ontodir=basic_ontologies)
self.tracker = self.system.store
self.insert_data ()
try:
# Boot the second set of ontologies
self.system.tracker_store_restart_with_new_ontologies (modified_ontologies)
except UnableToBootException, e:
self.fail (str(self.__class__) + " " + str(e))
self.validate_status ()
def setUpClass (self):
log ("Using %s as temp dir\n" % MINER_TMP_DIR)
if os.path.exists (MINER_TMP_DIR):
shutil.rmtree (MINER_TMP_DIR)
os.makedirs (MINER_TMP_DIR)
self.system = TrackerSystemAbstraction ()
self.system.set_up_environment (CONF_OPTIONS, None)
self.store = StoreHelper ()
self.store.start ()
self.miner_fs = MinerFsHelper ()
self.miner_fs.start ()
# GraphUpdated seems to not be emitted if the extractor isn't running
# even though the file resource still gets inserted - maybe because
# INSERT SILENT is used in the FS miner?
self.extractor = ExtractorHelper ()
self.extractor.start ()
def setUpClass (self):
#print "Starting the daemon in test mode"
self.__prepare_directories ()
self.system = TrackerSystemAbstraction ()
self.system.tracker_writeback_testing_start (CONF_OPTIONS)
def await_resource_extraction(url):
# Make sure a resource has been crawled by the FS miner and by
# tracker-extract. The extractor adds nie:contentCreated for
# image resources, so know once this property is set the
# extraction is complete.
self.system.store.await_resource_inserted('nfo:Image', url=url, required_property='nfo:width')
await_resource_extraction (self.get_test_filename_jpeg())
await_resource_extraction (self.get_test_filename_tiff())
await_resource_extraction (self.get_test_filename_png())
# Returns when ready
log ("Ready to go!")
def setUp (self):
# Create temp directory to monitor
if (os.path.exists (APPLICATIONS_TMP_DIR)):
shutil.rmtree (APPLICATIONS_TMP_DIR)
os.makedirs (APPLICATIONS_TMP_DIR)
# Use local directory if available. Installation otherwise.
if os.path.exists (os.path.join (os.getcwd (),
"test-apps-data")):
self.datadir = os.path.join (os.getcwd (),
"test-apps-data")
else:
self.datadir = os.path.join (cfg.DATADIR,
"tracker-tests",
"test-apps-data")
self.system = TrackerSystemAbstraction ()
self.system.tracker_all_testing_start (CONF_OPTIONS)
# Returns when ready
self.tracker = self.system.store
log ("Ready to go!")
class CommonTrackerApplicationTest(ut.TestCase):
def get_urn_count_by_url(self, url):
select = """
SELECT ?u WHERE { ?u nie:url \"%s\" }
""" % (url)
return len(self.tracker.query(select))
def get_test_image(self):
TEST_IMAGE = "test-image-1.jpg"
return TEST_IMAGE
def get_test_video(self):
TEST_VIDEO = "test-video-1.mp4"
return TEST_VIDEO
def get_test_music(self):
TEST_AUDIO = "test-music-1.mp3"
return TEST_AUDIO
def get_data_dir(self):
return self.datadir
def get_dest_dir(self):
return APPLICATIONS_TMP_DIR
def slowcopy_file_fd(self, src, fdest, rate=SLOWCOPY_RATE):
"""
@rate: bytes per 100ms
"""
log("Copying slowly\n '%s' to\n '%s'" % (src, fdest.name))
fsrc = open(src, 'rb')
buffer_ = fsrc.read(rate)
while (buffer_ != ""):
fdest.write(buffer_)
time.sleep(0.1)
buffer_ = fsrc.read(rate)
fsrc.close()
def slowcopy_file(self, src, dst, rate=SLOWCOPY_RATE):
"""
@rate: bytes per 100ms
"""
fdest = open(dst, 'wb')
self.slowcopy_file_fd(src, fdest, rate)
fdest.close()
@classmethod
def setUp(self):
# Create temp directory to monitor
if (os.path.exists(APPLICATIONS_TMP_DIR)):
shutil.rmtree(APPLICATIONS_TMP_DIR)
os.makedirs(APPLICATIONS_TMP_DIR)
# Use local directory if available. Installation otherwise.
if os.path.exists(os.path.join(os.getcwd(), "test-apps-data")):
self.datadir = os.path.join(os.getcwd(), "test-apps-data")
else:
self.datadir = os.path.join(cfg.DATADIR, "tracker-tests",
"test-apps-data")
self.system = TrackerSystemAbstraction()
self.system.tracker_all_testing_start(CONF_OPTIONS)
# Returns when ready
self.tracker = self.system.store
log("Ready to go!")
@classmethod
def tearDown(self):
#print "Stopping the daemon in test mode (Doing nothing now)"
self.system.tracker_all_testing_stop()
# Remove monitored directory
if (os.path.exists(APPLICATIONS_TMP_DIR)):
shutil.rmtree(APPLICATIONS_TMP_DIR)
def setUpClass(self):
self.system = TrackerSystemAbstraction()
self.system.tracker_store_testing_start()
self.tracker = self.system.store
class MinerResourceRemovalTest(ut.TestCase):
# Use the same instances of store and miner-fs for the whole test suite,
# because they take so long to do first-time init.
@classmethod
def setUpClass(self):
log("Using %s as temp dir\n" % MINER_TMP_DIR)
if os.path.exists(MINER_TMP_DIR):
shutil.rmtree(MINER_TMP_DIR)
os.makedirs(MINER_TMP_DIR)
self.system = TrackerSystemAbstraction()
self.system.set_up_environment(CONF_OPTIONS, None)
self.store = StoreHelper()
self.store.start()
# GraphUpdated seems to not be emitted if the extractor isn't running
# even though the file resource still gets inserted - maybe because
# INSERT SILENT is used in the FS miner?
self.extractor = ExtractorHelper()
self.extractor.start()
self.miner_fs = MinerFsHelper()
self.miner_fs.start()
@classmethod
def tearDownClass(self):
self.miner_fs.stop()
self.extractor.stop()
self.store.stop()
def setUp(self):
self.store.reset_graph_updates_tracking()
def tearDown(self):
self.system.unset_up_environment()
def create_test_content(self, file_urn, title):
sparql = "INSERT { \
_:ie a nmm:MusicPiece ; \
nie:title \"%s\" ; \
nie:isStoredAs <%s> \
} " % (title, file_urn)
self.store.update(sparql)
return self.store.await_resource_inserted(rdf_class='nmm:MusicPiece',
title=title)
def create_test_file(self, file_name):
file_path = get_test_path(file_name)
file = open(file_path, 'w')
file.write("Test")
file.close()
return self.store.await_resource_inserted(rdf_class='nfo:Document',
url=get_test_uri(file_name))
def assertResourceExists(self, urn):
if self.store.ask("ASK { <%s> a rdfs:Resource }" % urn) == False:
self.fail("Resource <%s> does not exist" % urn)
def assertResourceMissing(self, urn):
if self.store.ask("ASK { <%s> a rdfs:Resource }" % urn) == True:
self.fail("Resource <%s> should not exist" % urn)
def test_01_file_deletion(self):
"""
Ensure every logical resource (nie:InformationElement) contained with
in a file is deleted when the file is deleted.
"""
(file_1_id, file_1_urn) = self.create_test_file("test_1.txt")
(file_2_id, file_2_urn) = self.create_test_file("test_2.txt")
(ie_1_id, ie_1_urn) = self.create_test_content(file_1_urn,
"Test resource 1")
(ie_2_id, ie_2_urn) = self.create_test_content(file_2_urn,
"Test resource 2")
os.unlink(get_test_path("test_1.txt"))
self.store.await_resource_deleted(file_1_id)
self.store.await_resource_deleted (ie_1_id,
"Associated logical resource failed to be deleted " \
"when its containing file was removed.")
self.assertResourceMissing(file_1_urn)
self.assertResourceMissing(ie_1_urn)
self.assertResourceExists(file_2_urn)
self.assertResourceExists(ie_2_urn)
def test_02_removable_device_data(self):
"""
def setUpClass (self):
#print "Starting the daemon in test mode"
self.system = TrackerSystemAbstraction ()
self.system.tracker_store_testing_start ()
self.tracker = self.system.store
class CommonTrackerMinerTest(ut.TestCase):
def setUp(self):
self.workdir = cfg.create_monitored_test_dir()
self.indexed_dir = os.path.join(self.workdir, 'test-monitored')
# It's important that this directory exists BEFORE we start Tracker:
# it won't monitor an indexing root for changes if it doesn't exist,
# it'll silently ignore it instead. See the tracker_crawler_start()
# function.
ensure_dir_exists(self.indexed_dir)
self.system = TrackerSystemAbstraction(
settings={
'org.freedesktop.Tracker.Store': {
'graphupdated-delay': GLib.Variant('i', 100)
}
})
config = {
cfg.DCONF_MINER_SCHEMA: {
'index-recursive-directories':
GLib.Variant.new_strv([self.indexed_dir]),
'index-single-directories':
GLib.Variant.new_strv([]),
'index-optical-discs':
GLib.Variant.new_boolean(False),
'index-removable-devices':
GLib.Variant.new_boolean(False),
'throttle':
GLib.Variant.new_int32(5),
}
}
try:
self.system.tracker_miner_fs_testing_start(config)
except RuntimeError as e:
self.fail(e)
self.tracker = self.system.store
try:
self.create_test_data()
self.tracker.reset_graph_updates_tracking()
except Exception as e:
self.tearDown()
raise
def tearDown(self):
self.system.finish()
self.remove_test_data()
cfg.remove_monitored_test_dir(self.workdir)
def path(self, filename):
return os.path.join(self.workdir, filename)
def uri(self, filename):
return "file://" + os.path.join(self.workdir, filename)
def create_test_data(self):
monitored_files = [
'test-monitored/file1.txt', 'test-monitored/dir1/file2.txt',
'test-monitored/dir1/dir2/file3.txt'
]
unmonitored_files = ['test-no-monitored/file0.txt']
for tf in chain(monitored_files, unmonitored_files):
testfile = self.path(tf)
ensure_dir_exists(os.path.dirname(testfile))
with open(testfile, 'w') as f:
f.write(DEFAULT_TEXT)
for tf in monitored_files:
self.tracker.await_resource_inserted(NFO_DOCUMENT,
url=self.uri(tf))
def remove_test_data(self):
try:
shutil.rmtree(os.path.join(self.workdir, 'test-monitored'))
shutil.rmtree(os.path.join(self.workdir, 'test-no-monitored'))
except Exception as e:
log("Failed to remove temporary data dir: %s" % e)
def assertResourceExists(self, urn):
if self.tracker.ask("ASK { <%s> a rdfs:Resource }" % urn) == False:
self.fail("Resource <%s> does not exist" % urn)
def assertResourceMissing(self, urn):
if self.tracker.ask("ASK { <%s> a rdfs:Resource }" % urn) == True:
self.fail("Resource <%s> should not exist" % urn)
class CommonTrackerWritebackTest(ut.TestCase):
"""
Superclass to share methods. Shouldn't be run by itself.
Start all processes including writeback, miner pointing to WRITEBACK_TMP_DIR
"""
def setUp(self):
self.workdir = cfg.create_monitored_test_dir()
index_dirs = [self.workdir]
CONF_OPTIONS = {
cfg.DCONF_MINER_SCHEMA: {
'index-recursive-directories':
GLib.Variant.new_strv(index_dirs),
'index-single-directories': GLib.Variant.new_strv([]),
'index-optical-discs': GLib.Variant.new_boolean(False),
'index-removable-devices': GLib.Variant.new_boolean(False),
},
'org.freedesktop.Tracker.Store': {
'graphupdated-delay': GLib.Variant.new_int32(100)
}
}
self.system = TrackerSystemAbstraction()
self.system.tracker_writeback_testing_start(CONF_OPTIONS)
self.tracker = self.system.store
self.extractor = self.system.extractor
def tearDown(self):
self.system.finish()
for test_file in pathlib.Path(self.workdir).iterdir():
test_file.unlink()
cfg.remove_monitored_test_dir(self.workdir)
def datadir_path(self, filename):
"""Returns the full path to a writeback test file."""
datadir = os.path.join(os.path.dirname(__file__), '..', '..',
'test-writeback-data')
return pathlib.Path(os.path.join(datadir, filename))
def prepare_test_file(self, path, expect_mime_type, expect_property):
"""Copies a file into the test working directory.
The function waits until the file has been seen by the Tracker
miner before returning.
"""
log("Copying %s -> %s" % (path, self.workdir))
shutil.copy(path, self.workdir)
output_path = pathlib.Path(
os.path.join(self.workdir, os.path.basename(path)))
# Make sure a resource has been crawled by the FS miner and by
# tracker-extract. The extractor adds nie:contentCreated for
# image resources, so know once this property is set the
# extraction is complete.
self.system.store.await_resource_inserted(
expect_mime_type,
url=output_path.as_uri(),
required_property=expect_property)
return output_path
def prepare_test_audio(self, filename):
return self.prepare_test_file(
filename,
'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#Audio',
'nfo:duration')
def prepare_test_image(self, filename):
return self.prepare_test_file(
filename,
'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#Image',
'nfo:width')
def uri(self, filename):
return pathlib.Path(filename).as_uri()
def get_mtime(self, filename):
return os.stat(filename).st_mtime
def wait_for_file_change(self, filename, initial_mtime):
start = time.time()
while time.time() < start + 5:
mtime = os.stat(filename).st_mtime
if mtime > initial_mtime:
return
time.sleep(0.2)
raise Exception(
"Timeout waiting for %s to be updated (mtime has not changed)" %
filename)
def setUp (self):
self.system = TrackerSystemAbstraction ()
class CommonTrackerMinerTest(ut.TestCase):
def prepare_directories(self):
#
# ~/test-monitored/
# /file1.txt
# /dir1/
# /file2.txt
# /dir2/
# /file3.txt
#
#
# ~/test-no-monitored/
# /file0.txt
#
monitored_files = [
'test-monitored/file1.txt', 'test-monitored/dir1/file2.txt',
'test-monitored/dir1/dir2/file3.txt'
]
unmonitored_files = ['test-no-monitored/file0.txt']
def ensure_dir_exists(dirname):
if not os.path.exists(dirname):
os.makedirs(dirname)
for tf in chain(monitored_files, unmonitored_files):
testfile = path(tf)
ensure_dir_exists(os.path.dirname(testfile))
with open(testfile, 'w') as f:
f.write(DEFAULT_TEXT)
for tf in monitored_files:
self.tracker.await_resource_inserted('nfo:TextDocument',
url=uri(tf))
def setUp(self):
for d in ['test-monitored', 'test-no-monitored']:
dirname = path(d)
if os.path.exists(dirname):
shutil.rmtree(dirname)
os.makedirs(dirname)
self.system = TrackerSystemAbstraction()
self.system.tracker_miner_fs_testing_start(CONF_OPTIONS)
self.tracker = self.system.store
try:
self.prepare_directories()
self.tracker.reset_graph_updates_tracking()
except Exception as e:
self.tearDown()
raise
def tearDown(self):
self.system.tracker_miner_fs_testing_stop()
def assertResourceExists(self, urn):
if self.tracker.ask("ASK { <%s> a rdfs:Resource }" % urn) == False:
self.fail("Resource <%s> does not exist" % urn)
def assertResourceMissing(self, urn):
if self.tracker.ask("ASK { <%s> a rdfs:Resource }" % urn) == True:
self.fail("Resource <%s> should not exist" % urn)
def setUp(self):
self.system = TrackerSystemAbstraction()
def setUpClass(self):
#print "Starting the daemon in test mode"
self.system = TrackerSystemAbstraction()
self.system.tracker_store_testing_start()
self.tracker = self.system.store
class CommonTrackerWritebackTest(ut.TestCase):
"""
Superclass to share methods. Shouldn't be run by itself.
Start all processes including writeback, miner pointing to WRITEBACK_TMP_DIR
"""
@classmethod
def __prepare_directories(self):
if (os.path.exists(os.getcwd() + "/test-writeback-data")):
# Use local directory if available
datadir = os.getcwd() + "/test-writeback-data"
else:
datadir = os.path.join(cfg.DATADIR, "tracker-tests",
"test-writeback-data")
if not os.path.exists(WRITEBACK_TMP_DIR):
os.makedirs(WRITEBACK_TMP_DIR)
else:
if not os.path.isdir(WRITEBACK_TMP_DIR):
raise Exception("%s exists already and is not a directory" %
WRITEBACK_TMP_DIR)
for testfile in [TEST_FILE_JPEG, TEST_FILE_PNG, TEST_FILE_TIFF]:
origin = os.path.join(datadir, testfile)
log("Copying %s -> %s" % (origin, WRITEBACK_TMP_DIR))
shutil.copy(origin, WRITEBACK_TMP_DIR)
@classmethod
def setUpClass(self):
#print "Starting the daemon in test mode"
self.__prepare_directories()
self.system = TrackerSystemAbstraction()
self.system.tracker_writeback_testing_start(CONF_OPTIONS)
def await_resource_extraction(url):
# Make sure a resource has been crawled by the FS miner and by
# tracker-extract. The extractor adds nie:contentCreated for
# image resources, so know once this property is set the
# extraction is complete.
self.system.store.await_resource_inserted(
NFO_IMAGE, url=url, required_property='nfo:width')
await_resource_extraction(self.get_test_filename_jpeg())
await_resource_extraction(self.get_test_filename_tiff())
await_resource_extraction(self.get_test_filename_png())
# Returns when ready
log("Ready to go!")
@classmethod
def tearDownClass(self):
#print "Stopping the daemon in test mode (Doing nothing now)"
self.system.tracker_writeback_testing_stop()
log("Test finished")
@staticmethod
def get_test_filename_jpeg():
return uri(TEST_FILE_JPEG)
@staticmethod
def get_test_filename_tiff():
return uri(TEST_FILE_TIFF)
@staticmethod
def get_test_filename_png():
return uri(TEST_FILE_PNG)
def get_mtime(self, filename):
return os.stat(filename).st_mtime
def wait_for_file_change(self, filename, initial_mtime):
start = time.time()
while time.time() < start + 5:
mtime = os.stat(filename).st_mtime
if mtime > initial_mtime:
return
time.sleep(0.2)
raise Exception(
"Timeout waiting for %s to be updated (mtime has not changed)" %
filename)
class ExtractorDecoratorTest(ut.TestCase):
def setUp(self):
self.datadir = cfg.create_monitored_test_dir()
config = {
cfg.DCONF_MINER_SCHEMA: {
'enable-writeback': GLib.Variant.new_boolean(False),
'index-recursive-directories': GLib.Variant.new_strv([]),
'index-single-directories':
GLib.Variant.new_strv([self.datadir]),
'index-optical-discs': GLib.Variant.new_boolean(False),
'index-removable-devices': GLib.Variant.new_boolean(False),
},
'org.freedesktop.Tracker.Store': {
'graphupdated-delay': GLib.Variant('i', 100)
}
}
self.system = TrackerSystemAbstraction(config)
self.system.tracker_miner_fs_testing_start()
def tearDown(self):
self.system.finish()
cfg.remove_monitored_test_dir(self.datadir)
def test_reextraction(self):
"""Tests whether known files are still re-extracted on user request."""
miner_fs = self.system.miner_fs
store = self.system.store
# Insert a valid file and wait extraction of its metadata.
file_path = os.path.join(self.datadir, os.path.basename(VALID_FILE))
shutil.copy(VALID_FILE, file_path)
try:
file_id, file_urn = store.await_resource_inserted(
VALID_FILE_CLASS, title=VALID_FILE_TITLE)
# Remove a key piece of metadata.
# (Writeback must be disabled in the config so that the file
# itself is not changed).
store.update('DELETE { <%s> nie:title ?title }'
' WHERE { <%s> nie:title ?title }' %
(file_urn, file_urn))
store.await_property_changed(VALID_FILE_CLASS, file_id,
'nie:title')
assert not store.ask('ASK { <%s> nie:title ?title }' % file_urn)
log("Sending re-index request")
# Request re-indexing (same as `tracker index --file ...`)
miner_fs.index_file('file://' +
os.path.join(self.datadir, file_path))
# The extractor should reindex the file and re-add the metadata that we
# deleted, so we should see the nie:title property change.
store.await_property_changed(VALID_FILE_CLASS, file_id,
'nie:title')
title_result = store.query(
'SELECT ?title { <%s> nie:title ?title }' % file_urn)
assert len(title_result) == 1
self.assertEqual(title_result[0][0], VALID_FILE_TITLE)
finally:
os.remove(file_path)
class CommonTrackerApplicationTest (ut.TestCase):
def get_urn_count_by_url (self, url):
select = """
SELECT ?u WHERE { ?u nie:url \"%s\" }
""" % (url)
return len (self.tracker.query (select))
def get_test_image (self):
TEST_IMAGE = "test-image-1.jpg"
return TEST_IMAGE
def get_test_video (self):
TEST_VIDEO = "test-video-1.mp4"
return TEST_VIDEO
def get_test_music (self):
TEST_AUDIO = "test-music-1.mp3"
return TEST_AUDIO
def get_data_dir (self):
return self.datadir
def get_dest_dir (self):
return APPLICATIONS_TMP_DIR
def slowcopy_file_fd (self, src, fdest, rate=SLOWCOPY_RATE):
"""
@rate: bytes per 100ms
"""
log ("Copying slowly\n '%s' to\n '%s'" % (src, fdest.name))
fsrc = open (src, 'rb')
buffer_ = fsrc.read (rate)
while (buffer_ != ""):
fdest.write (buffer_)
time.sleep (0.1)
buffer_ = fsrc.read (rate)
fsrc.close ()
def slowcopy_file (self, src, dst, rate=SLOWCOPY_RATE):
"""
@rate: bytes per 100ms
"""
fdest = open (dst, 'wb')
self.slowcopy_file_fd (src, fdest, rate)
fdest.close ()
@classmethod
def setUp (self):
# Create temp directory to monitor
if (os.path.exists (APPLICATIONS_TMP_DIR)):
shutil.rmtree (APPLICATIONS_TMP_DIR)
os.makedirs (APPLICATIONS_TMP_DIR)
# Use local directory if available. Installation otherwise.
if os.path.exists (os.path.join (os.getcwd (),
"test-apps-data")):
self.datadir = os.path.join (os.getcwd (),
"test-apps-data")
else:
self.datadir = os.path.join (cfg.DATADIR,
"tracker-tests",
"test-apps-data")
self.system = TrackerSystemAbstraction ()
self.system.tracker_all_testing_start (CONF_OPTIONS)
# Returns when ready
self.tracker = self.system.store
log ("Ready to go!")
@classmethod
def tearDown (self):
#print "Stopping the daemon in test mode (Doing nothing now)"
self.system.tracker_all_testing_stop ()
# Remove monitored directory
if (os.path.exists (APPLICATIONS_TMP_DIR)):
shutil.rmtree (APPLICATIONS_TMP_DIR)
class CommonTrackerWritebackTest (ut.TestCase):
"""
Superclass to share methods. Shouldn't be run by itself.
Start all processes including writeback, miner pointing to WRITEBACK_TMP_DIR
"""
@classmethod
def __prepare_directories (self):
if (os.path.exists (os.getcwd() + "/test-writeback-data")):
# Use local directory if available
datadir = os.getcwd() + "/test-writeback-data"
else:
datadir = os.path.join (cfg.DATADIR, "tracker-tests",
"test-writeback-data")
if not os.path.exists(WRITEBACK_TMP_DIR):
os.makedirs(WRITEBACK_TMP_DIR)
else:
if not os.path.isdir(WRITEBACK_TMP_DIR):
raise Exception("%s exists already and is not a directory" % WRITEBACK_TMP_DIR)
for testfile in [TEST_FILE_JPEG, TEST_FILE_PNG,TEST_FILE_TIFF]:
origin = os.path.join (datadir, testfile)
log ("Copying %s -> %s" % (origin, WRITEBACK_TMP_DIR))
shutil.copy (origin, WRITEBACK_TMP_DIR)
@classmethod
def setUpClass (self):
#print "Starting the daemon in test mode"
self.__prepare_directories ()
self.system = TrackerSystemAbstraction ()
self.system.tracker_writeback_testing_start (CONF_OPTIONS)
def await_resource_extraction(url):
# Make sure a resource has been crawled by the FS miner and by
# tracker-extract. The extractor adds nie:contentCreated for
# image resources, so know once this property is set the
# extraction is complete.
self.system.store.await_resource_inserted('nfo:Image', url=url, required_property='nfo:width')
await_resource_extraction (self.get_test_filename_jpeg())
await_resource_extraction (self.get_test_filename_tiff())
await_resource_extraction (self.get_test_filename_png())
# Returns when ready
log ("Ready to go!")
@classmethod
def tearDownClass (self):
#print "Stopping the daemon in test mode (Doing nothing now)"
self.system.tracker_writeback_testing_stop ()
@staticmethod
def get_test_filename_jpeg ():
return uri (TEST_FILE_JPEG)
@staticmethod
def get_test_filename_tiff ():
return uri (TEST_FILE_TIFF)
@staticmethod
def get_test_filename_png ():
return uri (TEST_FILE_PNG)
def get_mtime (self, filename):
return os.stat(filename).st_mtime
def wait_for_file_change (self, filename, initial_mtime):
start = time.time()
while time.time() < start + 5:
mtime = os.stat(filename).st_mtime
if mtime > initial_mtime:
return
time.sleep(0.2)
raise Exception(
"Timeout waiting for %s to be updated (mtime has not changed)" %
filename)
class CommonTrackerMinerTest (ut.TestCase):
def prepare_directories (self):
#
# ~/test-monitored/
# /file1.txt
# /dir1/
# /file2.txt
# /dir2/
# /file3.txt
#
#
# ~/test-no-monitored/
# /file0.txt
#
monitored_files = [
'test-monitored/file1.txt',
'test-monitored/dir1/file2.txt',
'test-monitored/dir1/dir2/file3.txt'
]
unmonitored_files = [
'test-no-monitored/file0.txt'
]
def ensure_dir_exists(dirname):
if not os.path.exists(dirname):
os.makedirs(dirname)
for tf in chain(monitored_files, unmonitored_files):
testfile = path(tf)
ensure_dir_exists(os.path.dirname(testfile))
with open (testfile, 'w') as f:
f.write (DEFAULT_TEXT)
for tf in monitored_files:
self.tracker.await_resource_inserted(
'nfo:TextDocument', url=uri(tf))
def setUp (self):
for d in ['test-monitored', 'test-no-monitored']:
dirname = path(d)
if os.path.exists (dirname):
shutil.rmtree(dirname)
os.makedirs(dirname)
self.system = TrackerSystemAbstraction ()
self.system.tracker_miner_fs_testing_start (CONF_OPTIONS)
self.tracker = self.system.store
try:
self.prepare_directories ()
self.tracker.reset_graph_updates_tracking ()
except Exception as e:
self.tearDown ()
raise
def tearDown (self):
self.system.tracker_miner_fs_testing_stop ()
def assertResourceExists (self, urn):
if self.tracker.ask ("ASK { <%s> a rdfs:Resource }" % urn) == False:
self.fail ("Resource <%s> does not exist" % urn)
def assertResourceMissing (self, urn):
if self.tracker.ask ("ASK { <%s> a rdfs:Resource }" % urn) == True:
self.fail ("Resource <%s> should not exist" % urn)
class CommonTrackerApplicationTest(ut.TestCase):
def get_urn_count_by_url(self, url):
select = """
SELECT ?u WHERE { ?u nie:url \"%s\" }
""" % (url)
return len(self.tracker.query(select))
def get_test_image(self):
TEST_IMAGE = "test-image-1.jpg"
return TEST_IMAGE
def get_test_video(self):
TEST_VIDEO = "test-video-1.mp4"
return TEST_VIDEO
def get_test_music(self):
TEST_AUDIO = "test-music-1.mp3"
return TEST_AUDIO
def get_data_dir(self):
return self.datadir
def get_dest_dir(self):
return self.workdir
def slowcopy_file_fd(self, src, fdest, rate=SLOWCOPY_RATE):
"""
@rate: bytes per 100ms
"""
log("Copying slowly\n '%s' to\n '%s'" % (src, fdest.name))
fsrc = open(src, 'rb')
buffer_ = fsrc.read(rate)
while (buffer_ != b""):
fdest.write(buffer_)
time.sleep(0.1)
buffer_ = fsrc.read(rate)
fsrc.close()
def slowcopy_file(self, src, dst, rate=SLOWCOPY_RATE):
"""
@rate: bytes per 100ms
"""
fdest = open(dst, 'wb')
self.slowcopy_file_fd(src, fdest, rate)
fdest.close()
@classmethod
def setUp(self):
self.workdir = cfg.create_monitored_test_dir()
index_dirs = [self.workdir]
CONF_OPTIONS = {
cfg.DCONF_MINER_SCHEMA: {
'index-recursive-directories':
GLib.Variant.new_strv(index_dirs),
'index-single-directories': GLib.Variant.new_strv([]),
'index-optical-discs': GLib.Variant.new_boolean(False),
'index-removable-devices': GLib.Variant.new_boolean(False),
}
}
# Use local directory if available. Installation otherwise.
if os.path.exists(os.path.join(os.getcwd(), "test-apps-data")):
self.datadir = os.path.join(os.getcwd(), "test-apps-data")
else:
self.datadir = os.path.join(cfg.DATADIR, "tracker-tests",
"test-apps-data")
self.system = TrackerSystemAbstraction()
self.system.tracker_all_testing_start(CONF_OPTIONS)
self.tracker = self.system.store
@classmethod
def tearDown(self):
self.system.finish()
cfg.remove_monitored_test_dir(self.workdir)
class MinerResourceRemovalTest (ut.TestCase):
# Use the same instances of store and miner-fs for the whole test suite,
# because they take so long to do first-time init.
@classmethod
def setUpClass (self):
log ("Using %s as temp dir\n" % MINER_TMP_DIR)
if os.path.exists (MINER_TMP_DIR):
shutil.rmtree (MINER_TMP_DIR)
os.makedirs (MINER_TMP_DIR)
self.system = TrackerSystemAbstraction ()
self.system.set_up_environment (CONF_OPTIONS, None)
self.store = StoreHelper ()
self.store.start ()
# GraphUpdated seems to not be emitted if the extractor isn't running
# even though the file resource still gets inserted - maybe because
# INSERT SILENT is used in the FS miner?
self.extractor = ExtractorHelper ()
self.extractor.start ()
self.miner_fs = MinerFsHelper ()
self.miner_fs.start ()
@classmethod
def tearDownClass (self):
self.miner_fs.stop ()
self.extractor.stop ()
self.store.stop ()
def setUp (self):
self.store.reset_graph_updates_tracking ()
def tearDown (self):
self.system.unset_up_environment ()
def create_test_content (self, file_urn, title):
sparql = "INSERT { \
_:ie a nmm:MusicPiece ; \
nie:title \"%s\" ; \
nie:isStoredAs <%s> \
} " % (title, file_urn)
self.store.update (sparql)
return self.store.await_resource_inserted (rdf_class = 'nmm:MusicPiece',
title = title)
def create_test_file (self, file_name):
file_path = get_test_path (file_name)
file = open (file_path, 'w')
file.write ("Test")
file.close ()
return self.store.await_resource_inserted (rdf_class = 'nfo:Document',
url = get_test_uri (file_name));
def assertResourceExists (self, urn):
if self.store.ask ("ASK { <%s> a rdfs:Resource }" % urn) == False:
self.fail ("Resource <%s> does not exist" % urn)
def assertResourceMissing (self, urn):
if self.store.ask ("ASK { <%s> a rdfs:Resource }" % urn) == True:
self.fail ("Resource <%s> should not exist" % urn)
def test_01_file_deletion (self):
"""
Ensure every logical resource (nie:InformationElement) contained with
in a file is deleted when the file is deleted.
"""
(file_1_id, file_1_urn) = self.create_test_file ("test_1.txt")
(file_2_id, file_2_urn) = self.create_test_file ("test_2.txt")
(ie_1_id, ie_1_urn) = self.create_test_content (file_1_urn, "Test resource 1")
(ie_2_id, ie_2_urn) = self.create_test_content (file_2_urn, "Test resource 2")
os.unlink (get_test_path ("test_1.txt"))
self.store.await_resource_deleted (file_1_id)
self.store.await_resource_deleted (ie_1_id,
"Associated logical resource failed to be deleted " \
"when its containing file was removed.")
self.assertResourceMissing (file_1_urn)
self.assertResourceMissing (ie_1_urn)
self.assertResourceExists (file_2_urn)
self.assertResourceExists (ie_2_urn)
def test_02_removable_device_data (self):
"""
class MinerResourceRemovalTest (ut.TestCase):
graph_updated_handler_id = 0
# Use the same instances of store and miner-fs for the whole test suite,
# because they take so long to do first-time init.
@classmethod
def setUpClass (self):
log ("Using %s as temp dir\n" % MINER_TMP_DIR)
if os.path.exists (MINER_TMP_DIR):
shutil.rmtree (MINER_TMP_DIR)
os.makedirs (MINER_TMP_DIR)
self.system = TrackerSystemAbstraction ()
self.system.set_up_environment (CONF_OPTIONS, None)
self.store = StoreHelper ()
self.store.start ()
self.miner_fs = MinerFsHelper ()
self.miner_fs.start ()
# GraphUpdated seems to not be emitted if the extractor isn't running
# even though the file resource still gets inserted - maybe because
# INSERT SILENT is used in the FS miner?
self.extractor = ExtractorHelper ()
self.extractor.start ()
@classmethod
def tearDownClass (self):
self.store.bus._clean_up_signal_match (self.graph_updated_handler_id)
self.extractor.stop ()
self.miner_fs.stop ()
self.store.stop ()
def setUp (self):
self.inserts_list = []
self.deletes_list = []
self.inserts_match_function = None
self.deletes_match_function = None
self.match_timed_out = False
self.graph_updated_handler_id = self.store.bus.add_signal_receiver (self._graph_updated_cb,
signal_name = "GraphUpdated",
path = "/org/freedesktop/Tracker1/Resources",
dbus_interface = "org.freedesktop.Tracker1.Resources")
def tearDown (self):
self.system.unset_up_environment ()
# A system to follow GraphUpdated and make sure all changes are tracked.
# This code saves every change notification received, and exposes methods
# to await insertion or deletion of a certain resource which first check
# the list of events already received and wait for more if the event has
# not yet happened.
#
# FIXME: put this stuff in StoreHelper
def _timeout_cb (self):
self.match_timed_out = True
self.store.loop.quit ()
# Don't fail here, exceptions don't get propagated correctly
# from the GMainLoop
def _graph_updated_cb (self, class_name, deletes_list, inserts_list):
"""
Process notifications from tracker-store on resource changes.
"""
matched = False
if inserts_list is not None:
if self.inserts_match_function is not None:
# The match function will remove matched entries from the list
(matched, inserts_list) = self.inserts_match_function (inserts_list)
self.inserts_list += inserts_list
if deletes_list is not None:
if self.deletes_match_function is not None:
(matched, deletes_list) = self.deletes_match_function (deletes_list)
self.deletes_list += deletes_list
def await_resource_inserted (self, rdf_class, url = None, title = None):
"""
Block until a resource matching the parameters becomes available
"""
assert (self.inserts_match_function == None)
def match_cb (inserts_list, in_main_loop = True):
matched = False
filtered_list = []
known_subjects = set ()
#print "Got inserts: ", inserts_list, "\n"
# FIXME: this could be done in an easier way: build one query that filters
# based on every subject id in inserts_list, and returns the id of the one
# that matched :)
for insert in inserts_list:
id = insert[1]
if not matched and id not in known_subjects:
known_subjects.add (id)
where = " ?urn a %s " % rdf_class
if url is not None:
where += "; nie:url \"%s\"" % url
if title is not None:
where += "; nie:title \"%s\"" % title
query = "SELECT ?urn WHERE { %s FILTER (tracker:id(?urn) = %s)}" % (where, insert[1])
#print "%s\n" % query
result_set = self.store.query (query)
#print result_set, "\n\n"
if len (result_set) > 0:
matched = True
self.matched_resource_urn = result_set[0][0]
self.matched_resource_id = insert[1]
if not matched or id != self.matched_resource_id:
filtered_list += [insert]
if matched and in_main_loop:
glib.source_remove (self.graph_updated_timeout_id)
self.graph_updated_timeout_id = 0
self.inserts_match_function = None
self.store.loop.quit ()
return (matched, filtered_list)
self.matched_resource_urn = None
self.matched_resource_id = None
log ("Await new %s (%i existing inserts)" % (rdf_class, len (self.inserts_list)))
# Check the list of previously received events for matches
(existing_match, self.inserts_list) = match_cb (self.inserts_list, False)
if not existing_match:
self.graph_updated_timeout_id = glib.timeout_add_seconds (REASONABLE_TIMEOUT, self._timeout_cb)
self.inserts_match_function = match_cb
# Run the event loop until the correct notification arrives
self.store.loop.run ()
if self.match_timed_out:
self.fail ("Timeout waiting for resource: class %s, URL %s, title %s" % (rdf_class, url, title))
return (self.matched_resource_id, self.matched_resource_urn)
def await_resource_deleted (self, id, fail_message = None):
"""
Block until we are notified of a resources deletion
"""
assert (self.deletes_match_function == None)
def match_cb (deletes_list, in_main_loop = True):
matched = False
filtered_list = []
#print "Looking for %i in " % id, deletes_list, "\n"
for delete in deletes_list:
if delete[1] == id:
matched = True
else:
filtered_list += [delete]
if matched and in_main_loop:
glib.source_remove (self.graph_updated_timeout_id)
self.graph_updated_timeout_id = 0
self.deletes_match_function = None
self.store.loop.quit ()
return (matched, filtered_list)
log ("Await deletion of %i (%i existing)" % (id, len (self.deletes_list)))
(existing_match, self.deletes_list) = match_cb (self.deletes_list, False)
if not existing_match:
self.graph_updated_timeout_id = glib.timeout_add_seconds (REASONABLE_TIMEOUT, self._timeout_cb)
self.deletes_match_function = match_cb
# Run the event loop until the correct notification arrives
self.store.loop.run ()
if self.match_timed_out:
if fail_message is not None:
self.fail (fail_message)
else:
self.fail ("Resource %i has not been deleted." % id)
return
def create_test_content (self, file_urn, title):
sparql = "INSERT { \
_:ie a nmm:MusicPiece ; \
nie:title \"%s\" ; \
nie:isStoredAs <%s> \
} " % (title, file_urn)
self.store.update (sparql)
return self.await_resource_inserted (rdf_class = 'nmm:MusicPiece',
title = title)
def create_test_file (self, file_name):
file_path = get_test_path (file_name)
file = open (file_path, 'w')
file.write ("Test")
file.close ()
return self.await_resource_inserted (rdf_class = 'nfo:Document',
url = get_test_uri (file_name));
def assertResourceExists (self, urn):
if self.store.ask ("ASK { <%s> a rdfs:Resource }" % urn) == False:
self.fail ("Resource <%s> does not exist" % urn)
def assertResourceMissing (self, urn):
if self.store.ask ("ASK { <%s> a rdfs:Resource }" % urn) == True:
self.fail ("Resource <%s> should not exist" % urn)
def test_01_file_deletion (self):
"""
Ensure every logical resource (nie:InformationElement) contained with
in a file is deleted when the file is deleted.
"""
(file_1_id, file_1_urn) = self.create_test_file ("test_1.txt")
(file_2_id, file_2_urn) = self.create_test_file ("test_2.txt")
(ie_1_id, ie_1_urn) = self.create_test_content (file_1_urn, "Test resource 1")
(ie_2_id, ie_2_urn) = self.create_test_content (file_2_urn, "Test resource 2")
os.unlink (get_test_path ("test_1.txt"))
self.await_resource_deleted (file_1_id)
self.await_resource_deleted (ie_1_id,
"Associated logical resource failed to be deleted " \
"when its containing file was removed.")
self.assertResourceMissing (file_1_urn)
self.assertResourceMissing (ie_1_urn)
self.assertResourceExists (file_2_urn)
self.assertResourceExists (ie_2_urn)
def test_02_removable_device_data (self):
"""