def get(self):
index = Index(INDEX_DB_URL)
collection_name = request.args.get('collection_name')
start_time = request.args.get('start_time')
end_time = request.args.get('end_time')
start_time = timestamp_parser.parse_datetime(
start_time, default=timestamp_parser.min_datetime)
end_time = timestamp_parser.parse_datetime(
end_time, default=timestamp_parser.max_datetime)
# refresh if needed (might be time consuming)
scraper = CollectionRefreshScraper(index)
scraper.set_refresh_scope(collection_name, start_time, end_time)
driver = ScraperDriver(scraper, 4, 0.5)
driver.harvest()
scraper.sync_index()
# retrieve from index DB
result = index.get_granules(collection_name, start_time, end_time)
print("read index for %s" % collection_name, flush=True)
return result
def _setup_tag_invalid_meta_data_file(confirm_nuke):
exif = ExifEditor(Configuration())
index = Index(TEST_INDEX_LOCATION, exif)
core = TieCoreImpl(exif, index)
cli.run_cmd(["cp", READ_FILE, WRITE_FILE])
frontend = FrontendTest(confirm_nuke, [])
tie_main.run(core, RunOptions(["tag", "foo", WRITE_FILE]), frontend)
def post(self):
print("1 TRAP SIGURG on " + str(os.getpid()))
debug_thread = threading.Thread(target=debug_dump_watch)
debug_thread.start()
# get params
# collection_name = request.form.get('collection_name')
catalog_url = request.form.get('catalog_url')
if 'thredds.ucar.edu' in catalog_url:
Dataset.default_authority = 'edu.ucar.unidata'
if 'rda.ucar.edu' in catalog_url:
Dataset.default_authority = 'edu.ucar.rda'
job_id = str(uuid.uuid4())[:8] # short id for uniqueness
index = Index(INDEX_DB_URL)
output_dir = RECORDS_DIR + '/result.' + job_id
scraper = CollectionImportScraper(output_dir, index)
scraper.add_catalog(catalog_url=catalog_url)
driver = ScraperDriver(scraper, 40, 1)
driver.harvest()
scraper.sync_index()
# complete
print("indexing harvest complete", flush=True)
print("importing %s" % output_dir, flush=True)
print("")
PycswHelper().load_records(output_dir)
return output_dir
def main(*args):
try:
setup_sys_path()
configuration = config.load_user_config()
run_options = RunOptions(list(args[1:]))
frontend_type = run_options.frontend
front_end = ff.from_type(frontend_type)
index_root_dir = configuration.index_path
exif = ExifEditor(configuration)
index = Index(index_root_dir, exif)
core = TieCoreImpl(exif, index)
if run_options.action == Action.help:
print_usage()
else:
tie_main.run(core, run_options, front_end)
except ParseError as parse_error:
printerr("Error: " + parse_error.msg)
sys.exit(EXIT_CODE_PARSE_ERROR)
except InvalidMetaDataError as meta_data_error:
printerr("Error: " + meta_data_error.msg)
sys.exit(EXIT_CODE_INVALID_META_DATA)
except KeyboardInterrupt:
printerr("Application aborted by user")
sys.exit(EXIT_CODE_INVALID_META_DATA)
except FileNotFoundError:
# No need to print it. this is already done by subprocess
sys.exit(EXIT_CODE_FILE_NOT_FOUND)
except CalledProcessError:
# No need to print it. this is already done by subprocess
sys.exit(EXIT_CODE_UNKNOWN_SUBPROCESS_ERROR)
def setUp(self):
_remove_index()
self.exif = ee.ExifEditor(Configuration())
self.files_base_path = _path_to_linkname(os.path.abspath("../res"))
self.index = Index(TEST_INDEX_LOCATION, self.exif)
self.tie_core = TieCoreImpl(self.exif, self.index)
class TestTieCore(TestCase):
def setUp(self):
_remove_index()
self.exif = ee.ExifEditor(Configuration())
self.files_base_path = _path_to_linkname(os.path.abspath("../res"))
self.index = Index(TEST_INDEX_LOCATION, self.exif)
self.tie_core = TieCoreImpl(self.exif, self.index)
def tearDown(self):
_remove_index()
def test_query_all(self):
self._prepare_query_test()
files_2_3 = self.tie_core.query(
Query([QUERY_TAG_2, QUERY_TAG_3], MatchType.all))
self.assertEqual(
[os.path.abspath(QUERY_FILE_2)], files_2_3,
"Query all with 2 tags did not find the expected files")
files_3 = self.tie_core.query(Query([QUERY_TAG_3], MatchType.all))
self.assertEqual(
[os.path.abspath(QUERY_FILE_2),
os.path.abspath(QUERY_FILE_3)], files_3,
"Query all with 1 tag did not find the expected files")
_clean_after_query_test()
def test_query_all_empty_tags_list(self):
self._prepare_query_test()
result = self.tie_core.query(Query([], MatchType.all))
self.assertEqual([], result, "Query with no tags")
_clean_after_query_test()
def test_query_all_no_match(self):
self._prepare_query_test()
result = self.tie_core.query(
Query([QUERY_TAG_2, "some random non existant tag"],
MatchType.all))
self.assertEqual([], result, "Query with no result")
_clean_after_query_test()
def test_query_any(self):
self._prepare_query_test()
files = self.tie_core.query(
Query([QUERY_TAG_2, QUERY_TAG_3], MatchType.any))
self.assertEqual([
os.path.abspath(QUERY_FILE_1),
os.path.abspath(QUERY_FILE_2),
os.path.abspath(QUERY_FILE_3)
], files, "Query any did not find the expected files")
_clean_after_query_test()
def _prepare_query_test(self):
cli.run_cmd(["cp", READ_FILE_MD, QUERY_FILE_1])
cli.run_cmd(["cp", READ_FILE_MD, QUERY_FILE_2])
cli.run_cmd(["cp", READ_FILE_MD, QUERY_FILE_3])
cli.run_cmd(["cp", READ_FILE_MD, QUERY_FILE_4])
self.exif.set_meta_data(QUERY_FILE_1,
md.MetaData([QUERY_TAG_1, QUERY_TAG_2]))
self.exif.set_meta_data(QUERY_FILE_2,
md.MetaData([QUERY_TAG_2, QUERY_TAG_3]))
self.exif.set_meta_data(QUERY_FILE_3,
md.MetaData([QUERY_TAG_3, QUERY_TAG_4]))
self.exif.set_meta_data(QUERY_FILE_4, md.MetaData([QUERY_TAG_4]))
self.index.update(QUERY_FILE_1)
self.index.update(QUERY_FILE_2)
self.index.update(QUERY_FILE_3)
self.index.update(QUERY_FILE_4)
def test_list(self):
tags = self.tie_core.list([READ_FILE_MD])
self.assertEqual([TEST_READ_TAG_1.lower(),
TEST_READ_TAG_2.lower()], tags,
"listed tags do not match")
def test_list_multiple_files(self):
tags = self.tie_core.list([READ_FILE_MD, READ_FILE_MD_2])
self.assertEqual(
sorted([
TEST_READ_TAG_1.lower(),
TEST_READ_TAG_2.lower(),
TEST_READ_TAG_3.lower()
]), tags, "listed tags do not match")
def test_list_empty_raw(self):
cli.run_cmd(["cp", READ_FILE, WRITE_FILE])
self.exif._write_exif_field("Exif.Photo.UserComment", "", WRITE_FILE)
tags = self.tie_core.list([WRITE_FILE])
self.assertEqual([], tags, "listed tags do not match empty list")
os.remove(WRITE_FILE)
def test_list_empty_md(self):
cli.run_cmd(["cp", READ_FILE_MD, WRITE_FILE_MD])
self.tie_core.clear(WRITE_FILE_MD)
tags = self.tie_core.list([WRITE_FILE_MD])
self.assertEqual([], tags, "listed tags do not match empty list")
os.remove(WRITE_FILE_MD)
def test_list_invalid(self):
self.assertRaises(InvalidMetaDataError,
lambda: self.tie_core.list([READ_FILE]))
def test_list_not_found(self):
self.assertRaises(
FileNotFoundError,
lambda: self.tie_core.list(["../res/foobar.not.existant"]))
def test_tag(self):
cli.run_cmd(["cp", READ_FILE_MD, WRITE_FILE_MD])
added_tag1 = "New tag Ä"
added_tag2 = "Other tag Ä"
self.tie_core.tag(WRITE_FILE_MD, [added_tag1, added_tag2])
self.assertEqual(
sorted([
TEST_READ_TAG_1.lower(),
TEST_READ_TAG_2.lower(),
added_tag1.lower(),
added_tag2.lower()
]), self.tie_core.list([WRITE_FILE_MD]),
"Tags after adding did not match")
os.remove(WRITE_FILE_MD)
def test_tag_empty(self):
cli.run_cmd(["cp", READ_FILE_NO_EXIF, WRITE_FILE_NO_EXIF])
tag = "foo bar"
self.tie_core.tag(WRITE_FILE_NO_EXIF, [tag])
self.assertEqual([tag], self.tie_core.list([WRITE_FILE_NO_EXIF]),
"File without exif data not tagged")
os.remove(WRITE_FILE_NO_EXIF)
def test_tag_invalid(self):
self.assertRaises(
InvalidMetaDataError,
lambda: self.tie_core.tag(READ_FILE, [TEST_READ_TAG_1]))
def test_list_no_exif_data(self):
res = self.tie_core.list([READ_FILE_NO_EXIF])
self.assertEqual([], res,
"File without exif data does not show empty tag list")
def test_tag_duplicate(self):
cli.run_cmd(["cp", READ_FILE_MD, WRITE_FILE_MD])
added_tag1 = "New tag Ä"
self.tie_core.tag(WRITE_FILE_MD,
[added_tag1, added_tag1, TEST_READ_TAG_1])
self.assertEqual(
sorted([
TEST_READ_TAG_1.lower(),
TEST_READ_TAG_2.lower(),
added_tag1.lower()
]), self.tie_core.list([WRITE_FILE_MD]),
"Tags after duplicate adding did not match")
os.remove(WRITE_FILE_MD)
def test_tag_not_found(self):
self.assertRaises(
FileNotFoundError, lambda: self.tie_core.tag(
"../res/foobar.not.existant", [TEST_READ_TAG_1]))
def test_untag(self):
cli.run_cmd(["cp", READ_FILE_MD, WRITE_FILE_MD])
self.tie_core.untag(WRITE_FILE_MD, [TEST_READ_TAG_1])
self.assertEqual([TEST_READ_TAG_2.lower()],
self.tie_core.list([WRITE_FILE_MD]),
"Tags after removing did not match")
os.remove(WRITE_FILE_MD)
def test_untag_duplicate(self):
cli.run_cmd(["cp", READ_FILE_MD, WRITE_FILE_MD])
self.tie_core.untag(WRITE_FILE_MD, [TEST_READ_TAG_1])
self.tie_core.untag(WRITE_FILE_MD, [TEST_READ_TAG_1])
self.tie_core.untag(WRITE_FILE_MD, [TEST_READ_TAG_1, TEST_READ_TAG_1])
self.assertEqual([TEST_READ_TAG_2.lower()],
self.tie_core.list([WRITE_FILE_MD]),
"Tags after duplicate removing did not match")
os.remove(WRITE_FILE_MD)
def test_untag_invalid(self):
self.assertRaises(
InvalidMetaDataError,
lambda: self.tie_core.untag(READ_FILE, [TEST_READ_TAG_1]))
def test_untag_not_found(self):
self.assertRaises(
FileNotFoundError, lambda: self.tie_core.untag(
"../res/foobar.not.existant", [TEST_READ_TAG_1]))
def test_clear(self):
cli.run_cmd(["cp", READ_FILE_MD, WRITE_FILE_MD])
self.tie_core.clear(WRITE_FILE_MD)
tags_after_clear = self.tie_core.list([WRITE_FILE_MD])
self.assertEqual([], tags_after_clear,
"Tag list was not empty after clear")
os.remove(WRITE_FILE_MD)
def test_clear_invalid(self):
cli.run_cmd(["cp", READ_FILE, WRITE_FILE])
self.tie_core.clear(WRITE_FILE)
tags_after_clear = self.tie_core.list([WRITE_FILE])
self.assertEqual([], tags_after_clear,
"Tag list was not empty after clearing corrupt file")
os.remove(WRITE_FILE)
def test_clear_not_found(self):
self.assertRaises(
FileNotFoundError,
lambda: self.tie_core.clear("../res/foobar.not.existant"))
def test_index(self):
self.tie_core.update_index(READ_FILE_MD)
link_name = self.files_base_path + ":" + "read_md.jpg"
self.assertTrue(
os.path.islink(
os.path.join(TEST_INDEX_LOCATION, "tags",
TEST_READ_TAG_1.lower(), link_name)),
"no link in tagdir 1")
self.assertTrue(
os.path.islink(
os.path.join(TEST_INDEX_LOCATION, "tags",
TEST_READ_TAG_2.lower(), link_name)),
"no link in tagdir 2")
def test_list_all_tags(self):
cli.run_cmd(["cp", READ_FILE_MD, WRITE_FILE_MD])
self.tie_core.update_index(WRITE_FILE_MD)
tags = self.tie_core.list_all_tags()
self.assertEqual(
sorted([TEST_READ_TAG_1.lower(),
TEST_READ_TAG_2.lower()]), sorted(tags))
os.remove(WRITE_FILE_MD)
def contact():
return Index.show_contactpage(app)
def team():
return Index.show_teampage(app)
def about():
return Index.show_aboutpage(app)
def index():
return Index.show_indexpage(app)
def setUp(self):
_remove_index()
self.index = Index(TEST_INDEX_LOCATION, ee.ExifEditor(Configuration()))
self.files_base_path = _path_to_linkname(os.path.abspath("../res"))
class TestIndex(TestCase):
def setUp(self):
_remove_index()
self.index = Index(TEST_INDEX_LOCATION, ee.ExifEditor(Configuration()))
self.files_base_path = _path_to_linkname(os.path.abspath("../res"))
def tearDown(self):
_remove_index()
def test_initialization(self):
self.assertTrue(os.path.isdir(TEST_INDEX_LOCATION))
self.assertTrue(
os.path.isdir(os.path.join(TEST_INDEX_LOCATION, "tags")))
def test_update_file(self):
self.index.update("../res/read_md.jpg")
link_name = self.files_base_path + ":" + "read_md.jpg"
self.assertTrue(
os.path.islink(
os.path.join(TEST_INDEX_LOCATION, "tags",
TEST_READ_TAG_1.lower(), link_name)),
"no link in tagdir 1")
self.assertTrue(
os.path.islink(
os.path.join(TEST_INDEX_LOCATION, "tags",
TEST_READ_TAG_2.lower(), link_name)),
"no link in tagdir 2")
def test_list_tags(self):
cli.run_cmd(
["mkdir", "-p",
os.path.join(TEST_INDEX_LOCATION, "tags", "foo")])
cli.run_cmd(
["mkdir", "-p",
os.path.join(TEST_INDEX_LOCATION, "tags", "bar")])
tags = self.index.list_tags()
self.assertEqual(["bar", "foo"], tags, "tags list incorrect")
def test_list_files(self):
self.index.update(READ_FILE_MD)
files = self.index.list_files(TEST_READ_TAG_1)
self.assertEqual([os.path.abspath(READ_FILE_MD)], files,
"files list incorrect")
def test_update_file_invalid_md(self):
img = os.path.abspath("../res/read.jpg")
linkname = _path_to_linkname(img)
removed_tag_dir = os.path.join(TEST_INDEX_LOCATION, "tags",
"removed tag")
link_path = os.path.join(removed_tag_dir, linkname)
cli.run_cmd(["mkdir", "-p", removed_tag_dir])
cli.run_cmd(["ln", "-sf", img, link_path])
self.index.update("../res/read.jpg")
self.assertFalse(os.path.islink(link_path), "tag was not removed!")
def test_update_vanished_file(self):
vanished_file = "../res/vanished_file.jpg"
link_path = self._prepare_vanished_file(vanished_file, TEST_READ_TAG_2)
self.index.update(vanished_file)
self.assertFalse(os.path.islink(link_path),
"tag of vanished file was not removed!")
def test_update_vanished_file_with_implicit_update(self):
vanished_file = "../res/vanished_file.jpg"
link_path = self._prepare_vanished_file(vanished_file, TEST_READ_TAG_2)
self.index.update("../res/")
self.assertFalse(os.path.islink(link_path),
"tag of vanished file was not removed!")
def test_list_vanished_file(self):
vanished_file = "../res/vanished_file.jpg"
link_path = self._prepare_vanished_file(vanished_file, TEST_READ_TAG_2)
self.index.list_files(TEST_READ_TAG_2)
self.assertFalse(os.path.islink(link_path),
"tag of vanished file was not removed!")
def test_update_vanished_directory(self):
vanished_directory = "../res/vanished_directory"
vanished_file = os.path.join(vanished_directory,
"some_subdir/vanished_file.jpg")
link_path = self._prepare_vanished_file(vanished_file, TEST_READ_TAG_2)
self.index.update(vanished_directory)
self.assertFalse(os.path.islink(link_path),
"tag of file in vanished directory was not removed!")
def test_list_vanished_directory(self):
vanished_directory = "../res/vanished_directory"
vanished_file = os.path.join(vanished_directory,
"some_subdir/vanished_file.jpg")
link_path = self._prepare_vanished_file(vanished_file, TEST_READ_TAG_2)
self.index.list_files(TEST_READ_TAG_2)
self.assertFalse(os.path.islink(link_path),
"tag of file in vanished directory was not removed!")
def _prepare_vanished_file(self, path: str, tag_name: str) -> str:
img = os.path.abspath(path)
linkname = _path_to_linkname(img)
tag_dir = os.path.join(TEST_INDEX_LOCATION, "tags", tag_name)
link_path = os.path.join(tag_dir, linkname)
cli.run_cmd(["mkdir", "-p", tag_dir])
cli.run_cmd(["ln", "-sf", img, link_path])
return link_path
def test_update_dir(self):
self.index.update("../res/recursive.d")
link_name_root = self.files_base_path + ":recursive.d:" + "read_md.jpg"
link_name_lvl1 = self.files_base_path + ":recursive.d:level1:" + "read_md.jpg"
link_name_lvl2 = self.files_base_path + ":recursive.d:level1:level2:" + "read_md.jpg"
for link_name in [link_name_root, link_name_lvl1, link_name_lvl2]:
self.assertTrue(
os.path.islink(
os.path.join(TEST_INDEX_LOCATION, "tags",
TEST_READ_TAG_1.lower(), link_name)),
"Link " + link_name + " should exist in tagdir 1 but does not")
self.assertTrue(
os.path.islink(
os.path.join(TEST_INDEX_LOCATION, "tags",
TEST_READ_TAG_2.lower(), link_name)),
"Link " + link_name + " should exist in tagdir 2 but does not")
def test_removed_tag(self):
img = os.path.abspath("../res/read_md.jpg")
linkname = _path_to_linkname(img)
removed_tag_dir = os.path.join(TEST_INDEX_LOCATION, "tags",
"removed tag")
link_path = os.path.join(removed_tag_dir, linkname)
cli.run_cmd(["mkdir", "-p", removed_tag_dir])
cli.run_cmd(["ln", "-sf", img, link_path])
self.index.update("../res/read_md.jpg")
self.assertFalse(os.path.islink(link_path), "tag was not removed!")
# Execute again to test if this (incorrectly) raises a FileNotFoundError
self.index.update("../res/read_md.jpg")
def test_auto_remove_empty_tag_dir(self):
empty_tag_dir = os.path.join(TEST_INDEX_LOCATION, "tags",
"removed tag")
cli.run_cmd(["mkdir", "-p", empty_tag_dir])
self.index.update("../res/read_md.jpg")
self.assertFalse(os.path.isdir(empty_tag_dir),
"empty tag dir was not removed!")
def test_symlinks_followed(self):
self.index.update("../res/recursive.d/level1_symlinked/read_md.jpg")
link_name = self.files_base_path + ":recursive.d:level1:read_md.jpg"
self.assertTrue(
os.path.islink(
os.path.join(TEST_INDEX_LOCATION, "tags",
TEST_READ_TAG_1.lower(), link_name)),
"no link in tagdir 1")
self.assertTrue(
os.path.islink(
os.path.join(TEST_INDEX_LOCATION, "tags",
TEST_READ_TAG_2.lower(), link_name)),
"no link in tagdir 2")
def test_unfollowed_symlinks_corrected(self):
link_name_right = self.files_base_path + ":recursive.d:level1:read_md.jpg"
link_name_wrong = self.files_base_path + ":recursive.d:level1_symlinked:read_md.jpg"
cli.run_cmd([
"mkdir", "-p",
os.path.join(TEST_INDEX_LOCATION, "tags", TEST_READ_TAG_1.lower())
])
cli.run_cmd([
"mkdir", "-p",
os.path.join(TEST_INDEX_LOCATION, "tags", TEST_READ_TAG_2.lower())
])
cli.run_cmd([
"ln", "-sf", "../res/recursive.d/level1_symlinked/read_md.jpg",
os.path.join(TEST_INDEX_LOCATION, "tags", TEST_READ_TAG_1.lower(),
link_name_wrong)
])
cli.run_cmd([
"ln", "-sf", "../res/recursive.d/level1_symlinked/read_md.jpg",
os.path.join(TEST_INDEX_LOCATION, "tags", TEST_READ_TAG_2.lower(),
link_name_wrong)
])
self.index.update("../res/recursive.d/level1_symlinked/read_md.jpg")
self.assertTrue(
os.path.islink(
os.path.join(TEST_INDEX_LOCATION, "tags",
TEST_READ_TAG_1.lower(), link_name_right)),
"no link in tagdir 1")
self.assertTrue(
os.path.islink(
os.path.join(TEST_INDEX_LOCATION, "tags",
TEST_READ_TAG_2.lower(), link_name_right)),
"no link in tagdir 2")
self.assertFalse(
os.path.islink(
os.path.join(TEST_INDEX_LOCATION, "tags",
TEST_READ_TAG_1.lower(), link_name_wrong)),
"wrong link in tagdir 1")
self.assertFalse(
os.path.islink(
os.path.join(TEST_INDEX_LOCATION, "tags",
TEST_READ_TAG_2.lower(), link_name_wrong)),
"wrong link in tagdir 2")
def index():
print(os.path.dirname(os.path.realpath(__file__)))
return Index.show_index(app)
def name():
return Index.show_namepage(app)
def index(index_path):
return Index(index_path)