def test_add_data(self):
mem_kv = MemoryKeyValueStore()
kvds = KVSDataSet(mem_kv)
de1 = DataMemoryElement(six.b('bytes1'))
de2 = DataMemoryElement(six.b('bytes2'))
kvds.add_data(de1, de2)
# Check that appropriate keys and values are retrievable and located in
# used KV-store.
self.assertIn(de1.uuid(), mem_kv)
self.assertIn(de2.uuid(), mem_kv)
self.assertEqual(mem_kv.get(de1.uuid()), de1)
self.assertEqual(mem_kv.get(de2.uuid()), de2)
def test_add_data(self):
mem_kv = MemoryKeyValueStore()
kvds = KVSDataSet(mem_kv)
de1 = DataMemoryElement(six.b('bytes1'))
de2 = DataMemoryElement(six.b('bytes2'))
kvds.add_data(de1, de2)
# Check that appropriate keys and values are retrievable and located in
# used KV-store.
self.assertIn(de1.uuid(), mem_kv)
self.assertIn(de2.uuid(), mem_kv)
self.assertEqual(mem_kv.get(de1.uuid()), de1)
self.assertEqual(mem_kv.get(de2.uuid()), de2)
def test_add_data(self):
de = DataMemoryElement(six.b('some bytes'), 'text/plain', True)
expected_map = {de.uuid(): de}
dms = DataMemorySet()
dms.add_data(de)
self.assertEqual(dms._element_map, expected_map)
def dl_image(meta):
try:
c_type = meta['fields']['content_type'][0]
obj_stored_url = meta['fields']['obj_stored_url'][0]
obj_original_url = meta['fields']['obj_original_url'][0]
c_ext = m.guess_extension(c_type, strict=False)
if c_ext is None:
log.warn(
"Guessed 'None' extension for content-type '%s', "
"skipping.", c_type)
return None
save_dir = os.path.abspath(
os.path.expanduser(
os.path.join(output_dir, meta['index'], meta['doc_type'])))
save_file = meta['id'] + c_ext
save_path = os.path.join(save_dir, save_file)
# Save/write file if needed
if not os.path.isfile(save_path):
# First try 'stored' url, fallback on original
# Return None if failed to download anything
ok, r = try_download(obj_stored_url, stored_http_auth)
if not ok:
log.warn(
"Failed to download stored-data URL \"%s\" "
"(error=%s)", obj_stored_url, str(r))
ok, r = try_download(obj_original_url)
if not ok:
log.warn(
"Failed to download original URL \"%s\" "
"(error=%s)", obj_stored_url, str(r))
return None
# Assuming OK at this point
content = r.content
d = DataMemoryElement(content, c_type)
safe_create_dir(save_dir)
with open(save_path, 'wb') as out:
log.debug("Saving to file: '%s'", save_path)
out.write(content)
else:
d = DataFileElement(save_path)
return meta['id'], save_path, d.uuid()
except KeyError, ex:
log.error("Failed to find key %s in meta block: %s", str(ex), meta)
raise
def dl_image(meta):
try:
c_type = meta['fields']['content_type'][0]
obj_stored_url = meta['fields']['obj_stored_url'][0]
obj_original_url = meta['fields']['obj_original_url'][0]
c_ext = m.guess_extension(c_type, strict=False)
if c_ext is None:
log.warn("Guessed 'None' extension for content-type '%s', "
"skipping.", c_type)
return None
save_dir = os.path.abspath(os.path.expanduser(
os.path.join(output_dir, meta['index'], meta['doc_type'])
))
save_file = meta['id'] + c_ext
save_path = os.path.join(save_dir, save_file)
# Save/write file if needed
if not os.path.isfile(save_path):
# First try 'stored' url, fallback on original
# Return None if failed to download anything
ok, r = try_download(obj_stored_url, stored_http_auth)
if not ok:
log.warn("Failed to download stored-data URL \"%s\" "
"(error=%s)",
obj_stored_url, str(r))
ok, r = try_download(obj_original_url)
if not ok:
log.warn("Failed to download original URL \"%s\" "
"(error=%s)",
obj_stored_url, str(r))
return None
# Assuming OK at this point
content = r.content
d = DataMemoryElement(content, c_type)
safe_create_dir(save_dir)
with open(save_path, 'wb') as out:
log.debug("Saving to file: '%s'", save_path)
out.write(content)
else:
d = DataFileElement(save_path)
return meta['id'], save_path, d.uuid()
except KeyError, ex:
log.error("Failed to find key %s in meta block: %s",
str(ex), meta)
raise
