def test_load_dataset_tempfile(self):
"""
Test DataElement temporary file based context loader.
"""
# Creating separate element from global so we can mock it up.
e = DataFileElement(GH_IMAGE_FP, readonly=True)
e.write_temp = mock.MagicMock(wraps=e.write_temp)
e.clean_temp = mock.MagicMock(wraps=e.clean_temp)
e.get_bytes = mock.MagicMock(wraps=e.get_bytes)
# Using explicit patcher start/stop in order to avoid using ``patch``
# as a decorator because ``osgeo`` might not be defined when
# decorating the method.
patcher_gdal_open = mock.patch('smqtk.algorithms.image_io.gdal_io.gdal'
'.Open', wraps=osgeo.gdal.Open)
self.addCleanup(patcher_gdal_open.stop)
m_gdal_open = patcher_gdal_open.start()
with load_dataset_tempfile(e) as gdal_ds:
# noinspection PyUnresolvedReferences
e.write_temp.assert_called_once_with()
# noinspection PyUnresolvedReferences
e.get_bytes.assert_not_called()
m_gdal_open.assert_called_once()
assert gdal_ds.RasterCount == 3
assert gdal_ds.RasterXSize == 512
assert gdal_ds.RasterYSize == 600
# noinspection PyUnresolvedReferences
e.clean_temp.assert_called_once_with()
assert len(e._temp_filepath_stack) == 0
def test_writeTempOverride_diffDir(self, mock_open, mock_os_open,
mock_os_close, mock_fcntl, mock_scd):
source_filepath = '/path/to/file.png'
target_dir = '/some/other/dir'
d = DataFileElement(source_filepath)
fp = d.write_temp(temp_dir=target_dir)
ntools.assert_not_equal(fp, source_filepath)
ntools.assert_equal(os.path.dirname(fp), target_dir)
# subsequent call to write temp should not invoke creation of a new file
fp2 = d.write_temp()
ntools.assert_equal(fp2, source_filepath)
# request in same dir should return same path as first request with that
# directory
fp3 = d.write_temp(target_dir)
ntools.assert_equal(fp, fp3)
# request different target dir
target2 = '/even/different/path'
fp4 = d.write_temp(target2)
ntools.assert_equal(os.path.dirname(fp4), target2)
ntools.assert_not_equal(fp, fp4)
ntools.assert_equal(len(d._temp_filepath_stack), 2)
def test_get_bytes_no_file(self):
e = DataFileElement("/not/a/valid/path.txt", readonly=True)
# We currently expect, in the case where the filepath doesn't exist, to
# get the same bytes as if the file existed and were empty.
self.assertEqual(e.get_bytes(), six.b(""))
# read-only status should have no effect.
e = DataFileElement("/not/a/valid/path.txt", readonly=True)
self.assertEqual(e.get_bytes(), six.b(""))
def test_writeTempOverride(self, mock_DataElement_wt):
# no manual directory, should return the base filepath
expected_filepath = '/path/to/file.txt'
d = DataFileElement(expected_filepath)
fp = d.write_temp()
self.assertFalse(mock_DataElement_wt.called)
self.assertEqual(expected_filepath, fp)
def test_writeTempOverride(self, mock_DataElement_wt):
# no manual directory, should return the base filepath
expected_filepath = '/path/to/file.txt'
d = DataFileElement(expected_filepath)
fp = d.write_temp()
self.assertFalse(mock_DataElement_wt.called)
self.assertEqual(expected_filepath, fp)
def test_writeTempOverride_sameDir(self, mock_DataElement_wt):
expected_filepath = '/path/to/file.txt'
target_dir = '/path/to'
d = DataFileElement(expected_filepath)
fp = d.write_temp(temp_dir=target_dir)
self.assertFalse(mock_DataElement_wt.called)
self.assertEqual(fp, expected_filepath)
def test_writeTempOverride_sameDir(self, mock_DataElement_wt):
expected_filepath = "/path/to/file.txt"
target_dir = "/path/to"
d = DataFileElement(expected_filepath)
fp = d.write_temp(temp_dir=target_dir)
ntools.assert_false(mock_DataElement_wt.called)
ntools.assert_equal(fp, expected_filepath)
def iter_input_elements():
for f in args.input_files:
f = osp.expanduser(f)
if osp.isfile(f):
yield DataFileElement(f)
else:
log.debug("Expanding glob: %s" % f)
for g in glob.glob(f):
yield DataFileElement(g)
def test_cleanTemp(self):
# a write temp and clean temp should not affect original file
source_file = os.path.join(TEST_DATA_DIR, 'test_file.dat')
self.assertTrue(os.path.isfile(source_file))
d = DataFileElement(source_file)
d.write_temp()
self.assertEqual(len(d._temp_filepath_stack), 0)
d.clean_temp()
self.assertTrue(os.path.isfile(source_file))
def test_writeTempOverride_sameDir(self, mock_DataElement_wt):
expected_filepath = '/path/to/file.txt'
target_dir = '/path/to'
d = DataFileElement(expected_filepath)
fp = d.write_temp(temp_dir=target_dir)
self.assertFalse(mock_DataElement_wt.called)
self.assertEqual(fp, expected_filepath)
def iter_valid_elements():
for fp in file_paths:
dfe = DataFileElement(fp)
ct = dfe.content_type()
if ct in generator.valid_content_types():
valid_file_paths[fp] = ct
yield dfe
else:
invalid_file_paths[fp] = ct
def iter_valid_elements():
for fp in file_paths:
dfe = DataFileElement(fp)
ct = dfe.content_type()
if ct in generator.valid_content_types():
valid_file_paths[fp] = ct
yield dfe
else:
invalid_file_paths[fp] = ct
def test_set_bytes_writable(self, m_sfw):
# Using a relative filepath
test_path = 'foo'
test_bytes = six.b('test string of bytes')
e = DataFileElement(test_path)
e.set_bytes(test_bytes)
# File write function should be called
m_sfw.assert_called_once_with(test_path, test_bytes)
def setUpClass(cls):
cls.gh_image_fp = os.path.join(TEST_DATA_DIR, "grace_hopper.png")
cls.gh_file_element = DataFileElement(cls.gh_image_fp)
assert cls.gh_file_element.content_type() == 'image/png'
cls.gh_cropped_image_fp = \
os.path.join(TEST_DATA_DIR, 'grace_hopper.100x100+100+100.png')
cls.gh_cropped_file_element = DataFileElement(cls.gh_cropped_image_fp)
assert cls.gh_cropped_file_element.content_type() == 'image/png'
cls.gh_cropped_bbox = AxisAlignedBoundingBox([100, 100], [200, 200])
def test_set_bytes_writable(self, m_sfw):
# Using a relative filepath
test_path = 'foo'
test_bytes = six.b('test string of bytes')
e = DataFileElement(test_path)
e.set_bytes(test_bytes)
# File write function should be called
m_sfw.assert_called_once_with(test_path, test_bytes)
def test_configuration(self):
fp = os.path.join(TEST_DATA_DIR, "Lenna.png")
default_config = DataFileElement.get_default_config()
ntools.assert_equal(default_config, {'filepath': None})
default_config['filepath'] = fp
inst1 = DataFileElement.from_config(default_config)
ntools.assert_equal(default_config, inst1.get_config())
inst2 = DataFileElement.from_config(inst1.get_config())
ntools.assert_equal(inst1, inst2)
def test_configuration(self):
fp = os.path.join(TEST_DATA_DIR, "Lenna.png")
default_config = DataFileElement.get_default_config()
ntools.assert_equal(default_config, {"filepath": None})
default_config["filepath"] = fp
inst1 = DataFileElement.from_config(default_config)
ntools.assert_equal(default_config, inst1.get_config())
inst2 = DataFileElement.from_config(inst1.get_config())
ntools.assert_equal(inst1, inst2)
def test_writable_readonly_false(self):
e = DataFileElement('foo')
self.assertTrue(e.writable())
e = DataFileElement('foo', False)
self.assertTrue(e.writable())
e = DataFileElement('foo', readonly=False)
self.assertTrue(e.writable())
def test_configuration(self):
fp = os.path.join(TEST_DATA_DIR, "Lenna.png")
default_config = DataFileElement.get_default_config()
self.assertEqual(default_config,
{'filepath': None, 'readonly': False,
'explicit_mimetype': None})
default_config['filepath'] = fp
inst1 = DataFileElement.from_config(default_config)
self.assertEqual(default_config, inst1.get_config())
inst2 = DataFileElement.from_config(inst1.get_config())
self.assertEqual(inst1, inst2)
def is_valid_element(fp):
dfe = DataFileElement(fp)
ct = dfe.content_type()
if ct in generator.valid_content_types():
if not check_image or test_image_load(dfe):
return dfe
else:
return None
else:
log.debug("Skipping file (invalid content) type for "
"descriptor generator (fp='%s', ct=%s)",
fp, ct)
return None
def is_valid_element(fp):
dfe = DataFileElement(fp)
ct = dfe.content_type()
if ct in generator.valid_content_types():
if not check_image or test_image_load(dfe):
return dfe
else:
return None
else:
log.debug(
"Skipping file (invalid content) type for "
"descriptor generator (fp='%s', ct=%s)", str(fp), ct)
return None
def test_load_dataset_vsimem(self):
"""
Test that VSIMEM loading context
"""
if LooseVersion(osgeo.__version__).version[0] < 2:
pytest.skip("Skipping VSIMEM test because GDAL version < 2")
# Creating separate element from global so we can mock it up.
e = DataFileElement(GH_IMAGE_FP, readonly=True)
e.write_temp = mock.MagicMock(wraps=e.write_temp)
e.clean_temp = mock.MagicMock(wraps=e.clean_temp)
e.get_bytes = mock.MagicMock(wraps=e.get_bytes)
vsimem_path_re = re.compile(r'^/vsimem/\w+$')
# Using explicit patcher start/stop in order to avoid using ``patch``
# as a *decorator* because ``osgeo`` might not be defined when
# decorating the method.
patcher_gdal_open = mock.patch(
'smqtk.algorithms.image_io.gdal_io.gdal.Open',
wraps=osgeo.gdal.Open,
)
self.addCleanup(patcher_gdal_open.stop)
patcher_gdal_unlink = mock.patch(
'smqtk.algorithms.image_io.gdal_io.gdal.Unlink',
wraps=osgeo.gdal.Unlink,
)
self.addCleanup(patcher_gdal_unlink.stop)
m_gdal_open = patcher_gdal_open.start()
m_gdal_unlink = patcher_gdal_unlink.start()
with load_dataset_vsimem(e) as gdal_ds:
# noinspection PyUnresolvedReferences
e.write_temp.assert_not_called()
# noinspection PyUnresolvedReferences
e.get_bytes.assert_called_once_with()
m_gdal_open.assert_called_once()
ds_path = gdal_ds.GetDescription()
assert vsimem_path_re.match(ds_path)
assert gdal_ds.RasterCount == 3
assert gdal_ds.RasterXSize == 512
assert gdal_ds.RasterYSize == 600
m_gdal_unlink.assert_called_once_with(ds_path)
# noinspection PyUnresolvedReferences
e.clean_temp.assert_not_called()
assert len(e._temp_filepath_stack) == 0
def test_from_uri(self):
# will be absolute path
test_file_path = os.path.join(TEST_DATA_DIR, "test_file.dat")
print("Test file path:", test_file_path)
e = DataFileElement.from_uri(test_file_path)
ntools.assert_is_instance(e, DataFileElement)
ntools.assert_equal(e._filepath, test_file_path)
ntools.assert_equal(e.get_bytes(), '')
e = DataFileElement.from_uri('file://' + test_file_path)
ntools.assert_is_instance(e, DataFileElement)
ntools.assert_equal(e._filepath, test_file_path)
ntools.assert_equal(e.get_bytes(), '')
def test_from_uri(self):
# will be absolute path
test_file_path = os.path.join(TEST_DATA_DIR, "test_file.dat")
print("Test file path:", test_file_path)
e = DataFileElement.from_uri(test_file_path)
self.assertIsInstance(e, DataFileElement)
self.assertEqual(e._filepath, test_file_path)
self.assertEqual(e.get_bytes(), six.b(''))
e = DataFileElement.from_uri('file://' + test_file_path)
self.assertIsInstance(e, DataFileElement)
self.assertEqual(e._filepath, test_file_path)
self.assertEqual(e.get_bytes(), six.b(''))
def test_from_uri(self):
# will be absolute path
test_file_path = os.path.join(TEST_DATA_DIR, "test_file.dat")
print("Test file path:", test_file_path)
e = DataFileElement.from_uri(test_file_path)
self.assertIsInstance(e, DataFileElement)
self.assertEqual(e._filepath, test_file_path)
self.assertEqual(e.get_bytes(), six.b(''))
e = DataFileElement.from_uri('file://' + test_file_path)
self.assertIsInstance(e, DataFileElement)
self.assertEqual(e._filepath, test_file_path)
self.assertEqual(e.get_bytes(), six.b(''))
def iqr_ingest_file():
"""
Ingest the file with the given UID, getting the path from the
uploader.
:return: string of data/descriptor element's UUID
:rtype: str
"""
# TODO: Add status dict with a "GET" method branch for getting that
# status information.
# Start the ingest of a FID when POST
if flask.request.method == "POST":
with self.get_current_iqr_session() as iqrs:
fid = flask.request.form['fid']
self._log.debug(
"[%s::%s] Getting temporary filepath from "
"uploader module", iqrs.uuid, fid)
upload_filepath = self.mod_upload.get_path_for_id(fid)
self.mod_upload.clear_completed(fid)
self._log.debug("[%s::%s] Moving uploaded file", iqrs.uuid,
fid)
sess_upload = osp.join(self._iqr_work_dirs[iqrs.uuid],
osp.basename(upload_filepath))
os.rename(upload_filepath, sess_upload)
upload_data = DataFileElement(sess_upload)
uuid = upload_data.uuid()
self._iqr_example_data[iqrs.uuid][uuid] = upload_data
# Extend session ingest -- modifying
self._log.debug(
"[%s::%s] Adding new data to session "
"positives", iqrs.uuid, fid)
# iqrs.add_positive_data(upload_data)
try:
upload_descr = \
self._descriptor_generator.compute_descriptor(
upload_data, self._descr_elem_factory
)
except ValueError, ex:
return "Input Error: %s" % str(ex), 400
self._iqr_example_pos_descr[iqrs.uuid][uuid] = upload_descr
iqrs.adjudicate((upload_descr, ))
return str(uuid)
def test_writeTempOverride_diffDir(self, mock_DataElement_wt):
"""
Test that adding ``temp_dir`` parameter triggers call to parent class
"""
source_filepath = '/path/to/file.png'
target_dir = '/some/other/dir'
d = DataFileElement(source_filepath)
# Should call parent class write_temp since target is not the same dir
# that the source file is in.
mock_DataElement_wt.return_value = 'expected'
v = d.write_temp(temp_dir=target_dir)
self.assertEqual(v, 'expected')
mock_DataElement_wt.assert_called_with(target_dir)
def test_repr(self):
e = DataFileElement('foo')
self.assertEqual(repr(e),
"DataFileElement{filepath: foo, readonly: False, "
"explicit_mimetype: None}")
e = DataFileElement('bar', readonly=True)
self.assertEqual(repr(e),
"DataFileElement{filepath: bar, readonly: True, "
"explicit_mimetype: None}")
e = DataFileElement('baz', readonly=True, explicit_mimetype='some/type')
self.assertEqual(repr(e),
"DataFileElement{filepath: baz, readonly: True, "
"explicit_mimetype: some/type}")
def test_writeTempOverride_diffDir(self, mock_DataElement_wt):
"""
Test that adding ``temp_dir`` parameter triggers call to parent class
"""
source_filepath = '/path/to/file.png'
target_dir = '/some/other/dir'
d = DataFileElement(source_filepath)
# Should call parent class write_temp since target is not the same dir
# that the source file is in.
mock_DataElement_wt.return_value = 'expected'
v = d.write_temp(temp_dir=target_dir)
self.assertEqual(v, 'expected')
mock_DataElement_wt.assert_called_with(target_dir)
def test_configuration(self):
fp = os.path.join(TEST_DATA_DIR, "Lenna.png")
default_config = DataFileElement.get_default_config()
self.assertEqual(default_config, {
'filepath': None,
'readonly': False,
'explicit_mimetype': None
})
default_config['filepath'] = fp
inst1 = DataFileElement.from_config(default_config)
self.assertEqual(default_config, inst1.get_config())
inst2 = DataFileElement.from_config(inst1.get_config())
self.assertEqual(inst1, inst2)
def main():
parser = cli_parser()
args = parser.parse_args()
config = cli.utility_main_helper(default_config, args)
log = logging.getLogger(__name__)
output_filepath = args.output_filepath
overwrite = args.overwrite
if not args.input_file:
log.error("Failed to provide an input file path")
exit(1)
elif not os.path.isfile(args.input_file):
log.error("Given path does not point to a file.")
exit(1)
input_filepath = args.input_file
data_element = DataFileElement(input_filepath)
factory = DescriptorElementFactory.from_config(config['descriptor_factory'])
#: :type: smqtk.algorithms.descriptor_generator.DescriptorGenerator
cd = from_config_dict(config['content_descriptor'],
DescriptorGenerator.get_impls())
vec = generate_vector(log, cd, data_element, factory, overwrite)
if output_filepath:
numpy.save(output_filepath, vec)
else:
# Construct string, because numpy
s = []
# noinspection PyTypeChecker
for f in vec:
s.append('%15f' % f)
print(' '.join(s))
def test_set_bytes_readonly(self):
e = DataFileElement('foo', readonly=True)
self.assertRaises(
ReadOnlyError,
e.set_bytes,
six.b('some bytes')
)
def check_image(image_path):
if not os.path.exists(image_path):
log.warn('Invalid image path given (does not exist): %s', image_path)
return (False, False)
else:
dfe = DataFileElement(image_path)
return (is_valid_element(dfe, check_image=True), dfe)
def _step(self):
# grab image container from port using traits
in_img_c = self.grab_input_using_trait('image')
# If we're in test mode, just grab the image and
# push a fake descriptor without trying to use
# smqtk.
if not apply_descriptor_test_mode:
# Get image from conatiner
in_img = in_img_c.image()
# convert generic image to PIL image
pil_image = get_pil_image(in_img)
pix = np.array(pil_image)
# get image in acceptable format
# TBD use in memory transfer
pil_image.save( "file.png" )
test_data = DataFileElement("file.png")
result = self.generator.compute_descriptor(test_data, self.factory)
desc_list = result.vector().tolist()
# push list to output port
self.push_to_port_using_trait( 'vector', desc_list )
else:
desc_list = 4096 * [0.223] # Create fake descriptor in test mode
self.push_to_port_using_trait('vector', desc_list)
self._base_step()
def test_fromConfig(self):
fp = os.path.join(TEST_DATA_DIR, "Lenna.png")
c = {
"filepath": fp
}
df = DataFileElement.from_config(c)
ntools.assert_equal(df._filepath, fp)
def iqr_ingest_file():
"""
Ingest the file with the given UID, getting the path from the
uploader.
:return: string of data/descriptor element's UUID
:rtype: str
"""
# TODO: Add status dict with a "GET" method branch for getting that
# status information.
# Start the ingest of a FID when POST
if flask.request.method == "POST":
with self.get_current_iqr_session() as iqrs:
fid = flask.request.form['fid']
self._log.debug("[%s::%s] Getting temporary filepath from "
"uploader module", iqrs.uuid, fid)
upload_filepath = self.mod_upload.get_path_for_id(fid)
self.mod_upload.clear_completed(fid)
self._log.debug("[%s::%s] Moving uploaded file",
iqrs.uuid, fid)
sess_upload = osp.join(self._iqr_work_dirs[iqrs.uuid],
osp.basename(upload_filepath))
os.rename(upload_filepath, sess_upload)
upload_data = DataFileElement(sess_upload)
uuid = upload_data.uuid()
self._iqr_example_data[iqrs.uuid][uuid] = upload_data
# Extend session ingest -- modifying
self._log.debug("[%s::%s] Adding new data to session "
"positives", iqrs.uuid, fid)
# iqrs.add_positive_data(upload_data)
try:
upload_descr = \
self._descriptor_generator.compute_descriptor(
upload_data, self._descr_elem_factory
)
except ValueError, ex:
return "Input Error: %s" % str(ex), 400
self._iqr_example_pos_descr[iqrs.uuid][uuid] = upload_descr
iqrs.adjudicate((upload_descr,))
return str(uuid)
def test_configuration(self):
fp = os.path.join(TEST_DATA_DIR, "grace_hopper.png")
inst = DataFileElement(filepath=fp, readonly=True,
explicit_mimetype='foo/bar')
for i in configuration_test_helper(inst): # type: DataFileElement
assert i._filepath == fp
assert i._readonly is True
assert i._explicit_mimetype == 'foo/bar'
def test_fromConfig(self):
fp = os.path.join(TEST_DATA_DIR, "Lenna.png")
c = {
"filepath": fp
}
#: :type: DataFileElement
df = DataFileElement.from_config(c)
self.assertEqual(df._filepath, fp)
def data_file_element_iter():
"""
Helper iterator to produce DataFileElement instances from file paths
"""
for fp in file_paths:
dfe = DataFileElement(fp)
dfe_deque.append(dfe)
yield dfe
def iqr_ingest_file():
"""
Ingest the file with the given UID, getting the path from the
uploader.
:return: string of data/descriptor element's UUID
:rtype: str
"""
# TODO: Add status dict with a "GET" method branch for getting that
# status information.
fid = flask.request.form['fid']
sid = self.get_current_iqr_session()
self._log.debug(
"[%s::%s] Getting temporary filepath from "
"uploader module", sid, fid)
upload_filepath = self.mod_upload.get_path_for_id(fid)
self.mod_upload.clear_completed(fid)
self._log.debug("[%s::%s] Moving uploaded file", sid, fid)
sess_upload = osp.join(self._iqr_work_dirs[sid],
osp.basename(upload_filepath))
os.rename(upload_filepath, sess_upload)
# Record uploaded data as user example data for this session.
upload_data = DataFileElement(sess_upload)
uuid = upload_data.uuid()
self._iqr_example_data[sid][uuid] = upload_data
# Extend session ingest -- modifying
self._log.debug(
"[%s::%s] Adding new data to session "
"external positives", sid, fid)
data_b64 = base64.b64encode(upload_data.get_bytes())
data_ct = upload_data.content_type()
r = self._iqr_service.post('add_external_pos',
sid=sid,
base64=data_b64,
content_type=data_ct)
r.raise_for_status()
return str(uuid)
def is_valid(file_path):
e = DataFileElement(file_path)
if is_valid_element(
e,
valid_content_types=generator.valid_content_types(),
check_image=check_image):
return e
else:
return False
def iqr_ingest_file():
"""
Ingest the file with the given UID, getting the path from the
uploader.
:return: string of data/descriptor element's UUID
:rtype: str
"""
# TODO: Add status dict with a "GET" method branch for getting that
# status information.
fid = flask.request.form['fid']
sid = self.get_current_iqr_session()
self._log.debug("[%s::%s] Getting temporary filepath from "
"uploader module", sid, fid)
upload_filepath = self.mod_upload.get_path_for_id(fid)
self.mod_upload.clear_completed(fid)
self._log.debug("[%s::%s] Moving uploaded file",
sid, fid)
sess_upload = osp.join(self._iqr_work_dirs[sid],
osp.basename(upload_filepath))
os.rename(upload_filepath, sess_upload)
# Record uploaded data as user example data for this session.
upload_data = DataFileElement(sess_upload)
uuid = upload_data.uuid()
self._iqr_example_data[sid][uuid] = upload_data
# Extend session ingest -- modifying
self._log.debug("[%s::%s] Adding new data to session "
"external positives", sid, fid)
data_b64 = base64.b64encode(upload_data.get_bytes())
data_ct = upload_data.content_type()
r = self._iqr_service.post('add_external_pos', sid=sid,
base64=data_b64, content_type=data_ct)
r.raise_for_status()
return str(uuid)
def test_process_load_img(self):
# using image shape, meaning no transformation should occur
test_data_layer = 'data'
test_transformer = \
caffe.io.Transformer({test_data_layer: (1, 3, 600, 512)})
hopper_elem = DataFileElement(self.hopper_image_fp, readonly=True)
a_expected = numpy.asarray(PIL.Image.open(self.hopper_image_fp),
numpy.float32)
a = _process_load_img_array(
(hopper_elem, test_transformer, test_data_layer, None, None))
numpy.testing.assert_allclose(a, a_expected)
def test_invalid_datatype(self, _m_cdg_setupNetwork):
# Test that a data element with an incorrect content type raises an
# exception.
# Passing purposefully bag constructor parameters and ignoring
# Caffe network setup (above mocking).
# noinspection PyTypeChecker
g = CaffeDescriptorGenerator(None, None, None)
bad_element = DataFileElement(os.path.join(TEST_DATA_DIR,
'test_file.dat'),
readonly=True)
self.assertRaises(ValueError, g.compute_descriptor, bad_element)
def dl_image(meta):
try:
c_type = meta['fields']['content_type'][0]
obj_stored_url = meta['fields']['obj_stored_url'][0]
obj_original_url = meta['fields']['obj_original_url'][0]
c_ext = m.guess_extension(c_type, strict=False)
if c_ext is None:
log.warn(
"Guessed 'None' extension for content-type '%s', "
"skipping.", c_type)
return None
save_dir = os.path.abspath(
os.path.expanduser(
os.path.join(output_dir, meta['index'], meta['doc_type'])))
save_file = meta['id'] + c_ext
save_path = os.path.join(save_dir, save_file)
# Save/write file if needed
if not os.path.isfile(save_path):
# First try 'stored' url, fallback on original
# Return None if failed to download anything
ok, r = try_download(obj_stored_url, stored_http_auth)
if not ok:
log.warn(
"Failed to download stored-data URL \"%s\" "
"(error=%s)", obj_stored_url, str(r))
ok, r = try_download(obj_original_url)
if not ok:
log.warn(
"Failed to download original URL \"%s\" "
"(error=%s)", obj_stored_url, str(r))
return None
# Assuming OK at this point
content = r.content
d = DataMemoryElement(content, c_type)
safe_create_dir(save_dir)
with open(save_path, 'wb') as out:
log.debug("Saving to file: '%s'", save_path)
out.write(content)
else:
d = DataFileElement(save_path)
return meta['id'], save_path, d.uuid()
except KeyError, ex:
log.error("Failed to find key %s in meta block: %s", str(ex), meta)
raise
def test_writeTempOverride_diffDir(self, mock_open, mock_os_open, mock_os_close, mock_fcntl, mock_scd, mock_isfile):
source_filepath = "/path/to/file.png"
target_dir = "/some/other/dir"
d = DataFileElement(source_filepath)
fp = d.write_temp(temp_dir=target_dir)
# Custom side-effect for os.path.isfile for simulated files
simulate = True
def osp_isfile_side_effect(path):
if simulate and path == fp:
return True
else:
return False
mock_isfile.side_effect = osp_isfile_side_effect
ntools.assert_not_equal(fp, source_filepath)
ntools.assert_equal(os.path.dirname(fp), target_dir)
# subsequent call to write temp should not invoke creation of a new file
fp2 = d.write_temp()
ntools.assert_equal(fp2, source_filepath)
# request in same dir should return same path as first request with that
# directory
fp3 = d.write_temp(target_dir)
ntools.assert_equal(fp, fp3)
# request different target dir
target2 = "/even/different/path"
fp4 = d.write_temp(target2)
ntools.assert_equal(os.path.dirname(fp4), target2)
ntools.assert_not_equal(fp, fp4)
ntools.assert_equal(len(d._temp_filepath_stack), 2)
# Restore normal os.path.isfile functionality
simulate = False
def test_get_bytes_no_file(self):
e = DataFileElement("/not/a/valid/path.txt", readonly=True)
# We currently expect, in the case where the filepath doesn't exist, to
# get the same bytes as if the file existed and were empty.
self.assertEqual(e.get_bytes(), six.b(""))
# read-only status should have no effect.
e = DataFileElement("/not/a/valid/path.txt", readonly=True)
self.assertEqual(e.get_bytes(), six.b(""))
def test_compute_descriptor_from_url_hopper_description(self):
# Caffe AlexNet interaction test (Grace Hopper image)
# This is a long test since it has to download data for remote URIs
d = CaffeDescriptorGenerator(
self.alexnet_prototxt_elem,
self.alexnet_caffemodel_elem,
self.image_mean_proto_elem,
return_layer='fc7',
use_gpu=False,
)
hopper_elem = DataFileElement(self.hopper_image_fp, readonly=True)
expected_descr = numpy.load(self.hopper_alexnet_fc7_descr_fp)
descr = d.compute_descriptor(hopper_elem).vector()
numpy.testing.assert_allclose(descr, expected_descr, atol=1e-4)
def test_writable_readonly_false(self):
e = DataFileElement('foo')
self.assertTrue(e.writable())
e = DataFileElement('foo', False)
self.assertTrue(e.writable())
e = DataFileElement('foo', readonly=False)
self.assertTrue(e.writable())
def iqr_ingest_file():
"""
Ingest the file with the given UID, getting the path from the
uploader.
:return: string of data/descriptor element's UUID
:rtype: str
"""
# TODO: Add status dict with a "GET" method branch for getting that
# status information.
# Start the ingest of a FID when POST
if flask.request.method == "POST":
iqr_sess = self.get_current_iqr_session()
fid = flask.request.form['fid']
self.log.debug("[%s::%s] Getting temporary filepath from "
"uploader module", iqr_sess.uuid, fid)
upload_filepath = self.mod_upload.get_path_for_id(fid)
self.mod_upload.clear_completed(fid)
self.log.debug("[%s::%s] Moving uploaded file",
iqr_sess.uuid, fid)
sess_upload = osp.join(iqr_sess.work_dir,
osp.basename(upload_filepath))
os.rename(upload_filepath, sess_upload)
upload_data = DataFileElement(sess_upload)
upload_data.uuid()
# Extend session ingest -- modifying
self.log.debug("[%s::%s] Adding new data to session positives",
iqr_sess.uuid, fid)
iqr_sess.add_positive_data(upload_data)
return str(upload_data.uuid())
def test_get_bytes(self):
# Test with a known real file.
test_file_path = os.path.join(TEST_DATA_DIR, 'text_file')
e = DataFileElement(test_file_path)
self.assertEqual(e.get_bytes(), six.b("Some text content.\n"))
def test_writable_readonly_true(self):
e = DataFileElement('foo', True)
self.assertFalse(e.writable())
e = DataFileElement('foo', readonly=True)
self.assertFalse(e.writable())
def test_is_empty_file_not_exists(self):
e = DataFileElement('/no/exists')
self.assertTrue(e.is_empty())
def test_is_empty_file_zero_data(self):
e = DataFileElement(os.path.join(TEST_DATA_DIR, 'test_file.dat'))
self.assertTrue(e.is_empty())
def test_toConfig(self):
fp = os.path.join(TEST_DATA_DIR, "Lenna.png")
df = DataFileElement(fp)
c = df.get_config()
self.assertEqual(c['filepath'], fp)
def test_toConfig(self):
fp = os.path.join(TEST_DATA_DIR, "Lenna.png")
df = DataFileElement(fp)
c = df.get_config()
ntools.assert_equal(c["filepath"], fp)
def test_content_type_explicit_type(self):
ex_type = 'image/png'
d = DataFileElement('foo.txt', explicit_mimetype=ex_type)
self.assertEqual(d.content_type(), ex_type)
def test_content_type(self):
d = DataFileElement("foo.txt")
ntools.assert_equal(d.content_type(), "text/plain")
def test_is_empty_file_has_data(self):
e = DataFileElement(os.path.join(TEST_DATA_DIR, 'Lenna.png'))
self.assertFalse(e.is_empty())
def test_content_type(self):
d = DataFileElement('foo.txt')
self.assertEqual(d.content_type(), 'text/plain')
