def test_load_dataset_tempfile(self): """ Test DataElement temporary file based context loader. """ # Creating separate element from global so we can mock it up. e = DataFileElement(GH_IMAGE_FP, readonly=True) e.write_temp = mock.MagicMock(wraps=e.write_temp) e.clean_temp = mock.MagicMock(wraps=e.clean_temp) e.get_bytes = mock.MagicMock(wraps=e.get_bytes) # Using explicit patcher start/stop in order to avoid using ``patch`` # as a decorator because ``osgeo`` might not be defined when # decorating the method. patcher_gdal_open = mock.patch('smqtk.algorithms.image_io.gdal_io.gdal' '.Open', wraps=osgeo.gdal.Open) self.addCleanup(patcher_gdal_open.stop) m_gdal_open = patcher_gdal_open.start() with load_dataset_tempfile(e) as gdal_ds: # noinspection PyUnresolvedReferences e.write_temp.assert_called_once_with() # noinspection PyUnresolvedReferences e.get_bytes.assert_not_called() m_gdal_open.assert_called_once() assert gdal_ds.RasterCount == 3 assert gdal_ds.RasterXSize == 512 assert gdal_ds.RasterYSize == 600 # noinspection PyUnresolvedReferences e.clean_temp.assert_called_once_with() assert len(e._temp_filepath_stack) == 0
def test_writeTempOverride_diffDir(self, mock_open, mock_os_open, mock_os_close, mock_fcntl, mock_scd): source_filepath = '/path/to/file.png' target_dir = '/some/other/dir' d = DataFileElement(source_filepath) fp = d.write_temp(temp_dir=target_dir) ntools.assert_not_equal(fp, source_filepath) ntools.assert_equal(os.path.dirname(fp), target_dir) # subsequent call to write temp should not invoke creation of a new file fp2 = d.write_temp() ntools.assert_equal(fp2, source_filepath) # request in same dir should return same path as first request with that # directory fp3 = d.write_temp(target_dir) ntools.assert_equal(fp, fp3) # request different target dir target2 = '/even/different/path' fp4 = d.write_temp(target2) ntools.assert_equal(os.path.dirname(fp4), target2) ntools.assert_not_equal(fp, fp4) ntools.assert_equal(len(d._temp_filepath_stack), 2)
def test_get_bytes_no_file(self): e = DataFileElement("/not/a/valid/path.txt", readonly=True) # We currently expect, in the case where the filepath doesn't exist, to # get the same bytes as if the file existed and were empty. self.assertEqual(e.get_bytes(), six.b("")) # read-only status should have no effect. e = DataFileElement("/not/a/valid/path.txt", readonly=True) self.assertEqual(e.get_bytes(), six.b(""))
def test_writeTempOverride(self, mock_DataElement_wt): # no manual directory, should return the base filepath expected_filepath = '/path/to/file.txt' d = DataFileElement(expected_filepath) fp = d.write_temp() self.assertFalse(mock_DataElement_wt.called) self.assertEqual(expected_filepath, fp)
def test_writeTempOverride_sameDir(self, mock_DataElement_wt): expected_filepath = '/path/to/file.txt' target_dir = '/path/to' d = DataFileElement(expected_filepath) fp = d.write_temp(temp_dir=target_dir) self.assertFalse(mock_DataElement_wt.called) self.assertEqual(fp, expected_filepath)
def test_writeTempOverride_sameDir(self, mock_DataElement_wt): expected_filepath = "/path/to/file.txt" target_dir = "/path/to" d = DataFileElement(expected_filepath) fp = d.write_temp(temp_dir=target_dir) ntools.assert_false(mock_DataElement_wt.called) ntools.assert_equal(fp, expected_filepath)
def iter_input_elements(): for f in args.input_files: f = osp.expanduser(f) if osp.isfile(f): yield DataFileElement(f) else: log.debug("Expanding glob: %s" % f) for g in glob.glob(f): yield DataFileElement(g)
def test_cleanTemp(self): # a write temp and clean temp should not affect original file source_file = os.path.join(TEST_DATA_DIR, 'test_file.dat') self.assertTrue(os.path.isfile(source_file)) d = DataFileElement(source_file) d.write_temp() self.assertEqual(len(d._temp_filepath_stack), 0) d.clean_temp() self.assertTrue(os.path.isfile(source_file))
def iter_valid_elements(): for fp in file_paths: dfe = DataFileElement(fp) ct = dfe.content_type() if ct in generator.valid_content_types(): valid_file_paths[fp] = ct yield dfe else: invalid_file_paths[fp] = ct
def test_set_bytes_writable(self, m_sfw): # Using a relative filepath test_path = 'foo' test_bytes = six.b('test string of bytes') e = DataFileElement(test_path) e.set_bytes(test_bytes) # File write function should be called m_sfw.assert_called_once_with(test_path, test_bytes)
def setUpClass(cls): cls.gh_image_fp = os.path.join(TEST_DATA_DIR, "grace_hopper.png") cls.gh_file_element = DataFileElement(cls.gh_image_fp) assert cls.gh_file_element.content_type() == 'image/png' cls.gh_cropped_image_fp = \ os.path.join(TEST_DATA_DIR, 'grace_hopper.100x100+100+100.png') cls.gh_cropped_file_element = DataFileElement(cls.gh_cropped_image_fp) assert cls.gh_cropped_file_element.content_type() == 'image/png' cls.gh_cropped_bbox = AxisAlignedBoundingBox([100, 100], [200, 200])
def test_configuration(self): fp = os.path.join(TEST_DATA_DIR, "Lenna.png") default_config = DataFileElement.get_default_config() ntools.assert_equal(default_config, {'filepath': None}) default_config['filepath'] = fp inst1 = DataFileElement.from_config(default_config) ntools.assert_equal(default_config, inst1.get_config()) inst2 = DataFileElement.from_config(inst1.get_config()) ntools.assert_equal(inst1, inst2)
def test_configuration(self): fp = os.path.join(TEST_DATA_DIR, "Lenna.png") default_config = DataFileElement.get_default_config() ntools.assert_equal(default_config, {"filepath": None}) default_config["filepath"] = fp inst1 = DataFileElement.from_config(default_config) ntools.assert_equal(default_config, inst1.get_config()) inst2 = DataFileElement.from_config(inst1.get_config()) ntools.assert_equal(inst1, inst2)
def test_writable_readonly_false(self): e = DataFileElement('foo') self.assertTrue(e.writable()) e = DataFileElement('foo', False) self.assertTrue(e.writable()) e = DataFileElement('foo', readonly=False) self.assertTrue(e.writable())
def test_configuration(self): fp = os.path.join(TEST_DATA_DIR, "Lenna.png") default_config = DataFileElement.get_default_config() self.assertEqual(default_config, {'filepath': None, 'readonly': False, 'explicit_mimetype': None}) default_config['filepath'] = fp inst1 = DataFileElement.from_config(default_config) self.assertEqual(default_config, inst1.get_config()) inst2 = DataFileElement.from_config(inst1.get_config()) self.assertEqual(inst1, inst2)
def is_valid_element(fp): dfe = DataFileElement(fp) ct = dfe.content_type() if ct in generator.valid_content_types(): if not check_image or test_image_load(dfe): return dfe else: return None else: log.debug("Skipping file (invalid content) type for " "descriptor generator (fp='%s', ct=%s)", fp, ct) return None
def is_valid_element(fp): dfe = DataFileElement(fp) ct = dfe.content_type() if ct in generator.valid_content_types(): if not check_image or test_image_load(dfe): return dfe else: return None else: log.debug( "Skipping file (invalid content) type for " "descriptor generator (fp='%s', ct=%s)", str(fp), ct) return None
def test_load_dataset_vsimem(self): """ Test that VSIMEM loading context """ if LooseVersion(osgeo.__version__).version[0] < 2: pytest.skip("Skipping VSIMEM test because GDAL version < 2") # Creating separate element from global so we can mock it up. e = DataFileElement(GH_IMAGE_FP, readonly=True) e.write_temp = mock.MagicMock(wraps=e.write_temp) e.clean_temp = mock.MagicMock(wraps=e.clean_temp) e.get_bytes = mock.MagicMock(wraps=e.get_bytes) vsimem_path_re = re.compile(r'^/vsimem/\w+$') # Using explicit patcher start/stop in order to avoid using ``patch`` # as a *decorator* because ``osgeo`` might not be defined when # decorating the method. patcher_gdal_open = mock.patch( 'smqtk.algorithms.image_io.gdal_io.gdal.Open', wraps=osgeo.gdal.Open, ) self.addCleanup(patcher_gdal_open.stop) patcher_gdal_unlink = mock.patch( 'smqtk.algorithms.image_io.gdal_io.gdal.Unlink', wraps=osgeo.gdal.Unlink, ) self.addCleanup(patcher_gdal_unlink.stop) m_gdal_open = patcher_gdal_open.start() m_gdal_unlink = patcher_gdal_unlink.start() with load_dataset_vsimem(e) as gdal_ds: # noinspection PyUnresolvedReferences e.write_temp.assert_not_called() # noinspection PyUnresolvedReferences e.get_bytes.assert_called_once_with() m_gdal_open.assert_called_once() ds_path = gdal_ds.GetDescription() assert vsimem_path_re.match(ds_path) assert gdal_ds.RasterCount == 3 assert gdal_ds.RasterXSize == 512 assert gdal_ds.RasterYSize == 600 m_gdal_unlink.assert_called_once_with(ds_path) # noinspection PyUnresolvedReferences e.clean_temp.assert_not_called() assert len(e._temp_filepath_stack) == 0
def test_from_uri(self): # will be absolute path test_file_path = os.path.join(TEST_DATA_DIR, "test_file.dat") print("Test file path:", test_file_path) e = DataFileElement.from_uri(test_file_path) ntools.assert_is_instance(e, DataFileElement) ntools.assert_equal(e._filepath, test_file_path) ntools.assert_equal(e.get_bytes(), '') e = DataFileElement.from_uri('file://' + test_file_path) ntools.assert_is_instance(e, DataFileElement) ntools.assert_equal(e._filepath, test_file_path) ntools.assert_equal(e.get_bytes(), '')
def test_from_uri(self): # will be absolute path test_file_path = os.path.join(TEST_DATA_DIR, "test_file.dat") print("Test file path:", test_file_path) e = DataFileElement.from_uri(test_file_path) self.assertIsInstance(e, DataFileElement) self.assertEqual(e._filepath, test_file_path) self.assertEqual(e.get_bytes(), six.b('')) e = DataFileElement.from_uri('file://' + test_file_path) self.assertIsInstance(e, DataFileElement) self.assertEqual(e._filepath, test_file_path) self.assertEqual(e.get_bytes(), six.b(''))
def iqr_ingest_file(): """ Ingest the file with the given UID, getting the path from the uploader. :return: string of data/descriptor element's UUID :rtype: str """ # TODO: Add status dict with a "GET" method branch for getting that # status information. # Start the ingest of a FID when POST if flask.request.method == "POST": with self.get_current_iqr_session() as iqrs: fid = flask.request.form['fid'] self._log.debug( "[%s::%s] Getting temporary filepath from " "uploader module", iqrs.uuid, fid) upload_filepath = self.mod_upload.get_path_for_id(fid) self.mod_upload.clear_completed(fid) self._log.debug("[%s::%s] Moving uploaded file", iqrs.uuid, fid) sess_upload = osp.join(self._iqr_work_dirs[iqrs.uuid], osp.basename(upload_filepath)) os.rename(upload_filepath, sess_upload) upload_data = DataFileElement(sess_upload) uuid = upload_data.uuid() self._iqr_example_data[iqrs.uuid][uuid] = upload_data # Extend session ingest -- modifying self._log.debug( "[%s::%s] Adding new data to session " "positives", iqrs.uuid, fid) # iqrs.add_positive_data(upload_data) try: upload_descr = \ self._descriptor_generator.compute_descriptor( upload_data, self._descr_elem_factory ) except ValueError, ex: return "Input Error: %s" % str(ex), 400 self._iqr_example_pos_descr[iqrs.uuid][uuid] = upload_descr iqrs.adjudicate((upload_descr, )) return str(uuid)
def test_writeTempOverride_diffDir(self, mock_DataElement_wt): """ Test that adding ``temp_dir`` parameter triggers call to parent class """ source_filepath = '/path/to/file.png' target_dir = '/some/other/dir' d = DataFileElement(source_filepath) # Should call parent class write_temp since target is not the same dir # that the source file is in. mock_DataElement_wt.return_value = 'expected' v = d.write_temp(temp_dir=target_dir) self.assertEqual(v, 'expected') mock_DataElement_wt.assert_called_with(target_dir)
def test_repr(self): e = DataFileElement('foo') self.assertEqual(repr(e), "DataFileElement{filepath: foo, readonly: False, " "explicit_mimetype: None}") e = DataFileElement('bar', readonly=True) self.assertEqual(repr(e), "DataFileElement{filepath: bar, readonly: True, " "explicit_mimetype: None}") e = DataFileElement('baz', readonly=True, explicit_mimetype='some/type') self.assertEqual(repr(e), "DataFileElement{filepath: baz, readonly: True, " "explicit_mimetype: some/type}")
def test_configuration(self): fp = os.path.join(TEST_DATA_DIR, "Lenna.png") default_config = DataFileElement.get_default_config() self.assertEqual(default_config, { 'filepath': None, 'readonly': False, 'explicit_mimetype': None }) default_config['filepath'] = fp inst1 = DataFileElement.from_config(default_config) self.assertEqual(default_config, inst1.get_config()) inst2 = DataFileElement.from_config(inst1.get_config()) self.assertEqual(inst1, inst2)
def main(): parser = cli_parser() args = parser.parse_args() config = cli.utility_main_helper(default_config, args) log = logging.getLogger(__name__) output_filepath = args.output_filepath overwrite = args.overwrite if not args.input_file: log.error("Failed to provide an input file path") exit(1) elif not os.path.isfile(args.input_file): log.error("Given path does not point to a file.") exit(1) input_filepath = args.input_file data_element = DataFileElement(input_filepath) factory = DescriptorElementFactory.from_config(config['descriptor_factory']) #: :type: smqtk.algorithms.descriptor_generator.DescriptorGenerator cd = from_config_dict(config['content_descriptor'], DescriptorGenerator.get_impls()) vec = generate_vector(log, cd, data_element, factory, overwrite) if output_filepath: numpy.save(output_filepath, vec) else: # Construct string, because numpy s = [] # noinspection PyTypeChecker for f in vec: s.append('%15f' % f) print(' '.join(s))
def test_set_bytes_readonly(self): e = DataFileElement('foo', readonly=True) self.assertRaises( ReadOnlyError, e.set_bytes, six.b('some bytes') )
def check_image(image_path): if not os.path.exists(image_path): log.warn('Invalid image path given (does not exist): %s', image_path) return (False, False) else: dfe = DataFileElement(image_path) return (is_valid_element(dfe, check_image=True), dfe)
def _step(self): # grab image container from port using traits in_img_c = self.grab_input_using_trait('image') # If we're in test mode, just grab the image and # push a fake descriptor without trying to use # smqtk. if not apply_descriptor_test_mode: # Get image from conatiner in_img = in_img_c.image() # convert generic image to PIL image pil_image = get_pil_image(in_img) pix = np.array(pil_image) # get image in acceptable format # TBD use in memory transfer pil_image.save( "file.png" ) test_data = DataFileElement("file.png") result = self.generator.compute_descriptor(test_data, self.factory) desc_list = result.vector().tolist() # push list to output port self.push_to_port_using_trait( 'vector', desc_list ) else: desc_list = 4096 * [0.223] # Create fake descriptor in test mode self.push_to_port_using_trait('vector', desc_list) self._base_step()
def test_fromConfig(self): fp = os.path.join(TEST_DATA_DIR, "Lenna.png") c = { "filepath": fp } df = DataFileElement.from_config(c) ntools.assert_equal(df._filepath, fp)
def iqr_ingest_file(): """ Ingest the file with the given UID, getting the path from the uploader. :return: string of data/descriptor element's UUID :rtype: str """ # TODO: Add status dict with a "GET" method branch for getting that # status information. # Start the ingest of a FID when POST if flask.request.method == "POST": with self.get_current_iqr_session() as iqrs: fid = flask.request.form['fid'] self._log.debug("[%s::%s] Getting temporary filepath from " "uploader module", iqrs.uuid, fid) upload_filepath = self.mod_upload.get_path_for_id(fid) self.mod_upload.clear_completed(fid) self._log.debug("[%s::%s] Moving uploaded file", iqrs.uuid, fid) sess_upload = osp.join(self._iqr_work_dirs[iqrs.uuid], osp.basename(upload_filepath)) os.rename(upload_filepath, sess_upload) upload_data = DataFileElement(sess_upload) uuid = upload_data.uuid() self._iqr_example_data[iqrs.uuid][uuid] = upload_data # Extend session ingest -- modifying self._log.debug("[%s::%s] Adding new data to session " "positives", iqrs.uuid, fid) # iqrs.add_positive_data(upload_data) try: upload_descr = \ self._descriptor_generator.compute_descriptor( upload_data, self._descr_elem_factory ) except ValueError, ex: return "Input Error: %s" % str(ex), 400 self._iqr_example_pos_descr[iqrs.uuid][uuid] = upload_descr iqrs.adjudicate((upload_descr,)) return str(uuid)
def test_configuration(self): fp = os.path.join(TEST_DATA_DIR, "grace_hopper.png") inst = DataFileElement(filepath=fp, readonly=True, explicit_mimetype='foo/bar') for i in configuration_test_helper(inst): # type: DataFileElement assert i._filepath == fp assert i._readonly is True assert i._explicit_mimetype == 'foo/bar'
def test_fromConfig(self): fp = os.path.join(TEST_DATA_DIR, "Lenna.png") c = { "filepath": fp } #: :type: DataFileElement df = DataFileElement.from_config(c) self.assertEqual(df._filepath, fp)
def data_file_element_iter(): """ Helper iterator to produce DataFileElement instances from file paths """ for fp in file_paths: dfe = DataFileElement(fp) dfe_deque.append(dfe) yield dfe
def iqr_ingest_file(): """ Ingest the file with the given UID, getting the path from the uploader. :return: string of data/descriptor element's UUID :rtype: str """ # TODO: Add status dict with a "GET" method branch for getting that # status information. fid = flask.request.form['fid'] sid = self.get_current_iqr_session() self._log.debug( "[%s::%s] Getting temporary filepath from " "uploader module", sid, fid) upload_filepath = self.mod_upload.get_path_for_id(fid) self.mod_upload.clear_completed(fid) self._log.debug("[%s::%s] Moving uploaded file", sid, fid) sess_upload = osp.join(self._iqr_work_dirs[sid], osp.basename(upload_filepath)) os.rename(upload_filepath, sess_upload) # Record uploaded data as user example data for this session. upload_data = DataFileElement(sess_upload) uuid = upload_data.uuid() self._iqr_example_data[sid][uuid] = upload_data # Extend session ingest -- modifying self._log.debug( "[%s::%s] Adding new data to session " "external positives", sid, fid) data_b64 = base64.b64encode(upload_data.get_bytes()) data_ct = upload_data.content_type() r = self._iqr_service.post('add_external_pos', sid=sid, base64=data_b64, content_type=data_ct) r.raise_for_status() return str(uuid)
def is_valid(file_path): e = DataFileElement(file_path) if is_valid_element( e, valid_content_types=generator.valid_content_types(), check_image=check_image): return e else: return False
def iqr_ingest_file(): """ Ingest the file with the given UID, getting the path from the uploader. :return: string of data/descriptor element's UUID :rtype: str """ # TODO: Add status dict with a "GET" method branch for getting that # status information. fid = flask.request.form['fid'] sid = self.get_current_iqr_session() self._log.debug("[%s::%s] Getting temporary filepath from " "uploader module", sid, fid) upload_filepath = self.mod_upload.get_path_for_id(fid) self.mod_upload.clear_completed(fid) self._log.debug("[%s::%s] Moving uploaded file", sid, fid) sess_upload = osp.join(self._iqr_work_dirs[sid], osp.basename(upload_filepath)) os.rename(upload_filepath, sess_upload) # Record uploaded data as user example data for this session. upload_data = DataFileElement(sess_upload) uuid = upload_data.uuid() self._iqr_example_data[sid][uuid] = upload_data # Extend session ingest -- modifying self._log.debug("[%s::%s] Adding new data to session " "external positives", sid, fid) data_b64 = base64.b64encode(upload_data.get_bytes()) data_ct = upload_data.content_type() r = self._iqr_service.post('add_external_pos', sid=sid, base64=data_b64, content_type=data_ct) r.raise_for_status() return str(uuid)
def test_process_load_img(self): # using image shape, meaning no transformation should occur test_data_layer = 'data' test_transformer = \ caffe.io.Transformer({test_data_layer: (1, 3, 600, 512)}) hopper_elem = DataFileElement(self.hopper_image_fp, readonly=True) a_expected = numpy.asarray(PIL.Image.open(self.hopper_image_fp), numpy.float32) a = _process_load_img_array( (hopper_elem, test_transformer, test_data_layer, None, None)) numpy.testing.assert_allclose(a, a_expected)
def test_invalid_datatype(self, _m_cdg_setupNetwork): # Test that a data element with an incorrect content type raises an # exception. # Passing purposefully bag constructor parameters and ignoring # Caffe network setup (above mocking). # noinspection PyTypeChecker g = CaffeDescriptorGenerator(None, None, None) bad_element = DataFileElement(os.path.join(TEST_DATA_DIR, 'test_file.dat'), readonly=True) self.assertRaises(ValueError, g.compute_descriptor, bad_element)
def dl_image(meta): try: c_type = meta['fields']['content_type'][0] obj_stored_url = meta['fields']['obj_stored_url'][0] obj_original_url = meta['fields']['obj_original_url'][0] c_ext = m.guess_extension(c_type, strict=False) if c_ext is None: log.warn( "Guessed 'None' extension for content-type '%s', " "skipping.", c_type) return None save_dir = os.path.abspath( os.path.expanduser( os.path.join(output_dir, meta['index'], meta['doc_type']))) save_file = meta['id'] + c_ext save_path = os.path.join(save_dir, save_file) # Save/write file if needed if not os.path.isfile(save_path): # First try 'stored' url, fallback on original # Return None if failed to download anything ok, r = try_download(obj_stored_url, stored_http_auth) if not ok: log.warn( "Failed to download stored-data URL \"%s\" " "(error=%s)", obj_stored_url, str(r)) ok, r = try_download(obj_original_url) if not ok: log.warn( "Failed to download original URL \"%s\" " "(error=%s)", obj_stored_url, str(r)) return None # Assuming OK at this point content = r.content d = DataMemoryElement(content, c_type) safe_create_dir(save_dir) with open(save_path, 'wb') as out: log.debug("Saving to file: '%s'", save_path) out.write(content) else: d = DataFileElement(save_path) return meta['id'], save_path, d.uuid() except KeyError, ex: log.error("Failed to find key %s in meta block: %s", str(ex), meta) raise
def test_writeTempOverride_diffDir(self, mock_open, mock_os_open, mock_os_close, mock_fcntl, mock_scd, mock_isfile): source_filepath = "/path/to/file.png" target_dir = "/some/other/dir" d = DataFileElement(source_filepath) fp = d.write_temp(temp_dir=target_dir) # Custom side-effect for os.path.isfile for simulated files simulate = True def osp_isfile_side_effect(path): if simulate and path == fp: return True else: return False mock_isfile.side_effect = osp_isfile_side_effect ntools.assert_not_equal(fp, source_filepath) ntools.assert_equal(os.path.dirname(fp), target_dir) # subsequent call to write temp should not invoke creation of a new file fp2 = d.write_temp() ntools.assert_equal(fp2, source_filepath) # request in same dir should return same path as first request with that # directory fp3 = d.write_temp(target_dir) ntools.assert_equal(fp, fp3) # request different target dir target2 = "/even/different/path" fp4 = d.write_temp(target2) ntools.assert_equal(os.path.dirname(fp4), target2) ntools.assert_not_equal(fp, fp4) ntools.assert_equal(len(d._temp_filepath_stack), 2) # Restore normal os.path.isfile functionality simulate = False
def test_compute_descriptor_from_url_hopper_description(self): # Caffe AlexNet interaction test (Grace Hopper image) # This is a long test since it has to download data for remote URIs d = CaffeDescriptorGenerator( self.alexnet_prototxt_elem, self.alexnet_caffemodel_elem, self.image_mean_proto_elem, return_layer='fc7', use_gpu=False, ) hopper_elem = DataFileElement(self.hopper_image_fp, readonly=True) expected_descr = numpy.load(self.hopper_alexnet_fc7_descr_fp) descr = d.compute_descriptor(hopper_elem).vector() numpy.testing.assert_allclose(descr, expected_descr, atol=1e-4)
def iqr_ingest_file(): """ Ingest the file with the given UID, getting the path from the uploader. :return: string of data/descriptor element's UUID :rtype: str """ # TODO: Add status dict with a "GET" method branch for getting that # status information. # Start the ingest of a FID when POST if flask.request.method == "POST": iqr_sess = self.get_current_iqr_session() fid = flask.request.form['fid'] self.log.debug("[%s::%s] Getting temporary filepath from " "uploader module", iqr_sess.uuid, fid) upload_filepath = self.mod_upload.get_path_for_id(fid) self.mod_upload.clear_completed(fid) self.log.debug("[%s::%s] Moving uploaded file", iqr_sess.uuid, fid) sess_upload = osp.join(iqr_sess.work_dir, osp.basename(upload_filepath)) os.rename(upload_filepath, sess_upload) upload_data = DataFileElement(sess_upload) upload_data.uuid() # Extend session ingest -- modifying self.log.debug("[%s::%s] Adding new data to session positives", iqr_sess.uuid, fid) iqr_sess.add_positive_data(upload_data) return str(upload_data.uuid())
def test_get_bytes(self): # Test with a known real file. test_file_path = os.path.join(TEST_DATA_DIR, 'text_file') e = DataFileElement(test_file_path) self.assertEqual(e.get_bytes(), six.b("Some text content.\n"))
def test_writable_readonly_true(self): e = DataFileElement('foo', True) self.assertFalse(e.writable()) e = DataFileElement('foo', readonly=True) self.assertFalse(e.writable())
def test_is_empty_file_not_exists(self): e = DataFileElement('/no/exists') self.assertTrue(e.is_empty())
def test_is_empty_file_zero_data(self): e = DataFileElement(os.path.join(TEST_DATA_DIR, 'test_file.dat')) self.assertTrue(e.is_empty())
def test_toConfig(self): fp = os.path.join(TEST_DATA_DIR, "Lenna.png") df = DataFileElement(fp) c = df.get_config() self.assertEqual(c['filepath'], fp)
def test_toConfig(self): fp = os.path.join(TEST_DATA_DIR, "Lenna.png") df = DataFileElement(fp) c = df.get_config() ntools.assert_equal(c["filepath"], fp)
def test_content_type_explicit_type(self): ex_type = 'image/png' d = DataFileElement('foo.txt', explicit_mimetype=ex_type) self.assertEqual(d.content_type(), ex_type)
def test_content_type(self): d = DataFileElement("foo.txt") ntools.assert_equal(d.content_type(), "text/plain")
def test_is_empty_file_has_data(self): e = DataFileElement(os.path.join(TEST_DATA_DIR, 'Lenna.png')) self.assertFalse(e.is_empty())
def test_content_type(self): d = DataFileElement('foo.txt') self.assertEqual(d.content_type(), 'text/plain')