def test_open_yield(self): data_url = 'https://raw.github.com/BCCVL/BCCVL_Visualiser/master/bccvl_visualiser/tests/fixtures/hello_world.r' MyDataMover = FDataMover.get_data_mover_class() self.assertEqual(MyDataMover, LocalDataMover) expected_content = r"cat('Hello, world!\n')" content = None with MyDataMover.open(data_url=data_url) as f: content = f.read() self.assertEqual(content.strip(), expected_content.strip())
def test_open_yield(self): data_url = "https://raw.github.com/BCCVL/BCCVL_Visualiser/master/BCCVL_Visualiser/bccvl_visualiser/tests/fixtures/hello_world.r" MyDataMover = FDataMover.get_data_mover_class() self.assertEqual(MyDataMover, LocalDataMover) expected_content = r"cat('Hello, world!\n')" content = None with MyDataMover.open(data_url=data_url) as f: content = f.read() self.assertEqual(content.strip(), expected_content.strip())
def _download_data_to_file(self): log = logging.getLogger(__name__) mover = FDataMover.new_data_mover(self.data_file_path, data_url = self.data_url) # Make sure only one thread is trying to check for # the existance of, or trying to write the file # at any time. with self.__class__.MAPSCRIPT_RLOCK: mover.move_and_wait_for_completion() valid, problems = self._validate_file() if not valid: log.info("Deleting invalid file: %s", self.data_file_path) # Delete the file os.remove(self.data_file_path) raise ValueError("Problem validating file. Problems: %s" % (problems))
def _download_data_to_file(self): log = logging.getLogger(__name__) mover = FDataMover.new_data_mover(self.data_file_path, data_url = self.data_url) # Make sure only one thread is trying to check for # the existance of, or trying to write the file # at any time. try: mover.move_and_wait_for_completion() valid, problems = self._validate_file() except: # Invalid file if data move does not complete the move valid = False # Delete invalid file if not valid: log.info("Deleting invalid file: %s", self.data_file_path) os.remove(self.data_file_path) raise ValueError("Problem validating file. Problems: %s" % (problems))
def test_move_file_successfully_status(self): tmp = tempfile.gettempdir() tmp_file_path = os.path.join(tmp, "hello_world.html") url = "https://raw.github.com/BCCVL/BCCVL_Visualiser/master/BCCVL_Visualiser/bccvl_visualiser/tests/fixtures/hello_world.html" # Delete the file (if it exists) if os.path.exists(tmp_file_path): os.remove(tmp_file_path) # Create a DataMover object to move the file to the dest file path mover = FDataMover.new_data_mover(tmp_file_path, data_url=url) move_output = mover.move_file() move_job_id = move_output["id"] self.assertTrue(isinstance(move_job_id, int)) self.assertEqual(move_job_id, mover.job_id) move_status = mover.get_status() self.assertEqual(move_status["status"], "COMPLETED") self.assertEqual(move_status["id"], move_job_id)
def _download_data_to_file(self): log = logging.getLogger(__name__) mover = FDataMover.new_data_mover(self.data_file_path, data_url=self.data_url) # Make sure only one thread is trying to check for # the existance of, or trying to write the file # at any time. try: mover.move_and_wait_for_completion() valid, problems = self._validate_file() except: # Invalid file if data move does not complete the move valid = False # Delete invalid file if not valid: log.info("Deleting invalid file: %s", self.data_file_path) os.remove(self.data_file_path) raise ValueError("Problem validating file. Problems: %s" % (problems))
def test_move_file_successfully_status(self): tmp = tempfile.gettempdir() tmp_file_path = os.path.join(tmp, 'hello_world.html') url = 'https://raw.github.com/BCCVL/BCCVL_Visualiser/master/bccvl_visualiser/tests/fixtures/hello_world.html' # Delete the file (if it exists) if os.path.exists(tmp_file_path): os.remove(tmp_file_path) # Create a DataMover object to move the file to the dest file path mover = FDataMover.new_data_mover(tmp_file_path, data_url=url) move_output = mover.move_file() move_job_id = move_output['id'] self.assertTrue(isinstance(move_job_id, int)) self.assertEqual(move_job_id, mover.job_id) move_status = mover.get_status() self.assertEqual(move_status['status'], 'COMPLETED') self.assertEqual(move_status['id'], move_job_id)
def test_new_data_mover_from_data_id(self): # This isn't implemented yet with self.assertRaises(NotImplementedError): my_map = FDataMover.new_data_mover("tmp/a.csv", data_id="908h08h")
def test_new_data_mover_raises_on_bad_args(self): with self.assertRaises(ValueError): FDataMover.new_data_mover("/tmp/a.csv") with self.assertRaises(ValueError): FDataMover.new_data_mover("/tmp/a.csv", data_id="111", data_url="http://example.com")
def test_new_data_mover_from_data_id(self): # This isn't implemented yet with self.assertRaises(NotImplementedError): my_map = FDataMover.new_data_mover('tmp/a.csv', data_id="908h08h")
def test_new_data_mover_raises_on_bad_args(self): with self.assertRaises(ValueError): FDataMover.new_data_mover('/tmp/a.csv') with self.assertRaises(ValueError): FDataMover.new_data_mover('/tmp/a.csv', data_id="111", data_url="http://example.com")