def test_get_remote_shake_map(self): """Check that we can retrieve both input and output from ftp at once. """ shake_data = ShakeData(SHAKE_ID) expected_input_file = os.path.join( shakemap_zip_dir(), SHAKE_ID + '.inp.zip') expected_output_file = os.path.join( shakemap_zip_dir(), SHAKE_ID + '.out.zip') if os.path.exists(expected_input_file): os.remove(expected_input_file) if os.path.exists(expected_output_file): os.remove(expected_output_file) input_file, output_file = shake_data.fetch_event() message = ('Expected path for downloaded shakemap INP not received' '\nExpected: %s\nGot: %s' % (expected_output_file, output_file)) self.assertEqual(input_file, expected_input_file, message) message = ('Expected path for downloaded shakemap OUT not received' '\nExpected: %s\nGot: %s' % (expected_output_file, output_file)) self.assertEqual(output_file, expected_output_file, message) self.assertTrue(os.path.exists(expected_input_file)) self.assertTrue(os.path.exists(expected_output_file))
def setUp(self): """Copy our cached dataset from the fixture dir to the cache dir.""" # Run monkey patching to ftp_client run_monkey_patching_ftp_client() output_file = '20120726022003.out.zip' input_file = '20120726022003.inp.zip' output_path = os.path.abspath( os.path.join( os.path.dirname(__file__), '../fixtures/shake_data', output_file)) input_path = os.path.abspath( os.path.join( os.path.dirname(__file__), '../fixtures/shake_data', input_file)) shutil.copyfile( output_path, os.path.join(shakemap_zip_dir(), output_file)) shutil.copyfile( input_path, os.path.join(shakemap_zip_dir(), input_file))
def test_get_remote_shake_map(self): """Check that we can retrieve both input and output from ftp at once. """ shake_data = ShakeData(SHAKE_ID) expected_input_file = os.path.join( shakemap_zip_dir(), SHAKE_ID + '.inp.zip') expected_output_file = os.path.join( shakemap_zip_dir(), SHAKE_ID + '.out.zip') if os.path.exists(expected_input_file): os.remove(expected_input_file) if os.path.exists(expected_output_file): os.remove(expected_output_file) input_file, output_file = shake_data.fetch_event() message = ('Expected path for downloaded shakemap INP not received' '\nExpected: %s\nGot: %s' % (expected_output_file, output_file)) self.assertEqual(input_file, expected_input_file, message) message = ('Expected path for downloaded shakemap OUT not received' '\nExpected: %s\nGot: %s' % (expected_output_file, output_file)) self.assertEqual(output_file, expected_output_file, message) self.assertTrue(os.path.exists(expected_input_file)) self.assertTrue(os.path.exists(expected_output_file))
def cache_paths(self): """Return the paths to the inp and out files as expected locally. :return: Tuple consisting of inp and out local cache paths. :rtype: tuple (str, str) :raises: None """ input_file_name, output_file_name = self.file_names() input_file_path = os.path.join(shakemap_zip_dir(), input_file_name) output_file_path = os.path.join(shakemap_zip_dir(), output_file_name) return input_file_path, output_file_path
def cache_paths(self): """Return the paths to the inp and out files as expected locally. :return: Tuple consisting of inp and out local cache paths. :rtype: tuple (str, str) :raises: None """ input_file_name, output_file_name = self.file_names() input_file_path = os.path.join(shakemap_zip_dir(), input_file_name) output_file_path = os.path.join(shakemap_zip_dir(), output_file_name) return input_file_path, output_file_path
def test_shakemap_zip_dir(self): """Test we can get the shakemap zip dir.""" data_dir = shakemap_zip_dir() expected_dir = '%s/shakemaps-zipped' % INASAFE_WORK_DIR self.assertTrue(os.path.exists(expected_dir)) message = 'Got %s, Expectation %s' % (expected_dir, data_dir) self.assertEqual(data_dir, expected_dir, message)
def test_shakemap_zip_dir(self): """Test we can get the shakemap zip dir.""" data_dir = shakemap_zip_dir() expected_dir = os.path.join(INASAFE_WORK_DIR, 'shakemaps-zipped') self.assertTrue(os.path.exists(expected_dir)) message = 'Got %s, Expectation %s' % (expected_dir, data_dir) self.assertEqual(data_dir, expected_dir, message)
def test_get_shake_map_output(self): """Check that we can retrieve a shakemap 'out' input file.""" shake_data = ShakeData(SHAKE_ID) shakemap_file = shake_data.fetch_output() expected_file = os.path.join( shakemap_zip_dir(), SHAKE_ID + '.out.zip') message = 'Expected path for downloaded shakemap OUT not received' self.assertEqual(shakemap_file, expected_file, message)
def test_get_shake_map_output(self): """Check that we can retrieve a shakemap 'out' input file.""" shake_data = ShakeData(SHAKE_ID) shakemap_file = shake_data.fetch_output() expected_file = os.path.join( shakemap_zip_dir(), SHAKE_ID + '.out.zip') message = 'Expected path for downloaded shakemap OUT not received' self.assertEqual(shakemap_file, expected_file, message)
def test_get_latest_shake_map(self): """Check that we can retrieve the latest shake event.""" # Simply dont set the event id in the ctor to get the latest shake_data = ShakeData() input_file, output_file = shake_data.fetch_event() event_id = shake_data.event_id expected_input_file = os.path.join(shakemap_zip_dir(), event_id + '.inp.zip') expected_output_file = os.path.join(shakemap_zip_dir(), event_id + '.out.zip') message = ('Expected path for downloaded shakemap INP not received' '\nExpected: %s\nGot: %s' % (expected_output_file, output_file)) self.assertEqual(input_file, expected_input_file, message) message = ('Expected path for downloaded shakemap OUT not received' '\nExpected: %s\nGot: %s' % (expected_output_file, output_file)) self.assertEqual(output_file, expected_output_file, message)
def test_get_latest_shake_map(self): """Check that we can retrieve the latest shake event.""" # Simply dont set the event id in the ctor to get the latest shake_data = ShakeData() input_file, output_file = shake_data.fetch_event() event_id = shake_data.event_id expected_input_file = os.path.join(shakemap_zip_dir(), event_id + '.inp.zip') expected_output_file = os.path.join(shakemap_zip_dir(), event_id + '.out.zip') message = ('Expected path for downloaded shakemap INP not received' '\nExpected: %s\nGot: %s' % (expected_output_file, output_file)) self.assertEqual(input_file, expected_input_file, message) message = ('Expected path for downloaded shakemap OUT not received' '\nExpected: %s\nGot: %s' % (expected_output_file, output_file)) self.assertEqual(output_file, expected_output_file, message)
def setUp(self): """Copy our cached dataset from the fixture dir to the cache dir.""" # Run monkey patching to ftp_client run_monkey_patching_ftp_client() output_file = '20120726022003.out.zip' input_file = '20120726022003.inp.zip' shake_path = get_shake_test_data_path() output_path = os.path.abspath(os.path.join(shake_path, output_file)) input_path = os.path.abspath(os.path.join(shake_path, input_file)) shutil.copyfile( output_path, os.path.join(shakemap_zip_dir(), output_file)) shutil.copyfile( input_path, os.path.join(shakemap_zip_dir(), input_file))
def _fetch_file(self, event_file, retries=3): """Private helper to fetch a file from the ftp site. e.g. for event 20110413170148 this file would be fetched:: ftp://118.97.83.243/20110413170148.inp.zip and this local file created:: /tmp/realtime/20110413170148.inp.zip .. note:: If a cached copy of the file exits, the path to the cache copy will simply be returned without invoking any network requests. :param event_file: Filename on server e.g.20110413170148.inp.zip :type event_file: str :param retries: Number of reattempts that should be made in in case of network error etc. :type retries: int :return: A string for the dataset path on the local storage system. :rtype: str :raises: EventUndefinedError, NetworkError """ # Return the cache copy if it exists local_path = os.path.join(shakemap_zip_dir(), event_file) if os.path.exists(local_path): return local_path #Otherwise try to fetch it using ftp for counter in range(retries): last_error = None try: client = FtpClient() client.get_file(event_file, local_path) except NetworkError, e: last_error = e except:
def _fetch_file(self, event_file, retries=3): """Private helper to fetch a file from the ftp site. e.g. for event 20110413170148 this file would be fetched:: ftp://118.97.83.243/20110413170148.inp.zip and this local file created:: /tmp/realtime/20110413170148.inp.zip .. note:: If a cached copy of the file exits, the path to the cache copy will simply be returned without invoking any network requests. :param event_file: Filename on server e.g.20110413170148.inp.zip :type event_file: str :param retries: Number of reattempts that should be made in in case of network error etc. :type retries: int :return: A string for the dataset path on the local storage system. :rtype: str :raises: EventUndefinedError, NetworkError """ # Return the cache copy if it exists local_path = os.path.join(shakemap_zip_dir(), event_file) if os.path.exists(local_path): return local_path # Otherwise try to fetch it using ftp for counter in range(retries): last_error = None try: client = FtpClient() client.get_file(event_file, local_path) except NetworkError, e: last_error = e except: