def test_get_remote_shake_map(self):
"""Check that we can retrieve both input and output from ftp at once.
"""
shake_data = ShakeData(SHAKE_ID)
expected_input_file = os.path.join(
shakemap_zip_dir(), SHAKE_ID + '.inp.zip')
expected_output_file = os.path.join(
shakemap_zip_dir(), SHAKE_ID + '.out.zip')
if os.path.exists(expected_input_file):
os.remove(expected_input_file)
if os.path.exists(expected_output_file):
os.remove(expected_output_file)
input_file, output_file = shake_data.fetch_event()
message = ('Expected path for downloaded shakemap INP not received'
'\nExpected: %s\nGot: %s' %
(expected_output_file, output_file))
self.assertEqual(input_file, expected_input_file, message)
message = ('Expected path for downloaded shakemap OUT not received'
'\nExpected: %s\nGot: %s' %
(expected_output_file, output_file))
self.assertEqual(output_file, expected_output_file, message)
self.assertTrue(os.path.exists(expected_input_file))
self.assertTrue(os.path.exists(expected_output_file))
def setUp(self):
"""Copy our cached dataset from the fixture dir to the cache dir."""
# Run monkey patching to ftp_client
run_monkey_patching_ftp_client()
output_file = '20120726022003.out.zip'
input_file = '20120726022003.inp.zip'
output_path = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'../fixtures/shake_data',
output_file))
input_path = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'../fixtures/shake_data',
input_file))
shutil.copyfile(
output_path,
os.path.join(shakemap_zip_dir(),
output_file))
shutil.copyfile(
input_path,
os.path.join(shakemap_zip_dir(),
input_file))
def test_get_remote_shake_map(self):
"""Check that we can retrieve both input and output from ftp at once.
"""
shake_data = ShakeData(SHAKE_ID)
expected_input_file = os.path.join(
shakemap_zip_dir(), SHAKE_ID + '.inp.zip')
expected_output_file = os.path.join(
shakemap_zip_dir(), SHAKE_ID + '.out.zip')
if os.path.exists(expected_input_file):
os.remove(expected_input_file)
if os.path.exists(expected_output_file):
os.remove(expected_output_file)
input_file, output_file = shake_data.fetch_event()
message = ('Expected path for downloaded shakemap INP not received'
'\nExpected: %s\nGot: %s' %
(expected_output_file, output_file))
self.assertEqual(input_file, expected_input_file, message)
message = ('Expected path for downloaded shakemap OUT not received'
'\nExpected: %s\nGot: %s' %
(expected_output_file, output_file))
self.assertEqual(output_file, expected_output_file, message)
self.assertTrue(os.path.exists(expected_input_file))
self.assertTrue(os.path.exists(expected_output_file))
def cache_paths(self):
"""Return the paths to the inp and out files as expected locally.
:return: Tuple consisting of inp and out local cache paths.
:rtype: tuple (str, str)
:raises: None
"""
input_file_name, output_file_name = self.file_names()
input_file_path = os.path.join(shakemap_zip_dir(), input_file_name)
output_file_path = os.path.join(shakemap_zip_dir(), output_file_name)
return input_file_path, output_file_path
def cache_paths(self):
"""Return the paths to the inp and out files as expected locally.
:return: Tuple consisting of inp and out local cache paths.
:rtype: tuple (str, str)
:raises: None
"""
input_file_name, output_file_name = self.file_names()
input_file_path = os.path.join(shakemap_zip_dir(), input_file_name)
output_file_path = os.path.join(shakemap_zip_dir(), output_file_name)
return input_file_path, output_file_path
def test_shakemap_zip_dir(self):
"""Test we can get the shakemap zip dir."""
data_dir = shakemap_zip_dir()
expected_dir = '%s/shakemaps-zipped' % INASAFE_WORK_DIR
self.assertTrue(os.path.exists(expected_dir))
message = 'Got %s, Expectation %s' % (expected_dir, data_dir)
self.assertEqual(data_dir, expected_dir, message)
def test_shakemap_zip_dir(self):
"""Test we can get the shakemap zip dir."""
data_dir = shakemap_zip_dir()
expected_dir = os.path.join(INASAFE_WORK_DIR, 'shakemaps-zipped')
self.assertTrue(os.path.exists(expected_dir))
message = 'Got %s, Expectation %s' % (expected_dir, data_dir)
self.assertEqual(data_dir, expected_dir, message)
def test_get_shake_map_output(self):
"""Check that we can retrieve a shakemap 'out' input file."""
shake_data = ShakeData(SHAKE_ID)
shakemap_file = shake_data.fetch_output()
expected_file = os.path.join(
shakemap_zip_dir(), SHAKE_ID + '.out.zip')
message = 'Expected path for downloaded shakemap OUT not received'
self.assertEqual(shakemap_file, expected_file, message)
def test_get_shake_map_output(self):
"""Check that we can retrieve a shakemap 'out' input file."""
shake_data = ShakeData(SHAKE_ID)
shakemap_file = shake_data.fetch_output()
expected_file = os.path.join(
shakemap_zip_dir(), SHAKE_ID + '.out.zip')
message = 'Expected path for downloaded shakemap OUT not received'
self.assertEqual(shakemap_file, expected_file, message)
def test_get_latest_shake_map(self):
"""Check that we can retrieve the latest shake event."""
# Simply dont set the event id in the ctor to get the latest
shake_data = ShakeData()
input_file, output_file = shake_data.fetch_event()
event_id = shake_data.event_id
expected_input_file = os.path.join(shakemap_zip_dir(),
event_id + '.inp.zip')
expected_output_file = os.path.join(shakemap_zip_dir(),
event_id + '.out.zip')
message = ('Expected path for downloaded shakemap INP not received'
'\nExpected: %s\nGot: %s' %
(expected_output_file, output_file))
self.assertEqual(input_file, expected_input_file, message)
message = ('Expected path for downloaded shakemap OUT not received'
'\nExpected: %s\nGot: %s' %
(expected_output_file, output_file))
self.assertEqual(output_file, expected_output_file, message)
def test_get_latest_shake_map(self):
"""Check that we can retrieve the latest shake event."""
# Simply dont set the event id in the ctor to get the latest
shake_data = ShakeData()
input_file, output_file = shake_data.fetch_event()
event_id = shake_data.event_id
expected_input_file = os.path.join(shakemap_zip_dir(),
event_id + '.inp.zip')
expected_output_file = os.path.join(shakemap_zip_dir(),
event_id + '.out.zip')
message = ('Expected path for downloaded shakemap INP not received'
'\nExpected: %s\nGot: %s' %
(expected_output_file, output_file))
self.assertEqual(input_file, expected_input_file, message)
message = ('Expected path for downloaded shakemap OUT not received'
'\nExpected: %s\nGot: %s' %
(expected_output_file, output_file))
self.assertEqual(output_file, expected_output_file, message)
def setUp(self):
"""Copy our cached dataset from the fixture dir to the cache dir."""
# Run monkey patching to ftp_client
run_monkey_patching_ftp_client()
output_file = '20120726022003.out.zip'
input_file = '20120726022003.inp.zip'
shake_path = get_shake_test_data_path()
output_path = os.path.abspath(os.path.join(shake_path, output_file))
input_path = os.path.abspath(os.path.join(shake_path, input_file))
shutil.copyfile(
output_path,
os.path.join(shakemap_zip_dir(),
output_file))
shutil.copyfile(
input_path,
os.path.join(shakemap_zip_dir(),
input_file))
def _fetch_file(self, event_file, retries=3):
"""Private helper to fetch a file from the ftp site.
e.g. for event 20110413170148 this file would be fetched::
ftp://118.97.83.243/20110413170148.inp.zip
and this local file created::
/tmp/realtime/20110413170148.inp.zip
.. note:: If a cached copy of the file exits, the path to the cache
copy will simply be returned without invoking any network requests.
:param event_file: Filename on server e.g.20110413170148.inp.zip
:type event_file: str
:param retries: Number of reattempts that should be made in
in case of network error etc.
:type retries: int
:return: A string for the dataset path on the local storage system.
:rtype: str
:raises: EventUndefinedError, NetworkError
"""
# Return the cache copy if it exists
local_path = os.path.join(shakemap_zip_dir(), event_file)
if os.path.exists(local_path):
return local_path
#Otherwise try to fetch it using ftp
for counter in range(retries):
last_error = None
try:
client = FtpClient()
client.get_file(event_file, local_path)
except NetworkError, e:
last_error = e
except:
def _fetch_file(self, event_file, retries=3):
"""Private helper to fetch a file from the ftp site.
e.g. for event 20110413170148 this file would be fetched::
ftp://118.97.83.243/20110413170148.inp.zip
and this local file created::
/tmp/realtime/20110413170148.inp.zip
.. note:: If a cached copy of the file exits, the path to the cache
copy will simply be returned without invoking any network requests.
:param event_file: Filename on server e.g.20110413170148.inp.zip
:type event_file: str
:param retries: Number of reattempts that should be made in
in case of network error etc.
:type retries: int
:return: A string for the dataset path on the local storage system.
:rtype: str
:raises: EventUndefinedError, NetworkError
"""
# Return the cache copy if it exists
local_path = os.path.join(shakemap_zip_dir(), event_file)
if os.path.exists(local_path):
return local_path
# Otherwise try to fetch it using ftp
for counter in range(retries):
last_error = None
try:
client = FtpClient()
client.get_file(event_file, local_path)
except NetworkError, e:
last_error = e
except: