def test_download_path(self):
"""Test to download all directories and files under a path."""
sftp_client = SFtpClient(working_dir=temp_dir('realtime-test'))
self.assertIsNotNone(sftp_client)
# Download directories
shake_path = get_shake_test_data_path()
remote_path = os.path.abspath(os.path.join(shake_path, SHAKE_ID))
local_path = temp_dir('realtime-test')
sftp_client.download_path(remote_path, local_path)
# Check the local_path consist of SHAKE_ID folder
expected_dir = [SHAKE_ID]
actual_dir = os.listdir(local_path)
message = "In the local path, I got: %s dir, Expectation: %s dir" % (
actual_dir, expected_dir)
self.assertEqual(expected_dir, actual_dir, message)
# Inside that SHAKE_ID folder, there should be 'output' folder
expected_dir = ['output']
actual_dir = os.listdir(os.path.join(local_path, SHAKE_ID))
message = "In the local path, I got: %s dir, Expectation: %s dir" % (
actual_dir, expected_dir)
self.assertEqual(expected_dir, actual_dir, message)
# Inside that output folder, there should be 'grid.xml' file
expected_dir = ['grid.xml']
actual_dir = os.listdir(os.path.join(local_path, SHAKE_ID, 'output'))
message = "In the local path, I got: %s dir, Expectation: %s dir" % (
actual_dir, expected_dir)
self.assertEqual(expected_dir, actual_dir, message)
def test_download_path(self):
"""Test to download all directories and files under a path."""
sftp_client = SFtpClient(working_dir=temp_dir('realtime-test'))
self.assertIsNotNone(sftp_client)
# Download directories
shake_path = get_shake_test_data_path()
remote_path = os.path.abspath(
os.path.join(shake_path, SHAKE_ID))
local_path = temp_dir('realtime-test')
sftp_client.download_path(remote_path, local_path)
# Check the local_path consist of SHAKE_ID folder
expected_dir = [SHAKE_ID]
actual_dir = os.listdir(local_path)
message = "In the local path, I got: %s dir, Expectation: %s dir" % (
actual_dir, expected_dir)
self.assertEqual(expected_dir, actual_dir, message)
# Inside that SHAKE_ID folder, there should be 'output' folder
expected_dir = ['output']
actual_dir = os.listdir(os.path.join(local_path, SHAKE_ID))
message = "In the local path, I got: %s dir, Expectation: %s dir" % (
actual_dir, expected_dir)
self.assertEqual(expected_dir, actual_dir, message)
# Inside that output folder, there should be 'grid.xml' file
expected_dir = ['grid.xml']
actual_dir = os.listdir(os.path.join(local_path, SHAKE_ID, 'output'))
message = "In the local path, I got: %s dir, Expectation: %s dir" % (
actual_dir, expected_dir)
self.assertEqual(expected_dir, actual_dir, message)
def __init__(self,
event=None,
host=get_sftp_base_url(),
port=get_sftp_port(),
user_name=get_sftp_user_name(),
password=get_sftp_user_password(),
working_dir=get_sftp_base_path(),
force_flag=False):
"""Constructor for the SftpShakeData class.
:param event: A string representing the event id that this raster is
associated with. e.g. 20110413170148 (Optional).
**If no event id is supplied, a query will be made to the ftp
server, and the latest event id assigned.**
:type event: str
:param host: A string representing the ip address or host name of the
server from which the data should be retrieved. It assumes that
the data is in the root directory (Optional).
:type host: str
:param port: The port of the host.
:type port: int
"""
self.event_id = event
self.host = host
self.port = port
self.username = user_name
self.password = password
self.working_dir = working_dir
self.force_flag = force_flag
self.input_file_name = 'grid.xml'
self.sftp_client = SFtpClient(
self.host,
self.port,
self.username,
self.password,
self.working_dir)
if self.event_id is None:
try:
self.get_latest_event_id()
except (SFTPEmptyError, NetworkError, EventIdError):
raise
else:
# If we fetched it above using get_latest_event_id we assume it is
# already validated.
try:
self.validate_event()
except EventValidationError:
raise
# If event_id is still None after all the above, moan....
if self.event_id is None:
message = ('No id was passed to the constructor and the latest '
'id could not be retrieved from the server.')
LOGGER.exception('ShakeData initialisation failed')
raise EventIdError(message)
def test_download_path(self):
"""Test to download all directories and files under a path."""
my_ssh_client = SFtpClient(working_dir="shakemaps")
assert my_ssh_client is not None
remote_path = os.path.join(my_ssh_client.sftp.getcwd(), "20130113003746/output/grid.xml")
local_path = "/tmp/inasafe/"
print local_path
my_ssh_client.download_path(remote_path, local_path)
def test_download_path(self):
"""Test to download all directories and files under a path."""
my_ssh_client = SFtpClient(working_dir='shakemaps')
assert (my_ssh_client is not None)
remote_path = os.path.join(my_ssh_client.sftp.getcwd(),
'20130113003746/output/grid.xml')
local_path = '/tmp/inasafe/'
print local_path
my_ssh_client.download_path(remote_path, local_path)
def test_download_path(self):
"""Test to download all directories and files under a path
"""
my_ssh_client = SFtpClient(the_working_dir='shakemaps')
assert(my_ssh_client is not None)
remote_path = os.path.join(
my_ssh_client.sftp.getcwd(), '20130113003746/output/grid.xml')
local_path = '/tmp/inasafe/'
print local_path
my_ssh_client.download_path(remote_path, local_path)
def setUp(self):
"""Setup before each test."""
# Call sftp client monkey patching before running each tests
run_monkey_patching_sftp_client()
# Download files (which are local files) to realtime-test temp folder
# AG:
# So since we're using local data, in instantiating SFTPShakeData,
# please pass the working dir to the local dir
sftp_client = SFtpClient(working_dir=temp_dir('realtime-test'))
local_path = temp_dir('realtime-test')
remote_path = os.path.abspath(
os.path.join(get_shake_test_data_path(), SHAKE_ID))
sftp_client.download_path(remote_path, local_path)
def setUp(self):
"""Setup before each test."""
# Call sftp client monkey patching before running each tests
run_monkey_patching_sftp_client()
# Download files (which are local files) to realtime-test temp folder
# AG:
# So since we're using local data, in instantiating SFTPShakeData,
# please pass the working dir to the local dir
sftp_client = SFtpClient(working_dir=temp_dir('realtime-test'))
local_path = temp_dir('realtime-test')
remote_path = os.path.abspath(
os.path.join(get_shake_test_data_path(), SHAKE_ID))
sftp_client.download_path(remote_path, local_path)
def test_get_listing(self):
"""Test get_listing if it's working correctly."""
sftp_client = SFtpClient(working_dir=temp_dir('realtime-test'))
self.assertIsNotNone(sftp_client)
# Download directories
shake_path = get_shake_test_data_path()
remote_path = os.path.abspath(os.path.join(shake_path, SHAKE_ID))
local_path = temp_dir('realtime-test')
sftp_client.download_path(remote_path, local_path)
event_ids = sftp_client.get_listing(function=is_event_id)
expected_event_ids = [SHAKE_ID]
message = 'In the local path I got %s dir, Expectation %s dir'
self.assertEqual(event_ids, expected_event_ids, message)
def test_get_listing(self):
"""Test get_listing if it's working correctly."""
sftp_client = SFtpClient(working_dir=temp_dir('realtime-test'))
self.assertIsNotNone(sftp_client)
# Download directories
shake_path = get_shake_test_data_path()
remote_path = os.path.abspath(
os.path.join(shake_path, SHAKE_ID))
local_path = temp_dir('realtime-test')
sftp_client.download_path(remote_path, local_path)
event_ids = sftp_client.get_listing(function=is_event_id)
expected_event_ids = [SHAKE_ID]
message = 'In the local path I got %s dir, Expectation %s dir'
self.assertEqual(event_ids, expected_event_ids, message)
if 'INASAFE_LOCALE' in os.environ:
locale_option = os.environ['INASAFE_LOCALE']
else:
locale_option = 'en'
if len(sys.argv) > 2:
sys.exit(
'Usage:\n%s [optional shakeid]\nor\n%s --list\nor%s --run-all' % (
sys.argv[0], sys.argv[0], sys.argv[0]))
elif len(sys.argv) == 2:
print('Processing shakemap %s' % sys.argv[1])
event_option = sys.argv[1]
if event_option in '--list':
sftp_client = SFtpClient()
dir_listing = sftp_client.get_listing(function=is_event_id)
for event in dir_listing:
print event
sys.exit(0)
elif event_option in '--run-all':
#
# Caution, this code path gets memory leaks, use the
# batch file approach rather!
#
sftp_client = SFtpClient()
dir_listing = sftp_client.get_listing()
for event in dir_listing:
print 'Processing %s' % event
# noinspection PyBroadException
try:
def test_get_list_events(self):
"""Test to get all event ids."""
my_ssh_client = SFtpClient()
assert (my_ssh_client is not None)
if 'INASAFE_LOCALE' in os.environ:
locale_option = os.environ['INASAFE_LOCALE']
else:
locale_option = 'en'
if len(sys.argv) > 2:
sys.exit(
'Usage:\n%s [optional shakeid]\nor\n%s --list\nor%s --run-all' %
(sys.argv[0], sys.argv[0], sys.argv[0]))
elif len(sys.argv) == 2:
print('Processing shakemap %s' % sys.argv[1])
event_option = sys.argv[1]
if event_option in '--list':
sftp_client = SFtpClient()
dir_listing = sftp_client.get_listing(function=is_event_id)
for event in dir_listing:
print event
sys.exit(0)
elif event_option in '--run-all':
#
# Caution, this code path gets memory leaks, use the
# batch file approach rather!
#
sftp_client = SFtpClient()
dir_listing = sftp_client.get_listing()
for event in dir_listing:
print 'Processing %s' % event
# noinspection PyBroadException
try:
class SftpShakeData:
"""A class for retrieving, reading data from shake files.
Shake files are provided on server and can be accessed using SSH protocol.
The shake files currently located under BASE_PATH directory in a folder
named by the event id (which represent the timestamp of the event of the
shake)
There are numerous files in that directory but there is only really one
that we are interested in:
* grid.xml - which contains all the metadata pertaining to the event
It's located at output/grid.xml under each event directory
The remaining files are fetched for completeness and possibly use in the
future.
This class provides a high level interface for retrieving this data and
then extracting various by products from it
Note :
* inspired by shake_data.py but modified according to SSH protocol
"""
def __init__(self,
event=None,
host=get_sftp_base_url(),
port=get_sftp_port(),
user_name=get_sftp_user_name(),
password=get_sftp_user_password(),
working_dir=get_sftp_base_path(),
force_flag=False):
"""Constructor for the SftpShakeData class.
:param event: A string representing the event id that this raster is
associated with. e.g. 20110413170148 (Optional).
**If no event id is supplied, a query will be made to the ftp
server, and the latest event id assigned.**
:type event: str
:param host: A string representing the ip address or host name of the
server from which the data should be retrieved. It assumes that
the data is in the root directory (Optional).
:type host: str
:param port: The port of the host.
:type port: int
"""
self.event_id = event
self.host = host
self.port = port
self.username = user_name
self.password = password
self.working_dir = working_dir
self.force_flag = force_flag
self.input_file_name = 'grid.xml'
self.sftp_client = SFtpClient(
self.host,
self.port,
self.username,
self.password,
self.working_dir)
if self.event_id is None:
try:
self.get_latest_event_id()
except (SFTPEmptyError, NetworkError, EventIdError):
raise
else:
# If we fetched it above using get_latest_event_id we assume it is
# already validated.
try:
self.validate_event()
except EventValidationError:
raise
# If event_id is still None after all the above, moan....
if self.event_id is None:
message = ('No id was passed to the constructor and the latest '
'id could not be retrieved from the server.')
LOGGER.exception('ShakeData initialisation failed')
raise EventIdError(message)
def reconnect_sftp(self):
"""Reconnect to the server."""
self.sftp_client = SFtpClient(
self.host, self.username, self.password, self.working_dir)
def validate_event(self):
"""Check that the event associated with this instance exists.
This will check either in the local event cache or on the remote ftp
site.
:return: True if valid, False if not.
:rtype: bool
:raises: NetworkError
"""
# First check local cache
if self.is_cached():
return True
else:
return self.is_on_server()
def is_cached(self):
"""Check the event associated with this instance exists in cache.
:return: True if locally cached, otherwise False.
:rtype: bool
:raises: None
"""
xml_file_path = self.cache_paths()
if os.path.exists(xml_file_path):
return True
else:
LOGGER.debug('%s is not cached' % xml_file_path)
return False
def cache_paths(self):
"""Return the paths to the inp and out files as expected locally.
:return: The grid.xml local cache path.
:rtype: str
"""
xml_file_name = self.input_file_name
xml_file_path = os.path.join(
shakemap_cache_dir(), self.event_id, xml_file_name)
return xml_file_path
def is_on_server(self):
"""Check the event associated with this instance exists on the server.
:return: True if valid, False if not.
:raises: NetworkError
"""
remote_xml_path = os.path.join(
self.sftp_client.working_dir_path, self.event_id)
return self.sftp_client.path_exists(remote_xml_path)
def get_list_event_ids(self):
"""Get all event id indicated by folder in remote_path."""
dirs = self.sftp_client.get_listing(function=is_event_id)
if len(dirs) == 0:
raise SFTPEmptyError(
'The SFTP directory does not contain any shakemaps.')
return dirs
def get_latest_event_id(self):
"""Return latest event id."""
try:
event_ids = self.get_list_event_ids()
except SFTPEmptyError:
raise
now = datetime.now()
now = int(
'%04d%02d%02d%02d%02d%02d' %
(now.year, now.month, now.day, now.hour, now.minute, now.second))
if event_ids is not None:
event_ids.sort()
latest_event_id = now + 1
while int(latest_event_id) > now:
if len(event_ids) < 1:
raise EventIdError('Latest Event Id could not be obtained')
latest_event_id = event_ids.pop()
self.event_id = latest_event_id
return self.event_id
def fetch_file(self, retries=3):
"""Private helper to fetch a file from the sftp site.
.. note:: If a cached copy of the file exits, the path to the cache
copy will simply be returned without invoking any network requests.
:param retries: The number of reattempts that should be made in
case of e.g network error.
e.g. for event 20110413170148 this file would be fetched::
20110413170148 directory
:type retries: int
:return: A string for the dataset path on the local storage system.
:rtype: str
:raises: EventUndefinedError, NetworkError
"""
local_path = os.path.join(shakemap_cache_dir(), self.event_id)
local_parent_path = os.path.join(local_path, 'output')
xml_file = os.path.join(local_parent_path, self.input_file_name)
if os.path.exists(xml_file):
return local_path
# fetch from sftp
trials = [i + 1 for i in xrange(retries)]
remote_path = os.path.join(
self.sftp_client.working_dir_path, self.event_id)
xml_remote_path = os.path.join(
remote_path, 'output', self.input_file_name)
for counter in trials:
last_error = None
try:
make_directory(local_path)
make_directory(os.path.join(local_path, 'output'))
self.sftp_client.download_path(
xml_remote_path, local_parent_path)
except NetworkError, e:
last_error = e
except:
def reconnect_sftp(self):
"""Reconnect to the server."""
self.sftp_client = SFtpClient(
self.host, self.username, self.password, self.working_dir)