def test_download_path(self):
"""Test to download all directories and files under a path
"""
my_ssh_client = SFtpClient(the_working_dir='shakemaps')
assert(my_ssh_client is not None)
remote_path = os.path.join(
my_ssh_client.sftp.getcwd(), '20130113003746/output/grid.xml')
local_path = '/tmp/inasafe/'
print local_path
my_ssh_client.download_path(remote_path, local_path)
def test_download_path(self):
"""Test to download all directories and files under a path
"""
my_ssh_client = SFtpClient(the_working_dir='shakemaps')
assert (my_ssh_client is not None)
remote_path = os.path.join(my_ssh_client.sftp.getcwd(),
'20130113003746/output/grid.xml')
local_path = '/tmp/inasafe/'
print local_path
my_ssh_client.download_path(remote_path, local_path)
def __init__(self,
theEvent=None,
theHost=defaultHost,
theUserName=defUserName,
thePassword=defPassword,
theWorkingDir=defWorkDir,
theForceFlag=False):
"""Constructor for the SftpShakeData class
Args:
* theEvent - (Optional) a string representing the event id
that this raster is associated with. e.g. 20110413170148.
**If no event id is supplied, a query will be made to the
ftp server, and the latest event id assigned.**
* theHost - (Optional) a string representing the ip address
or host name of the server from which the data should be
retrieved. It assumes that the data is in the root directory.
Returns:
None
Raises:
None
"""
self.eventId = theEvent
self.host = theHost
self.username = theUserName
self.password = thePassword
self.workdir = theWorkingDir
self.forceFlag = theForceFlag
self.sftpclient = SFtpClient(self.host, self.username, self.password,
self.workdir)
if self.eventId is None:
try:
self.getLatestEventId()
except NetworkError:
raise
else:
# If we fetched it above using getLatestEventId we assume it is
# already validated.
try:
self.validateEvent()
except EventValidationError:
raise
# If eventId is still None after all the above, moan....
if self.eventId is None:
myMessage = ('No id was passed to the constructor and the '
'latest id could not be retrieved from the'
'server.')
LOGGER.exception('ShakeData initialisation failed')
raise EventIdError(myMessage)
def __init__(self,
event=None,
host=default_host,
user_name=def_user_name,
password=def_password,
working_dir=def_work_dir,
force_flag=False):
"""Constructor for the SftpShakeData class
:param event: (Optional) a string representing the event id
that this raster is associated with. e.g. 20110413170148.
**If no event id is supplied, a query will be made to the
ftp server, and the latest event id assigned.**
:param host: (Optional) a string representing the ip address
or host name of the server from which the data should be
retrieved. It assumes that the data is in the root directory.
"""
self.event_id = event
self.host = host
self.username = user_name
self.password = password
self.workdir = working_dir
self.force_flag = force_flag
self.sftpclient = SFtpClient(self.host, self.username, self.password,
self.workdir)
if self.event_id is None:
try:
self.get_latest_event_id()
except NetworkError:
raise
else:
# If we fetched it above using get_latest_event_id we assume it is
# already validated.
try:
self.validate_event()
except EventValidationError:
raise
# If event_id is still None after all the above, moan....
if self.event_id is None:
message = ('No id was passed to the constructor and the '
'latest id could not be retrieved from the'
'server.')
LOGGER.exception('ShakeData initialisation failed')
raise EventIdError(message)
def __init__(self, theEvent=None, theHost=defaultHost,
theUserName=defUserName, thePassword=defPassword,
theWorkingDir=defWorkDir, theForceFlag=False):
"""Constructor for the SftpShakeData class
Args:
* theEvent - (Optional) a string representing the event id
that this raster is associated with. e.g. 20110413170148.
**If no event id is supplied, a query will be made to the
ftp server, and the latest event id assigned.**
* theHost - (Optional) a string representing the ip address
or host name of the server from which the data should be
retrieved. It assumes that the data is in the root directory.
Returns:
None
Raises:
None
"""
self.eventId = theEvent
self.host = theHost
self.username = theUserName
self.password = thePassword
self.workdir = theWorkingDir
self.forceFlag = theForceFlag
self.sftpclient = SFtpClient(
self.host, self.username, self.password, self.workdir)
if self.eventId is None:
try:
self.getLatestEventId()
except NetworkError:
raise
else:
# If we fetched it above using getLatestEventId we assume it is
# already validated.
try:
self.validateEvent()
except EventValidationError:
raise
# If eventId is still None after all the above, moan....
if self.eventId is None:
myMessage = ('No id was passed to the constructor and the '
'latest id could not be retrieved from the'
'server.')
LOGGER.exception('ShakeData initialisation failed')
raise EventIdError(myMessage)
def __init__(
self,
event=None,
host=default_host,
user_name=def_user_name,
password=def_password,
working_dir=def_work_dir,
force_flag=False,
):
"""Constructor for the SftpShakeData class
:param event: (Optional) a string representing the event id
that this raster is associated with. e.g. 20110413170148.
**If no event id is supplied, a query will be made to the
ftp server, and the latest event id assigned.**
:param host: (Optional) a string representing the ip address
or host name of the server from which the data should be
retrieved. It assumes that the data is in the root directory.
"""
self.event_id = event
self.host = host
self.username = user_name
self.password = password
self.workdir = working_dir
self.force_flag = force_flag
self.sftpclient = SFtpClient(self.host, self.username, self.password, self.workdir)
if self.event_id is None:
try:
self.get_latest_event_id()
except NetworkError:
raise
else:
# If we fetched it above using get_latest_event_id we assume it is
# already validated.
try:
self.validate_event()
except EventValidationError:
raise
# If event_id is still None after all the above, moan....
if self.event_id is None:
message = (
"No id was passed to the constructor and the " "latest id could not be retrieved from the" "server."
)
LOGGER.exception("ShakeData initialisation failed")
raise EventIdError(message)
if 'INASAFE_LOCALE' in os.environ:
myLocale = os.environ['INASAFE_LOCALE']
else:
myLocale = 'en'
if len(sys.argv) > 2:
sys.exit('Usage:\n%s [optional shakeid]\nor\n%s --list' % (
sys.argv[0], sys.argv[0]))
elif len(sys.argv) == 2:
print('Processing shakemap %s' % sys.argv[1])
myEventId = sys.argv[1]
if myEventId in '--list':
# myFtpClient = FtpClient()
mySftpClient = SFtpClient()
# myListing = myFtpClient.getListing()
myListing = mySftpClient.getListing(my_func=is_event_id)
for myEvent in myListing:
print myEvent
sys.exit(0)
elif myEventId in '--run-all':
#
# Caution, this code path gets memory leaks, use the
# batch file approach rather!
#
myFtpClient = FtpClient()
myListing = myFtpClient.getListing()
for myEvent in myListing:
if 'out' not in myEvent:
continue
class SftpShakeData:
"""A class for retrieving, reading data from shakefiles.
Shake files are provide on server and can be accessed using SSH protocol.
The shape files currently located under shakemaps directory in a folder
named by the event id (which represent the timestamp of the event of the
shake)
There are numerous files in that directory but there is only really one
that we are interested in:
* grid.xml - which contains all the metadata pertaining to the event
It's located in under output/grid.xml under each event directory
The remaining files are fetched for completeness and possibly use in the
future.
This class provides a high level interface for retrieving this data and
then extracting various by products from it
Note :
* inspired by shake_data.py but modified according to SSH protocol
"""
def __init__(self,
theEvent=None,
theHost=defaultHost,
theUserName=defUserName,
thePassword=defPassword,
theWorkingDir=defWorkDir,
theForceFlag=False):
"""Constructor for the SftpShakeData class
Args:
* theEvent - (Optional) a string representing the event id
that this raster is associated with. e.g. 20110413170148.
**If no event id is supplied, a query will be made to the
ftp server, and the latest event id assigned.**
* theHost - (Optional) a string representing the ip address
or host name of the server from which the data should be
retrieved. It assumes that the data is in the root directory.
Returns:
None
Raises:
None
"""
self.eventId = theEvent
self.host = theHost
self.username = theUserName
self.password = thePassword
self.workdir = theWorkingDir
self.forceFlag = theForceFlag
self.sftpclient = SFtpClient(self.host, self.username, self.password,
self.workdir)
if self.eventId is None:
try:
self.getLatestEventId()
except NetworkError:
raise
else:
# If we fetched it above using getLatestEventId we assume it is
# already validated.
try:
self.validateEvent()
except EventValidationError:
raise
# If eventId is still None after all the above, moan....
if self.eventId is None:
myMessage = ('No id was passed to the constructor and the '
'latest id could not be retrieved from the'
'server.')
LOGGER.exception('ShakeData initialisation failed')
raise EventIdError(myMessage)
def reconnectSFTP(self):
"""Reconnect to the server
"""
self.sftpclient = SFtpClient(self.host, self.username, self.password,
self.workdir)
def validateEvent(self):
"""Check that the event associated with this instance exists either
in the local event cache, or on the remote ftp site.
Args: None
Returns: True if valid, False if not
Raises: NetworkError
"""
# First check local cache
if self.isCached():
return True
else:
return self.isOnServer()
def isCached(self):
"""Check the event associated with this instance exists in cache.
Args: None
Returns: True if locally cached, False if not
Raises: None
"""
myXMLFilePath = self.cachePaths()
if os.path.exists(myXMLFilePath):
return True
else:
LOGGER.debug('%s is not cached' % myXMLFilePath)
return False
def cachePaths(self):
"""Return the paths to the inp and out files as expected locally.
Args: None
Returns:
str: grid.xml local cache paths.
Raises:
None
"""
myXMLFileName = self.fileName()
myXMLFilePath = os.path.join(shakemapCacheDir(), self.eventId,
myXMLFileName)
return myXMLFilePath
def fileName(self):
"""Return file names for the inp and out files based on the event id.
for this class, only the grid.xml only
Args: None
Returns:
str: grid.xml
Raises:
None
"""
return 'grid.xml'
def isOnServer(self):
"""Check the event associated with this instance exists on the server.
Args: None
Returns: True if valid, False if not
Raises: NetworkError
"""
myRemoteXMLPath = os.path.join(self.sftpclient.workdir_path,
self.eventId)
return self.sftpclient.is_path_exist(myRemoteXMLPath)
def get_list_event_ids(self):
"""Get all event id indicated by folder in remote_path
"""
dirs = self.sftpclient.getListing(my_func=is_event_id)
if len(dirs) == 0:
raise Exception('List event is empty')
return dirs
def getLatestEventId(self):
"""Return latest event id
"""
event_ids = self.get_list_event_ids()
latest_event_id = None
if event_ids is not None:
event_ids.sort()
latest_event_id = event_ids[-1]
if latest_event_id is None:
raise EventIdError('Latest Event Id could not be obtained')
self.eventId = latest_event_id
return self.eventId
def fetchFile(self, theRetries=3):
"""Private helper to fetch a file from the sftp site.
e.g. for event 20110413170148 this file would be fetched::
20110413170148 directory
.. note:: If a cached copy of the file exits, the path to the cache
copy will simply be returned without invoking any network requests.
Args:
* theEventFile: str - filename on server e.g.20110413170148.inp.zip
* theRetries: int - number of reattempts that should be made in
in case of network error etc.
Returns:
str: A string for the dataset path on the local storage system.
Raises:
EventUndefinedError, NetworkError
"""
myLocalPath = os.path.join(shakemapCacheDir(), self.eventId)
myLocalParentPath = os.path.join(myLocalPath, 'output')
myXMLFile = os.path.join(myLocalParentPath, self.fileName())
if os.path.exists(myXMLFile):
return myLocalPath
# fetch from sftp
trials = [i + 1 for i in xrange(theRetries)]
remote_path = os.path.join(self.sftpclient.workdir_path, self.eventId)
myXMLRemotePath = os.path.join(remote_path, 'output', self.fileName())
for my_counter in trials:
myLastError = None
try:
mkDir(myLocalPath)
mkDir(os.path.join(myLocalPath, 'output'))
self.sftpclient.download_path(myXMLRemotePath,
myLocalParentPath)
except NetworkError, e:
myLastError = e
except:
def reconnectSFTP(self):
"""Reconnect to the server
"""
self.sftpclient = SFtpClient(self.host, self.username, self.password,
self.workdir)
class SftpShakeData:
"""A class for retrieving, reading data from shakefiles.
Shake files are provide on server and can be accessed using SSH protocol.
The shape files currently located under shakemaps directory in a folder
named by the event id (which represent the timestamp of the event of the
shake)
There are numerous files in that directory but there is only really one
that we are interested in:
* grid.xml - which contains all the metadata pertaining to the event
It's located in under output/grid.xml under each event directory
The remaining files are fetched for completeness and possibly use in the
future.
This class provides a high level interface for retrieving this data and
then extracting various by products from it
Note :
* inspired by shake_data.py but modified according to SSH protocol
"""
def __init__(self, theEvent=None, theHost=defaultHost,
theUserName=defUserName, thePassword=defPassword,
theWorkingDir=defWorkDir, theForceFlag=False):
"""Constructor for the SftpShakeData class
Args:
* theEvent - (Optional) a string representing the event id
that this raster is associated with. e.g. 20110413170148.
**If no event id is supplied, a query will be made to the
ftp server, and the latest event id assigned.**
* theHost - (Optional) a string representing the ip address
or host name of the server from which the data should be
retrieved. It assumes that the data is in the root directory.
Returns:
None
Raises:
None
"""
self.eventId = theEvent
self.host = theHost
self.username = theUserName
self.password = thePassword
self.workdir = theWorkingDir
self.forceFlag = theForceFlag
self.sftpclient = SFtpClient(
self.host, self.username, self.password, self.workdir)
if self.eventId is None:
try:
self.getLatestEventId()
except NetworkError:
raise
else:
# If we fetched it above using getLatestEventId we assume it is
# already validated.
try:
self.validateEvent()
except EventValidationError:
raise
# If eventId is still None after all the above, moan....
if self.eventId is None:
myMessage = ('No id was passed to the constructor and the '
'latest id could not be retrieved from the'
'server.')
LOGGER.exception('ShakeData initialisation failed')
raise EventIdError(myMessage)
def reconnectSFTP(self):
"""Reconnect to the server
"""
self.sftpclient = SFtpClient(
self.host, self.username, self.password, self.workdir)
def validateEvent(self):
"""Check that the event associated with this instance exists either
in the local event cache, or on the remote ftp site.
Args: None
Returns: True if valid, False if not
Raises: NetworkError
"""
# First check local cache
if self.isCached():
return True
else:
return self.isOnServer()
def isCached(self):
"""Check the event associated with this instance exists in cache.
Args: None
Returns: True if locally cached, False if not
Raises: None
"""
myXMLFilePath = self.cachePaths()
if os.path.exists(myXMLFilePath):
return True
else:
LOGGER.debug('%s is not cached' % myXMLFilePath)
return False
def cachePaths(self):
"""Return the paths to the inp and out files as expected locally.
Args: None
Returns:
str: grid.xml local cache paths.
Raises:
None
"""
myXMLFileName = self.fileName()
myXMLFilePath = os.path.join(
shakemapCacheDir(), self.eventId, myXMLFileName)
return myXMLFilePath
def fileName(self):
"""Return file names for the inp and out files based on the event id.
for this class, only the grid.xml only
Args: None
Returns:
str: grid.xml
Raises:
None
"""
return 'grid.xml'
def isOnServer(self):
"""Check the event associated with this instance exists on the server.
Args: None
Returns: True if valid, False if not
Raises: NetworkError
"""
myRemoteXMLPath = os.path.join(
self.sftpclient.workdir_path, self.eventId)
return self.sftpclient.is_path_exist(myRemoteXMLPath)
def get_list_event_ids(self):
"""Get all event id indicated by folder in remote_path
"""
dirs = self.sftpclient.getListing(my_func=is_event_id)
if len(dirs) == 0:
raise Exception('List event is empty')
return dirs
def getLatestEventId(self):
"""Return latest event id
"""
event_ids = self.get_list_event_ids()
latest_event_id = None
if event_ids is not None:
event_ids.sort()
latest_event_id = event_ids[-1]
if latest_event_id is None:
raise EventIdError('Latest Event Id could not be obtained')
self.eventId = latest_event_id
return self.eventId
def fetchFile(self, theRetries=3):
"""Private helper to fetch a file from the sftp site.
:param theRetries: int - number of reattempts that should be made in
in case of network error etc.
e.g. for event 20110413170148 this file would be fetched::
20110413170148 directory
.. note:: If a cached copy of the file exits, the path to the cache
copy will simply be returned without invoking any network requests.
Returns:
str: A string for the dataset path on the local storage system.
Raises:
EventUndefinedError, NetworkError
"""
myLocalPath = os.path.join(shakemapCacheDir(), self.eventId)
myLocalParentPath = os.path.join(myLocalPath, 'output')
myXMLFile = os.path.join(myLocalParentPath, self.fileName())
if os.path.exists(myXMLFile):
return myLocalPath
# fetch from sftp
trials = [i + 1 for i in xrange(theRetries)]
remote_path = os.path.join(self.sftpclient.workdir_path, self.eventId)
myXMLRemotePath = os.path.join(remote_path, 'output', self.fileName())
for my_counter in trials:
myLastError = None
try:
mkDir(myLocalPath)
mkDir(os.path.join(myLocalPath, 'output'))
self.sftpclient.download_path(
myXMLRemotePath, myLocalParentPath)
except NetworkError, e:
myLastError = e
except:
if 'INASAFE_LOCALE' in os.environ:
myLocale = os.environ['INASAFE_LOCALE']
else:
myLocale = 'en'
if len(sys.argv) > 2:
sys.exit('Usage:\n%s [optional shakeid]\nor\n%s --list' % (
sys.argv[0], sys.argv[0]))
elif len(sys.argv) == 2:
print('Processing shakemap %s' % sys.argv[1])
myEventId = sys.argv[1]
if myEventId in '--list':
# myFtpClient = FtpClient()
mySftpClient = SFtpClient()
# myListing = myFtpClient.getListing()
myListing = mySftpClient.getListing(my_func=is_event_id)
for myEvent in myListing:
print myEvent
sys.exit(0)
elif myEventId in '--run-all':
#
# Caution, this code path gets memory leaks, use the
# batch file approach rather!
#
myFtpClient = FtpClient()
myListing = myFtpClient.getListing()
for myEvent in myListing:
if 'out' not in myEvent:
continue
def test_get_list_events(self):
"""Test to get all event ids
"""
my_ssh_client = SFtpClient()
assert (my_ssh_client is not None)
class SftpShakeData:
"""A class for retrieving, reading data from shakefiles.
Shake files are provide on server and can be accessed using SSH protocol.
The shape files currently located under shakemaps directory in a folder
named by the event id (which represent the timestamp of the event of the
shake)
There are numerous files in that directory but there is only really one
that we are interested in:
* grid.xml - which contains all the metadata pertaining to the event
It's located in under output/grid.xml under each event directory
The remaining files are fetched for completeness and possibly use in the
future.
This class provides a high level interface for retrieving this data and
then extracting various by products from it
Note :
* inspired by shake_data.py but modified according to SSH protocol
"""
def __init__(
self,
event=None,
host=default_host,
user_name=def_user_name,
password=def_password,
working_dir=def_work_dir,
force_flag=False,
):
"""Constructor for the SftpShakeData class
:param event: (Optional) a string representing the event id
that this raster is associated with. e.g. 20110413170148.
**If no event id is supplied, a query will be made to the
ftp server, and the latest event id assigned.**
:param host: (Optional) a string representing the ip address
or host name of the server from which the data should be
retrieved. It assumes that the data is in the root directory.
"""
self.event_id = event
self.host = host
self.username = user_name
self.password = password
self.workdir = working_dir
self.force_flag = force_flag
self.sftpclient = SFtpClient(self.host, self.username, self.password, self.workdir)
if self.event_id is None:
try:
self.get_latest_event_id()
except NetworkError:
raise
else:
# If we fetched it above using get_latest_event_id we assume it is
# already validated.
try:
self.validate_event()
except EventValidationError:
raise
# If event_id is still None after all the above, moan....
if self.event_id is None:
message = (
"No id was passed to the constructor and the " "latest id could not be retrieved from the" "server."
)
LOGGER.exception("ShakeData initialisation failed")
raise EventIdError(message)
def reconnect_sftp(self):
"""Reconnect to the server."""
self.sftpclient = SFtpClient(self.host, self.username, self.password, self.workdir)
def validate_event(self):
"""Check that the event associated with this instance exists either
in the local event cache, or on the remote ftp site.
:return: True if valid, False if not
:raises: NetworkError
"""
# First check local cache
if self.is_cached():
return True
else:
return self.is_on_server()
def is_cached(self):
"""Check the event associated with this instance exists in cache.
Args: None
Returns: True if locally cached, False if not
Raises: None
"""
xml_file_path = self.cache_paths()
if os.path.exists(xml_file_path):
return True
else:
LOGGER.debug("%s is not cached" % xml_file_path)
return False
def cache_paths(self):
"""Return the paths to the inp and out files as expected locally.
:return: grid.xml local cache paths.
:rtype: str
"""
xml_file_name = self.file_name()
xml_file_path = os.path.join(shakemap_cache_dir(), self.event_id, xml_file_name)
return xml_file_path
# noinspection PyMethodMayBeStatic
def file_name(self):
"""Return file names for the inp and out files based on the event id.
For this class, only the grid.xml that is used.
:return: grid.xml
:rtype: str
"""
return "grid.xml"
def is_on_server(self):
"""Check the event associated with this instance exists on the server.
:return: True if valid, False if not
:raises: NetworkError
"""
remote_xml_path = os.path.join(self.sftpclient.workdir_path, self.event_id)
return self.sftpclient.is_path_exist(remote_xml_path)
def get_list_event_ids(self):
"""Get all event id indicated by folder in remote_path
"""
dirs = self.sftpclient.get_listing(my_func=is_event_id)
if len(dirs) == 0:
raise Exception("List event is empty")
return dirs
def get_latest_event_id(self):
"""Return latest event id.
"""
event_ids = self.get_list_event_ids()
now = datetime.now()
now = int("%04d%02d%02d%02d%02d%02d" % (now.year, now.month, now.day, now.hour, now.minute, now.second))
if event_ids is not None:
event_ids.sort()
latest_event_id = now + 1
while int(latest_event_id) > now:
if len(event_ids) < 1:
raise EventIdError("Latest Event Id could not be obtained")
latest_event_id = event_ids.pop()
self.event_id = latest_event_id
return self.event_id
def fetch_file(self, retries=3):
"""Private helper to fetch a file from the sftp site.
:param retries: int - number of reattempts that should be made in
in case of network error etc.
e.g. for event 20110413170148 this file would be fetched::
20110413170148 directory
.. note:: If a cached copy of the file exits, the path to the cache
copy will simply be returned without invoking any network requests.
:return: A string for the dataset path on the local storage system.
:rtype: str
:raises: EventUndefinedError, NetworkError
"""
local_path = os.path.join(shakemap_cache_dir(), self.event_id)
local_parent_path = os.path.join(local_path, "output")
xml_file = os.path.join(local_parent_path, self.file_name())
if os.path.exists(xml_file):
return local_path
# fetch from sftp
trials = [i + 1 for i in xrange(retries)]
remote_path = os.path.join(self.sftpclient.workdir_path, self.event_id)
xml_remote_path = os.path.join(remote_path, "output", self.file_name())
for counter in trials:
last_error = None
try:
mk_dir(local_path)
mk_dir(os.path.join(local_path, "output"))
self.sftpclient.download_path(xml_remote_path, local_parent_path)
except NetworkError, e:
last_error = e
except:
if 'INASAFE_LOCALE' in os.environ:
my_locale = os.environ['INASAFE_LOCALE']
else:
my_locale = 'en'
if len(sys.argv) > 2:
sys.exit('Usage:\n%s [optional shakeid]\nor\n%s --list' %
(sys.argv[0], sys.argv[0]))
elif len(sys.argv) == 2:
print('Processing shakemap %s' % sys.argv[1])
my_event_id = sys.argv[1]
if my_event_id in '--list':
# ftp_client = FtpClient()
sftp_client = SFtpClient()
# myListing = ftp_client.get_listing()
dir_listing = sftp_client.get_listing(my_func=is_event_id)
for event in dir_listing:
print event
sys.exit(0)
elif my_event_id in '--run-all':
#
# Caution, this code path gets memory leaks, use the
# batch file approach rather!
#
ftp_client = FtpClient()
dir_listing = ftp_client.get_listing()
for event in dir_listing:
if 'out' not in event:
continue
def reconnectSFTP(self):
"""Reconnect to the server
"""
self.sftpclient = SFtpClient(
self.host, self.username, self.password, self.workdir)
LOGGER.info("-------------------------------------------")
if "INASAFE_LOCALE" in os.environ:
my_locale = os.environ["INASAFE_LOCALE"]
else:
my_locale = "en"
if len(sys.argv) > 2:
sys.exit("Usage:\n%s [optional shakeid]\nor\n%s --list" % (sys.argv[0], sys.argv[0]))
elif len(sys.argv) == 2:
print ("Processing shakemap %s" % sys.argv[1])
my_event_id = sys.argv[1]
if my_event_id in "--list":
# ftp_client = FtpClient()
sftp_client = SFtpClient()
# myListing = ftp_client.get_listing()
dir_listing = sftp_client.get_listing(my_func=is_event_id)
for event in dir_listing:
print event
sys.exit(0)
elif my_event_id in "--run-all":
#
# Caution, this code path gets memory leaks, use the
# batch file approach rather!
#
ftp_client = FtpClient()
dir_listing = ftp_client.get_listing()
for event in dir_listing:
if "out" not in event:
continue
class SftpShakeData:
"""A class for retrieving, reading data from shakefiles.
Shake files are provide on server and can be accessed using SSH protocol.
The shape files currently located under shakemaps directory in a folder
named by the event id (which represent the timestamp of the event of the
shake)
There are numerous files in that directory but there is only really one
that we are interested in:
* grid.xml - which contains all the metadata pertaining to the event
It's located in under output/grid.xml under each event directory
The remaining files are fetched for completeness and possibly use in the
future.
This class provides a high level interface for retrieving this data and
then extracting various by products from it
Note :
* inspired by shake_data.py but modified according to SSH protocol
"""
def __init__(self,
event=None,
host=default_host,
user_name=def_user_name,
password=def_password,
working_dir=def_work_dir,
force_flag=False):
"""Constructor for the SftpShakeData class
:param event: (Optional) a string representing the event id
that this raster is associated with. e.g. 20110413170148.
**If no event id is supplied, a query will be made to the
ftp server, and the latest event id assigned.**
:param host: (Optional) a string representing the ip address
or host name of the server from which the data should be
retrieved. It assumes that the data is in the root directory.
"""
self.event_id = event
self.host = host
self.username = user_name
self.password = password
self.workdir = working_dir
self.force_flag = force_flag
self.sftpclient = SFtpClient(self.host, self.username, self.password,
self.workdir)
if self.event_id is None:
try:
self.get_latest_event_id()
except NetworkError:
raise
else:
# If we fetched it above using get_latest_event_id we assume it is
# already validated.
try:
self.validate_event()
except EventValidationError:
raise
# If event_id is still None after all the above, moan....
if self.event_id is None:
message = ('No id was passed to the constructor and the '
'latest id could not be retrieved from the'
'server.')
LOGGER.exception('ShakeData initialisation failed')
raise EventIdError(message)
def reconnect_sftp(self):
"""Reconnect to the server."""
self.sftpclient = SFtpClient(self.host, self.username, self.password,
self.workdir)
def validate_event(self):
"""Check that the event associated with this instance exists either
in the local event cache, or on the remote ftp site.
:return: True if valid, False if not
:raises: NetworkError
"""
# First check local cache
if self.is_cached():
return True
else:
return self.is_on_server()
def is_cached(self):
"""Check the event associated with this instance exists in cache.
Args: None
Returns: True if locally cached, False if not
Raises: None
"""
xml_file_path = self.cache_paths()
if os.path.exists(xml_file_path):
return True
else:
LOGGER.debug('%s is not cached' % xml_file_path)
return False
def cache_paths(self):
"""Return the paths to the inp and out files as expected locally.
:return: grid.xml local cache paths.
:rtype: str
"""
xml_file_name = self.file_name()
xml_file_path = os.path.join(shakemap_cache_dir(), self.event_id,
xml_file_name)
return xml_file_path
#noinspection PyMethodMayBeStatic
def file_name(self):
"""Return file names for the inp and out files based on the event id.
For this class, only the grid.xml that is used.
:return: grid.xml
:rtype: str
"""
return 'grid.xml'
def is_on_server(self):
"""Check the event associated with this instance exists on the server.
:return: True if valid, False if not
:raises: NetworkError
"""
remote_xml_path = os.path.join(self.sftpclient.workdir_path,
self.event_id)
return self.sftpclient.is_path_exist(remote_xml_path)
def get_list_event_ids(self):
"""Get all event id indicated by folder in remote_path
"""
dirs = self.sftpclient.get_listing(my_func=is_event_id)
if len(dirs) == 0:
raise Exception('List event is empty')
return dirs
def get_latest_event_id(self):
"""Return latest event id.
"""
event_ids = self.get_list_event_ids()
now = datetime.now()
now = int(
'%04d%02d%02d%02d%02d%02d' %
(now.year, now.month, now.day, now.hour, now.minute, now.second))
if event_ids is not None:
event_ids.sort()
latest_event_id = now + 1
while int(latest_event_id) > now:
if len(event_ids) < 1:
raise EventIdError('Latest Event Id could not be obtained')
latest_event_id = event_ids.pop()
self.event_id = latest_event_id
return self.event_id
def fetch_file(self, retries=3):
"""Private helper to fetch a file from the sftp site.
:param retries: int - number of reattempts that should be made in
in case of network error etc.
e.g. for event 20110413170148 this file would be fetched::
20110413170148 directory
.. note:: If a cached copy of the file exits, the path to the cache
copy will simply be returned without invoking any network requests.
:return: A string for the dataset path on the local storage system.
:rtype: str
:raises: EventUndefinedError, NetworkError
"""
local_path = os.path.join(shakemap_cache_dir(), self.event_id)
local_parent_path = os.path.join(local_path, 'output')
xml_file = os.path.join(local_parent_path, self.file_name())
if os.path.exists(xml_file):
return local_path
# fetch from sftp
trials = [i + 1 for i in xrange(retries)]
remote_path = os.path.join(self.sftpclient.workdir_path, self.event_id)
xml_remote_path = os.path.join(remote_path, 'output', self.file_name())
for counter in trials:
last_error = None
try:
mk_dir(local_path)
mk_dir(os.path.join(local_path, 'output'))
self.sftpclient.download_path(xml_remote_path,
local_parent_path)
except NetworkError, e:
last_error = e
except:
