myLocale = os.environ['INASAFE_LOCALE'] else: myLocale = 'en' if len(sys.argv) > 2: sys.exit('Usage:\n%s [optional shakeid]\nor\n%s --list' % ( sys.argv[0], sys.argv[0])) elif len(sys.argv) == 2: print('Processing shakemap %s' % sys.argv[1]) myEventId = sys.argv[1] if myEventId in '--list': # myFtpClient = FtpClient() mySftpClient = SFtpClient() # myListing = myFtpClient.getListing() myListing = mySftpClient.getListing(my_func=is_event_id) for myEvent in myListing: print myEvent sys.exit(0) elif myEventId in '--run-all': # # Caution, this code path gets memory leaks, use the # batch file approach rather! # myFtpClient = FtpClient() myListing = myFtpClient.getListing() for myEvent in myListing: if 'out' not in myEvent: continue myEvent = myEvent.replace('ftp://118.97.83.243/', '') myEvent = myEvent.replace('.out.zip', '')
class SftpShakeData: """A class for retrieving, reading data from shakefiles. Shake files are provide on server and can be accessed using SSH protocol. The shape files currently located under shakemaps directory in a folder named by the event id (which represent the timestamp of the event of the shake) There are numerous files in that directory but there is only really one that we are interested in: * grid.xml - which contains all the metadata pertaining to the event It's located in under output/grid.xml under each event directory The remaining files are fetched for completeness and possibly use in the future. This class provides a high level interface for retrieving this data and then extracting various by products from it Note : * inspired by shake_data.py but modified according to SSH protocol """ def __init__(self, theEvent=None, theHost=defaultHost, theUserName=defUserName, thePassword=defPassword, theWorkingDir=defWorkDir, theForceFlag=False): """Constructor for the SftpShakeData class Args: * theEvent - (Optional) a string representing the event id that this raster is associated with. e.g. 20110413170148. **If no event id is supplied, a query will be made to the ftp server, and the latest event id assigned.** * theHost - (Optional) a string representing the ip address or host name of the server from which the data should be retrieved. It assumes that the data is in the root directory. Returns: None Raises: None """ self.eventId = theEvent self.host = theHost self.username = theUserName self.password = thePassword self.workdir = theWorkingDir self.forceFlag = theForceFlag self.sftpclient = SFtpClient( self.host, self.username, self.password, self.workdir) if self.eventId is None: try: self.getLatestEventId() except NetworkError: raise else: # If we fetched it above using getLatestEventId we assume it is # already validated. try: self.validateEvent() except EventValidationError: raise # If eventId is still None after all the above, moan.... if self.eventId is None: myMessage = ('No id was passed to the constructor and the ' 'latest id could not be retrieved from the' 'server.') LOGGER.exception('ShakeData initialisation failed') raise EventIdError(myMessage) def reconnectSFTP(self): """Reconnect to the server """ self.sftpclient = SFtpClient( self.host, self.username, self.password, self.workdir) def validateEvent(self): """Check that the event associated with this instance exists either in the local event cache, or on the remote ftp site. Args: None Returns: True if valid, False if not Raises: NetworkError """ # First check local cache if self.isCached(): return True else: return self.isOnServer() def isCached(self): """Check the event associated with this instance exists in cache. Args: None Returns: True if locally cached, False if not Raises: None """ myXMLFilePath = self.cachePaths() if os.path.exists(myXMLFilePath): return True else: LOGGER.debug('%s is not cached' % myXMLFilePath) return False def cachePaths(self): """Return the paths to the inp and out files as expected locally. Args: None Returns: str: grid.xml local cache paths. Raises: None """ myXMLFileName = self.fileName() myXMLFilePath = os.path.join( shakemapCacheDir(), self.eventId, myXMLFileName) return myXMLFilePath def fileName(self): """Return file names for the inp and out files based on the event id. for this class, only the grid.xml only Args: None Returns: str: grid.xml Raises: None """ return 'grid.xml' def isOnServer(self): """Check the event associated with this instance exists on the server. Args: None Returns: True if valid, False if not Raises: NetworkError """ myRemoteXMLPath = os.path.join( self.sftpclient.workdir_path, self.eventId) return self.sftpclient.is_path_exist(myRemoteXMLPath) def get_list_event_ids(self): """Get all event id indicated by folder in remote_path """ dirs = self.sftpclient.getListing(my_func=is_event_id) if len(dirs) == 0: raise Exception('List event is empty') return dirs def getLatestEventId(self): """Return latest event id """ event_ids = self.get_list_event_ids() latest_event_id = None if event_ids is not None: event_ids.sort() latest_event_id = event_ids[-1] if latest_event_id is None: raise EventIdError('Latest Event Id could not be obtained') self.eventId = latest_event_id return self.eventId def fetchFile(self, theRetries=3): """Private helper to fetch a file from the sftp site. :param theRetries: int - number of reattempts that should be made in in case of network error etc. e.g. for event 20110413170148 this file would be fetched:: 20110413170148 directory .. note:: If a cached copy of the file exits, the path to the cache copy will simply be returned without invoking any network requests. Returns: str: A string for the dataset path on the local storage system. Raises: EventUndefinedError, NetworkError """ myLocalPath = os.path.join(shakemapCacheDir(), self.eventId) myLocalParentPath = os.path.join(myLocalPath, 'output') myXMLFile = os.path.join(myLocalParentPath, self.fileName()) if os.path.exists(myXMLFile): return myLocalPath # fetch from sftp trials = [i + 1 for i in xrange(theRetries)] remote_path = os.path.join(self.sftpclient.workdir_path, self.eventId) myXMLRemotePath = os.path.join(remote_path, 'output', self.fileName()) for my_counter in trials: myLastError = None try: mkDir(myLocalPath) mkDir(os.path.join(myLocalPath, 'output')) self.sftpclient.download_path( myXMLRemotePath, myLocalParentPath) except NetworkError, e: myLastError = e except:
class SftpShakeData: """A class for retrieving, reading data from shakefiles. Shake files are provide on server and can be accessed using SSH protocol. The shape files currently located under shakemaps directory in a folder named by the event id (which represent the timestamp of the event of the shake) There are numerous files in that directory but there is only really one that we are interested in: * grid.xml - which contains all the metadata pertaining to the event It's located in under output/grid.xml under each event directory The remaining files are fetched for completeness and possibly use in the future. This class provides a high level interface for retrieving this data and then extracting various by products from it Note : * inspired by shake_data.py but modified according to SSH protocol """ def __init__(self, theEvent=None, theHost=defaultHost, theUserName=defUserName, thePassword=defPassword, theWorkingDir=defWorkDir, theForceFlag=False): """Constructor for the SftpShakeData class Args: * theEvent - (Optional) a string representing the event id that this raster is associated with. e.g. 20110413170148. **If no event id is supplied, a query will be made to the ftp server, and the latest event id assigned.** * theHost - (Optional) a string representing the ip address or host name of the server from which the data should be retrieved. It assumes that the data is in the root directory. Returns: None Raises: None """ self.eventId = theEvent self.host = theHost self.username = theUserName self.password = thePassword self.workdir = theWorkingDir self.forceFlag = theForceFlag self.sftpclient = SFtpClient(self.host, self.username, self.password, self.workdir) if self.eventId is None: try: self.getLatestEventId() except NetworkError: raise else: # If we fetched it above using getLatestEventId we assume it is # already validated. try: self.validateEvent() except EventValidationError: raise # If eventId is still None after all the above, moan.... if self.eventId is None: myMessage = ('No id was passed to the constructor and the ' 'latest id could not be retrieved from the' 'server.') LOGGER.exception('ShakeData initialisation failed') raise EventIdError(myMessage) def reconnectSFTP(self): """Reconnect to the server """ self.sftpclient = SFtpClient(self.host, self.username, self.password, self.workdir) def validateEvent(self): """Check that the event associated with this instance exists either in the local event cache, or on the remote ftp site. Args: None Returns: True if valid, False if not Raises: NetworkError """ # First check local cache if self.isCached(): return True else: return self.isOnServer() def isCached(self): """Check the event associated with this instance exists in cache. Args: None Returns: True if locally cached, False if not Raises: None """ myXMLFilePath = self.cachePaths() if os.path.exists(myXMLFilePath): return True else: LOGGER.debug('%s is not cached' % myXMLFilePath) return False def cachePaths(self): """Return the paths to the inp and out files as expected locally. Args: None Returns: str: grid.xml local cache paths. Raises: None """ myXMLFileName = self.fileName() myXMLFilePath = os.path.join(shakemapCacheDir(), self.eventId, myXMLFileName) return myXMLFilePath def fileName(self): """Return file names for the inp and out files based on the event id. for this class, only the grid.xml only Args: None Returns: str: grid.xml Raises: None """ return 'grid.xml' def isOnServer(self): """Check the event associated with this instance exists on the server. Args: None Returns: True if valid, False if not Raises: NetworkError """ myRemoteXMLPath = os.path.join(self.sftpclient.workdir_path, self.eventId) return self.sftpclient.is_path_exist(myRemoteXMLPath) def get_list_event_ids(self): """Get all event id indicated by folder in remote_path """ dirs = self.sftpclient.getListing(my_func=is_event_id) if len(dirs) == 0: raise Exception('List event is empty') return dirs def getLatestEventId(self): """Return latest event id """ event_ids = self.get_list_event_ids() latest_event_id = None if event_ids is not None: event_ids.sort() latest_event_id = event_ids[-1] if latest_event_id is None: raise EventIdError('Latest Event Id could not be obtained') self.eventId = latest_event_id return self.eventId def fetchFile(self, theRetries=3): """Private helper to fetch a file from the sftp site. e.g. for event 20110413170148 this file would be fetched:: 20110413170148 directory .. note:: If a cached copy of the file exits, the path to the cache copy will simply be returned without invoking any network requests. Args: * theEventFile: str - filename on server e.g.20110413170148.inp.zip * theRetries: int - number of reattempts that should be made in in case of network error etc. Returns: str: A string for the dataset path on the local storage system. Raises: EventUndefinedError, NetworkError """ myLocalPath = os.path.join(shakemapCacheDir(), self.eventId) myLocalParentPath = os.path.join(myLocalPath, 'output') myXMLFile = os.path.join(myLocalParentPath, self.fileName()) if os.path.exists(myXMLFile): return myLocalPath # fetch from sftp trials = [i + 1 for i in xrange(theRetries)] remote_path = os.path.join(self.sftpclient.workdir_path, self.eventId) myXMLRemotePath = os.path.join(remote_path, 'output', self.fileName()) for my_counter in trials: myLastError = None try: mkDir(myLocalPath) mkDir(os.path.join(myLocalPath, 'output')) self.sftpclient.download_path(myXMLRemotePath, myLocalParentPath) except NetworkError, e: myLastError = e except: