def get_latest_event_id(self):
"""Query the ftp server and determine the latest event id.
:return: A string containing a valid event id.
:raises: NetworkError
"""
ftp_client = FtpClient()
try:
ftp_client_list = ftp_client.get_listing()
ftp_client_list.sort(key=lambda x: x.lower())
except NetworkError:
raise
now = datetime.now()
now = int(
'%04d%02d%02d%02d%02d%02d' % (
now.year, now.month, now.day, now.hour, now.minute, now.second
))
event_id = now + 1
while int(event_id) > now:
if len(ftp_client_list) < 1:
raise EventIdError('Latest Event Id could not be obtained')
event_id = ftp_client_list.pop().split('/')[-1].split('.')[0]
if event_id is None:
raise EventIdError('Latest Event Id could not be obtained')
self.event_id = event_id
def get_latest_event_id(self):
"""Query the ftp server and determine the latest event id.
:return: A string containing a valid event id.
:raises: NetworkError
"""
ftp_client = FtpClient()
try:
ftp_client_list = ftp_client.get_listing()
ftp_client_list.sort(key=lambda x: x.lower())
except NetworkError:
raise
now = datetime.now()
now = int(
'%04d%02d%02d%02d%02d%02d' %
(now.year, now.month, now.day, now.hour, now.minute, now.second))
event_id = now + 1
while int(event_id) > now:
if len(ftp_client_list) < 1:
raise EventIdError('Latest Event Id could not be obtained')
event_id = ftp_client_list.pop().split('/')[-1].split('.')[0]
if event_id is None:
raise EventIdError('Latest Event Id could not be obtained')
self.event_id = event_id
def testGetFile(self):
"""Test that the ftp client can fetch a file ok"""
myClient = FtpClient()
myListing = myClient.getListing()
# Make it a single string
myListing = "\n".join(myListing)
myMessage = "Expected outcome:\n%s\nActual outcome:\n%s" % (myListing, self._expectedMatches)
for myExpectedFile in self._expectedMatches:
assert re.search(myExpectedFile, myListing), myMessage
def test_get_file(self):
"""Test that the ftp client can fetch a file ok"""
client = FtpClient()
local_path = os.path.join(temp_dir('realtime-test'),
'20120726022003.inp.zip')
client.get_file('20120726022003.inp.zip', local_path)
message = 'Function get_file is not working correctly.'
self.assertTrue(os.path.exists(local_path), message)
def testGetDirectoryListing(self):
"""Check if we can get a nice directory listing"""
myClient = FtpClient()
myListing = myClient.getListing()
# Make it a single string
myListing = "\n".join(myListing)
myMessage = "Expected this list:\n%s\nTo contain these items:\n%s" % (myListing, self._expectedMatches)
for myExpectedFile in self._expectedMatches:
assert re.search(myExpectedFile, myListing), myMessage
def test_get_file(self):
"""Test that the ftp client can fetch a file ok"""
client = FtpClient()
local_path = os.path.join(
temp_dir('realtime-test'),
'20120726022003.inp.zip')
client.get_file('20120726022003.inp.zip', local_path)
message = 'Function get_file is not working correctly.'
self.assertTrue(os.path.exists(local_path), message)
def test_get_directory_listing(self):
"""Check if we can get a nice directory listing"""
client = FtpClient()
file_list = client.get_listing()
#Make it a single string
file_list = '\n'.join(file_list)
message = ('Expected this list:\n%s\nTo contain these items:\n%s' %
(file_list, self._expectedMatches))
for expected_file in self._expectedMatches:
assert re.search(expected_file, file_list), message
def test_get_file(self):
"""Test that the ftp client can fetch a file ok"""
client = FtpClient()
file_list = client.get_listing()
#Make it a single string
file_list = '\n'.join(file_list)
message = ('Expected outcome:\n%s\nActual outcome:\n%s' %
(file_list, self._expectedMatches))
for expected_file in self._expectedMatches:
assert re.search(expected_file, file_list), message
def is_on_server(self):
"""Check the event associated with this instance exists on the server.
:return: True if valid, False if not
:raises: NetworkError
"""
input_file_name, output_file_name = self.file_names()
file_list = [input_file_name, output_file_name]
ftp_client = FtpClient()
return ftp_client.has_files(file_list)
def test_get_file(self):
"""Test that the ftp client can fetch a file ok"""
client = FtpClient()
file_list = client.get_listing()
#Make it a single string
file_list = '\n'.join(file_list)
message = (
'Expected outcome:\n%s\nActual outcome:\n%s' %
(file_list, self._expectedMatches))
for expected_file in self._expectedMatches:
assert re.search(expected_file, file_list), message
def test_get_directory_listing(self):
"""Check if we can get a nice directory listing"""
client = FtpClient()
file_list = client.get_listing()
#Make it a single string
file_list = '\n'.join(file_list)
message = (
'Expected this list:\n%s\nTo contain these items:\n%s' %
(file_list, self._expectedMatches))
for expected_file in self._expectedMatches:
assert re.search(expected_file, file_list), message
def testGetFile(self):
"""Test that the ftp client can fetch a file ok"""
myClient = FtpClient()
myListing = myClient.getListing()
#Make it a single string
myListing = '\n'.join(myListing)
myMessage = (
'Expected outcome:\n%s\nActual outcome:\n%s' %
(myListing, self._expectedMatches))
for myExpectedFile in self._expectedMatches:
assert re.search(myExpectedFile, myListing), myMessage
def testGetDirectoryListing(self):
"""Check if we can get a nice directory listing"""
myClient = FtpClient()
myListing = myClient.getListing()
#Make it a single string
myListing = '\n'.join(myListing)
myMessage = (
'Expected this list:\n%s\nTo contain these items:\n%s' %
(myListing, self._expectedMatches))
for myExpectedFile in self._expectedMatches:
assert re.search(myExpectedFile, myListing), myMessage
def test_get_directory_listing(self):
"""Check if we can get a nice directory listing"""
client = FtpClient()
file_list = client.get_listing()
# Make it a single string
file_list = '\n'.join(file_list)
expected_output = ['20120726022003.inp.zip', '20120726022003.out.zip']
message = ('Expected this list:\n%s\nTo contain these items:\n%s' %
(file_list, expected_output))
for expected_file in expected_output:
assert re.search(expected_file, file_list), message
def is_on_server(self):
"""Check the event associated with this instance exists on the server.
:return: True if valid, False if not
:raises: NetworkError
"""
input_file_name, output_file_name = self.file_names()
file_list = [input_file_name, output_file_name]
ftp_client = FtpClient()
return ftp_client.has_files(file_list)
def test_get_directory_listing(self):
"""Check if we can get a nice directory listing"""
client = FtpClient()
file_list = client.get_listing()
# Make it a single string
file_list = '\n'.join(file_list)
expected_output = ['20120726022003.inp.zip', '20120726022003.out.zip']
message = (
'Expected this list:\n%s\nTo contain these items:\n%s' %
(file_list, expected_output))
for expected_file in expected_output:
assert re.search(expected_file, file_list), message
def _fetch_file(self, event_file, retries=3):
"""Private helper to fetch a file from the ftp site.
e.g. for event 20110413170148 this file would be fetched::
ftp://118.97.83.243/20110413170148.inp.zip
and this local file created::
/tmp/realtime/20110413170148.inp.zip
.. note:: If a cached copy of the file exits, the path to the cache
copy will simply be returned without invoking any network requests.
:param event_file: Filename on server e.g.20110413170148.inp.zip
:type event_file: str
:param retries: Number of reattempts that should be made in
in case of network error etc.
:type retries: int
:return: A string for the dataset path on the local storage system.
:rtype: str
:raises: EventUndefinedError, NetworkError
"""
# Return the cache copy if it exists
local_path = os.path.join(shakemap_zip_dir(), event_file)
if os.path.exists(local_path):
return local_path
#Otherwise try to fetch it using ftp
for counter in range(retries):
last_error = None
try:
client = FtpClient()
client.get_file(event_file, local_path)
except NetworkError, e:
last_error = e
except:
def _fetch_file(self, event_file, retries=3):
"""Private helper to fetch a file from the ftp site.
e.g. for event 20110413170148 this file would be fetched::
ftp://118.97.83.243/20110413170148.inp.zip
and this local file created::
/tmp/realtime/20110413170148.inp.zip
.. note:: If a cached copy of the file exits, the path to the cache
copy will simply be returned without invoking any network requests.
:param event_file: Filename on server e.g.20110413170148.inp.zip
:type event_file: str
:param retries: Number of reattempts that should be made in
in case of network error etc.
:type retries: int
:return: A string for the dataset path on the local storage system.
:rtype: str
:raises: EventUndefinedError, NetworkError
"""
# Return the cache copy if it exists
local_path = os.path.join(shakemap_zip_dir(), event_file)
if os.path.exists(local_path):
return local_path
# Otherwise try to fetch it using ftp
for counter in range(retries):
last_error = None
try:
client = FtpClient()
client.get_file(event_file, local_path)
except NetworkError, e:
last_error = e
except:
def testHasFiles(self):
"""Test that the ftp client can check if a list of file exists"""
myClient = FtpClient()
myFiles = ["20120726022003.inp.zip", "20120726022003.out.zip"]
myMessage = "Expected that %s exist on the server" % myFiles
self.assertTrue(myClient.hasFiles(myFiles), myMessage)
def testHasFile(self):
"""Test that the ftp client can check if a file exists"""
myClient = FtpClient()
myFile = '20120726022003.inp.zip'
myMessage = ('Expected that %s exists on the server' % myFile)
self.assertTrue(myClient.hasFile(myFile), myMessage)
def testHasFile(self):
"""Test that the ftp client can check if a file exists"""
myClient = FtpClient()
myFile = "20120726022003.inp.zip"
myMessage = "Expected that %s exists on the server" % myFile
self.assertTrue(myClient.hasFile(myFile), myMessage)
def testHasFiles(self):
"""Test that the ftp client can check if a list of file exists"""
myClient = FtpClient()
myFiles = ['20120726022003.inp.zip', '20120726022003.out.zip']
myMessage = ('Expected that %s exist on the server' % myFiles)
self.assertTrue(myClient.hasFiles(myFiles), myMessage)
def test_has_files(self):
"""Test that the ftp client can check if a list of file exists"""
client = FtpClient()
input_files = ['20120726022003.inp.zip', '20120726022003.out.zip']
message = ('Expected that %s exist on the server' % input_files)
self.assertTrue(client.has_files(input_files), message)
def test_has_file(self):
"""Test that the ftp client can check if a file exists"""
client = FtpClient()
input_file = '20120726022003.inp.zip'
message = ('Expected that %s exists on the server' % input_file)
self.assertTrue(client.has_file(input_file), message)
def test_has_file(self):
"""Test that the ftp client can check if a file exists"""
client = FtpClient()
input_file = '20120726022003.inp.zip'
message = ('Expected that %s exists on the server' % input_file)
self.assertTrue(client.has_file(input_file), message)
def test_has_files(self):
"""Test that the ftp client can check if a list of file exists"""
client = FtpClient()
input_files = ['20120726022003.inp.zip', '20120726022003.out.zip']
message = ('Expected that %s exist on the server' % input_files)
self.assertTrue(client.has_files(input_files), message)
