def test_download_path(self):
"""Test to download all directories and files under a path."""
sftp_client = SFtpClient(working_dir=temp_dir('realtime-test'))
self.assertIsNotNone(sftp_client)
# Download directories
shake_path = get_shake_test_data_path()
remote_path = os.path.abspath(
os.path.join(shake_path, SHAKE_ID))
local_path = temp_dir('realtime-test')
sftp_client.download_path(remote_path, local_path)
# Check the local_path consist of SHAKE_ID folder
expected_dir = [SHAKE_ID]
actual_dir = os.listdir(local_path)
message = "In the local path, I got: %s dir, Expectation: %s dir" % (
actual_dir, expected_dir)
self.assertEqual(expected_dir, actual_dir, message)
# Inside that SHAKE_ID folder, there should be 'output' folder
expected_dir = ['output']
actual_dir = os.listdir(os.path.join(local_path, SHAKE_ID))
message = "In the local path, I got: %s dir, Expectation: %s dir" % (
actual_dir, expected_dir)
self.assertEqual(expected_dir, actual_dir, message)
# Inside that output folder, there should be 'grid.xml' file
expected_dir = ['grid.xml']
actual_dir = os.listdir(os.path.join(local_path, SHAKE_ID, 'output'))
message = "In the local path, I got: %s dir, Expectation: %s dir" % (
actual_dir, expected_dir)
self.assertEqual(expected_dir, actual_dir, message)
def test_download_path(self):
"""Test to download all directories and files under a path."""
sftp_client = SFtpClient(working_dir=temp_dir('realtime-test'))
self.assertIsNotNone(sftp_client)
# Download directories
shake_path = get_shake_test_data_path()
remote_path = os.path.abspath(os.path.join(shake_path, SHAKE_ID))
local_path = temp_dir('realtime-test')
sftp_client.download_path(remote_path, local_path)
# Check the local_path consist of SHAKE_ID folder
expected_dir = [SHAKE_ID]
actual_dir = os.listdir(local_path)
message = "In the local path, I got: %s dir, Expectation: %s dir" % (
actual_dir, expected_dir)
self.assertEqual(expected_dir, actual_dir, message)
# Inside that SHAKE_ID folder, there should be 'output' folder
expected_dir = ['output']
actual_dir = os.listdir(os.path.join(local_path, SHAKE_ID))
message = "In the local path, I got: %s dir, Expectation: %s dir" % (
actual_dir, expected_dir)
self.assertEqual(expected_dir, actual_dir, message)
# Inside that output folder, there should be 'grid.xml' file
expected_dir = ['grid.xml']
actual_dir = os.listdir(os.path.join(local_path, SHAKE_ID, 'output'))
message = "In the local path, I got: %s dir, Expectation: %s dir" % (
actual_dir, expected_dir)
self.assertEqual(expected_dir, actual_dir, message)
def test_constructor(self):
"""Test create shake data."""
try:
event_one = SftpShakeData(working_dir=temp_dir('realtime-test'))
event_two = SftpShakeData(event=SHAKE_ID,
working_dir=temp_dir('realtime-test'))
self.assertIsNotNone(event_one)
self.assertIsNotNone(event_two)
except:
raise
def test_constructor(self):
"""Test create shake data."""
try:
event_one = SftpShakeData(working_dir=temp_dir('realtime-test'))
event_two = SftpShakeData(
event=SHAKE_ID,
working_dir=temp_dir('realtime-test'))
self.assertIsNotNone(event_one)
self.assertIsNotNone(event_two)
except:
raise
def setUp(self):
"""Setup before each test."""
# Call sftp client monkey patching before running each tests
run_monkey_patching_sftp_client()
# Download files (which are local files) to realtime-test temp folder
# AG:
# So since we're using local data, in instantiating SFTPShakeData,
# please pass the working dir to the local dir
sftp_client = SFtpClient(working_dir=temp_dir('realtime-test'))
local_path = temp_dir('realtime-test')
remote_path = os.path.abspath(
os.path.join(get_shake_test_data_path(), SHAKE_ID))
sftp_client.download_path(remote_path, local_path)
def setUp(self):
"""Setup before each test."""
# Call sftp client monkey patching before running each tests
run_monkey_patching_sftp_client()
# Download files (which are local files) to realtime-test temp folder
# AG:
# So since we're using local data, in instantiating SFTPShakeData,
# please pass the working dir to the local dir
sftp_client = SFtpClient(working_dir=temp_dir('realtime-test'))
local_path = temp_dir('realtime-test')
remote_path = os.path.abspath(
os.path.join(get_shake_test_data_path(), SHAKE_ID))
sftp_client.download_path(remote_path, local_path)
def test_get_listing(self):
"""Test get_listing if it's working correctly."""
sftp_client = SFtpClient(working_dir=temp_dir('realtime-test'))
self.assertIsNotNone(sftp_client)
# Download directories
shake_path = get_shake_test_data_path()
remote_path = os.path.abspath(os.path.join(shake_path, SHAKE_ID))
local_path = temp_dir('realtime-test')
sftp_client.download_path(remote_path, local_path)
event_ids = sftp_client.get_listing(function=is_event_id)
expected_event_ids = [SHAKE_ID]
message = 'In the local path I got %s dir, Expectation %s dir'
self.assertEqual(event_ids, expected_event_ids, message)
def test_sorted_impacted_cities(self):
"""Test getting impacted cities sorted by mmi then population."""
shake_id = '20120726022003'
shake_event = ShakeEvent(shake_id)
table = shake_event.sorted_impacted_cities()
file_path = unique_filename(
prefix='test_sorted_impacted_cities',
suffix='.txt',
dir=temp_dir('test'))
cities_file = file(file_path, 'w')
cities_file.writelines(str(table))
cities_file.close()
table = str(table).replace(', \'', ',\n\'')
table += '\n'
fixture_path = os.path.join(
data_dir(), 'tests', 'test_sorted_impacted_cities.txt')
cities_file = file(fixture_path)
expected_string = cities_file.read()
cities_file.close()
expected_string = expected_string.replace(', \'', ',\n\'')
self.max_diff = None
self.assertEqual(expected_string, table)
def testSortedImpactedCities(self):
"""Test getting impacted cities sorted by mmi then population."""
myShakeId = '20120726022003'
myShakeEvent = ShakeEvent(myShakeId)
myTable = myShakeEvent.sortedImpactedCities()
myFilePath = unique_filename(
prefix='testSortedImpactedCities',
suffix='.txt',
dir=temp_dir('test'))
myFile = file(myFilePath, 'wt')
myFile.writelines(str(myTable))
myFile.close()
myTable = str(myTable).replace(', \'', ',\n\'')
myTable += '\n'
myFixturePath = os.path.join(
dataDir(), 'tests', 'testSortedImpactedCities.txt')
myFile = file(myFixturePath, 'rt')
myExpectedString = myFile.read()
myFile.close()
myExpectedString = myExpectedString.replace(', \'', ',\n\'')
self.maxDiff = None
self.assertEqual(myExpectedString, myTable)
def test_sorted_impacted_cities(self):
"""Test getting impacted cities sorted by mmi then population."""
shake_event = ShakeEvent(SHAKE_ID, data_is_local_flag=True)
table = shake_event.sorted_impacted_cities()
file_path = unique_filename(
prefix='test_sorted_impacted_cities',
suffix='.txt',
dir=temp_dir('test'))
cities_file = file(file_path, 'w')
cities_file.writelines(str(table))
cities_file.close()
table = str(table).replace(', \'', ',\n\'')
table += '\n'
fixture_path = os.path.join(
data_dir(), 'tests', 'test_sorted_impacted_cities.txt')
cities_file = file(fixture_path)
expected_string = cities_file.read()
cities_file.close()
expected_string = expected_string.replace(', \'', ',\n\'')
self.max_diff = None
message = 'Expectation:\n%s, Got\n%s' % (expected_string, table)
self.assertEqual(expected_string, table, message)
def test_get_listing(self):
"""Test get_listing if it's working correctly."""
sftp_client = SFtpClient(working_dir=temp_dir('realtime-test'))
self.assertIsNotNone(sftp_client)
# Download directories
shake_path = get_shake_test_data_path()
remote_path = os.path.abspath(
os.path.join(shake_path, SHAKE_ID))
local_path = temp_dir('realtime-test')
sftp_client.download_path(remote_path, local_path)
event_ids = sftp_client.get_listing(function=is_event_id)
expected_event_ids = [SHAKE_ID]
message = 'In the local path I got %s dir, Expectation %s dir'
self.assertEqual(event_ids, expected_event_ids, message)
def testSortedImpactedCities(self):
"""Test getting impacted cities sorted by mmi then population."""
myShakeId = '20120726022003'
myShakeEvent = ShakeEvent(myShakeId)
myTable = myShakeEvent.sortedImpactedCities()
myFilePath = unique_filename(
prefix='testSortedImpactedCities',
suffix='.txt',
dir=temp_dir('test'))
myFile = file(myFilePath, 'wt')
myFile.writelines(str(myTable))
myFile.close()
myTable = str(myTable).replace(', \'', ',\n\'')
myTable += '\n'
myFixturePath = os.path.join(
dataDir(), 'tests', 'testSortedImpactedCities.txt')
myFile = file(myFixturePath, 'rt')
myExpectedString = myFile.read()
myFile.close()
myExpectedString = myExpectedString.replace(', \'', ',\n\'')
self.maxDiff = None
self.assertEqual(myExpectedString, myTable)
def test_sorted_impacted_cities(self):
"""Test getting impacted cities sorted by mmi then population."""
shake_event = ShakeEvent(SHAKE_ID, data_is_local_flag=True)
table = shake_event.sorted_impacted_cities()
file_path = unique_filename(
prefix='test_sorted_impacted_cities',
suffix='.txt',
dir=temp_dir('test'))
cities_file = file(file_path, 'w')
cities_file.writelines(str(table))
cities_file.close()
table = str(table).replace(', \'', ',\n\'')
table += '\n'
fixture_path = os.path.join(
data_dir(), 'tests', 'test_sorted_impacted_cities.txt')
cities_file = file(fixture_path)
expected_string = cities_file.read()
cities_file.close()
expected_string = expected_string.replace(', \'', ',\n\'')
self.max_diff = None
message = 'Expectation:\n%s, Got\n%s' % (expected_string, table)
self.assertEqual(expected_string, table, message)
def test_extract(self):
"""Test extracting data to be used in earth quake realtime."""
sftp_shake_data = SftpShakeData(working_dir=temp_dir('realtime-test'))
sftp_shake_data.extract()
final_grid_xml_file = os.path.join(sftp_shake_data.extract_dir(),
'grid.xml')
self.assertTrue(os.path.exists(final_grid_xml_file),
'grid.xml not found')
def test_fetch_file(self):
"""Test fetch data."""
sftp_shake_data = SftpShakeData(working_dir=temp_dir('realtime-test'))
local_path = sftp_shake_data.fetch_file()
expected_path = os.path.join(shakemap_cache_dir(), SHAKE_ID)
message = 'File should be fetched to %s, I got %s' % (expected_path,
local_path)
self.assertEqual(local_path, expected_path, message)
def test_fetch_file(self):
"""Test fetch data."""
sftp_shake_data = SftpShakeData(working_dir=temp_dir('realtime-test'))
local_path = sftp_shake_data.fetch_file()
expected_path = os.path.join(shakemap_cache_dir(), SHAKE_ID)
message = 'File should be fetched to %s, I got %s' % (
expected_path, local_path)
self.assertEqual(local_path, expected_path, message)
def test_extract(self):
"""Test extracting data to be used in earth quake realtime."""
sftp_shake_data = SftpShakeData(working_dir=temp_dir('realtime-test'))
sftp_shake_data.extract()
final_grid_xml_file = os.path.join(
sftp_shake_data.extract_dir(), 'grid.xml')
self.assertTrue(
os.path.exists(final_grid_xml_file), 'grid.xml not found')
def test_get_list_event_ids(self):
"""Test get list event id."""
sftp_shake_data = SftpShakeData(working_dir=temp_dir('realtime-test'))
list_id = sftp_shake_data.get_list_event_ids()
expected_list_id = [SHAKE_ID]
message = 'I got %s for the event ID in the server, Expectation %s' % (
list_id, expected_list_id)
self.assertEqual(list_id, expected_list_id, message)
def test_get_list_event_ids(self):
"""Test get list event id."""
sftp_shake_data = SftpShakeData(working_dir=temp_dir('realtime-test'))
list_id = sftp_shake_data.get_list_event_ids()
expected_list_id = [SHAKE_ID]
message = 'I got %s for the event ID in the server, Expectation %s' % (
list_id, expected_list_id)
self.assertEqual(list_id, expected_list_id, message)
def test_get_latest_event_id(self):
"""Test get latest event id."""
sftp_shake_data = SftpShakeData(working_dir=temp_dir('realtime-test'))
latest_id = sftp_shake_data.get_latest_event_id()
# The latest event ID should be = SHAKE_ID since there's only one
expected_event_id = SHAKE_ID
message = 'I got %s for this latest event id, Expectation %s' % (
latest_id, expected_event_id)
self.assertEqual(expected_event_id, latest_id, message)
def test_get_latest_event_id(self):
"""Test get latest event id."""
sftp_shake_data = SftpShakeData(working_dir=temp_dir('realtime-test'))
latest_id = sftp_shake_data.get_latest_event_id()
# The latest event ID should be = SHAKE_ID since there's only one
expected_event_id = SHAKE_ID
message = 'I got %s for this latest event id, Expectation %s' % (
latest_id, expected_event_id)
self.assertEqual(expected_event_id, latest_id, message)
def test_get_file(self):
"""Test that the ftp client can fetch a file ok"""
client = FtpClient()
local_path = os.path.join(temp_dir('realtime-test'),
'20120726022003.inp.zip')
client.get_file('20120726022003.inp.zip', local_path)
message = 'Function get_file is not working correctly.'
self.assertTrue(os.path.exists(local_path), message)
def test_get_file(self):
"""Test that the ftp client can fetch a file ok"""
client = FtpClient()
local_path = os.path.join(
temp_dir('realtime-test'),
'20120726022003.inp.zip')
client.get_file('20120726022003.inp.zip', local_path)
message = 'Function get_file is not working correctly.'
self.assertTrue(os.path.exists(local_path), message)
def test_reconnect_sftp(self):
"""Test to reconnect SFTP."""
sftp_shake_data = SftpShakeData(working_dir=temp_dir('realtime-test'))
sftp_client = sftp_shake_data.sftp_client
sftp_shake_data.reconnect_sftp()
new_sftp_client = sftp_shake_data.sftp_client
message = 'Oh no, we got the same sftp client after reconnecting!'
self.assertNotEqual(sftp_client, new_sftp_client, message)
message = 'Oh dear, the new sftp object is None after reconnecting'
self.assertIsNotNone(new_sftp_client, message)
def test_reconnect_sftp(self):
"""Test to reconnect SFTP."""
sftp_shake_data = SftpShakeData(working_dir=temp_dir('realtime-test'))
sftp_client = sftp_shake_data.sftp_client
sftp_shake_data.reconnect_sftp()
new_sftp_client = sftp_shake_data.sftp_client
message = 'Oh no, we got the same sftp client after reconnecting!'
self.assertNotEqual(sftp_client, new_sftp_client, message)
message = 'Oh dear, the new sftp object is None after reconnecting'
self.assertIsNotNone(new_sftp_client, message)
def test_local_cities(self):
"""Test that we can retrieve the cities local to the event"""
shake_id = '20120726022003'
shake_event = ShakeEvent(shake_id)
# Get teh mem layer
cities_layer = shake_event.local_cities_memory_layer()
provider = cities_layer.dataProvider()
expected_feature_count = 6
self.assertEquals(provider.featureCount(), expected_feature_count)
strings = []
request = QgsFeatureRequest()
for feature in cities_layer.getFeatures(request):
# fetch map of attributes
attributes = cities_layer.dataProvider().attributeIndexes()
for attribute_key in attributes:
strings.append("%d: %s\n" % (
attribute_key, feature[attribute_key].toString()))
strings.append('------------------\n')
LOGGER.debug('Mem table:\n %s' % strings)
file_path = unique_filename(prefix='test_local_cities',
suffix='.txt',
dir=temp_dir('test'))
cities_file = file(file_path, 'w')
cities_file.writelines(strings)
cities_file.close()
fixture_path = os.path.join(data_dir(),
'tests',
'test_local_cities.txt')
cities_file = file(fixture_path)
expected_string = cities_file.readlines()
cities_file.close()
diff = difflib.unified_diff(expected_string, strings)
diff_list = list(diff)
diff_string = ''
for _, myLine in enumerate(diff_list):
diff_string += myLine
message = ('Diff is not zero length:\n'
'Control file: %s\n'
'Test file: %s\n'
'Diff:\n%s' %
(fixture_path,
file_path,
diff_string))
self.assertEqual(diff_string, '', message)
def testLocalCities(self):
"""Test that we can retrieve the cities local to the event"""
myShakeId = '20120726022003'
myShakeEvent = ShakeEvent(myShakeId)
# Get teh mem layer
myCitiesLayer = myShakeEvent.localCitiesMemoryLayer()
myProvider = myCitiesLayer.dataProvider()
myFeature = QgsFeature()
myAttributes = myProvider.attributeIndexes()
myProvider.select(myAttributes)
myExpectedFeatureCount = 6
self.assertEquals(myProvider.featureCount(), myExpectedFeatureCount)
myStrings = []
while myProvider.nextFeature(myFeature):
# fetch map of attributes
myAttributes = myFeature.attributeMap()
for (myKey, myValue) in myAttributes.iteritems():
myStrings.append("%d: %s\n" % (myKey, myValue.toString()))
myStrings.append('------------------\n')
LOGGER.debug('Mem table:\n %s' % myStrings)
myFilePath = unique_filename(prefix='testLocalCities',
suffix='.txt',
dir=temp_dir('test'))
myFile = file(myFilePath, 'wt')
myFile.writelines(myStrings)
myFile.close()
myFixturePath = os.path.join(dataDir(), 'tests', 'testLocalCities.txt')
myFile = file(myFixturePath, 'rt')
myExpectedString = myFile.readlines()
myFile.close()
myDiff = difflib.unified_diff(myStrings, myExpectedString)
myDiffList = list(myDiff)
myDiffString = ''
for _, myLine in enumerate(myDiffList):
myDiffString += myLine
myMessage = ('Diff is not zero length:\n'
'Control file: %s\n'
'Test file: %s\n'
'Diff:\n%s'
% (myFixturePath,
myFilePath,
myDiffString))
self.assertEqual(myDiffString, '', myMessage)
def testLocalCities(self):
"""Test that we can retrieve the cities local to the event"""
myShakeId = '20120726022003'
myShakeEvent = ShakeEvent(myShakeId)
# Get teh mem layer
myCitiesLayer = myShakeEvent.localCitiesMemoryLayer()
myProvider = myCitiesLayer.dataProvider()
myExpectedFeatureCount = 6
self.assertEquals(myProvider.featureCount(), myExpectedFeatureCount)
myStrings = []
myRequest = QgsFeatureRequest()
for myFeature in myCitiesLayer.getFeatures(myRequest):
# fetch map of attributes
myAttributes = myCitiesLayer.dataProvider().attributeIndexes()
for myKey in myAttributes:
myStrings.append("%d: %s\n" % (
myKey, myFeature[myKey].toString()))
myStrings.append('------------------\n')
LOGGER.debug('Mem table:\n %s' % myStrings)
myFilePath = unique_filename(prefix='testLocalCities',
suffix='.txt',
dir=temp_dir('test'))
myFile = file(myFilePath, 'wt')
myFile.writelines(myStrings)
myFile.close()
myFixturePath = os.path.join(dataDir(), 'tests', 'testLocalCities.txt')
myFile = file(myFixturePath, 'rt')
myExpectedString = myFile.readlines()
myFile.close()
myDiff = difflib.unified_diff(myExpectedString, myStrings)
myDiffList = list(myDiff)
myDiffString = ''
for _, myLine in enumerate(myDiffList):
myDiffString += myLine
myMessage = ('Diff is not zero length:\n'
'Control file: %s\n'
'Test file: %s\n'
'Diff:\n%s'
% (myFixturePath,
myFilePath,
myDiffString))
self.assertEqual(myDiffString, '', myMessage)
def tearDown(self):
"""Action after each test is called."""
# Delete the files that we make in the init for the shake data
shutil.rmtree(temp_dir('realtime-test'))
def setUp(self):
"""Setup before each test call."""
run_monkey_patching_sftp_client()
# Make temp dir
temp_dir('realtime-test')
def test_is_on_server(self):
"""Test to check if a event is in server."""
sftp_shake_data = SftpShakeData(working_dir=temp_dir('realtime-test'))
message = 'Event does not exist in the server.'
self.assertTrue(sftp_shake_data.is_on_server(), message)
def setUp(self):
"""Run before calling each test."""
run_monkey_patching_ftp_client()
temp_dir('realtime-test')
def test_is_on_server(self):
"""Test to check if a event is in server."""
sftp_shake_data = SftpShakeData(working_dir=temp_dir('realtime-test'))
message = 'Event does not exist in the server.'
self.assertTrue(sftp_shake_data.is_on_server(), message)
def setUp(self):
"""Run before calling each test."""
run_monkey_patching_ftp_client()
temp_dir('realtime-test')
def tearDown(self):
"""Action after each test is called."""
# Delete the files that we make in the init for the shake data
shutil.rmtree(temp_dir('realtime-test'))
def setUp(self):
"""Setup before each test call."""
run_monkey_patching_sftp_client()
# Make temp dir
temp_dir('realtime-test')
def tearDown(self):
"""Action after each test call."""
# Delete temp dir
shutil.rmtree(temp_dir('realtime-test'))
def tearDown(self):
"""Action after each test is called."""
# Delete temp dir
shutil.rmtree(temp_dir('realtime-test'))