def get_results(year_range, results_name):
'''Gets the results for the given years (list)'''
results_manager = StormtracksResultsManager(results_name)
max_index = 0
fields = []
best_tracks = []
for year in year_range:
print(year)
all_fields = results_manager.get_result(year, 'all_fields')
all_fields['t_anom'] = all_fields.t9950 - all_fields.t850
best_track_matches = results_manager.get_result(
year, 'best_track_matches')
all_fields.index += max_index
best_track_matches.index += max_index
max_index = all_fields.index.max() + 1
fields.append(all_fields)
best_tracks.append(best_track_matches)
print('Concatenating')
fields = pd.concat(fields)
best_tracks = pd.concat(best_tracks)
print('Joining')
combined_fields_matches = fields.join(best_tracks)
return combined_fields_matches
def get_results(year_range, results_name):
'''Gets the results for the given years (list)'''
results_manager = StormtracksResultsManager(results_name)
max_index = 0
fields = []
best_tracks = []
for year in year_range:
print(year)
all_fields = results_manager.get_result(year, 'all_fields')
all_fields['t_anom'] = all_fields.t9950 - all_fields.t850
best_track_matches = results_manager.get_result(year, 'best_track_matches')
all_fields.index += max_index
best_track_matches.index += max_index
max_index = all_fields.index.max() + 1
fields.append(all_fields)
best_tracks.append(best_track_matches)
print('Concatenating')
fields = pd.concat(fields)
best_tracks = pd.concat(best_tracks)
print('Joining')
combined_fields_matches = fields.join(best_tracks)
return combined_fields_matches
def do_vort_tracking(self, year, ensemble_member):
log.info(
'Received request for vort_tracking for year {0} ensemble {1}'.
format(year, ensemble_member))
if year in self.best_tracks_by_year.keys():
best_tracks = self.best_tracks_by_year[year]
else:
log.info('Loading best_tracks for year {0}'.format(year))
ibt = IbtracsData(verbose=False)
best_tracks = ibt.load_ibtracks_year(year)
self.best_tracks_by_year[year] = best_tracks
results_manager = StormtracksResultsManager('pyro_vort_tracking')
start = time.time()
c20data = C20Data(year, verbose=False)
gdata = GlobalEnsembleMember(c20data, ensemble_member)
log.debug('Processing')
vort_finder = VortmaxFinder(gdata)
vort_finder.find_vort_maxima(dt.datetime(year, 6, 1),
dt.datetime(year, 12, 1))
tracker = VortmaxNearestNeighbourTracker()
tracker.track_vort_maxima(vort_finder.vortmax_time_series)
matches = match_vort_tracks_by_date_to_best_tracks(
tracker.vort_tracks_by_date, best_tracks)
# Quick to execute, no need to store.
# good_matches = [ma for ma in matches.values() if ma.av_dist() < 5 and ma.overlap > 6]
log.debug('Saving data')
results_manager.add_result(year, ensemble_member,
'vortmax_time_series',
vort_finder.vortmax_time_series)
results_manager.add_result(year, ensemble_member,
'vort_tracks_by_date',
tracker.vort_tracks_by_date)
results_manager.add_result(year, ensemble_member, 'matches', matches)
results_manager.save()
end = time.time()
log.info('Found {0} matches in {1}s'.format(len(matches.values()),
end - start))
response = {
'status': 'complete',
'time_taken': end - start,
}
return response
def list_output(full=False):
from stormtracks.load_settings import settings
from stormtracks.results import StormtracksResultsManager
cprint('Using output dir: {}'.format(settings.OUTPUT_DIR), 'green', attrs=['bold'])
cprint(' Tracking results: {}'.format(settings.RESULTS), 'green', attrs=['bold'])
results_manager = StormtracksResultsManager(settings.RESULTS)
results_years = results_manager.list_years()
for year in results_years:
results = results_manager.list_results(year)
for result_name in results:
cprint(' Tracking results year: {}; {}'.
format(year, result_name), 'blue', attrs=['bold'])
def analyse_year(year=2005, results_name='demo'):
start = dt.datetime.now()
results_manager = StormtracksResultsManager(results_name)
try:
all_fields = results_manager.get_result(year, 'all_fields')
best_track_matches = results_manager.get_result(year, 'best_track_matches')
except ResultNotFound:
print('Could not load all_fields and best_track_matches for {}'.format(year))
print('Run download_2005.py and process_2005.py first')
exit(1)
combined_fields_matches = all_fields.join(best_track_matches)
# return combined_fields_matches
analyse_combined(combined_fields_matches)
end = dt.datetime.now()
print('Analysed {} in {}'.format(year, end - start))
def process_year(year=2005, results_name='demo'):
start = dt.datetime.now()
log.info('Processing year, results: {}, {}'.format(year, results_name))
start_date = dt.datetime(year, 6, 1)
end_date = dt.datetime(year, 12, 1)
results_manager = StormtracksResultsManager(results_name)
# Run through 20CR looking for vortmax. Collect all fields for each vortmax, save as pandas DataFrame.
c20data = C20Data(year)
finder = VortmaxFinder(c20data, False)
df_year = finder.find_vort_maxima(start_date, end_date)
results_manager.save_result(year, 'all_fields', df_year)
# Match each best track to the corresponding vortmax from 20CR.
ib = ibtracsdata.IbtracsData()
ib.load_ibtracks_year(year)
best_track_matches = matching.simple_matching(ib.best_tracks, df_year)
results_manager.save_result(year, 'best_track_matches', best_track_matches)
end = dt.datetime.now()
log.info('Processed {} in {}'.format(year, end - start))
return df_year, best_track_matches
def analyse_year(year=2005, results_name='demo'):
start = dt.datetime.now()
results_manager = StormtracksResultsManager(results_name)
try:
all_fields = results_manager.get_result(year, 'all_fields')
best_track_matches = results_manager.get_result(
year, 'best_track_matches')
except ResultNotFound:
print('Could not load all_fields and best_track_matches for {}'.format(
year))
print('Run download_2005.py and process_2005.py first')
exit(1)
combined_fields_matches = all_fields.join(best_track_matches)
# return combined_fields_matches
analyse_combined(combined_fields_matches)
end = dt.datetime.now()
print('Analysed {} in {}'.format(year, end - start))
def do_field_collection_analysis(self, year, ensemble_member):
log.info(
'Received request for field collection analysis for year {0} ensemble {1}'
.format(year, ensemble_member))
analysis = StormtracksAnalysis(year)
log.info('Made analysis object')
results_manager = StormtracksResultsManager(
'pyro_field_collection_analysis')
log.info('Got results manager')
try:
results_manager.get_result(year, ensemble_member, 'cyclones')
log.info('Results already created')
response = {
'status': 'complete',
'extra_info': 'results already created',
}
except:
start = time.time()
cyclones = analysis.run_individual_field_collection(
ensemble_member)
log.info('Found cyclones')
results_manager.add_result(year, ensemble_member, 'cyclones',
cyclones)
results_manager.save()
log.info('Saved results')
end = time.time()
response = {
'status': 'complete',
'time_taken': end - start,
}
return response
def main(task_name,
years,
num_ensemble_members=56,
use_best_config=True,
compress=True):
'''Established communication with all pyro_workers
N.B. pyro_nameserver must be set up first, and pyro workers must be running
on each of the servers defined by pyro_settings.worker servers (either by running
pyro_start.py or by manually starting them).
Starts off by finding each of the pyro_workers, then generates a task schedule before
handing over to run_tasks(...).
'''
log.info('Calling from {0}'.format(socket.gethostname()))
log.info('Running task {0}'.format(task_name))
workers = {}
free_workers = copy.copy(pyro_settings.worker_servers)
for server_name in free_workers:
log.info('Adding server {0}'.format(server_name))
worker_proxy = Pyro4.Proxy(
'PYRONAME:stormtracks.worker_{0}'.format(server_name))
async_worker_proxy = Pyro4. async (worker_proxy)
workers[server_name] = (worker_proxy, async_worker_proxy)
if task_name == 'vort_tracking':
results_manager = StormtracksResultsManager('pyro_vort_tracking')
elif task_name == 'tracking_analysis':
results_manager = StormtracksResultsManager('pyro_tracking_analysis')
elif task_name == 'field_collection_analysis':
results_manager = StormtracksResultsManager(
'pyro_field_collection_analysis')
else:
raise Exception('Task {0} not known'.format(task_name))
for year in years:
log.info('Running for year {0}'.format(year))
if task_name == 'vort_tracking':
task_provider = PyroTaskSchedule(
year, year, num_ensemble_members=num_ensemble_members)
elif task_name == 'tracking_analysis':
if use_best_config:
config = {
'pressure_level': 850,
'scale': 3,
'tracker': 'nearest_neighbour'
}
task_provider = PyroTrackingAnalysis(
year,
num_ensemble_members=num_ensemble_members,
config=config)
else:
task_provider = PyroTrackingAnalysis(
year, num_ensemble_members=num_ensemble_members)
elif task_name == 'field_collection_analysis':
task_provider = PyroFieldCollectionAnalysis(
year, num_ensemble_members=num_ensemble_members)
run_tasks(year,
task_provider,
workers,
free_workers,
task_name=task_name)
log.info('Tasks complete')
if compress:
log.info('Task complete, compressing year')
if task_name == 'vort_tracking':
results_manager.compress_year(year, delete=True)
elif task_name == 'tracking_analysis':
results_manager.compress_year(year, delete=True)
elif task_name == 'field_collection_analysis':
results_manager.compress_year(year, delete=True)
# Compress/delete tracking results now I'm finished with them too.
tracking_results_manager = StormtracksResultsManager(
'pyro_tracking_analysis')
tracking_results_manager.compress_year(year, delete=True)
log.info('Year {0} complete'.format(year))
def compress_year_output(year):
srm = StormtracksResultsManager('aws_tracking_analysis')
compressed_filename = srm.compress_year(year, delete=True)
return compressed_filename
def setUp(self):
self.srm = StormtracksResultsManager('_functional_test')
class TestResultsSave:
def __init__(self):
self.year = 1860
self.bad_year = 1066
self.ensemble_member = 70
self.result_key = 'dummy_result'
self.payload = {'one': 2}
def setUp(self):
self.srm = StormtracksResultsManager('_functional_test')
def test_0_add(self):
self.srm.add_result(self.year, self.ensemble_member, self.result_key,
self.payload)
def test_1_save(self):
self.srm.add_result(self.year, self.ensemble_member, self.result_key,
self.payload)
self.srm.save()
def test_2_get(self):
result = self.srm.get_result(self.year, self.ensemble_member,
self.result_key)
assert result['one'] == 2
def test_3_list_year(self):
assert self.year in self.srm.list_years()
def test_5_delete(self):
self.srm.delete(self.year, self.ensemble_member, self.result_key)
@raises(ResultNotFound)
def test_6_get_non_existent(self):
self.srm.get_result(self.bad_year, self.ensemble_member,
self.result_key)
@raises(OSError)
def test_7_delete_non_existent(self):
self.srm.delete(self.bad_year, self.ensemble_member, self.result_key)
def compress_year_output(year):
srm = StormtracksResultsManager('aws_tracking_analysis')
compressed_filename = srm.compress_year(year, delete=True)
return compressed_filename
def do_tracking_analysis(self, year, ensemble_member, config):
log.info(
'Received request for tracking analysis for year {0} ensemble {1}'.
format(year, ensemble_member))
analysis = StormtracksAnalysis(year)
good_matches_key = analysis.good_matches_key(config)
vort_tracks_by_date_key = analysis.vort_tracks_by_date_key(config)
log.info('Analysis data {0}'.format(good_matches_key))
results_manager = StormtracksResultsManager('pyro_tracking_analysis')
try:
results_manager.get_result(year, ensemble_member, good_matches_key)
results_manager.get_result(year, ensemble_member,
vort_tracks_by_date_key)
log.info('Results already created')
response = {
'status': 'complete',
'extra_info': 'results already created',
}
except:
log.info('Creating results')
start = time.time()
good_matches, vort_tracks_by_date = \
analysis.run_individual_tracking_matching_analysis(ensemble_member, config)
results_manager.add_result(year, ensemble_member, good_matches_key,
good_matches)
results_manager.add_result(year, ensemble_member,
vort_tracks_by_date_key,
vort_tracks_by_date)
results_manager.save()
end = time.time()
response = {
'status': 'complete',
'time_taken': end - start,
}
return response