def download_stations(channels, resp_file_folder, station_xml_folder,
dataless_seed_folder, logfile, arclink_user,
get_station_filename_fct):
"""
Convenience function downloading station information for all channels in
the channels filename list. It will only download what does not exist yet.
It will first attempt to download datalessSEED and RESP files from ArcLink
and then, do the same for all missing data from IRIS, except it will
attempt to download StationXML and RESP files.
:param channels: A list of filenames pointing to all channels.
:param resp_file_folder: The folder where the RESP file are stored.
:param station_XML: The folder where the StationXML files are stored.
:param dataless_seed_folder: The folder where the dataless SEED files are
stored.
:param get_station_filename_fct: Function get_station_filename_fct(network,
station, location, channel, format) the simply returns the path where
the given file should be written to.
"""
# Init logger.
logger = ColoredLogger(log_filename=logfile, debug=True)
# Log some basic information
logger.info(70 * "=")
logger.info(70 * "=")
logger.info("Starting to download station files for %i missing "
"channels..." % len(channels))
def save_station_file(memfile, network, station, location, channel,
format):
"""
Callback function saving a single file given as a StringIO instance.
"""
filename = get_station_filename_fct(network, station, location,
channel, format)
memfile.seek(0, 0)
with open(filename, "wb") as open_file:
open_file.write(memfile.read())
# Now download all the missing stations files.
successful_downloads = \
lasif.download_helpers.stations.download_station_files(
channels, save_station_fct=save_station_file,
arclink_user=arclink_user, logger=logger)
print "Done. Successfully downloaded %i station files." % \
successful_downloads
def launch_processing(data_generator, log_filename=None, waiting_time=4.0,
process_params=None):
"""
Launch the parallel processing.
:param data_generator: A generator yielding file information as required.
:param log_filename: If given, a log will be written to that file.
:param waiting_time: The time spent sleeping after the initial message has
been printed. Useful if the user should be given the chance to cancel
the processing.
:param process_params: If given, the processing parameters will be written
to the logfile.
"""
logger = ColoredLogger(log_filename=log_filename)
logger.info("Launching preprocessing using all processes...\n"
"This might take a while. Press Ctrl + C to cancel.\n")
# Give the user some time to read the message.
time.sleep(waiting_time)
results = parallel_map(preprocess_file,
({"file_info": i} for i in data_generator),
verbose=1, pre_dispatch="all")
# Keep track of all files.
successful_file_count = 0
warning_file_count = 0
failed_file_count = 0
total_file_count = len(results)
for result in results:
if result.exception is not None:
filename = result.func_args["file_info"]["data_path"]
msg = "Exception processing file '%s'. %s\n%s" % (
filename, result.exception, result.traceback)
logger.error(msg)
failed_file_count += 1
elif result.warnings:
warning_file_count += 1
else:
successful_file_count += 1
return {
"failed_file_count": failed_file_count,
"warning_file_count": warning_file_count,
"total_file_count": total_file_count,
"successful_file_count": successful_file_count}
def download_waveforms(
min_latitude, max_latitude, min_longitude, max_longitude,
rotation_axis, rotation_angle_in_degree, starttime, endtime,
arclink_user, channel_priority_list, logfile, download_folder,
waveform_format="mseed"):
"""
Convenience function downloading all waveform files in the specified
spherical section domain.
"""
# Init logger.
logger = ColoredLogger(log_filename=logfile, debug=True)
# Log some basic information
logger.info(70 * "=")
logger.info(70 * "=")
logger.info("Starting waveform downloads...")
def filter_location_fct(latitude, longitude):
"""
Simple function checking if a geographic point is placed inside a
rotated spherical section. It simple rotates the point and checks if it
is inside the unrotated domain. The domain specification are passed in
as a closure.
Returns True or False.
"""
# Rotate the station and check if it is still in bounds.
r_lat, r_lng = rotations.rotate_lat_lon(
latitude, longitude, rotation_axis, -1.0 *
rotation_angle_in_degree)
# Check if in bounds. If not continue.
if not (min_latitude <= r_lat <= max_latitude) or \
not (min_longitude <= r_lng <= max_longitude):
return False
return True
# Get the maximum bounds of the domain.
min_lat, max_lat, min_lng, max_lng = _get_maximum_bounds(
min_latitude, max_latitude, min_longitude, max_longitude,
rotation_axis, rotation_angle_in_degree)
# Get the availability.
channels = get_availability(
min_lat, max_lat, min_lng, max_lng, starttime, endtime, arclink_user,
channel_priority_list=channel_priority_list, logger=logger,
filter_location_fct=filter_location_fct)
if not channels:
msg = "No matching channels found. Program will terminate."
logger.critical(msg)
sys.exit(1)
def get_channel_filename(channel_id):
"""
Get the filename given a seed channel id.
File format and other things will be available via closures.
"""
filename = "%s.%s" % (channel_id, waveform_format.lower())
return os.path.join(download_folder, filename)
# Now get a list of those channels that still need downloading.
channels_to_download = []
for chan in channels.iterkeys():
filename = get_channel_filename(chan)
if os.path.exists(filename):
continue
channels_to_download.append(chan)
def save_channel(trace):
"""
Save a single trace.
"""
filename = get_channel_filename(trace.id)
if not os.path.exists(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
try:
trace.write(filename, format=waveform_format)
except Exception as e:
msg = "Error writing Trace. This is likely due to corrupted data."
msg += "Error message: %s" % e.message
logger.error(msg)
logger.info("Attempting to download %i (missing) waveform channels..." %
len(channels_to_download))
# Download in chunks of 100 channels.
channels_to_download.sort()
CHUNK_SIZE = 100
current_position = 1
successful_downloads = 0
for chunk in (channels_to_download[_i: _i + CHUNK_SIZE] for _i in xrange(0,
len(channels_to_download), CHUNK_SIZE)):
logger.info(70 * "=")
logger.info("Starting download for channels %i to %i..." %
(current_position, current_position + CHUNK_SIZE))
logger.info(70 * "=")
current_position += CHUNK_SIZE
successful_downloads += \
lasif.download_helpers.waveforms.download_waveforms(
chunk, starttime, endtime, 0.95, save_trace_fct=save_channel,
arclink_user=arclink_user, logger=logger)
print "Done. Successfully downloaded %i waveform channels." % \
successful_downloads
