def test_channel_in_parser(self):
"""
Tests if a given channel is part of a Parser object.
"""
starttime = UTCDateTime(2007, 2, 12, 10, 30, 28, 197700)
endtime = UTCDateTime(2007, 2, 12, 11, 35, 28, 197700)
channel_id = "ES.ECAL..HHE"
# An empty file should of course not contain much.
parser_object = Parser(
os.path.join(self.data_dir, "channelless_datalessSEED"))
self.assertFalse(
utils.channel_in_parser(parser_object, channel_id, starttime,
endtime))
# Now read a file that actually contains data.
channel_id = "IU.PAB.00.BHE"
starttime = UTCDateTime(1999, 2, 18, 10, 0)
endtime = UTCDateTime(2009, 8, 13, 19, 0)
parser_object = Parser(os.path.join(self.data_dir, "dataless.IU_PAB"))
# This is an exact fit of the start and end times in this file.
self.assertTrue(
utils.channel_in_parser(parser_object, channel_id, starttime,
endtime))
# Now try some others that do not fit.
self.assertFalse(
utils.channel_in_parser(parser_object, channel_id, starttime - 1,
endtime))
self.assertFalse(
utils.channel_in_parser(parser_object, channel_id, starttime,
endtime + 1))
self.assertFalse(
utils.channel_in_parser(parser_object, channel_id + "x", starttime,
endtime))
self.assertFalse(
utils.channel_in_parser(parser_object, channel_id, starttime - 200,
starttime - 100))
self.assertFalse(
utils.channel_in_parser(parser_object, channel_id, endtime + 100,
endtime + 200))
# And some that do fit.
self.assertTrue(
utils.channel_in_parser(parser_object, channel_id, starttime,
starttime + 10))
self.assertTrue(
utils.channel_in_parser(parser_object, channel_id, endtime - 100,
endtime))
def test_channel_in_parser():
"""
Tests if a given channel is part of a Parser object.
"""
starttime = UTCDateTime(2007, 2, 12, 10, 30, 28, 197700)
endtime = UTCDateTime(2007, 2, 12, 11, 35, 28, 197700)
channel_id = "ES.ECAL..HHE"
# An empty file should of course not contain much.
parser_object = Parser(os.path.join(data_dir,
"channelless_datalessSEED"))
assert utils.channel_in_parser(parser_object, channel_id,
starttime, endtime) is False
# Now read a file that actually contains data.
channel_id = "IU.PAB.00.BHE"
starttime = UTCDateTime(1999, 2, 18, 10, 0)
endtime = UTCDateTime(2009, 8, 13, 19, 0)
parser_object = Parser(os.path.join(data_dir,
"dataless.IU_PAB"))
# This is an exact fit of the start and end times in this file.
assert utils.channel_in_parser(
parser_object, channel_id, starttime, endtime) is True
# Now try some others that do not fit.
assert utils.channel_in_parser(
parser_object, channel_id, starttime - 1, endtime) is False
assert utils.channel_in_parser(
parser_object, channel_id, starttime, endtime + 1) is False
assert utils.channel_in_parser(
parser_object, channel_id + "x", starttime, endtime) is False
assert utils.channel_in_parser(
parser_object, channel_id, starttime - 200, starttime - 100) is False
assert utils.channel_in_parser(
parser_object, channel_id, endtime + 100, endtime + 200) is False
# And some that do fit.
assert utils.channel_in_parser(
parser_object, channel_id, starttime, starttime + 10) is True
assert utils.channel_in_parser(
parser_object, channel_id, endtime - 100, endtime) is True
def run(self):
while True:
try:
channel = self.queue.get(False)
except Queue.Empty:
break
network = channel["network"]
station = channel["station"]
location = channel["location"]
chan = channel["channel"]
starttime = channel["starttime"]
endtime = channel["endtime"]
channel_id = "%s.%s.%s.%s" % (network, station, location, chan)
time.sleep(0.5)
if logger:
logger.debug("Starting ArcLink download for %s..." % channel_id)
# Telnet sometimes has issues...
success = False
for _i in xrange(3):
try:
arc_client = obspy.arclink.Client(user=arclink_user, timeout=30)
success = True
break
except:
time.sleep(0.3)
if success is False:
msg = " A problem occured initializing ArcLink. Try " "again later"
logger.error(msg)
failed_downloads.put(channel)
continue
try:
memfile = StringIO.StringIO()
arc_client.saveResponse(
memfile,
channel["network"],
channel["station"],
channel["location"],
channel["channel"],
starttime=channel["starttime"],
endtime=channel["endtime"],
format="SEED",
)
except Exception as e:
msg = "While downloading %s [%s to %s]: %s" % (
channel_id,
channel["starttime"],
channel["endtime"],
str(e),
)
logger.error(msg)
failed_downloads.put(channel)
continue
memfile.seek(0, 0)
# Read the file again and perform a sanity check.
try:
parser = Parser(memfile)
except:
msg = ("Arclink did not return a valid dataless SEED file " "for channel %s [%s-%s]") % (
channel_id,
starttime,
endtime,
)
logger.error(msg)
failed_downloads.put(channel)
continue
if not utils.channel_in_parser(parser, channel_id, starttime, endtime):
msg = (
"Arclink returned a valid dataless SEED file "
"for channel %s [%s to %s], but it does not actually "
" contain data for the requested channel and time "
"frame."
) % (channel_id, starttime, endtime)
logger.error(msg)
failed_downloads.put(channel)
continue
memfile.seek(0, 0)
save_station_fct(
memfile,
channel["network"],
channel["station"],
channel["location"],
channel["channel"],
format="datalessSEED",
)
successful_downloads.put(channel)
if logger:
logger.info(
"Successfully downloaded dataless SEED for "
"channel %s.%s.%s.%s from ArcLink."
% (channel["network"], channel["station"], channel["location"], channel["channel"])
)
def run(self):
while True:
try:
channel = self.queue.get(False)
except Queue.Empty:
break
network = channel["network"]
station = channel["station"]
location = channel["location"]
chan = channel["channel"]
starttime = channel["starttime"]
endtime = channel["endtime"]
channel_id = "%s.%s.%s.%s" % (network, station, location, chan)
time.sleep(0.5)
if logger:
logger.debug("Starting ArcLink download for %s..." %
channel_id)
# Telnet sometimes has issues...
success = False
for _i in xrange(3):
try:
arc_client = obspy.arclink.Client(user=arclink_user,
timeout=30)
success = True
break
except:
time.sleep(0.3)
if success is False:
msg = (" A problem occured initializing ArcLink. Try "
"again later")
logger.error(msg)
failed_downloads.put(channel)
continue
try:
memfile = StringIO.StringIO()
arc_client.saveResponse(memfile,
channel["network"],
channel["station"],
channel["location"],
channel["channel"],
starttime=channel["starttime"],
endtime=channel["endtime"],
format="SEED")
except Exception as e:
msg = "While downloading %s [%s to %s]: %s" % (
channel_id, channel["starttime"], channel["endtime"],
str(e))
logger.error(msg)
failed_downloads.put(channel)
continue
memfile.seek(0, 0)
# Read the file again and perform a sanity check.
try:
parser = Parser(memfile)
except:
msg = ("Arclink did not return a valid dataless SEED file "
"for channel %s [%s-%s]") % (channel_id, starttime,
endtime)
logger.error(msg)
failed_downloads.put(channel)
continue
if not utils.channel_in_parser(parser, channel_id, starttime,
endtime):
msg = ("Arclink returned a valid dataless SEED file "
"for channel %s [%s to %s], but it does not actually "
" contain data for the requested channel and time "
"frame.") % \
(channel_id, starttime, endtime)
logger.error(msg)
failed_downloads.put(channel)
continue
memfile.seek(0, 0)
save_station_fct(memfile,
channel["network"],
channel["station"],
channel["location"],
channel["channel"],
format="datalessSEED")
successful_downloads.put(channel)
if logger:
logger.info("Successfully downloaded dataless SEED for "
"channel %s.%s.%s.%s from ArcLink." %
(channel["network"], channel["station"],
channel["location"], channel["channel"]))