def test_real_spade_pickup_data_bad_out_dir(self):
hsp = HsSpader.HsSpader()
# initialize directory parts
category = "XXX"
timetag = "12345678_987654"
host = "ichub01"
# initialize HitSpool file parameters
firstnum = 11
numfiles = 3
# create directory and files
MockHitspool.create_copy_files(category, timetag, host, firstnum,
numfiles, real_stuff=True)
# don't check DEBUG/INFO log messages
self.setLogLevel(logging.INFO)
# pylint: disable=anomalous-backslash-in-string
# this is meant to be an illegal directory path on all OSes
outdir = "/xxx/yyy/zzz\/\/foo::bar"
# add all expected log messages
self.expect_log_message("Preparation for SPADE Pickup of HS data"
" started manually via HsSpader...")
try:
hsp.spade_pickup_data(outdir, timetag, outdir)
self.fail("Should not be able to create %s" % outdir)
except OSError as err:
if err.errno != errno.EACCES:
raise
def test_real_spade_pickup_data_empty_dir(self):
hsp = HsSpader.HsSpader()
# initialize directory parts
category = "XXX"
timetag = "12345678_987654"
host = "ichub01"
# initialize HitSpool file parameters
firstnum = 11
numfiles = 3
# create directory and files
MockHitspool.create_copy_files(category, timetag, host, firstnum,
numfiles, real_stuff=True)
# don't check DEBUG/INFO log messages
self.setLogLevel(logging.INFO)
outdir = tempfile.mkdtemp()
# add all expected log messages
self.expect_log_message("Preparation for SPADE Pickup of HS data"
" started manually via HsSpader...")
self.expect_log_message("no HS data found for this alert time "
"pattern.")
hsp.spade_pickup_data(outdir, timetag, outdir)
def tearDown(self):
try:
super(HsSpaderTest, self).tearDown()
finally:
# clear lingering files
try:
MockHitspool.destroy()
except:
pass
def test_copy_sn_alert(self):
sender = FailableSender()
self.set_sender(sender)
# initialize HitSpool file parameters
firstnum = 11
numfiles = 3
# create fake directory paths
hsdir = MockHitspool.create_copy_files("HESE",
"12345678_987654",
"ichub01",
firstnum,
numfiles,
real_stuff=True)
usrdir = os.path.join(MockHitspool.COPY_DIR, "UserCopy")
# don't check DEBUG/INFO log messages
self.setLogLevel(logging.WARN)
# run it!
sender.move_to_destination_dir(hsdir, usrdir)
# files should have been moved!
self.assertTrue(sender.moved_files(), "Should have moved files")
# make sure 0MQ communications checked out
sender.validate()
def test_no_move(self):
sender = FailableSender()
self.set_sender(sender)
# initialize HitSpool file parameters
firstnum = 11
numfiles = 3
# create fake directory paths
hsdir = MockHitspool.create_copy_files("ANON",
"12345678_987654",
"ichub01",
firstnum,
numfiles,
real_stuff=True)
if hsdir.endswith('/'):
usrdir = os.path.dirname(hsdir[:-1])
else:
usrdir = os.path.dirname(hsdir)
# don't check DEBUG/INFO log messages
self.setLogLevel(logging.WARN)
# run it!
sender.move_to_destination_dir(hsdir, usrdir)
# make sure no files moved
self.assertFalse(sender.moved_files(), "Should not have moved files")
# make sure 0MQ communications checked out
sender.validate()
def __init__(self, req_id, req_type, username, start_ticks, stop_ticks,
timetag, host, firstfile, numfiles):
if start_ticks is not None and \
not isinstance(start_ticks, numbers.Number):
raise TypeError("Start time %s<%s> is not number" %
(start_ticks, type(start_ticks).__name__))
if stop_ticks is not None and \
not isinstance(stop_ticks, numbers.Number):
raise TypeError("Stop time %s<%s> is not number" %
(stop_ticks, type(stop_ticks).__name__))
self.__req_id = req_id
self.__start_ticks = start_ticks
self.__stop_ticks = stop_ticks
self.__username = username
self.__host = host
self.__firstfile = firstfile
self.__numfiles = numfiles
# create initial directory
category = self.__get_category(req_type)
self.__hsdir = MockHitspool.create_copy_files(category,
timetag,
host,
firstfile,
numfiles,
real_stuff=True)
# build final directory path
if self.USRDIR is None:
self.create_user_dir()
self.__destdir = os.path.join(self.USRDIR,
"%s_%s_%s" % (category, timetag, host))
def test_spade_data_nonstandard_prefix(self):
sender = FailableSender()
self.set_sender(sender)
# initialize directory parts
category = "SomeCategory"
timetag = "12345678_987654"
host = "ichub01"
# initialize HitSpool file parameters
firstnum = 11
numfiles = 3
# don't check DEBUG/INFO log messages
self.setLogLevel(logging.WARN)
mybase = "%s_%s_%s" % (category, timetag, host)
mytar = "%s%s" % (mybase, HsSender.HsSender.TAR_SUFFIX)
if HsSender.HsSender.WRITE_META_XML:
mysem = "%s%s" % (mybase, HsSender.HsSender.META_SUFFIX)
else:
mysem = "%s%s" % (mybase, HsSender.HsSender.SEM_SUFFIX)
# create real directories
hsdir = MockHitspool.create_copy_files(category,
timetag,
host,
firstnum,
numfiles,
real_stuff=True)
# clean up test files
for fnm in (mytar, mysem):
tmppath = os.path.join(sender.HS_SPADE_DIR, fnm)
if os.path.exists(tmppath):
os.unlink(tmppath)
# run it!
(tarname, semname) \
= sender.spade_pickup_data(hsdir, mybase, prefix=category)
self.assertEqual(
mytar, tarname, "Expected tarfile to be named \"%s\" not \"%s\"" %
(mytar, tarname))
self.assertEqual(
mysem, semname,
"Expected semaphore to be named \"%s\" not \"%s\"" %
(mysem, semname))
# make sure 0MQ communications checked out
sender.validate()
def tearDown(self):
try:
super(HsSenderTest, self).tearDown()
finally:
found_error = False
self.restore_copy_path()
# clear lingering files
try:
MockHitspool.destroy()
except:
traceback.print_exc()
found_error = True
try:
MockRequestBuilder.destroy()
except:
traceback.print_exc()
found_error = True
# close all sockets
found_error |= self.close_all_senders()
# get rid of HsSender's state database
dbpath = RequestMonitor.get_db_path()
if os.path.exists(dbpath):
try:
os.unlink(dbpath)
except:
traceback.print_exc()
found_error = True
if found_error:
self.fail("Found one or more errors during tear-down")
def test_real_spade_pickup_data(self):
hsp = HsSpader.HsSpader()
# initialize directory parts
category = "XXX"
timetag = "12345678_987654"
host = "ichub01"
# initialize HitSpool file parameters
firstnum = 11
numfiles = 3
# create directory and files
hsdir = MockHitspool.create_copy_files(category, timetag, host,
firstnum, numfiles,
real_stuff=True)
# don't check DEBUG log messages
self.setLogLevel(logging.INFO)
basename = "HS_SNALERT_%s_%s" % (timetag, host)
tarname = basename + HsSpader.HsSpader.TAR_SUFFIX
outdir = tempfile.mkdtemp()
# add all expected log messages
self.expect_log_message("Preparation for SPADE Pickup of HS data"
" started manually via HsSpader...")
self.expect_log_message("found HS data:\n%s" % [hsdir, ])
self.expect_log_message("data: %s_%s_%s will be tarred to: %s" %
(category, timetag, host, tarname))
self.expect_log_message("Preparation for SPADE Pickup of %s DONE" %
basename)
for hub in range(2, 87):
self.expect_log_message("no or ambiguous HS data found for"
" ichub%02d in this directory." % hub)
for hub in range(1, 12):
self.expect_log_message("no or ambiguous HS data found for"
" ithub%02d in this directory." % hub)
hsp.spade_pickup_data(MockHitspool.COPY_DIR, timetag, outdir)
def test_spade_data_fail_sem(self):
sender = FailableSender()
self.set_sender(sender)
sender.fail_create_sem_file()
# initialize directory parts
category = "SNALERT"
timetag = "12345678_987654"
host = "ichub07"
# initialize HitSpool file parameters
firstnum = 11
numfiles = 3
# create bad directory name
hsdir = MockHitspool.create_copy_files(category,
timetag,
host,
firstnum,
numfiles,
real_stuff=False)
# don't check DEBUG/INFO log messages
self.setLogLevel(logging.WARN)
# add all expected log messages
self.expect_log_message("Fake Touch Error")
self.expect_log_message("Please put the data manually in the SPADE"
" directory. Use HsSpader.py, for example.")
# run it!
result = sender.spade_pickup_data(hsdir, "ignored", prefix=category)
self.assertIsNone(
result, "spade_pickup_data() should return None,"
" not %s" % str(result))
# make sure 0MQ communications checked out
sender.validate()
def test_spade_pickup_data(self):
sender = MySender(verbose=False)
self.set_sender(sender)
# initialize directory parts
category = "SNALERT"
timetag = "12345678_987654"
host = "ichub01"
# initialize HitSpool file parameters
firstnum = 11
numfiles = 3
# create real directories
hsdir = MockHitspool.create_copy_files(category,
timetag,
host,
firstnum,
numfiles,
real_stuff=True)
# set SPADE path to something which exists everywhere
sender.HS_SPADE_DIR = tempfile.mkdtemp(prefix="SPADE_")
mybase = "%s_%s_%s" % (category, timetag, host)
mytar = "HS_%s%s" % (mybase, HsSender.HsSender.TAR_SUFFIX)
if HsSender.HsSender.WRITE_META_XML:
mysem = "HS_%s%s" % (mybase, HsSender.HsSender.META_SUFFIX)
else:
mysem = "HS_%s%s" % (mybase, HsSender.HsSender.SEM_SUFFIX)
# create intermediate directory
movetop = tempfile.mkdtemp(prefix="Intermediate_")
movedir = os.path.join(movetop, mybase)
os.makedirs(movedir)
# copy hitspool files to intermediate directory
shutil.copytree(hsdir, os.path.join(movedir, os.path.basename(hsdir)))
# don't check DEBUG/INFO log messages
self.setLogLevel(logging.WARN)
# add all expected log messages
# clean up test files
for fnm in (mytar, mysem):
tmppath = os.path.join(sender.HS_SPADE_DIR, fnm)
if os.path.exists(tmppath):
os.unlink(tmppath)
# run it!
(tarname, semname) = sender.spade_pickup_data(movedir,
mybase,
prefix=category)
self.assertEqual(
mytar, tarname, "Expected tarfile to be named \"%s\" not \"%s\"" %
(mytar, tarname))
self.assertEqual(
mysem, semname,
"Expected semaphore to be named \"%s\" not \"%s\"" %
(mysem, semname))
sempath = os.path.join(sender.HS_SPADE_DIR, semname)
self.assertTrue(os.path.exists(sempath),
"Semaphore file %s was not created" % sempath)
tarpath = os.path.join(sender.HS_SPADE_DIR, tarname)
self.assertTrue(tarfile.is_tarfile(tarpath),
"Tar file %s was not created" % tarpath)
# read in contents of tarfile
tar = tarfile.open(tarpath)
names = []
for fnm in tar.getnames():
if fnm == mybase:
continue
if fnm.startswith(mybase):
fnm = fnm[len(mybase) + 1:]
names.append(fnm)
tar.close()
# validate the list
MockRequestBuilder.check_hitspool_file_list(names, firstnum, numfiles)
# make sure 0MQ communications checked out
sender.validate()