def fix_message_dict(mdict, allow_old_format=False):
if mdict is None:
return None
if not isinstance(mdict, dict):
raise HsException("Message is not a dictionary: \"%s\"<%s>" %
(mdict, type(mdict)))
for prefix in ("start", "stop"):
found = False
for suffix in ("ticks", "time"):
if prefix + "_" + suffix in mdict:
found = True
break
if not found:
raise HsException("Dictionary is missing %s_time or %s_ticks" %
(prefix, prefix))
# check for mandatory fields
if "msgtype" not in mdict:
mdict["msgtype"] = INITIAL
if "request_id" not in mdict:
if mdict["msgtype"] != INITIAL:
raise HsException("No request ID found in %s" % str(mdict))
mdict["request_id"] = ID.generate()
if "copy_dir" not in mdict:
mdict["copy_dir"] = None
if "extract" not in mdict:
mdict["extract"] = False
if "hubs" not in mdict:
mdict["hubs"] = ""
return mdict
def __insert_detail(self, request_id, username, prefix, start_ticks,
stop_ticks, dest_dir, hubs, extract, phase):
if start_ticks is None:
raise HsException("Start time cannot be None")
elif not isinstance(start_ticks, numbers.Number):
raise HsException("Start time must be number, not %s(%s)" %
(type(start_ticks), str(start_ticks)))
if stop_ticks is None:
raise HsException("Stop time cannot be None")
elif not isinstance(stop_ticks, numbers.Number):
raise HsException("Stop time must be number, not %s(%s)" %
(type(stop_ticks), str(stop_ticks)))
if self.__sqlconn is None:
self.__sqlconn = self.__open_database()
with self.__sqlconn:
cursor = self.__sqlconn.cursor()
# add details to DB
cursor.execute(
"insert or replace into request_details"
"(id, username, prefix, start_time, stop_time,"
" destination, hubs, extract, phase)"
" values (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(request_id, username, prefix, start_ticks, stop_ticks,
dest_dir, "" if hubs is None else hubs, 1 if extract else 0,
phase))
# add details to cache
with self.__reqlock:
self.__requests[request_id][self.DETAIL_KEY] \
= (username, prefix, start_ticks, stop_ticks, dest_dir,
hubs, extract, phase)
def ticks_to_utc(ticks):
"Convert an integral DAQ tick value to a time object"
if ticks is None:
raise HsException("No tick value specified")
if not isinstance(ticks, numbers.Number):
raise HsException("Tick value %s should be number, not %s" %
(ticks, type(ticks).__name__))
return jan1_by_year() + datetime.timedelta(seconds=ticks / 1E10)
def move_file(cls, src, dst):
if not os.path.exists(dst):
raise HsException("Directory \"%s\" does not exist,"
" cannot move \"%s\"" % (dst, src))
try:
shutil.move(src, dst)
except Exception as sex:
raise HsException("Cannot move \"%s\" to \"%s\": %s" %
(src, dst, sex))
def hardlink(cls, filename, targetdir):
path = os.path.join(targetdir, os.path.basename(filename))
if os.path.exists(path):
raise HsException("File \"%s\" already exists" % path)
try:
os.link(filename, path)
except Exception as err:
raise HsException("Cannot link \"%s\" to \"%s\": %s" %
(filename, targetdir, err))
def copy(self, source_list, request=None, update_status=None):
if source_list is None or len(source_list) == 0:
raise HsException("No source specified")
failed = []
for src in source_list:
rtncode = self.copy_one(src)
if rtncode != 0:
logging.error("failed to copy %s to \"%s\" (rtn=%d,"
" unknown=%d)", src, self.__target, rtncode,
self.__unknown_count)
failed.append(src)
self.__unknown_count = 0
if update_status is not None and request is not None:
update_status(request.copy_dir, request.destination_dir,
HsMessage.WORKING)
result = self.summarize()
if result is not None:
(_, size, _, _, _) = result
if self.__size is None:
self.__size = size
else:
self.__size += size
return failed
def split_rsync_path(self, rsync_path):
"""
Return a tuple containing (user, host, path) pieces of the rsync path,
filling in default values for any missing pieces
"""
if rsync_path is None:
user = None
host = None
path = self.DEFAULT_COPY_PATH
else:
mtch = self.COPY_PATH_PAT.match(rsync_path)
if mtch is None:
raise HsException("Bad copy path \"%s\"" % rsync_path)
user = mtch.group(2)
host = mtch.group(4)
path = mtch.group(5)
if user is None:
user = self.rsync_user
if host is None:
host = self.rsync_host
return (user, host, path)
def __write_tarfile_internal(sourcedir, sourcefiles, tarname):
extra = ""
if tarname.endswith(".gz") or tarname.endswith(".tgz"):
extra = ":gz"
elif tarname.endswith(".bz2"):
extra = ":bz2"
try:
tar = tarfile.open(tarname, "w" + extra)
except Exception as err:
raise HsException("Cannot create \"%s\" in \"%s\": %s" %
(tarname, sourcedir, err))
if isinstance(sourcefiles, list):
sourcelist = sourcefiles
else:
sourcelist = [sourcefiles, ]
try:
for fnm in sourcelist:
try:
tar.add(fnm)
except Exception as err:
logging.error("Failed to add \"%s\" to \"%s\": %s",
fnm, os.path.join(sourcedir, tarname), err)
finally:
tar.close()
def assemble_email_dict(address_list,
header,
message,
description="HsInterface Data Request",
prio=2,
short_subject=True,
quiet=True):
if address_list is None or len(address_list) == 0:
raise HsException("No addresses specified")
notifies = []
for email in address_list:
ndict = {
"receiver": email,
"notifies_txt": message,
"notifies_header": header,
}
notifies.append(ndict)
now = datetime.datetime.now()
return {
"service": "HSiface",
"varname": "alert",
"prio": prio,
"time": now.strftime("%Y-%m-%d %H:%M:%S"),
"value": {
"condition": header,
"desc": description,
"notifies": notifies,
"short_subject": "true" if short_subject else "false",
"quiet": "true" if quiet else "false",
},
}
def __init__(self, cmd=None, rmt_user=None, rmt_host=None,
rmt_dir=None, rmt_subdir=None, make_remote_dir=False):
if rmt_host is None or rmt_host == "":
raise HsException("No remote host specified")
if rmt_dir is None or rmt_dir == "":
raise HsException("No remote directory specified")
if make_remote_dir:
self.make_remote_directory(rmt_user, rmt_host, rmt_dir)
self.__target = self.build_target(rmt_user, rmt_host, rmt_dir,
rmt_subdir)
self.__cmd = cmd
self.__size = None
self.__unknown_count = 0
def dict_to_object(xdict, expected_fields, objtype):
"""
Convert a dictionary (which must have the expected keys) into
a named tuple
"""
if not isinstance(xdict, dict):
raise HsException("Bad object \"%s\"<%s>" % (xdict, type(xdict)))
missing = []
for k in expected_fields:
if k not in xdict:
missing.append(k)
if len(missing) > 0:
raise HsException("Missing fields %s from %s" %
(tuple(missing), xdict))
return namedtuple(objtype, list(xdict.keys()))(**xdict)
def __receive(cls, sock):
mdict = sock.recv_json()
if mdict is None:
return None
if not isinstance(mdict, dict):
raise HsException("Received %s(%s), not dictionary" %
(mdict, type(mdict).__name__))
return HsMessage.from_dict(mdict)
def string_to_ticks(timestr, is_ns=False):
"Convert a time string to an integral DAQ tick value"
if timestr is None:
raise HsException("Found null value for start/stop time in %s" %
(timestr, ))
multiplier = 10 if is_ns else 1
if isinstance(timestr, numbers.Number):
return int(timestr * multiplier)
if isinstance(timestr, (str, unicode)):
if timestr.isdigit():
try:
return int(timestr) * multiplier
except:
raise HsException("Cannot convert \"%s\" to ticks" %
(timestr, ))
try:
utc = datetime.datetime.strptime(timestr, TIME_FORMAT)
except ValueError:
# Python date parser can only handle milliseconds
if timestr.find(".") > 0:
short = re.sub(r"(\.\d{6})\d+", r"\1", timestr)
try:
utc = datetime.datetime.strptime(short, TIME_FORMAT)
except:
raise HsException("Cannot convert \"%s\" to datetime" %
(timestr, ))
elif TIME_FORMAT.endswith(".%f"):
shortfmt = TIME_FORMAT[:-3]
try:
utc = datetime.datetime.strptime(timestr, shortfmt)
except:
raise HsException("Cannot convert \"%s\" to datetime" %
(timestr, ))
return utc_to_ticks(utc)
raise HsException("Cannot convert %s(%s) to ticks" %
(type(timestr).__name__, timestr))
def split_rsync_host_and_path(rsync_path):
"Remove leading 'user@host:' from rsync path"
if not isinstance(rsync_path, (str, unicode)):
raise HsException("Illegal rsync path \"%s\"<%s>" %
(rsync_path, type(rsync_path)))
parts = rsync_path.split(":", 1)
if len(parts) > 1 and parts[0].find("/") < 0:
return parts
# either no embedded colons or colons are part of the path
return "", rsync_path
def receive_request(self, sock):
"""
Receive the next request, validate it,
then push it to the processor thread
"""
req_dict = sock.recv_json()
if req_dict is None:
return
if not isinstance(req_dict, dict):
raise HsException("JSON message should be a dict: \"%s\"<%s>" %
(req_dict, type(req_dict)))
if "ping" in req_dict:
# reply to ping from sender
self.__ping_watcher.update()
self.sender.send_json({"pingback": self.fullhost})
return
# ensure 'start_time' and 'stop_time' are present and numbers
for tkey in ("start_ticks", "stop_ticks"):
if tkey not in req_dict:
raise HsException("Request does not contain '%s'" % (tkey, ))
elif not isinstance(req_dict[tkey], numbers.Number):
raise HsException("Request '%s' should be a number, not %s" %
(tkey, type(req_dict[tkey]).__name__))
# ensure 'extract' field is present and is a boolean value
req_dict["extract"] = "extract" in req_dict and \
req_dict["extract"] is True
alert_flds = ("request_id", "username", "start_ticks", "stop_ticks",
"destination_dir", "prefix", "extract")
req = HsUtil.dict_to_object(req_dict, alert_flds, 'WorkerRequest')
logging.info("HsWorker queued request:\n"
"%s\nfrom Publisher", str(req))
self.__req_thread.push(req)
def process_one_message(self, sock, force_spade=False):
mdict = sock.recv_json()
if mdict is None:
return False
if not isinstance(mdict, dict):
raise HsException("Received %s(%s), not dictionary" %
(mdict, type(mdict).__name__))
if sock == self.__reporter and "pingback" in mdict:
self.__ping_manager.add_reply(mdict["pingback"])
return True
error = True
try:
msg = HsMessage.from_dict(mdict, allow_old_format=True)
if msg is None:
return False
error = False
except HsException:
raise
except:
logging.exception("Cannot receive message from %s",
sock.identity)
found = False
if not error:
try:
self.__monitor.add_message(msg, force_spade)
found = True
except:
logging.exception("Received bad message %s", str(msg))
error = True
if sock == self.__alert_socket:
if error:
rtnmsg = "ERROR"
else:
rtnmsg = "DONE"
# reply to requester:
# added \0 to fit C/C++ zmq message termination
try:
answer = sock.send(rtnmsg + "\0")
if answer is not None:
logging.error("Failed sending %s to requester: %s",
rtnmsg, answer)
except zmq.ZMQError:
logging.exception("Cannot return \"%s\" to %s", rtnmsg,
sock.identity)
return found
def build_target(self, rmt_user, rmt_host, rmt_dir, rmt_subdir):
if self.__use_daemon:
# rsync daemon maps hitspool/ to /mnt/data/pdaqlocal/HsDataCopy/
target = '%s@%s::hitspool/%s/' % \
(rmt_user, rmt_host, rmt_subdir)
else:
target = rmt_dir
if target is None or target == "":
raise HsException("No target specified")
if target[-1] != "/":
# make sure `rsync` knows the target should be a directory
target += "/"
return target
def __convert_message_dict(cls, olddict, live_msg=False):
mdict = olddict.copy()
# fix time-based fields
for prefix in ("start", "stop"):
key = prefix + "_ticks"
if key not in mdict:
raise HsException("Message is missing \"%s_ticks\"" %
(prefix, ))
if not isinstance(mdict[key], numbers.Number):
raise HsException("Message field %s type is %s, not number" %
(key, type(mdict[key]).__name__))
if live_msg:
if key.endswith("_ticks"):
del mdict[key]
mdict[prefix + "_time"] \
= str(DAQTime.ticks_to_utc(mdict[key]))
# return new dictionary
return mdict
def __expect_live_status(cls,
sender,
mdict,
status=None,
success=None,
failed=None):
# if no status was specified, use success/failed to derive it
if status is None:
if success is not None:
if failed is not None:
status = HsUtil.STATUS_PARTIAL
else:
status = HsUtil.STATUS_SUCCESS
elif failed is not None:
status = HsUtil.STATUS_FAIL
else:
raise HsException("Must specify status or success/failed")
livedict = mdict.copy()
# delete non-Live fields
for fld in ("copy_dir", "host", "version", "msgtype", "extract",
"hubs"):
if fld in livedict:
del livedict[fld]
# fix time-based fields
for prefix in ("start", "stop"):
tick_key = prefix + "_ticks"
time_key = prefix + "_time"
if tick_key in livedict:
utc = DAQTime.ticks_to_utc(livedict[tick_key])
livedict[time_key] = str(utc)
del livedict[tick_key]
elif time_key not in livedict:
raise ValueError("Message has neither \"%s_ticks\" nor"
" \"%s_time\"" % (prefix, prefix))
# add a few Live-specific fields
livedict["status"] = status
livedict["update_time"] = TIME_PAT
if success is not None:
livedict["success"] = success
if failed is not None:
livedict["failed"] = failed
sender.i3socket.add_expected_message(livedict,
service="hitspool",
varname="hsrequest_info",
prio=1,
time=TIME_PAT)
def write_tarfile(self, sourcedir, sourcefiles, tarname):
if not os.path.exists(sourcedir):
raise HsException("Source directory \"%s\" does not exist" %
sourcedir)
curdir = os.getcwd()
# build tarfile inside sourcedir
os.chdir(sourcedir)
try:
self.__write_tarfile_internal(sourcedir, sourcefiles, tarname)
finally:
# move back to original directory
os.chdir(curdir)
def __make_staging_dir(self, timetag):
if self.is_cluster_sps or self.is_cluster_spts:
tmpdir = "/mnt/data/pdaqlocal/tmp"
else:
tmpdir = os.path.join(self.TEST_HUB_DIR, "tmp")
if not os.path.exists(tmpdir):
try:
os.makedirs(tmpdir)
except OSError:
# this should only happen inside unit tests
pass
if not os.path.isdir(tmpdir):
raise HsException("Found non-directory at %s" % tmpdir)
return tempfile.mkdtemp(suffix=timetag, dir=tmpdir)
def init_logging(cls, logfile=None, basename=None, basehost=None,
level=logging.INFO, both=False):
if logfile is None:
if basename is not None and basehost is not None:
logfile = os.path.join(HsBase.DEFAULT_LOG_PATH,
"%s_%s.log" % (basename, basehost))
elif basename is not None or basehost is not None:
print("Not logging to file (basehost=%s, basename=%s)" %
(basehost, basename), file=sys.stderr)
if logfile is not None:
logdir = os.path.dirname(logfile)
if not os.path.exists(logdir):
os.makedirs(logdir)
elif not os.path.isdir(logdir):
raise HsException("Base log directory \"%s\" isn't"
" a directory!" % logdir)
logger = logging.getLogger()
if len(logger.handlers) > 0:
# any logging done before this point creates a StreamHandler
# which causes basicConfig() to be ignored in Python 2.6
logger.handlers = []
if logfile is not None:
# set up logging to file
ten_mb = 1024 * 1024 * 10
rot = logging.handlers.RotatingFileHandler(logfile,
maxBytes=ten_mb,
backupCount=5)
longfmt = "%(asctime)s %(levelname)s %(message)s"
rot.setFormatter(logging.Formatter(fmt=longfmt,
datefmt="%Y-%m-%d %H:%M:%S"))
logger.addHandler(rot)
if logfile is None or both:
# set up logging to console
out = logging.StreamHandler()
out.setFormatter(logging.Formatter("%(message)s"))
logger.addHandler(out)
# set log level
logger.setLevel(level)
return logfile
def make_remote_directory(cls, rmt_user, rmt_host, rmt_dir):
if rmt_user is None:
rstr = rmt_host
else:
rstr = "%s@%s" % (rmt_user, rmt_host)
cmd = ["ssh", rstr, "if [ ! -d \"%s\" ]; then mkdir -p \"%s\"; fi" %
(rmt_dir, rmt_dir)]
for _ in range(3):
# this fails occasionally, possibly due to a wave of processes
# all trying to create subdirectories in the same directory
rtncode = subprocess.call(cmd)
if rtncode == 0:
return
# wait for this wave to die down before trying again
time.sleep(0.1)
raise HsException("Cannot create %s:%s (rtncode=%d)" %
(rstr, rmt_dir, rtncode))
def get_watchee(self):
"""
Depending on which machines this HsWatcher runs,
determine the processes it is responsible to watch.
Watcher at 2ndbuild --> HsSender
Watcher at expcont --> HsPublisher
Watcher at hub --> HsWorker
"""
if "2ndbuild" in self.fullhost:
return self.create_watchee("HsSender")
if "expcont" in self.fullhost:
return self.create_watchee("HsPublisher")
if "hub" in self.fullhost or "scube" in self.fullhost:
return self.create_watchee("HsWorker")
if "david" in self.fullhost:
return self.create_watchee("HsWorker")
raise HsException("Unrecognized host \"%s\"" % self.fullhost)
def hardlink(self, filename, targetdir):
if self.__fail_hardlink:
raise HsException("Fake Hardlink Error")
if len(self.__link_paths) == 0:
raise Exception("Unexpected hardlink from \"%s\" to \"%s\"" %
(filename, targetdir))
expfile, expdir, exptag = self.__link_paths.pop(0)
if not targetdir.startswith(expdir) or \
not targetdir.endswith(exptag):
if filename != expfile:
raise Exception("Expected to link \"%s\" to \"%s\", not"
" \"%s/*/%s\" to \"%s\"" %
(expfile, expdir, exptag, filename, targetdir))
raise Exception("Expected to link \"%s\" to \"%s/*/%s\", not to"
" \"%s\"" % (expfile, expdir, exptag, targetdir))
elif filename != expfile:
raise Exception("Expected to link \"%s\" to \"%s/*/%s\", not"
" \"%s\"" % (expfile, expdir, exptag, filename))
return 0
def touch_file(cls, name, times=None):
try:
with open(name, "a"):
os.utime(name, times)
except Exception as err:
raise HsException("Failed to 'touch' \"%s\": %s" % (name, err))
def write_tarfile(self, sourcedir, sourcefiles, tarname):
if self.__fail_tar_file > 0:
self.__fail_tar_file -= 1
raise HsException("Fake Tar Error")
def logline(self, idx):
if self.__loglines is None or len(self.__loglines) <= idx:
raise HsException("Bad log index %d (%d available)" %
(idx, 0 if self.__loglines is None else
len(self.__loglines)))
return self.__loglines[idx]
def write_sem(self, spadedir, basename):
if self.__fail_touch_file:
raise HsException("Fake Touch Error")
return "fake.sem"
def write_meta_xml(self, spadedir, basename, start_ticks, stop_ticks):
# use the tarfile creation time as the DIF_Creation_Date value
tarpath = os.path.join(spadedir, basename + self.TAR_SUFFIX)
try:
tarstamp = os.path.getmtime(tarpath)
except OSError:
raise HsException("Cannot write metadata file:"
" %s does not exist" % tarpath)
tartime = datetime.datetime.fromtimestamp(tarstamp)
if stop_ticks is not None:
stop_utc = DAQTime.ticks_to_utc(stop_ticks)
else:
# set stop time to the time the tarfile was written
stop_utc = tartime
if start_ticks is not None:
start_utc = DAQTime.ticks_to_utc(start_ticks)
else:
# set start time to midnight
start_utc = datetime.datetime(stop_utc.year, stop_utc.month,
stop_utc.day)
root = etree.Element("DIF_Plus")
root.set("{http://www.w3.org/2001/XMLSchema-instance}"
"noNamespaceSchemaLocation", "IceCubeDIFPlus.xsd")
# Metadata specification is at:
# https://docushare.icecube.wisc.edu/dsweb/Get/Document-20546/metadata_specification.pdf
xmldict = (
("DIF", (
("Entry_ID", basename),
("Entry_Title", "Hitspool_data"),
("Parameters",
"SPACE SCIENCE > Astrophysics > Neutrinos"),
("ISO_Topic_Category", "geoscientificinformation"),
("Data_Center", (
("Data_Center_Name",
"UWI-MAD/A3RI > Antarctic Astronomy and Astrophysics"
" Research Institute, University of Wisconsin, Madison"),
("Personnel", (
("Role", "Data Center Contact"),
("Email", "*****@*****.**"),
)),
)),
("Summary", "Hitspool data"),
("Metadata_Name", "[CEOS IDN DIF]"),
("Metadata_Version", "9.4"),
("Personnel", (
("Role", "Technical Contact"),
("First_Name", "Dave"),
("Last_Name", "Glowacki"),
("Email", "*****@*****.**"),
)),
("Sensor_Name", "ICECUBE > IceCube"),
("Source_Name",
"EXPERIMENTAL > Data with an instrumentation based"
" source"),
("DIF_Creation_Date", tartime.strftime("%Y-%m-%d")),
)),
("Plus", (
("Start_DateTime", start_utc.strftime("%Y-%m-%dT%H:%M:%S")),
("End_DateTime", stop_utc.strftime("%Y-%m-%dT%H:%M:%S")),
("Category", "internal-system"),
("Subcategory", "hit-spooling"),
)),
)
self.__convert_tuples_to_xml(root, xmldict)
metaname = basename + self.META_SUFFIX
with open(os.path.join(spadedir, metaname), "w") as out:
line = etree.tostring(root)
try:
# Python3 needs to convert XML bytes to a string
line = line.decode("utf-8")
except:
pass
out.write(line)
return metaname