def get(self, url, default=None, retry=True):
"Get HTML content"
target_url = self._db_url + "/" + km3db.compat.unquote(url)
try:
f = self.opener.open(target_url)
except km3db.compat.HTTPError as e:
if e.code == 403:
if retry:
log.error(
"Access forbidden, your session has expired. "
"Deleting the cookie ({}) and retrying once.".format(
COOKIE_FILENAME))
else:
log.critical("Access forbidden. Giving up...")
return default
time.sleep(1)
self.reset()
os.remove(COOKIE_FILENAME)
return self.get(url, default=default, retry=False)
log.error("HTTP error: {}\n"
"Target URL: {}".format(e, target_url))
return default
try:
content = f.read()
except km3db.compat.IncompleteRead as icread:
log.error("Incomplete data received from the DB.")
content = icread.partial
log.debug("Got {0} bytes of data.".format(len(content)))
return content.decode("utf-8")
def runtable(det_id,
n=5,
run_range=None,
target=None,
compact=False,
sep="\t",
regex=None):
"""Print the run table of the last `n` runs for given detector"""
runs = km3db.StreamDS(container="nt").get("runs", detid=det_id)
if run_range is not None:
try:
from_run, to_run = [int(r) for r in run_range.split("-")]
except ValueError:
log.critical("Please specify a valid range (e.g. 3100-3200)!")
raise SystemExit
else:
runs = [r for r in runs if from_run <= r.run <= to_run]
if regex is not None:
try:
pattern = re.compile(regex)
except re.error:
log.error("Invalid regex!")
return
else:
runs = [
r for r in runs if re.match(pattern, r.runsetupname)
or re.match(pattern, r.runsetupid)
]
if target is not None:
runs = [r for r in runs if r.jobtarget == target.capitalize()]
if n is not None:
runs = runs[-n:]
if not runs:
log.warning("No runs found.")
return
if compact:
attributes = ["run", "runsetupname"]
header = sep.join(attributes)
def lineformatter(entry):
return sep.join([str(getattr(entry, attr)) for attr in attributes])
else:
header = sep.join(runs[0]._fields) # the dataset is homogenious
def lineformatter(entry):
return sep.join(map(str, entry))
print(header)
for entry in runs:
print(lineformatter(entry))
def _request_session_cookie(self):
"""Request cookie for permanent session."""
# Next, try the configuration file according to
# the specification described here:
# https://wiki.km3net.de/index.php/Database#Scripting_access
if os.path.exists(COOKIE_FILENAME):
log.info("Using cookie from %s", COOKIE_FILENAME)
with open(COOKIE_FILENAME) as fobj:
content = fobj.read()
return content.split()[-1].strip()
# The cookie can also be set via the environment
cookie = os.getenv("KM3NET_DB_COOKIE")
if cookie is not None:
log.info("Using cookie from env ($KM3NET_DB_COOKIE)")
return cookie
username = os.getenv("KM3NET_DB_USERNAME")
password = os.getenv("KM3NET_DB_PASSWORD")
if username is None or password is None:
# Last resort: we ask interactively
username = km3db.compat.user_input(
"Please enter your KM3NeT DB username: "******"Password: "******"Using credentials from env ($KM3NET_DB_USERNAME and "
"$KM3NET_DB_PASSWORD)")
target_url = self._login_url + "?usr={0}&pwd={1}&persist=y".format(
username, password)
cookie = km3db.compat.urlopen(target_url).read()
# Unicode madness
try:
cookie = str(cookie, "utf-8") # Python 3
except TypeError:
cookie = str(cookie) # Python 2
cookie = cookie.split("sid=")[-1]
if not _cookie_sid_pattern.match(cookie):
message = "Wrong username or password."
log.critical(message)
raise AuthenticationError(message)
log.info("Writing session cookie to %s", COOKIE_FILENAME)
with open(COOKIE_FILENAME, "w") as fobj:
fobj.write(".in2p3.fr\tTRUE\t/\tTRUE\t0\tsid\t{}".format(cookie))
return cookie
def opener(self):
"A reusable connection manager"
if self._opener is None:
log.debug("Creating connection handler")
opener = km3db.compat.build_opener()
cookie = self.session_cookie
if cookie is None:
log.critical("Could not connect to database.")
return
opener.addheaders.append(("Cookie", "sid=" + cookie))
self._opener = opener
else:
log.debug("Reusing connection manager")
return self._opener
def get(self, stream, fmt="txt", container=None, renamemap=None, **kwargs):
"""Retrieve the data for a given stream manually
Parameters
==========
stream: str
Name of the stream (e.g. detectors)
fmt: str ("txt", "text", "bin")
Retrieved raw data format, depends on the stream type
container: str or None
The container to wrap the returned data, as specified in
`StreamDS`.
"""
sel = "".join(["&{0}={1}".format(k, v) for (k, v) in kwargs.items()])
url = "streamds/{0}.{1}?{2}".format(stream, fmt, sel[1:])
data = self._db.get(url)
if not data:
log.error("No data found at URL '%s'." % url)
return
if data.startswith("ERROR"):
log.error(data)
return
if container is None and self._default_container is not None:
container = self._default_container
try:
if container == "pd":
return topandas(data)
if container == "nt":
return tonamedtuples(stream.capitalize(),
data,
renamemap=renamemap)
except ValueError:
log.critical("Unable to convert data to container type '{}'. "
"Database response: {}".format(container, data))
else:
return data