def get(self, url, default=None, retry=True): "Get HTML content" target_url = self._db_url + "/" + km3db.compat.unquote(url) try: f = self.opener.open(target_url) except km3db.compat.HTTPError as e: if e.code == 403: if retry: log.error( "Access forbidden, your session has expired. " "Deleting the cookie ({}) and retrying once.".format( COOKIE_FILENAME)) else: log.critical("Access forbidden. Giving up...") return default time.sleep(1) self.reset() os.remove(COOKIE_FILENAME) return self.get(url, default=default, retry=False) log.error("HTTP error: {}\n" "Target URL: {}".format(e, target_url)) return default try: content = f.read() except km3db.compat.IncompleteRead as icread: log.error("Incomplete data received from the DB.") content = icread.partial log.debug("Got {0} bytes of data.".format(len(content))) return content.decode("utf-8")
def runtable(det_id, n=5, run_range=None, target=None, compact=False, sep="\t", regex=None): """Print the run table of the last `n` runs for given detector""" runs = km3db.StreamDS(container="nt").get("runs", detid=det_id) if run_range is not None: try: from_run, to_run = [int(r) for r in run_range.split("-")] except ValueError: log.critical("Please specify a valid range (e.g. 3100-3200)!") raise SystemExit else: runs = [r for r in runs if from_run <= r.run <= to_run] if regex is not None: try: pattern = re.compile(regex) except re.error: log.error("Invalid regex!") return else: runs = [ r for r in runs if re.match(pattern, r.runsetupname) or re.match(pattern, r.runsetupid) ] if target is not None: runs = [r for r in runs if r.jobtarget == target.capitalize()] if n is not None: runs = runs[-n:] if not runs: log.warning("No runs found.") return if compact: attributes = ["run", "runsetupname"] header = sep.join(attributes) def lineformatter(entry): return sep.join([str(getattr(entry, attr)) for attr in attributes]) else: header = sep.join(runs[0]._fields) # the dataset is homogenious def lineformatter(entry): return sep.join(map(str, entry)) print(header) for entry in runs: print(lineformatter(entry))
def _request_session_cookie(self): """Request cookie for permanent session.""" # Next, try the configuration file according to # the specification described here: # https://wiki.km3net.de/index.php/Database#Scripting_access if os.path.exists(COOKIE_FILENAME): log.info("Using cookie from %s", COOKIE_FILENAME) with open(COOKIE_FILENAME) as fobj: content = fobj.read() return content.split()[-1].strip() # The cookie can also be set via the environment cookie = os.getenv("KM3NET_DB_COOKIE") if cookie is not None: log.info("Using cookie from env ($KM3NET_DB_COOKIE)") return cookie username = os.getenv("KM3NET_DB_USERNAME") password = os.getenv("KM3NET_DB_PASSWORD") if username is None or password is None: # Last resort: we ask interactively username = km3db.compat.user_input( "Please enter your KM3NeT DB username: "******"Password: "******"Using credentials from env ($KM3NET_DB_USERNAME and " "$KM3NET_DB_PASSWORD)") target_url = self._login_url + "?usr={0}&pwd={1}&persist=y".format( username, password) cookie = km3db.compat.urlopen(target_url).read() # Unicode madness try: cookie = str(cookie, "utf-8") # Python 3 except TypeError: cookie = str(cookie) # Python 2 cookie = cookie.split("sid=")[-1] if not _cookie_sid_pattern.match(cookie): message = "Wrong username or password." log.critical(message) raise AuthenticationError(message) log.info("Writing session cookie to %s", COOKIE_FILENAME) with open(COOKIE_FILENAME, "w") as fobj: fobj.write(".in2p3.fr\tTRUE\t/\tTRUE\t0\tsid\t{}".format(cookie)) return cookie
def opener(self): "A reusable connection manager" if self._opener is None: log.debug("Creating connection handler") opener = km3db.compat.build_opener() cookie = self.session_cookie if cookie is None: log.critical("Could not connect to database.") return opener.addheaders.append(("Cookie", "sid=" + cookie)) self._opener = opener else: log.debug("Reusing connection manager") return self._opener
def get(self, stream, fmt="txt", container=None, renamemap=None, **kwargs): """Retrieve the data for a given stream manually Parameters ========== stream: str Name of the stream (e.g. detectors) fmt: str ("txt", "text", "bin") Retrieved raw data format, depends on the stream type container: str or None The container to wrap the returned data, as specified in `StreamDS`. """ sel = "".join(["&{0}={1}".format(k, v) for (k, v) in kwargs.items()]) url = "streamds/{0}.{1}?{2}".format(stream, fmt, sel[1:]) data = self._db.get(url) if not data: log.error("No data found at URL '%s'." % url) return if data.startswith("ERROR"): log.error(data) return if container is None and self._default_container is not None: container = self._default_container try: if container == "pd": return topandas(data) if container == "nt": return tonamedtuples(stream.capitalize(), data, renamemap=renamemap) except ValueError: log.critical("Unable to convert data to container type '{}'. " "Database response: {}".format(container, data)) else: return data