def init_runtime(remote, local, local_only): while True: try: opts = { "cache": { "preload": ["nodes", "services"] }, "proxy": { "defer_update": True } } if local_only: urls = [{"default": True, "url": local}] log.debug("Connecting to UNIS instance(s): {}".format(local)) else: urls = [{"url": local}, {"default": True, "url": remote}] log.debug( "Connecting to UNIS instance(s): {}".format(remote + ',' + local)) rt = Runtime(urls, **opts) if local_only: rt.exnodes.addCallback(file_cb) return rt except (ConnectionError, TimeoutError) as exp: log.warn( "Could not contact UNIS servers {}, retrying...".format(urls)) time.sleep(5)
def _files(): for f in bottle.request.files.keys(): dat = bottle.request.files.get(f) path = os.path.join(d, dat.filename) dat.save(path, overwrite=True) LOCAL_DEPOT = {s.accessPoint: {"enabled": True}} try: with libdlt.Session(s.unis_url, bs="5m", depots=LOCAL_DEPOT, threads=1) as sess: res = sess.upload(path) if not hasattr(s, 'uploaded_exnodes'): s.extendSchema('uploaded_exnodes', []) s.uploaded_exnodes.append(res.exnode) except ValueError as e: log.warn(e)
def _do_upload(self, src): LOCAL_DEPOT = { "ibp://{}:6714".format(socket.getfqdn()): { "enabled": True } } try: with libdlt.Session(self._local, bs="5m", depots=LOCAL_DEPOT, threads=1) as sess: res = sess.upload(src) except ValueError as e: log.warn(e) if not hasattr(self._s, 'uploaded_exnodes'): self._s.extendSchema('uploaded_exnodes', [res.exnode]) else: self._s.uploaded_exnodes.append(res.exnode)
def local_download(sess, exnodes): for f in exnodes: if not len(f.extents): continue fpath = os.path.join(DOWNLOAD_DIR, f.name) if os.path.exists(fpath) and os.path.getsize(fpath) == f.size: log.debug("File exists: {}, skipping!".format(f.name)) continue log.info("Downloading: {} ({} bytes)".format(f.name, f.size)) try: result = sess.download(f.selfRef, fpath) res, diff, dsize = result.exnode, result.time, result.t_size except Exception as e: log.error("Could not download file: {}".format(e)) continue if dsize != res.size: log.warn("WARNING: {}: transferred {} of {} bytes \ (check depot file)".format(res.name, dsize, res.size)) else: log.info("{0} ({1} {2:.2f} MB/s) {3}".format( res.name, res.size, res.size / 1e6 / diff, res.selfRef))
def __init__(self): self.sock = gps3.GPSDSocket() self.stream = gps3.DataStream() # GPS3 throws no exceptions and returns no status, # so we have to capture stdout and do string matching, sigh f = io.StringIO() with redirect_stderr(f): self.sock.connect() if "Connection refused" in f.getvalue(): log.warn( "Could not connect to GPSD socket, continuing with GPS_BOX") self.sock = None else: log.info("Connected to GPSD socket") self.sock.watch() # we will use the GPS_BOX setting to determine if # queries will return locations within a specified range if len(GPS_BOX) == 4: self.gps_box = box(GPS_BOX[0], GPS_BOX[1], GPS_BOX[2], GPS_BOX[3]) log.info("Created GPS box") else: log.warn("Invalid GPS_BOX size, using GPS_DEFAULT")