def init_runtime(remote, local, local_only):
while True:
try:
opts = {
"cache": {
"preload": ["nodes", "services"]
},
"proxy": {
"defer_update": True
}
}
if local_only:
urls = [{"default": True, "url": local}]
log.debug("Connecting to UNIS instance(s): {}".format(local))
else:
urls = [{"url": local}, {"default": True, "url": remote}]
log.debug(
"Connecting to UNIS instance(s): {}".format(remote + ',' +
local))
rt = Runtime(urls, **opts)
if local_only:
rt.exnodes.addCallback(file_cb)
return rt
except (ConnectionError, TimeoutError) as exp:
log.warn(
"Could not contact UNIS servers {}, retrying...".format(urls))
time.sleep(5)
def _watcher(self, cfg):
while True:
naddrs = self._get_addrs()
if set(naddrs) != set(self.addrs) or self.first:
log.debug(
"iface diff: {}".format(set(naddrs) - set(self.addrs)))
istr = ""
for i in naddrs:
istr += "{}:6714;".format(i)
self.addrs = naddrs
self.update_ibp_server(istr)
self.first = False
time.sleep(5)
def local_download(sess, exnodes):
for f in exnodes:
if not len(f.extents):
continue
fpath = os.path.join(DOWNLOAD_DIR, f.name)
if os.path.exists(fpath) and os.path.getsize(fpath) == f.size:
log.debug("File exists: {}, skipping!".format(f.name))
continue
log.info("Downloading: {} ({} bytes)".format(f.name, f.size))
try:
result = sess.download(f.selfRef, fpath)
res, diff, dsize = result.exnode, result.time, result.t_size
except Exception as e:
log.error("Could not download file: {}".format(e))
continue
if dsize != res.size:
log.warn("WARNING: {}: transferred {} of {} bytes \
(check depot file)".format(res.name, dsize, res.size))
else:
log.info("{0} ({1} {2:.2f} MB/s) {3}".format(
res.name, res.size, res.size / 1e6 / diff, res.selfRef))
def _sync(self, rt):
while True:
nds = []
n5 = int((time.time() - SYNC_INTERVAL) * 1e6)
n10 = int((time.time() - SERVICE_THRESH) * 1e6)
for n in rt.nodes:
if n.ts > n5:
nds.append(n)
if len(nds):
log.debug("Time to sync: {}".format(([n.name for n in nds])))
for s in rt.services:
if s.ts > n10 and s.serviceType in STYPES:
# push all known nodes if we haven't
# seen this service before
if s.name not in self.pushed:
nds = rt.nodes
# build a list of nodes for a single post
nlist = []
for n in nds:
repr = n.to_JSON()
del repr['selfRef']
nlist.append(repr)
# finally post the node list to the endpoint
url = "http://{}:9000/nodes".format(s.name)
log.debug("Syncing to {}".format(url))
try:
self._do_post(url, json.dumps(nlist))
self.pushed.append(s.name)
except:
pass
time.sleep(SYNC_INTERVAL)
def _srv_cb(self, s, ev):
log.debug("Service update: {}".format(s.name))