def assign_pops(args):
r = connection.Redis()
if args.reset:
log.info("Resetting processed_links")
if r.llen("delayed_job:unassigned_links") == 0:
r.rename("delayed_job:processed_links", "delayed_job:unassigned_links")
else:
while r.rpoplpush("delayed_job:processed_links",
"delayed_job:unassigned_links"):
pass
r.delete("delayed_job:unassigned_link_fails")
return
if args.process_failed:
log.info("Processing failed links")
dbkeys.mutex_popjoin().acquire()
_assign_pops("delayed_job:unassigned_link_fails",
"delayed_job:unassigned_link_fails2",
no_add_processed=True)
if r.exists("delayed_job:unassigned_link_fails2"):
r.rename("delayed_job:unassigned_link_fails2",
"delayed_job:unassigned_link_fails")
dbkeys.mutex_popjoin().release()
log.info("Complete")
return
_assign_pops("delayed_job:unassigned_links",
"delayed_job:unassigned_link_fails")
def parse(args):
# We don't use this, but it configures the singleton
connection.Redis(structures.ConnectionInfo(**args.redis))
try:
aslookup = preprocess.MaxMindGeoIPReader.Instance()
with open(args.trace) as trace_in:
tracehops = []
seenset = set()
numtraces = 0
log.info("Processed trace: %s" % str(numtraces))
for line in trace_in:
if line.split()[0] == "traceroute":
numtraces += 1
if numtraces % 1000 == 0:
log.info("\r\x1b[K" + "Processed trace: %s" % str(numtraces))
try:
newpairs, removed = TraceParser.parse(tracehops)
if removed is not None:
log.debug("Removed %s" % removed)
except EmptyTraceError:
tracehops = [line]
continue
if not args.dump and dbkeys.mutex_popjoin().is_locked():
log.debug("Waiting for popjoin lock")
dbkeys.mutex_popjoin().wait()
if args.dump:
for pair in newpairs:
print_unless_seen(pair[0], seenset)
print_unless_seen(pair[1], seenset)
else:
load_link_pairs(newpairs, geoipdb=aslookup)
tracehops = [line]
else:
tracehops.append(line)
except IOError as e:
log.error("Error: {0}".format(e))
raise SilentExit()
except DataError as e:
log.error("Error: {0}".format(e))
raise SilentExit()
def process_delayed_joins(args):
log.info("Processing delayed joins")
r = connection.Redis()
if r.llen("delayed_job:unassigned_link_fails") > 0:
sys.stderr.write("Have unassigned links. "
"Run assign_pops --process_failed\n")
raise SilentExit()
# Now we process any joins that need to happen. First we lock.
error_ctr = 0
dbkeys.mutex_popjoin().acquire()
try:
joinlist = preprocess_joins()
inprocess = list()
x = len(joinlist)
log.info("Joining pop pairs: %d".format(x))
fh = None
if args.log_joins:
fh = logging.FileHandler(args.log_joins, mode='w')
fh.setLevel(logging.DEBUG)
fh.setFormatter(logging.Formatter('%(message)s'))
log.addHandler(fh)
timer = ProgressTimer(x)
for i, to_join in enumerate(joinlist):
inprocess.append(to_join)
log.info("Joining %s to %s\n" % (to_join[1], to_join[0]))
try:
joined = join_pops(r, to_join[0], to_join[1])
except redis_errors as e:
log.error("Encountered error while processing: {0}. [{1}]\n"
.format(to_join, e))
joinlist.insert(0, inprocess.pop())
error_ctr += 1
continue
else:
if joined is not None:
log.info("Joined %s to %s\n" % (joined[1], joined[0]))
if (r.sismember(dbkeys.POP.list(), to_join[1])
or r.exists(dbkeys.POP.members(to_join[1]))):
if descend_target_chain(r, to_join[0]) != to_join[1]:
raise Exception("Join Failed in ways it should not have...")
else:
log.info("Did not join {0} to {1} because {2} had "
"previously been joined to {3}\n"
.format(to_join[1], to_join[0], to_join[0], to_join[1]))
timer.tick(1)
x = len(joinlist) - i
sys.stderr.write("{newl} {0} joins left {1}\n".format(
x,
Color.wrapformat("[{0} seconds to finish]",
Color.OKBLUE, timer.eta()),
newl=Color.NEWL))
r.delete('delayed_job:popjoins')
r.delete('delayed_job:popjoins:inprocess')
r.delete(r.keys(dbkeys.POP.joined("*")))
log.info("Joined pops with %d errors while processing" % error_ctr)
if fh is not None:
log.removeHandler(fh)
except KeyboardInterrupt:
pass
finally:
dbkeys.mutex_popjoin().release()
