def dorun():
'''
Handle the `run` command.
'''
loader = load_tests()
if config.tags:
suites = []
for tag in config.tags:
suites.extend(loader.suites_with_tag(tag))
else:
suites = loader.suites
# Create directory to save junit and internal results in.
mkdir_p(config.result_path)
with open(joinpath(config.result_path, 'pickle'), 'w') as result_file,\
open(joinpath(config.result_path, 'junit.xml'), 'w') as junit_f:
junit_logger = result.JUnitLogger(junit_f, result_file)
console_logger = result.ConsoleLogger()
loggers = (junit_logger, console_logger)
log.display(separator())
log.bold('Running Tests')
log.display('')
if config.uid:
test_item = loader.get_uid(config.uid)
results = Runner.run_items(test_item)
else:
testrunner = Runner(suites, loggers)
results = testrunner.run()
def draw_phenogenon(**kwargs):
'''
output pdf to visualise phenogenon heatmaps for each hpo/moi
'''
import itertools
import plotly.graph_objs as go
import plotly.io as pio
# log transform p values
def log_transform(x): return x if np.isnan(x) else -math.log(x)
log_transform = np.vectorize(log_transform)
for moi in ('r', 'd'):
# make folder
outdir = os.path.join(kwargs['heatmap_outdir'], moi)
helper.mkdir_p(outdir)
for hpo in kwargs['hpos'][moi]:
data = kwargs['data']['phenogenon'][moi][hpo]
# get file name:
outfile = os.path.join(
outdir, '{}.pdf'.format(hpo.replace(':', '_')))
# get zmax
zmax = min(kwargs['zmax'], max(
list(itertools.chain.from_iterable([i for i in data]))))
# get title
title = '{}-{}'.format(kwargs['hpo_db'][hpo]['name'][0], 'recessive' if moi
== 'r' else 'dominant')
# get trace
trace = {
'z': log_transform(data),
'x': np.arange(0, kwargs['gnomad_cutoff'], kwargs['gnomad_step']),
'y': np.arange(0, 60, kwargs['cadd_step']),
'connectgaps': False,
'type': 'heatmap',
'showscale': True,
'zmin': 0,
'zauto': False
}
layout = {
'title': title,
'xaxis': {'title': 'GF'},
'yaxis': {'title': 'CADD_phred'}
}
fig = go.Figure(data=[trace], layout=layout)
pio.write_image(fig, outfile)
def run_test(self, testobj, fixtures=None):
'''
Run the given test.
This performs the bulkwork of the testing framework.
1. Handle setup for all fixtures required for the specific test.
2. Run the test.
3. Teardown the fixtures for the test which are tied locally to the\
test.
'''
outdir = test_results_output_path(testobj)
mkdir_p(outdir)
fstdout_name = joinpath(outdir, config.constants.system_err_name)
fstderr_name = joinpath(outdir, config.constants.system_out_name)
# Capture the output into a file.
with tee(fstderr_name, stderr=True, stdout=False),\
tee(fstdout_name, stderr=False, stdout=True):
return self._run_test(testobj, fstdout_name, fstderr_name,
fixtures)
def main():
global opts
parser = create_parser()
opts, _args = parser.parse_args()
if opts.database is None:
parser.parse_args(['-h'])
opts.output = opts.output + str(datetime.now())
resource.setrlimit(resource.RLIMIT_AS, (2147483648L, 2147483648L))
Pyro4.config.HMAC_KEY = 'choffnes-cunha-javed-owning'
sys.excepthook = Pyro4.util.excepthook
ns = Pyro4.naming.locateNS('128.208.4.106', 51555)
# get the livedb object:
uri = ns.lookup('livedb.main')
global api
api = Pyro4.Proxy(uri)
mkdir_p(opts.logdir)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
loghandler = logging.handlers.RotatingFileHandler(opts.logfile,
maxBytes=128*1024*1024, backupCount=5)
loghandler.setFormatter(formatter)
logger.addHandler(loghandler)
sys.stdout = open(opts.output, 'w')
sys.stderr = sys.stdout
global conn
c = create_db()
global feeds
feeds = ['ftr', 'rv', 'rtr']
num2pfx = dict()
global pfxToLoc
pfxToLoc = dict()
for feed in feeds:
pfxToLoc[feed] = api.get_prefix_to_locations(feed)
global num2VPseen
num2VPseen = dict()
for num in PREFIX_RANGE:
num2VPseen[num] = dict()
global pfx2PoisonList
pfx2PoisonList = dict()
for num in PREFIX_RANGE:
pfx2PoisonList[num] = list()
pfx2PoisonList[num].append(RIOT_ASN)
global pfx2VPBeingPoisoned
pfx2VPBeingPoisoned = dict()
global pfx2VP
pfx2VP = defaultdict(list)
global pfx2VPAlreadyPoisoned
pfx2VPAlreadyPoisoned = dict()
for num in PREFIX_RANGE:
pfx2VPAlreadyPoisoned[num] = set()
global pfx2VPASNAlreadyDone
pfx2VPASNAlreadyDone = dict()
for num in PREFIX_RANGE:
pfx2VPASNAlreadyDone[num] = set()
global pfx2PoisVPNotSeenCount
pfx2PoisVPNotSeenCount = dict()
for num in PREFIX_RANGE:
pfx2PoisVPNotSeenCount[num] = 0
global vpPois_TR_path
vpPois_TR_path = dict()
for num in PREFIX_RANGE:
vpPois_TR_path[num] = TR_path_object('','','','','','','')
global vpPois_seen_prevTR
vpPois_seen_prevTR = dict()
for num in PREFIX_RANGE:
vpPois_seen_prevTR[num] = False
#signal.signal(signal.SIGUSR1, lambda sig, stack: traceback.print_stack(stack))
signal.signal(signal.SIGUSR1, printStack)
for num in PREFIX_RANGE:
num2pfx[num] = Prefix(num, RIOT_ASN, "announced")
num2pfx[num].up()
announce_null(num2pfx[num])
soft_reset(num2pfx)
wait_cmd(POISON_INTERVAL_2)
#initial poisoning so that the path shows up in the data feeds
pfx_idx = 0
for mux in MUX_NAMES:
pfx = num2pfx[PREFIX_RANGE[pfx_idx]]
pfx.poisonList([RIOT_ASN])
announce_sentinel(pfx,mux)
pfx_idx += 1
soft_reset(num2pfx)
global last_poison_time
last_poison_time = int(time.time())
wait_cmd(POISON_INTERVAL)
poisonRound = 0
sent_used = True
global num_ftr_query
num_ftr_query = NUM_FTR
while True:
if num_ftr_query>=NUM_FTR:
for num in PREFIX_RANGE:
if poisonRound%SENT_ROUND==0 and sent_used is True:
populateDB(c,num,poisonRound,SENTINEL,"")
else:
populateDB(c,num,poisonRound,TEST,pfx2VPBeingPoisoned[num].rv)
conn.commit()
print "VPNotSeenCount: " + str(pfx2PoisVPNotSeenCount[num])
'''
print "vantage points:"
for v in pfx2VP[num]:
print v
'''
print "vp's already poisoned: " + str(pfx2VPAlreadyPoisoned[num])
print "vp ASes already done: " + str(pfx2VPASNAlreadyDone[num])
pfx2PoisonList[num] = getNextPoisoning(num,poisonRound,sent_used)
print "poison_list: " + str(pfx2PoisonList[num])
print "sentinel used: " + str(sent_used)
pfx = num2pfx[num]
pfx.poisonList(list(pfx2PoisonList[num]))
for mux in MUX_NAMES:
if PREFIX_RANGE.index(num)==MUX_NAMES.index(mux):
pfx.update_route_map1(mux, True)
else:
pfx.update_route_map1(mux,False)
pfx.up()
if poisonRound%SENT_ROUND != 0:
poisonRound += 1
else:
if sent_used is False:
#we just sent sentinels out
sent_used = True
else:
sent_used = False
poisonRound += 1
sys.stdout.flush()
soft_reset(num2pfx)
last_poison_time = int(time.time())
num_ftr_query = 1
wait_cmd(FTR_INTERVAL)
else:
for num in PREFIX_RANGE:
if poisonRound%SENT_ROUND==0 and sent_used is True:
populateDB_ftr(c,num,poisonRound,"")
else:
populateDB_ftr(c,num,poisonRound,pfx2VPBeingPoisoned[num].rv)
conn.commit()
num_ftr_query += 1
wait_cmd(FTR_INTERVAL)
def main():
global opts
parser = create_parser()
opts, _args = parser.parse_args()
if opts.database is None or opts.refasn is None:
parser.parse_args(['-h'])
resource.setrlimit(resource.RLIMIT_AS, (2147483648L, 2147483648L))
Pyro4.config.HMAC_KEY = 'choffnes-cunha-javed-owning'
sys.excepthook = Pyro4.util.excepthook
ns = Pyro4.naming.locateNS('128.208.4.106', 51556)
# get the livedb object:
uri = ns.lookup('livedb.main')
global api
api = Pyro4.Proxy(uri)
mkdir_p(opts.logdir)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
loghandler = logging.handlers.RotatingFileHandler(opts.logfile,
maxBytes=128*1024*1024, backupCount=5)
loghandler.setFormatter(formatter)
logger.addHandler(loghandler)
sys.stdout = open(opts.output, 'w')
sys.stderr = sys.stdout
conn = sqlite3.connect(opts.database + str(datetime.now()))
c = conn.cursor()
#observedPath is the raw path that you see both feeds: lists if ASs from ftr and list of ASes from rv (W in case of withdrawn rv path)
#filteredASPath is the filtered AS path
#asn: the ASN that we're trying to measure
c.execute("create table if not exists ASGraph (prefix text, unix_time int, dtime text, asn int, \
prepend_round int, mux text, feed text, observedPath text, ipPath text, vpoint text)")
c.execute("create index idx_pfx on ASGraph(prefix)")
c.execute("create index idx_asn on ASGraph(asn)")
c.execute("create index idx_pfx_path on ASGraph(prefix,observedPath)")
c.execute("create index idx_prepend on ASGraph(prepend_round)")
global feeds
feeds = ['rv', 'ftr']
num2pfx = dict()
#announce 250-255 from all muxes
for num in PREFIX_RANGE:
num2pfx[num] = Prefix(num, 47065, "announced")
#initial poisoning so that the path shows up in the data feeds
for num in PREFIX_RANGE:
pfx = num2pfx[num]
for mux in MUX_NAMES:
pfx.update_route_map1(mux, True)
pfx.up()
soft_reset(num2pfx)
wait_cmd(1800)
global pfxToLoc
pfxToLoc = dict()
for feed in feeds:
pfxToLoc[feed] = api.get_prefix_to_locations(feed)
global num2VPseen
num2VPseen = dict()
for num in PREFIX_RANGE:
num2VPseen[num] = dict()
global num2pfxMux
num2pfxMux = dict()
global numPrepend
numPrepend = 0
#for refasn store the initial path for each prefix
for num in PREFIX_RANGE:
populateDB(c,num,opts.refasn)
conn.commit()
numPrepend = 1
while True:
for num in PREFIX_RANGE:
mux = num2pfxMux[num]
print "mux for prefix " + str(num) + ": " + mux
pfx = num2pfx[num]
assert mux in MUX_NAMES
pfx.set_num_prepend1(numPrepend,mux)
pfx.up()
soft_reset(num2pfx)
wait_cmd(1800)
for num in PREFIX_RANGE:
populateDB(c,num,opts.refasn)
conn.commit()
numPrepend += 1
def main():
#Parameters
show_useful_ngrams = False
best_cv_score = 0.
#Where to store files measuring how models performed
directory = "data/"
helper.mkdir_p(directory)
print "Extracting tweets from database"
tweets = helper.extract_tweets(users, max_users = -1, max_tweets = 1, min_tweets = 1)
#Can split out CV randomly or by specific terms
print "Splitting out CV"
#split_tweets = helper.split_out_cv_by_terms(tweets, users, cv_terms = ["husband", "wife"], train_terms = [])
split_tweets = helper.split_out_cv_random(tweets)
train_tweets = split_tweets["train"]
cv_tweets = split_tweets["cv"]
out_string = 'Running with ' + str(len(train_tweets)) + ' training tweets and ' + str(len(cv_tweets)) + ' cross-validation tweets'
sys.stdout.write(out_string + '\n')
#Build Ngrams for train tweets, make this general
print "Building Train Ngrams"
#ngrams maps the actual ngram to a unique id
ngrams = {}
helper.extract_ngrams(train_tweets, ngrams)
print "Number of Ngrams " + str(len(ngrams))
counts = [2, 6, 10]
min_sig = [0.2, 0.7, 1.2]
filename = directory + "Scores"
if show_useful_ngrams:
print_useful_ngrams(ngrams, 100, type = "T")
print "Building Cross-Validation Ngrams"
#useful_ngrams maps the ngram id to the column it occupies in matrix
useful_ngrams = find_useful_ngrams(ngrams, min_count=5, min_significance=0.5)
helper.extract_ngrams(cv_tweets, ngrams, for_cv_tweets = True)
with open('data/ngrams.pkl', 'wb') as fid:
cPickle.dump({"ngrams": ngrams, "useful_ngrams": useful_ngrams}, fid)
print "Building training matrix"
train_vars = helper.build_Xy(train_tweets, useful_ngrams)
X_train = train_vars["X"]
y_train = train_vars["y"]
print "Building cross-validation matrix"
cv_vars = helper.build_Xy(cv_tweets, useful_ngrams)
X_cv = cv_vars["X"]
y_cv = cv_vars["y"]
logit_score = logistic_regression(X_train, y_train, X_cv, y_cv, train_tweets, cv_tweets, directory + "Logit")
if logit_score > best_cv_score:
best_cv_score = logit_score
nb_score = naive_bayes(X_train, y_train, X_cv, y_cv, train_tweets, cv_tweets, directory + "MultiNB")
if nb_score > nb_score:
best_cv_score = nb_score
print('Best CV score: %(score)f'% {"score":best_cv_score})
def main():
global opts
parser = create_parser()
opts, _args = parser.parse_args()
if opts.database is None or opts.blacklist is None:
parser.parse_args(['-h'])
resource.setrlimit(resource.RLIMIT_AS, (2147483648L, 2147483648L))
Pyro4.config.HMAC_KEY = 'choffnes-cunha-javed-owning'
sys.excepthook = Pyro4.util.excepthook
ns = Pyro4.naming.locateNS('128.208.4.106', 51556)
# get the livedb object:
uri = ns.lookup('livedb.main')
global api
api = Pyro4.Proxy(uri)
mkdir_p(opts.logdir)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
loghandler = logging.handlers.RotatingFileHandler(opts.logfile,
maxBytes=128*1024*1024, backupCount=5)
loghandler.setFormatter(formatter)
logger.addHandler(loghandler)
sys.stdout = open(opts.output, 'w')
sys.stderr = sys.stdout
global conn
c = create_db()
global feeds
feeds = ['rv', 'ftr', 'rtr']
num2pfx = dict()
global pfxToLoc
pfxToLoc = dict()
for feed in feeds:
pfxToLoc[feed] = api.get_prefix_to_locations(feed)
global num2VPseen
num2VPseen = dict()
for num in PREFIX_RANGE:
num2VPseen[num] = dict()
global pfx2PoisonSet
pfx2PoisonSet = dict()
for num in PREFIX_RANGE:
pfx2PoisonSet[num] = set()
pfx2PoisonSet[num].add(RIOT_ASN)
global pfx2VPBeingPoisoned
pfx2VPBeingPoisoned = dict()
global pfx2VP
pfx2VP = defaultdict(list)
global sent_idx
sent_idx = 0
for num in PREFIX_RANGE:
num2pfx[num] = Prefix(num, RIOT_ASN, "announced")
num2pfx[num].up()
announce_null(num2pfx[num])
soft_reset(num2pfx)
wait_cmd(POISON_INTERVAL_2)
#initial poisoning so that the path shows up in the data feeds
pfx_idx = 0
for mux in MUX_NAMES:
pfx = num2pfx[PREFIX_RANGE[pfx_idx]]
pfx.poisonList([RIOT_ASN])
announce_sentinel(pfx,mux)
pfx_idx += 1
#last pfx is the sentinel: for now starting with the first mux
announce_sentinel(num2pfx[PREFIX_RANGE[-1]],MUX_NAMES[sent_idx])
soft_reset(num2pfx)
wait_cmd(POISON_INTERVAL)
poisonRound = 0
while True:
if poisonRound==0:
vpJustPoisoned = ""
else:
vpJustPoisoned = pfx2VPBeingPoisoned[PREFIX_RANGE[sent_idx]].rv
populateDB(c,PREFIX_RANGE[-1],poisonRound,SENTINEL,vpJustPoisoned)
conn.commit()
for num in PREFIX_RANGE[0:len(PREFIX_RANGE)-1]:
if poisonRound==0:
vpJustPoisoned = ""
else:
vpJustPoisoned = pfx2VPBeingPoisoned[num].rv
populateDB(c,num,poisonRound,TEST,vpJustPoisoned)
conn.commit()
print "vantage points:"
for v in pfx2VP[num]:
print v
pfx2PoisonSet[num] = getNextPoisoning(num)
print "pset: " + str(pfx2PoisonSet[num])
pfx = num2pfx[num]
pfx.poisonList(list(pfx2PoisonSet[num]))
for mux in MUX_NAMES:
if PREFIX_RANGE.index(num)==MUX_NAMES.index(mux):
pfx.update_route_map1(mux, True)
else:
pfx.update_route_map1(mux,False)
pfx.up()
sent_idx = (sent_idx+1)%len(MUX_NAMES)
announce_sentinel(num2pfx[PREFIX_RANGE[-1]],MUX_NAMES[sent_idx])
poisonRound += 1
sys.stdout.flush()
soft_reset(num2pfx)
wait_cmd(POISON_INTERVAL)
def main():
global opts
parser = create_parser()
opts, _args = parser.parse_args()
opts.output = opts.output + str(datetime.now())
resource.setrlimit(resource.RLIMIT_AS, (2147483648L, 2147483648L))
Pyro4.config.HMAC_KEY = 'choffnes-cunha-javed-owning'
sys.excepthook = Pyro4.util.excepthook
ns = Pyro4.naming.locateNS('128.208.4.106', 51556)
# get the livedb object:
uri = ns.lookup('livedb.main')
global api
api = Pyro4.Proxy(uri)
mkdir_p(opts.logdir)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
loghandler = logging.handlers.RotatingFileHandler(opts.logfile,
maxBytes=128*1024*1024, backupCount=5)
loghandler.setFormatter(formatter)
logger.addHandler(loghandler)
global feeds
feeds = ['ftr', 'rv', 'rtr']
num2pfx = dict()
global pfxToLoc
pfxToLoc = dict()
for feed in feeds:
pfxToLoc[feed] = api.get_prefix_to_locations(feed)
for num in PREFIX_RANGE:
num2pfx[num] = Prefix(num, RIOT_ASN, "announced")
num2pfx[num].up()
announce_null(num2pfx[num])
soft_reset(num2pfx)
wait_cmd(POISON_INTERVAL_2)
#initial poisoning so that the path shows up in the data feeds
pfx_idx = 0
for mux in MUX_NAMES:
pfx = num2pfx[PREFIX_RANGE[pfx_idx]]
pfx.poisonList([RIOT_ASN])
announce_sentinel(pfx,mux)
pfx_idx += 1
soft_reset(num2pfx)
wait_cmd(POISON_INTERVAL)
numPaths = 0
tstamp = int(time.time())
for feed in feeds:
locs = pfxToLoc[feed][pfx]
for rv in locs:
update = api.get_path(feed, pfx, rv, 'last_update')
if feed=='rv':
if update.time < START_TIME:
continue
vp = str(rv).split(',')
if len(vp)<2:
continue
elif feed=='ftr' or feed=='rtr':
if update.time < tstamp-(30*60):
continue
numPaths += 1
print numPaths
def main():
global opts
parser = create_parser()
opts, _args = parser.parse_args()
resource.setrlimit(resource.RLIMIT_AS, (2147483648L, 2147483648L))
Pyro4.config.HMAC_KEY = 'choffnes-cunha-javed-owning'
sys.excepthook = Pyro4.util.excepthook
ns = Pyro4.naming.locateNS('128.208.4.106', 51556)
# get the livedb object:
uri = ns.lookup('livedb.main')
global api
api = Pyro4.Proxy(uri)
mkdir_p(opts.logdir)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
loghandler = logging.handlers.RotatingFileHandler(opts.logfile,
maxBytes=128*1024*1024, backupCount=5)
loghandler.setFormatter(formatter)
logger.addHandler(loghandler)
sys.stdout = open(opts.output, 'w')
sys.stderr = sys.stdout
db_file = opts.database + str(datetime.now())
#db_file = "../data/mike_data/mike.db2012-04-25 00:08:59.710008"
conn = sqlite3.connect(db_file)
c = conn.cursor()
c.execute("create table if not exists ASGraph (prefix text, unix_time int, dtime text, \
poison_round int, poisonedAS int, feed text, observedPath text, filteredASPath, vpoint text, thisVPoisoned int)")
c.execute("create index idx_pfx on ASGraph(prefix)")
c.execute("create index idx_asn on ASGraph(poisonedAS)")
c.execute("create index idx_pfx_path on ASGraph(prefix,observedPath)")
c.execute("create index idx_pfx_filpath on ASGraph(prefix,filteredASPath)")
c.execute("create index idx_prepend on ASGraph(poison_round)")
c.execute("create index idx_vpoint on ASGraph(vpoint)")
num2pfx = dict()
for num in PREFIX_RANGE:
num2pfx[num] = Prefix(num, 47065, "announced")
global feeds
#feeds = ['rv', 'ftr']
feeds = ['rv']
global num2PoisonSet
num2PoisonSet = dict()
for num in PREFIX_RANGE:
num2PoisonSet[num] = set()
num2PoisonSet[num].add(47065)
global num2VPoint
num2VPoint = dict()
for num in PREFIX_RANGE:
num2VPoint[num] = "0"
global ASPoisoned
ASPoisoned = dict()
for num in PREFIX_RANGE:
pfx = num2pfx[num]
pfx.poisonList([47065])
pfx.up()
soft_reset(num2pfx)
wait_cmd(1800)
global poisonRound
poisonRound = 0
global pfxToLoc
pfxToLoc = dict()
for feed in feeds:
pfxToLoc[feed] = api.get_prefix_to_locations(feed)
while True:
for num in PREFIX_RANGE:
populateDB(c,num)
conn.commit()
#wait_cmd(1800)
for num in PREFIX_RANGE:
pfx = num2pfx[num]
f = False
while f is False:
f = getPoisonings(c,num)
pfx.poisonList(list(num2PoisonSet[num]))
pfx.up()
soft_reset(num2pfx)
poisonRound += 1
wait_cmd(3600)
def main():
global opts
parser = create_parser()
opts, _args = parser.parse_args()
resource.setrlimit(resource.RLIMIT_AS, (2147483648L, 2147483648L))
Pyro4.config.HMAC_KEY = 'choffnes-cunha-javed-owning'
sys.excepthook = Pyro4.util.excepthook
ns = Pyro4.naming.locateNS('128.208.4.106', 51556)
# get the livedb object:
uri = ns.lookup('livedb.main')
global api
api = Pyro4.Proxy(uri)
mkdir_p(opts.logdir)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
loghandler = logging.handlers.RotatingFileHandler(opts.logfile,
maxBytes=128*1024*1024, backupCount=5)
loghandler.setFormatter(formatter)
logger.addHandler(loghandler)
sys.stdout = open(opts.output, 'w')
sys.stderr = sys.stdout
global num2PoisonCandidates
num2PoisonCandidates = dict()
for num in PREFIX_RANGE:
num2PoisonCandidates[num] = list()
global num2commonAS
num2commonAS = dict()
num2commonAS[241] = [101,73]
num2commonAS[243] = [6939,2381]
num2commonAS[245] = [174,2637]
num2commonAS[247] = [7922,88]
num2commonAS[249] = [209,2722,12148]
global num2startPoisIdx
num2startPoisIdx = dict()
num2startPoisIdx[241] = 3
num2startPoisIdx[243] = 2
num2startPoisIdx[245] = 2
num2startPoisIdx[247] = 2
num2startPoisIdx[249] = 3
conn = sqlite3.connect('data/imc4.db')
c = conn.cursor()
#c.execute("drop table ASGraph ")
c.execute("create table ASGraph (prefix text, unix_time int, dtime text, asn int, edge int, \
bfs_level int, poison_round int, startTime text, poisoning text, feed text, observedPath text, vpoint text)")
c.execute("create index idx_pfx on ASGraph(prefix)")
c.execute("create index idx_asn on ASGraph(asn)")
c.execute("create index idx_pfx_path on ASGraph(prefix,observedPath)")
c.execute("create index idx_edge on ASGraph(asn,edge)")
global feeds
feeds = ['rv', 'ftr']
num2pfx = dict()
for num in PREFIX_RANGE:
num2pfx[num] = Prefix(num, 47065, "announced")
global num2PoisonSet
num2PoisonSet = dict()
for num in PREFIX_RANGE:
num2PoisonSet[num] = set()
num2PoisonSet[num].add(47065)
global startTime
startTime = str(datetime.now())
#initial poisoning so that the path shows up in the data feeds
for num in PREFIX_RANGE:
pfx = num2pfx[num]
pfx.poisonList([47065])
pfx.up()
soft_reset(num2pfx)
wait_cmd(1800)
global num2BFSLevel
num2BFSLevel = dict()
for num in PREFIX_RANGE:
num2BFSLevel[num] = 1
global num2PoisonRound
num2PoisonRound = dict()
for num in PREFIX_RANGE:
num2PoisonRound[num] = 1
global pfxToLoc
pfxToLoc = dict()
for feed in feeds:
pfxToLoc[feed] = api.get_prefix_to_locations(feed)
while True:
for num in PREFIX_RANGE:
populateDB(c,num)
conn.commit()
#wait_cmd(1800)
for num in PREFIX_RANGE:
pfx = num2pfx[num]
getPoisonings(c,num)
pfx.poisonList(list(num2PoisonSet[num]))
pfx.up()
soft_reset(num2pfx)
for num in PREFIX_RANGE:
num2PoisonRound[num] += 1
#for num in PREFIX_RANGE:
# logging.info('prefix_getPoisonings 184.164.%d.0/24 %s %s', num, num2PoisonSet[num])
wait_cmd(1800)
'''
def make_vm_directory(self, group_vm, subid):
vm_dir = self.vm_dir(group_vm, subid)
helper.mkdir_p(vm_dir)
def make_directories(self, group_vm):
self.logger.debug('Start method: make_directries of Group VM[%s]' % group_vm.group_name())
download_dir = self.group_download_dir(group_vm)
helper.mkdir_p(download_dir)
for subid in group_vm.subids():
self.make_vm_directory(group_vm, subid)
def __init__(self, stdout, stderr):
helper.mkdir_p(os.path.dirname(stdout))
helper.mkdir_p(os.path.dirname(stderr))
self.stdout = open(stdout, 'w')
self.stderr = open(stderr, 'w')
def main():
global opts
parser = create_parser()
opts, _args = parser.parse_args()
resource.setrlimit(resource.RLIMIT_AS, (2147483648L, 2147483648L))
Pyro4.config.HMAC_KEY = 'choffnes-cunha-javed-owning'
sys.excepthook = Pyro4.util.excepthook
ns = Pyro4.naming.locateNS('128.208.4.106', 51556)
# get the livedb object:
uri = ns.lookup('livedb.main')
global api
api = Pyro4.Proxy(uri)
mkdir_p(opts.logdir)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
loghandler = logging.handlers.RotatingFileHandler(opts.logfile,
maxBytes=128*1024*1024, backupCount=5)
loghandler.setFormatter(formatter)
logger.addHandler(loghandler)
#sys.stdout = open(opts.output, 'w')
#sys.stderr = sys.stdout
global num2PoisonCandidates
num2PoisonCandidates = dict()
for num in PREFIX_RANGE:
num2PoisonCandidates[num] = list()
global num2commonAS
num2commonAS = dict()
num2commonAS[241] = [101,73]
num2commonAS[243] = [6939,2381]
num2commonAS[245] = [174,2637]
num2commonAS[247] = [7922,88]
num2commonAS[249] = [209,2722,12148]
global num2startPoisIdx
num2startPoisIdx = dict()
num2startPoisIdx[241] = 3
num2startPoisIdx[243] = 2
num2startPoisIdx[245] = 2
num2startPoisIdx[247] = 2
num2startPoisIdx[249] = 3
conn = sqlite3.connect('data/imc3.db')
c = conn.cursor()
global feeds
feeds = ['rv', 'ftr']
global num2PoisonSet
num2PoisonSet = dict()
for num in PREFIX_RANGE:
num2PoisonSet[num] = set()
num2PoisonSet[num].add(47065)
global num2BFSLevel
num2BFSLevel = dict()
for num in PREFIX_RANGE:
num2BFSLevel[num] = 1
global num2PoisonRound
num2PoisonRound = dict()
for num in PREFIX_RANGE:
num2PoisonRound[num] = 1
global pfxToLoc
pfxToLoc = dict()
for feed in feeds:
pfxToLoc[feed] = api.get_prefix_to_locations(feed)
i = 1
while (i < 5):
for num in PREFIX_RANGE:
getPoisonings(c,num)
for num in PREFIX_RANGE:
num2PoisonRound[num] += 1
i += 1
'''
while True:
for num in PREFIX_RANGE:
populateDB(c,num)
conn.commit()
#wait_cmd(1800)
for num in PREFIX_RANGE:
pfx = num2pfx[num]
getPoisonings(c,num)
pfx.poisonList(list(num2PoisonSet[num]))
pfx.up()
soft_reset(num2pfx)
for num in PREFIX_RANGE:
num2PoisonRound[num] += 1
#for num in PREFIX_RANGE:
# logging.info('prefix_getPoisonings 184.164.%d.0/24 %s %s', num, num2PoisonSet[num])
wait_cmd(1800)
'''
'''
parser.add_argument('-o', '--outputpath', default='./wiki_tokenized', type=str, help='output path')
parser.add_argument('-p', '--separator', default='[,.;]', type=str, help='characters that separate sentences')
parser.add_argument('-n', '--number', default=-1, type=int, help='number of gram files we use')
parser.add_argument('-a', '--alphabet', default='abcdefghijklmnopqrstuvwxyz -',
type=str, help='supported alphabet')
parser.add_argument('--cores', default=6, type=int, help='number of concurrent threads')
args = parser.parse_args()
print(args)
# Begin Extract List of files
articles = [f for f in listdir(args.inputpath) if isfile(join(args.inputpath, f))]
articles.sort()
print len(articles), "article files found."
mkdir_p(join(args.outputpath + "/readable_articles"))
mkdir_p(join(args.outputpath + "/articles"))
grams = [f for f in listdir(args.grampath) if isfile(join(args.grampath, f))]
grams.sort()
print len(grams), "gram files found."
# End Extract List of files
N = args.number
if (N <= 0) or (N >= len(grams)): N = len(grams) # N is the number of gram files we use, after sorted by filename
print "Loading grams dictionary..."
Dict = {} # Global dictionary for gram-files
gram_file_count = 0
for gram_file in grams:
load_data_from_json(Dict, join(args.grampath, gram_file))
def __init__(self, stdout, stderr):
helper.mkdir_p(os.path.dirname(stdout))
helper.mkdir_p(os.path.dirname(stderr))
self.stdout = open(stdout, 'w')
self.stderr = open(stderr, 'w')
plt.savefig(figurename, dpi=300)
plt.close()
if __name__ == '__main__':
'''
'''
# Get config details
results_folder, title, smooth, methods = helper.read_config_PLOT()
rawdata_folder = results_folder + '/rawdata/'
for evaluation in ['FMAX', 'WFMAX', 'SMIN', 'NSMIN']:
for onto in ['BPO', 'CCO', 'MFO']:
for Type in ['LK', 'NK']:
for mode in ['partial', 'full']:
specific_title = '%s_%s_%s_%s_%s.png' % (onto, title, Type,
mode, evaluation)
print('\nPlotting %s\n' % title)
helper.mkdir_p('./plots')
result_list = []
for method in methods:
fields = method.split('_')
taxon = fields[2]
r = getResults(method, onto, taxon, Type, mode,
evaluation, results_folder)
result_list.append(r)
plotMultiple(specific_title, result_list, smooth)
else:
sys.exit()
def main():
global opts
parser = create_parser()
opts, _args = parser.parse_args()
resource.setrlimit(resource.RLIMIT_AS, (2147483648L, 2147483648L))
Pyro4.config.HMAC_KEY = "choffnes-cunha-javed-owning"
sys.excepthook = Pyro4.util.excepthook
ns = Pyro4.naming.locateNS("128.208.4.106", 51556)
uri = ns.lookup("livedb.main")
global api
api = Pyro4.Proxy(uri)
mkdir_p(opts.logdir)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(message)s")
loghandler = logging.handlers.RotatingFileHandler(opts.logfile, maxBytes=128 * 1024 * 1024, backupCount=5)
loghandler.setFormatter(formatter)
logger.addHandler(loghandler)
global feeds
feeds = ["rv", "ftr"]
global pfxToLoc
pfxToLoc = dict()
for feed in feeds:
pfxToLoc[feed] = api.get_prefix_to_locations(feed)
global num2PoisonAS
num2PoisonAS = dict()
for num in PREFIX_RANGE:
num2PoisonAS[num] = 47065
num2ASPoisonedThisRound = defaultdict(list)
global pfx2VP
pfx2VP = defaultdict(list)
global pfx2VPtoPoison
pfx2VPtoPoison = dict() # the VP index to be poisoned
for num in PREFIX_RANGE:
pfx2VPtoPoison[num] = 0
tstamp = int(time.time())
for num in PREFIX_RANGE:
pfx = "184.164." + str(num) + ".0/24"
for feed in feeds:
locs = pfxToLoc[feed][pfx]
for rv in locs:
update = api.get_path(feed, pfx, rv, "last_update")
if not update.path:
continue
if feed == "rv":
print update
if update.time < (START_TIME - 10 * 3600):
continue
vp = str(rv).split(",")
if len(vp) < 2:
continue
if feed == "ftr":
if update.time < tstamp - (30 * 60):
continue
path = ""
for asn in update.path:
path += str(asn) + " "
path = path.split()
tup = getFilteredPath(path, feed)
if tup is None:
continue
filpath = tup[0]
mux = tup[1]
vPoint = VP(pfx, filpath, feed, mux, str(rv))
pfx2VP[num].append(vPoint)
"""
def main():
global opts
parser = create_parser()
opts, _args = parser.parse_args()
if opts.database is None or opts.blacklist is None:
parser.parse_args(["-h"])
opts.output = opts.output + str(datetime.now())
resource.setrlimit(resource.RLIMIT_AS, (2147483648L, 2147483648L))
Pyro4.config.HMAC_KEY = "choffnes-cunha-javed-owning"
sys.excepthook = Pyro4.util.excepthook
ns = Pyro4.naming.locateNS("128.208.4.106", 51556)
# get the livedb object:
uri = ns.lookup("livedb.main")
global api
api = Pyro4.Proxy(uri)
mkdir_p(opts.logdir)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(message)s")
loghandler = logging.handlers.RotatingFileHandler(opts.logfile, maxBytes=128 * 1024 * 1024, backupCount=5)
loghandler.setFormatter(formatter)
logger.addHandler(loghandler)
sys.stdout = open(opts.output, "w")
sys.stderr = sys.stdout
global conn
c = create_db()
global feeds
feeds = ["ftr", "rv", "rtr"]
num2pfx = dict()
global pfxToLoc
pfxToLoc = dict()
for feed in feeds:
pfxToLoc[feed] = api.get_prefix_to_locations(feed)
global num2VPseen
num2VPseen = dict()
for num in PREFIX_RANGE:
num2VPseen[num] = dict()
global pfx2PoisonList
pfx2PoisonList = dict()
for num in PREFIX_RANGE:
pfx2PoisonList[num] = list()
pfx2PoisonList[num].append(RIOT_ASN)
global pfx2VPBeingPoisoned
pfx2VPBeingPoisoned = dict()
global pfx2VP
pfx2VP = defaultdict(list)
"""
global pfx2VPOrder
pfx2VPOrder = defaultdict(list)
"""
global pfx2VPAlreadyPoisoned
pfx2VPAlreadyPoisoned = dict()
for num in PREFIX_RANGE:
pfx2VPAlreadyPoisoned[num] = set()
global pfx2PoisVPNotSeenCount
pfx2PoisVPNotSeenCount = dict()
for num in PREFIX_RANGE:
pfx2PoisVPNotSeenCount[num] = 0
for num in PREFIX_RANGE:
num2pfx[num] = Prefix(num, RIOT_ASN, "announced")
num2pfx[num].up()
announce_null(num2pfx[num])
soft_reset(num2pfx)
wait_cmd(POISON_INTERVAL_2)
# initial poisoning so that the path shows up in the data feeds
pfx_idx = 0
for mux in MUX_NAMES:
pfx = num2pfx[PREFIX_RANGE[pfx_idx]]
pfx.poisonList([RIOT_ASN])
announce_sentinel(pfx, mux)
pfx_idx += 1
soft_reset(num2pfx)
wait_cmd(POISON_INTERVAL)
poisonRound = 0
sent_used = True
while True:
for num in PREFIX_RANGE:
if poisonRound % SENT_ROUND == 0 and sent_used is True:
vpJustPoisoned = ""
populateDB(c, num, poisonRound, SENTINEL, "")
else:
vpJustPoisoned = pfx2VPBeingPoisoned[num].rv
populateDB(c, num, poisonRound, TEST, vpJustPoisoned)
conn.commit()
print "vantage points:"
for v in pfx2VP[num]:
print v
print "vp's already poisoned: " + str(pfx2VPAlreadyPoisoned[num])
pfx2PoisonList[num] = getNextPoisoning(num, poisonRound, sent_used)
print "poison_list: " + str(pfx2PoisonList[num])
print "sentinel used: " + str(sent_used)
pfx = num2pfx[num]
pfx.poisonList(list(pfx2PoisonList[num]))
for mux in MUX_NAMES:
if PREFIX_RANGE.index(num) == MUX_NAMES.index(mux):
pfx.update_route_map1(mux, True)
else:
pfx.update_route_map1(mux, False)
pfx.up()
if poisonRound % SENT_ROUND != 0:
poisonRound += 1
else:
if sent_used is False:
# we just sent sentinels out
sent_used = True
else:
sent_used = False
poisonRound += 1
sys.stdout.flush()
soft_reset(num2pfx)
wait_cmd(POISON_INTERVAL)
def main():
global opts
parser = create_parser()
opts, _args = parser.parse_args()
if opts.database is None:
parser.parse_args(['-h'])
resource.setrlimit(resource.RLIMIT_AS, (2147483648L, 2147483648L))
Pyro4.config.HMAC_KEY = 'choffnes-cunha-javed-owning'
sys.excepthook = Pyro4.util.excepthook
ns = Pyro4.naming.locateNS('128.208.4.106', 51556)
uri = ns.lookup('livedb.main')
global api
api = Pyro4.Proxy(uri)
mkdir_p(opts.logdir)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(message)s')
loghandler = logging.handlers.RotatingFileHandler(opts.logfile,
maxBytes=128*1024*1024, backupCount=5)
loghandler.setFormatter(formatter)
logger.addHandler(loghandler)
sys.stdout = open(opts.output, 'w')
sys.stderr = sys.stdout
global conn
c = create_db()
global feeds
feeds = ['rv', 'ftr']
num2pfx = dict()
global pfxToLoc
pfxToLoc = dict()
for feed in feeds:
pfxToLoc[feed] = api.get_prefix_to_locations(feed)
global num2VPseen
num2VPseen = dict()
for num in PREFIX_RANGE:
num2VPseen[num] = dict()
for num in PREFIX_RANGE_SENT:
num2VPseen[num] = dict()
global num2PoisonAS
num2PoisonAS = dict()
for num in PREFIX_RANGE:
num2PoisonAS[num] = RIOT_ASN
num2ASPoisonedThisRound = defaultdict(list)
global pfx2VP
pfx2VP = defaultdict(list)
global pfx2VPtoPoison
pfx2VPtoPoison = dict() #the VP index to be poisoned
for num in PREFIX_RANGE:
pfx2VPtoPoison[num] = -1
for num in PREFIX_RANGE:
num2pfx[num] = Prefix(num, RIOT_ASN, "announced")
pfx = num2pfx[num]
pfx.poisonList([RIOT_ASN])
pfx.up()
for num in PREFIX_RANGE:
pfx = num2pfx[num]
for mux in MUX_NAMES:
if PREFIX_RANGE.index(num)==MUX_NAMES.index(mux):
pfx.update_route_map1(mux, True)
else:
pfx.update_route_map1(mux,False)
pfx.up()
for num in PREFIX_RANGE_SENT:
num2pfx[num] = Prefix(num, RIOT_ASN, "announced")
pfx = num2pfx[num]
pfx.poisonList([RIOT_ASN])
pfx.up()
for num in PREFIX_RANGE_SENT:
pfx = num2pfx[num]
for mux in MUX_NAMES:
if PREFIX_RANGE_SENT.index(num)==MUX_NAMES.index(mux):
pfx.update_route_map1(mux, True)
else:
pfx.update_route_map1(mux,False)
pfx.up()
soft_reset(num2pfx)
wait_cmd(POISON_INTERVAL)
poisonRound = 0
while True:
for num in PREFIX_RANGE_SENT:
populateDB(c,num,poisonRound,SENTINEL,"")
conn.commit()
for num in PREFIX_RANGE:
if poisonRound==0:
vpJustPoisoned = ""
else:
vpJustPoisoned = pfx2VP[num][pfx2VPtoPoison[num]].rv
populateDB(c,num,poisonRound,TEST,vpJustPoisoned)
conn.commit()
if poisonRound==0:
for num in PREFIX_RANGE:
pfx2VPtoPoison[num] = 0
pfx = num2pfx[num]
print num2ASPoisonedThisRound[num]
vp2poison = pfx2VP[num][pfx2VPtoPoison[num]]
print "vp to poison: " + str(pfx2VPtoPoison[num]) + " " + str(vp2poison)
num2PoisonAS[num] = getNextPoisoning(vp2poison, num2ASPoisonedThisRound[num])
if num2PoisonAS[num] > 0 and num2PoisonAS[num] not in opts.blacklist:
num2ASPoisonedThisRound[num].append(num2PoisonAS[num])
print "as to poison: " + str(num2PoisonAS[num])
print "new poisonidx: " + str(vp2poison.poisonIdx)
if vp2poison.poisonIdx==vp2poison.startingPoisonIdx:
if (pfx2VPtoPoison[num]+1)==len(pfx2VP[num]):
num2ASPoisonedThisRound[num] = []
pfx2VPtoPoison[num] = (pfx2VPtoPoison[num]+1)%len(pfx2VP[num])
print "new vp to poison: " + str(pfx2VPtoPoison[num])
if num2PoisonAS[num] > 0 and num2PoisonAS[num] not in opts.blacklist:
pfx.poisonList([num2PoisonAS[num]])
else:
pfx.poisonList([RIOT_ASN])
for mux in MUX_NAMES:
if PREFIX_RANGE.index(num)==MUX_NAMES.index(mux):
pfx.update_route_map1(mux, True)
else:
pfx.update_route_map1(mux,False)
pfx.up()
sys.stdout.flush()
soft_reset(num2pfx)
poisonRound += 1
wait_cmd(POISON_INTERVAL)