def getMergeAndSort(savePath, localPath, label=None):
tmp_path = "/tmp/tmp-spark"
try:
os.mkdirs(tmp_path)
except:
pass
hdfs.get(savePath, tmp_path)
cmdMerge = """find %s -name "*" -print0 | xargs -0 cat >> /tmp/tmp-spark""" % tmp_path
print cmdMerge
os.system(cmdMerge)
cmdSort = "sort -k1,1 /tmp/tmp-spark > {0}".format(
os.path.join(localPath,
localPath.split("/")[-1] + ".tsv"))
os.system(cmdSort)
if (label is not None):
cmd = "sed -i '1s/^/%s\\n/' %s" % (label, path + ".tsv")
print cmd
os.system(cmd)
cmdErase = "rm /tmp/tmp-spark"
os.system(cmdErase)
try:
shutil.rmtree(tmp_path)
except:
pass
def copy_to_local(hdfs_path, local_path, overwrite=False, project=None):
"""
Copies a path from HDFS project to local filesystem
Args:
:local_path: the path on the local filesystem to copy
:hdfs_path: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS).
:overwrite: a boolean flag whether to overwrite if the path already exists in HDFS
:project: name of the project, defaults to the current HDFS user's project
"""
if project == None:
project = project_name()
if "PDIR" in os.environ:
full_local = os.environ['PDIR'] + '/' + local_path
else:
full_local = os.getcwd() + '/' + local_path
project_hdfs_path = _expand_path(hdfs_path, project=project)
sub_path = hdfs_path.find("hdfs:///Projects/" + project)
rel_path = hdfs_path[sub_path + 1:]
if overwrite:
split = rel_path.split('/')
filename = split[len(split) - 1]
full_local_path = full_local + '/' + filename
if os.path.isdir(full_local_path):
shutil.rmtree(full_local_path)
elif os.path.isfile(full_local_path):
os.remove(full_local_path)
hdfs.get(project_hdfs_path, full_local)
def test_method(self):
self.logger.info( ('-'*20 + " %s " + '-'*20), self.test_name)
self.setup()
success = False
try:
self.logger.info("running %s program", self.test_name)
self.run_program(self.make_hdfs_input_path(), self.make_hdfs_output_path())
self.logger.info("now going to process output")
self.logger.debug("hdfs.get(%s, %s)", self.make_hdfs_output_path(), self.output_dir)
hdfs.get(self.make_hdfs_output_path(), self.output_dir)
self.process_output()
success = True
except Exception as e:
self.logger.error("*"*72)
self.logger.error("Test %s raised an exception" % self.test_name)
self.logger.error(e)
self.logger.error("*"*72)
finally:
self.logger.info("cleaning up")
self.clean_up()
self.logger.info( '-'*(42 + len(self.test_name)) ) # close the test section with a horizontal line
self.show_test_msg(success)
return success
def get(self):
src = self.hdfs_paths[0]
dest = hdfs.path.split(self.local_paths[0])[-1]
hdfs.dump(self.data, src)
hdfs.get(src, dest)
with open(dest) as fi:
rdata = fi.read()
self.assertEqual(rdata, self.data)
def get(self):
src = self.hdfs_paths[0]
dest = hdfs.path.split(self.local_paths[0])[-1]
hdfs.dump(self.data, src, mode="wb")
hdfs.get(src, dest, mode="wb")
with open(dest, 'rb') as fi:
rdata = fi.read()
self.assertEqual(rdata, self.data)
def runSparkNumASesInROAs(sc, ip_type):
roa_prefix_asn = "/hdfs-to-local-path/rpki/ripe/ripe-new-objects/roa-prefix-asn/*"
savePath = "/hdfs-to-local-path/rpki/results/roas-covering-AScnt-%s" % ip_type
localPath = "/home/tjchung/research/rpki/src/spark/results/roas-covering-AScnt-%s" % ip_type
try:
hdfs.rmr(savePath)
except:
pass
tals = [
"apnic", "apnic-iana", "apnic-afrinic", "apnic-arin", "apnic-lacnic",
"apnic-ripe", "lacnic", "ripencc", "arin", "afrinic", "localcert"
]
k = sc.textFile(roa_prefix_asn)\
.filter(lambda line: "#" not in line)\
.map(lambda line: line.rstrip().split("\t"))\
.filter(lambda (time, prefix_addr, prefix_len, maxlen, asID, num_ips, cc, tal): isIPv4v6(prefix_addr, ip_type))\
.distinct()\
.map(lambda (time, prefix_addr, prefix_len, maxlen, asID, num_ips, cc, tal): ((time, tal), asID))\
.groupByKey()\
.map(lambda ( (time, tal), num_ases): (time, tal, len(set(num_ases))))
sqlContext = SQLContext(sc)
df = sqlContext.createDataFrame(k, ['date', 'tal', 'num_ASes'])
grouped = df.rdd\
.map(lambda row: (row.date, (row.tal, row.num_ASes)))\
.groupByKey()
def make_row(kv):
k, vs = kv # time: [(-1, cnt), (0, cnt), (1, cnt)] ...
tmp = dict(list(vs) + [("date", k)])
return Row(**{k: tmp.get(k, 0) for k in ["date"] + tals})
reshaped = sqlContext.createDataFrame(grouped.map(make_row))
k = reshaped.rdd\
.map(lambda row: (row['date'], row["apnic"], row["apnic-iana"], row["apnic-afrinic"], row["apnic-arin"], row["apnic-lacnic"], row["apnic-ripe"], row["lacnic"], row["ripencc"], row["arin"], row["afrinic"], row["localcert"]))\
.map(toTSV)
k.saveAsTextFile(savePath)
try:
shutil.rmtree(localPath)
except:
pass
try:
os.makedirs(localPath)
except:
pass
hdfs.get(savePath, localPath)
mergeAndSort(localPath)
def copy_from_hdfs_to_local(self,src_hdfs_location,dest_local_location="../output/"):
if src_hdfs_location=="":
print "No source specified"
return False
elif self.handle.exists(src_hdfs_location)==False:
print "File does not exist"
return False
hdfs.get(src_hdfs_location,dest_local_location)
return True
def copy_to_local(hdfs_path, local_path, overwrite=False, project=None):
"""
Copies a path from HDFS project to local filesystem. If there is not enough space on the local scratch directory, an exception is thrown.
Raises:
IOError if there is not enough space to localize the file/directory in HDFS to the scratch directory ($PDIR)
Args:
:local_path: the path on the local filesystem to copy
:hdfs_path: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS).
:overwrite: a boolean flag whether to overwrite if the path already exists in HDFS
:project: name of the project, defaults to the current HDFS user's project
Returns:
the full local pathname of the file/dir
"""
import os
import pydoop.hdfs.path as path
if project == None:
project = project_name()
if "PDIR" in os.environ:
full_local = os.environ['PDIR'] + '/' + local_path
else:
full_local = os.getcwd() + '/' + local_path
project_hdfs_path = _expand_path(hdfs_path, project=project)
sub_path = hdfs_path.find("hdfs:///Projects/" + project)
rel_path = hdfs_path[sub_path + 1:]
# Get the amount of free space on the local drive
stat = os.statvfs(full_local)
free_space_bytes = stat.f_bsize * stat.f_bavail
hdfs_size = path.getsize(project_hdfs_path)
if (hdfs_size > free_space_bytes):
raise IOError(
"Not enough local free space available on scratch directory: %s" %
path)
if overwrite:
split = rel_path.split('/')
filename = split[len(split) - 1]
full_local_path = full_local + '/' + filename
if os.path.isdir(full_local_path):
shutil.rmtree(full_local_path)
elif os.path.isfile(full_local_path):
os.remove(full_local_path)
hdfs.get(project_hdfs_path, full_local)
return full_local
def runSparkNumPrefixWithMaxlen(sc, ip_type="ipv4"):
roa_prefix_asn = "/hdfs-to-local-path/rpki/ripe/ripe-new-objects/roa-prefix-asn/*"
localPath = "/home/tjchung/research/rpki/src/spark/results/roa-prefix-with-maxlength"
savePath = "/hdfs-to-local-path/rpki/results/roa-prefix-with-maxlength"
try:
hdfs.rmr(savePath)
except:
pass
k = sc.textFile(roa_prefix_asn)\
.filter(lambda line: "#" not in line)\
.map(lambda line: line.rstrip().split("\t"))\
.filter(lambda (time, prefix_addr, prefix_len, maxlen, asID, num_ips, cc, tal): isIPv4v6(prefix_addr, ip_type) )\
.map(lambda (time, prefix_addr, prefix_len, maxlen, asID, num_ips, cc, tal): (time, prefix_addr, prefix_len, maxlen, asID, num_ips, cc, tal))\
.distinct()\
.map(lambda (time, prefix_addr, prefix_len, maxlen, asID, num_ips, cc, tal): ((time, str(int( (prefix_len != maxlen) and maxlen != "None" ))), 1))\
.reduceByKey(lambda a, b: a+ b)\
.map(lambda ((time, hasMaxlen), cnt): (time, hasMaxlen, cnt))
sqlContext = SQLContext(sc)
df = sqlContext.createDataFrame(k, ['date', 'hasMaxlen', 'cnt'])
grouped = df.rdd\
.map(lambda row: (row.date, (row.hasMaxlen, row.cnt)))\
.groupByKey()
def make_row(kv):
k, vs = kv
tmp = dict(list(vs) + [("date", k)])
return Row(**{k: tmp.get(k, 0)
for k in ["date", "0", "1"]}) # 1 means has a maxlen
reshaped = sqlContext.createDataFrame(grouped.map(make_row))
k = reshaped.rdd\
.map(lambda row: (row['date'], row['0'], row['1']))\
.map(toTSV)
k.saveAsTextFile(savePath)
try:
shutil.rmtree(localPath)
except:
pass
try:
os.makedirs(localPath)
except:
pass
hdfs.get(savePath, localPath)
mergeAndSort(localPath)
def runSparkClassifyHijackingUniquePrefixDuration(sc, dataset, ip_type):
readPath = "/spark-hdfs-path/rpki/results/rpki-enabled-unique-prefix-asn-%s/%s" % (
ip_type, dataset)
savePath = "/spark-hdfs-path/rpki/results/rpki-enabled-unique-prefix-classify-hijack-duration-%s/%s" % (
ip_type, dataset)
localPath = "/local-spark-result-path/research/rpki/src/spark/results/rpki-unique-prefix-classify-hijack-duration-%s/%s" % (
ip_type, dataset)
try:
hdfs.rmr(savePath)
except:
pass
k = sc.textFile(readPath)\
.map(lambda v: parseVerifyLineUniquePrefix(v))\
.filter(lambda v: v is not None)\
.filter(lambda v: notDataError(dataset, v))\
.filter(lambda v: isIPv4v6(v, ip_type))\
.filter(lambda v: classifyBGPAdvSparse(v) == "rpki-invalid")\
.filter(lambda v: ip_type == "ipv6" or not isLargerSlash24(v))\
.filter(lambda v: onlyHijackAttempt(v))\
.map(lambda v: ( (classifyHijack(v), v['prefix_addr'], v['prefix_len'], v['origin_as']), v['time']))\
.groupByKey()\
.map(lambda ((classifyHijack, prefix_addr, prefix_len, origin), list_of_time): (classifyHijack, prefix_addr, prefix_len, origin, len(set(list_of_time))))\
.map(toTSV)\
.saveAsTextFile(savePath)
try:
shutil.rmtree(localPath)
except:
pass
try:
os.makedirs(localPath)
except:
pass
hdfs.get(savePath, localPath)
mergeAndSort(localPath)
def runSparkClassifyHijackingUniquePrefixList(sc, dataset, ip_type):
readPath = "/spark-hdfs-path/rpki/results/rpki-enabled-unique-prefix-asn-%s/%s" % (
ip_type, dataset)
savePath = "/spark-hdfs-path/rpki/results/rpki-enabled-unique-prefix-classify-hijack-list-%s/%s" % (
ip_type, dataset)
localPath = "/local-spark-result-path/research/rpki/src/spark/results/rpki-unique-prefix-classify-hijack-list-%s/%s" % (
ip_type, dataset)
try:
hdfs.rmr(savePath)
except:
pass
k = sc.textFile(readPath)\
.map(lambda v: parseVerifyLineUniquePrefix(v))\
.filter(lambda v: v is not None)\
.filter(lambda v: notDataError(dataset, v))\
.filter(lambda v: isIPv4v6(v, ip_type))\
.filter(lambda v: classifyBGPAdvSparse(v) == "rpki-invalid")\
.filter(lambda v: ip_type == "ipv6" or not isLargerSlash24(v))\
.filter(lambda v: onlyHijackAttempt(v))\
.map(lambda v: ((v['time'], classifyHijack(v)), json.dumps(v)))\
.map(toTSV)\
.saveAsTextFile(savePath)
try:
shutil.rmtree(localPath)
except:
pass
try:
os.makedirs(localPath)
except:
pass
hdfs.get(savePath, localPath)
mergeAndSort(localPath)
def test_method(self):
"""
"main" method
"""
self.options = self.parser.parse_args()
if self.options.debug:
self.logger.setLevel(logging.DEBUG)
self.logger.info( ('-'*20 + " %s " + '-'*20), self.test_name)
self.setup()
self.logger.debug("setup complete")
success = False
try:
self.logger.info("running %s program", self.test_name)
hdfs_input = self.make_hdfs_input_path()
hdfs_output = self.make_hdfs_output_path()
self.logger.debug("hdfs input path: %s", hdfs_input)
self.logger.debug("hdfs output path: %s", hdfs_output)
self.run_program(hdfs_input, hdfs_output)
self.logger.info("now going to process output")
self.logger.debug("hdfs.get(%s, %s)", self.make_hdfs_output_path(), self.output_dir)
hdfs.get(self.make_hdfs_output_path(), self.output_dir)
self.process_output()
success = True
except Exception as e:
self.logger.error("*"*72)
self.logger.error("Test %s raised an exception" % self.test_name)
self.logger.error(e)
self.logger.error("*"*72)
finally:
self.logger.info("cleaning up")
self.clean_up()
self.logger.info( '-'*(42 + len(self.test_name)) ) # close the test section with a horizontal line
self.show_test_msg(success)
return success
def copy_to_local(hdfs_path, local_path="", overwrite=False, project=None):
"""
Copies a directory or file from a HDFS project to a local private scratch directory. If there is not enough space on the local scratch directory, an exception is thrown.
If the local file exists, and the hdfs file and the local file are the same size in bytes, return 'ok' immediately.
If the local directory tree exists, and the hdfs subdirectory and the local subdirectory have the same files and directories, return 'ok' immediately.
For example, if you execute:
>>> copy_to_local("Resources/my_data")
This will copy the directory my_data from the Resources dataset in your project to the current working directory on the path ./my_data
Raises:
IOError if there is not enough space to localize the file/directory in HDFS to the scratch directory ($PDIR)
Args:
:hdfs_path: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS).
:local_path: the relative or full path to a directory on the local filesystem to copy to (relative to a scratch directory $PDIR), defaults to $CWD
:overwrite: a boolean flag whether to overwrite if the path already exists in the local scratch directory.
:project: name of the project, defaults to the current HDFS user's project
Returns:
the full local pathname of the file/dir
"""
if project == None:
project = project_name()
if local_path.startswith(os.getcwd()):
local_dir = local_path
else:
local_dir = os.getcwd() + '/' + local_path
if not os.path.isdir(local_dir):
raise IOError("You need to supply the path to a local directory. This is not a local dir: %s" % local_dir)
filename = path.basename(hdfs_path)
full_local = local_dir + "/" + filename
project_hdfs_path = _expand_path(hdfs_path, project=project)
# Get the amount of free space on the local drive
stat = os.statvfs(local_dir)
free_space_bytes = stat.f_bsize * stat.f_bavail
hdfs_size = path.getsize(project_hdfs_path)
if os.path.isfile(full_local) and not overwrite:
sz = os.path.getsize(full_local)
if hdfs_size == sz:
print("File " + project_hdfs_path + " is already localized, skipping download...")
return full_local
else:
os.remove(full_local)
if os.path.isdir(full_local) and not overwrite:
try:
localized = _is_same_directory(full_local, project_hdfs_path)
if localized:
print("Full directory subtree already on local disk and unchanged. Set overwrite=True to force download")
return full_local
else:
shutil.rmtree(full_local)
except Exception as e:
print("Failed while checking directory structure to avoid re-downloading dataset, falling back to downloading")
print(e)
shutil.rmtree(full_local)
if hdfs_size > free_space_bytes:
raise IOError("Not enough local free space available on scratch directory: %s" % local_path)
if overwrite:
if os.path.isdir(full_local):
shutil.rmtree(full_local)
elif os.path.isfile(full_local):
os.remove(full_local)
print("Started copying " + project_hdfs_path + " to local disk on path " + local_dir + "\n")
hdfs.get(project_hdfs_path, local_dir)
print("Finished copying\n")
return full_local
import os
from bs4 import BeautifulSoup
conn=happybase.Connection('127.0.0.1')
crawlTbl=conn.table('crawls')
ANET=125
for BNET in range(33,40):
for CNET in range(0,255):
for DNET in range(0,255):
FNAME =str(ANET)+"."+str(BNET)+"."+str(CNET)+"."+str(DNET)+"_root20150114.htm"
GETFILE="crawls/"+str(ANET)+"/"+str(BNET)+"/"+FNAME
print FNAME
try:
hdfs.get(GETFILE,FNAME)
except IOError:
print BNET,CNET,DNET
continue
soup=BeautifulSoup(open(FNAME,'r'))
for anchor in soup.find_all('a'):
link = anchor.get('href')
for key, data in table.rows([FNAME]):
print key,data
break
FNAME.close()
def index():
n = 'NongNghiep.mp4'
d = str(uuid4())
hdfs.get('/video/{}'.format(n), 'static/{}/{}'.format(d, n))
return render_template('index.html', video='static/{}/{}'.format(d, n))
def copy_to_local(hdfs_path, local_path, overwrite=False, project=None):
"""
Copies a directory or file from a HDFS project to a local private scratch directory. If there is not enough space on the local scratch directory, an exception is thrown.
If the local file exists, and the hdfs file and the local file are the same size in bytes, return 'ok' immediately.
If the local directory tree exists, and the hdfs subdirectory and the local subdirectory have the same files and directories, and the files are the same size in bytes, return 'ok' immediately.
Raises:
IOError if there is not enough space to localize the file/directory in HDFS to the scratch directory ($PDIR)
Args:
:local_path: the relative or full path to a directory on the local filesystem to copy to (relative to a scratch directory $PDIR)
:hdfs_path: You can specify either a full hdfs pathname or a relative one (relative to your Project's path in HDFS).
:overwrite: a boolean flag whether to overwrite if the path already exists in the local scratch directory.
:project: name of the project, defaults to the current HDFS user's project
Returns:
the full local pathname of the file/dir
"""
if project == None:
project = project_name()
if "PDIR" in os.environ:
local_dir = os.environ['PDIR'] + '/' + local_path
else:
local_dir = os.getcwd() + '/' + local_path
if os.path.isdir(local_dir) == False:
raise IOError(
"You need to supply the path to a local directory. This is not a local dir: %s"
% local_dir)
filename = path.basename(hdfs_path)
full_local = local_dir + "/" + filename
project_hdfs_path = _expand_path(hdfs_path, project=project)
sub_path = hdfs_path.find("hdfs:///Projects/" + project)
rel_path = hdfs_path[sub_path + 1:]
# Get the amount of free space on the local drive
stat = os.statvfs(local_dir)
free_space_bytes = stat.f_bsize * stat.f_bavail
hdfs_size = path.getsize(project_hdfs_path)
if os.path.isfile(full_local) and overwrite == False:
sz = os.path.getsize(full_local)
if (hdfs_size == sz):
return full_local
if os.path.isdir(full_local) and overwrite == False:
if FsTree().check(full_local, project_hdfs_path) == True:
print(
"Full directory subtree already on local disk and unchanged.")
return full_local
if (hdfs_size > free_space_bytes):
raise IOError(
"Not enough local free space available on scratch directory: %s" %
path)
if overwrite:
if os.path.isdir(full_local):
shutil.rmtree(full_local)
elif os.path.isfile(full_local):
os.remove(full_local)
hdfs.get(project_hdfs_path, local_dir)
return full_local
import pydoop.hdfs as hdfs
import os
from bs4 import BeautifulSoup
conn = happybase.Connection('127.0.0.1')
crawlTbl = conn.table('crawls')
ANET = 125
for BNET in range(33, 40):
for CNET in range(0, 255):
for DNET in range(0, 255):
FNAME = str(ANET) + "." + str(BNET) + "." + str(CNET) + "." + str(
DNET) + "_root20150114.htm"
GETFILE = "crawls/" + str(ANET) + "/" + str(BNET) + "/" + FNAME
print FNAME
try:
hdfs.get(GETFILE, FNAME)
except IOError:
print BNET, CNET, DNET
continue
soup = BeautifulSoup(open(FNAME, 'r'))
for anchor in soup.find_all('a'):
link = anchor.get('href')
for key, data in table.rows([FNAME]):
print key, data
break
FNAME.close()
os.remove(FNAME)
def fetch(savePath, dstPath, saveDir):
hdfs.get(savePath, os.path.join(dstPath, saveDir))
mergeAndSort(os.path.join(dstPath, saveDir))
def runSparkROAsIPPercentage(sc, year, ip_type): # only support v4
def getNumIPs(list_of_ips, ip_type):
s = []
for (prefix_addr, prefix_len) in list_of_ips:
if (ip_type == "ipv4"):
prefix = IPv4Network("%s/%s" % (prefix_addr, prefix_len))
else:
prefix = IPv6Network("%s/%s" % (prefix_addr, prefix_len))
s.append(prefix)
return str(sum(map(lambda v: v.num_addresses, collapse_addresses(s))))
tals = ["apnic", "lacnic", "ripencc", "arin", "afrinic"]
nro_stats = "/hdfs-to-local-path/rpki/nrostats-withdate/nrostats-%s*-v4.csv" % year
roa_prefix_asn = "/hdfs-to-local-path/rpki/ripe/ripe-new-objects/roa-prefix-asn/%s/*" % year
savePath = "/hdfs-to-local-path/rpki/results/roas-covering-IPcnt-%s/%s" % (
ip_type, year)
localPath = "/home/tjchung/research/rpki/src/spark/results/roas-covering-IPcnt-%s/%s" % (
ip_type, year)
try:
hdfs.rmr(savePath)
except:
pass
a = sc.textFile(nro_stats)\
.map(lambda v: parseNRO(v, ip_type))\
.filter(lambda v: v is not None)\
.reduceByKey(lambda a, b: a+ b)\
.map(lambda ((time, rir), num_ips): ( (time, rir), str(num_ips)))
k = sc.textFile(roa_prefix_asn)\
.filter(lambda line: "#" not in line)\
.map(lambda line: line.rstrip().split("\t"))\
.filter(lambda (time, prefix_addr, prefix_len, maxlen, asID, num_ips, cc, tal): isIPv4v6(prefix_addr, ip_type) and tal != "localcert")\
.map(lambda (time, prefix_addr, prefix_len, maxlen, asID, num_ips, cc, tal): (time, prefix_addr, prefix_len, maxlen, asID, num_ips, cc, tal))\
.distinct()\
.map(lambda (time, prefix_addr, prefix_len, maxlen, asID, num_ips, cc, tal): ((time, tal), (prefix_addr, prefix_len)))\
.groupByKey()\
.map(lambda ( (time, tal), list_ip_prefixes): ((time, tal), getNumIPs(list_ip_prefixes, ip_type)))\
.join(a)\
.map(lambda ((time, tal), (num_rpki_ips, all_ips)): (time, tal, "%s\t%s" % (num_rpki_ips, all_ips)))
sqlContext = SQLContext(sc)
df = sqlContext.createDataFrame(k, ['date', 'tal', 'num_ips'])
grouped = df.rdd\
.map(lambda row: (row.date, (row.tal, row.num_ips)))\
.groupByKey()
def make_row(kv):
k, vs = kv # time: [(-1, cnt), (0, cnt), (1, cnt)] ...
tmp = dict(list(vs) + [("date", k)])
return Row(**{k: tmp.get(k, "0\t0") for k in ["date"] + tals})
reshaped = sqlContext.createDataFrame(grouped.map(make_row))
k = reshaped.rdd\
.map(lambda row: (row['date'], row["apnic"], row["lacnic"], row["ripencc"], row["arin"], row["afrinic"]))\
.map(toTSV)
k.saveAsTextFile(savePath)
try:
shutil.rmtree(localPath)
except:
pass
try:
os.makedirs(localPath)
except:
pass
hdfs.get(savePath, localPath)
mergeAndSort(localPath)
def runSparkPercentageASesInROAs(sc, ip_type):
caida_as_org_days = [
'20110420', '20110701', '20111003', '20120105', '20120401', '20120629',
'20121002', '20130101', '20130401', '20130701', '20131001', '20140401',
'20140701', '20141001', '20150101', '20150701', '20151001', '20160101',
'20160401', '20160701', '20161001', '20170101', '20170401', '20170701',
'20171001', '20180101', '20180401', '20180703', '20181001', '20190101'
]
d = []
for year in range(2011, 2020):
d += createDates(year)
caida_as_org_days = [d[0]] + caida_as_org_days + [d[-1]]
as_org_days = dict.fromkeys(d, 0)
min_date = datetime.strptime(d[0], "%Y%m%d")
max_date = datetime.strptime(d[-1], "%Y%m%d")
set_days_scope(caida_as_org_days, as_org_days, min_date, max_date)
roa_prefix_asn = "/hdfs-to-local-path/rpki/ripe/ripe-new-objects/roa-prefix-asn/*"
savePath = "/hdfs-to-local-path/rpki/results/roas-covering-AScnt-%s" % ip_type
localPath = "/home/tjchung/research/rpki/src/spark/results/roas-covering-AScnt-%s" % ip_type
try:
hdfs.rmr(savePath)
except:
pass
tals = ["apnic", "lacnic", "ripencc", "arin", "afrinic"]
a = runSparkGetTotalASNs(sc)
k = sc.textFile(roa_prefix_asn)\
.filter(lambda line: "#" not in line)\
.map(lambda line: line.rstrip().split("\t"))\
.filter(lambda (time, prefix_addr, prefix_len, maxlen, asID, num_ips, cc, tal):\
isIPv4v6(prefix_addr, ip_type) and \
tal != "localcert")\
.map(lambda (time, prefix_addr, prefix_len, maxlen, asID, num_ips, cc, tal): ((time, tal.split("-")[0]), asID))\
.distinct()\
.groupByKey()\
.map(lambda ( (time, tal), num_ases): ( (as_org_days[time], tal), (time, len(set(num_ases)))))\
.join(a)\
.map(lambda ((time, tal), ((real_time, num_activated_asns), all_asns)): (real_time, tal, "%s\t%s" % (num_activated_asns, all_asns)))
sqlContext = SQLContext(sc)
df = sqlContext.createDataFrame(k, ['date', 'tal', 'num_ASes'])
grouped = df.rdd\
.map(lambda row: (row.date, (row.tal, row.num_ASes)))\
.groupByKey()
def make_row(kv):
k, vs = kv
tmp = dict(list(vs) + [("date", k)])
return Row(**{k: tmp.get(k, "0\t0") for k in ["date"] + tals})
reshaped = sqlContext.createDataFrame(grouped.map(make_row))
k = reshaped.rdd\
.map(lambda row: (row['date'], row["apnic"], row["lacnic"], row["ripencc"], row["arin"], row["afrinic"]))\
.map(toTSV)
k.saveAsTextFile(savePath)
try:
shutil.rmtree(localPath)
except:
pass
try:
os.makedirs(localPath)
except:
pass
hdfs.get(savePath, localPath)
mergeAndSort(localPath,
label="\t".join(
["#apnic", "lacnic", "ripencc", "arin", "afrinic"]))
def runSparkNumROAs(sc):
def parse(line):
time_tal, filename, _, skid, akid, ee, roa = line.rstrip().split(",")
time = time_tal[:8]
tal = time_tal[9:-4]
#time, tal = time_tal.replace(".txt", "").split("-")
return (time, tal, filename)
roas = "/hdfs-to-local-path/rpki/ripe/ripe-new-objects/roas/*"
savePath = "/hdfs-to-local-path/rpki/results/num-roas"
localPath = "/home/tjchung/research/rpki/src/spark/results/num-roas"
try:
hdfs.rmr(savePath)
except:
pass
tals = [
"apnic", "apnic-iana", "apnic-afrinic", "apnic-arin", "apnic-lacnic",
"apnic-ripe", "lacnic", "ripencc", "arin", "afrinic", "localcert"
]
k = sc.textFile(roas)\
.map(parse)\
.distinct()\
.map(lambda (time, tal, filename): ((time, tal), 1))\
.reduceByKey(lambda a, b : a +b )\
.map(lambda ( (time, tal), num_roas): (time, tal, num_roas))
sqlContext = SQLContext(sc)
df = sqlContext.createDataFrame(k, ['date', 'tal', 'num_roas'])
grouped = df.rdd\
.map(lambda row: (row.date, (row.tal, row.num_roas)))\
.groupByKey()
def make_row(kv):
k, vs = kv # time: [(-1, cnt), (0, cnt), (1, cnt)] ...
tmp = dict(list(vs) + [("date", k)])
return Row(**{k: tmp.get(k, 0) for k in ["date"] + tals})
reshaped = sqlContext.createDataFrame(grouped.map(make_row))
k = reshaped.rdd\
.map(lambda row: (row['date'], row["apnic"], row["apnic-iana"], row["apnic-afrinic"], row["apnic-arin"], row["apnic-lacnic"], row["apnic-ripe"], row["lacnic"], row["ripencc"], row["arin"], row["afrinic"], row["localcert"]))\
.map(toTSV)
k.saveAsTextFile(savePath)
try:
shutil.rmtree(localPath)
except:
pass
try:
os.makedirs(localPath)
except:
pass
hdfs.get(savePath, localPath)
mergeAndSort(localPath)
def runSparkCalcRPKIEnabledAdv(sc, dataset, ip_type, year):
print "runSparkCalcRPKIEnabledAdv", dataset, year
"""
It calculates the *number* of BGP announcements that
(1) can't be verified against RPKI
(2) can be verified against RPKI and invalid
(3) can be verified against RPKI and valid
Note:
BGP announcements are distinct based on three tuples:
(peer_ip, prefix_addr, as_path) given on a date
"""
readPath = "/spark-hdfs-path/rpki/results/bgp-verify-nometa/%s/%s*" % (
dataset, year)
savePath = "/spark-hdfs-path/rpki/results/rpki-enabled-adv-%s/%s/%s" % (
ip_type, dataset, year)
localPath = "/local-spark-result-path/research/rpki/src/spark/results/rpki-enabled-adv-%s/%s/%s" % (
ip_type, dataset, year)
try:
hdfs.rmr(savePath)
except:
pass
isJson = False
if (dataset == "akamai-public-prefix"): isJson = True
hasMeta = False
k = sc.textFile(readPath)\
.map(lambda v: parseVerifyline(v, hasMeta, isJson))\
.filter(lambda v: v is not None)\
.filter(lambda v: isIPv4v6(v, ip_type))\
.map(lambda j: ( (j['time'], classifyBGPAdvSparse(j)), 1))\
.reduceByKey(lambda a, b: a + b)\
.map(lambda ( (time, rpki_type), cnt): (time, rpki_type, cnt))
sqlContext = SQLContext(sc)
df = sqlContext.createDataFrame(k, ['timestamp', 'rpkiType', 'cnt'])
grouped = df.rdd\
.map(lambda row: (row.timestamp, (row.rpkiType, row.cnt)))\
.groupByKey()
def make_row(kv):
k, vs = kv # time: [(-1, cnt), (0, cnt), (1, cnt)] ...
tmp = dict(list(vs) + [("timestamp", k)])
return Row(
**{
k: tmp.get(k, 0)
for k in
["timestamp", "non-rpki", "rpki-invalid", "rpki-valid"]
})
reshaped = sqlContext.createDataFrame(grouped.map(make_row))
k = reshaped.rdd\
.map(lambda row: (row['timestamp'], row['non-rpki'], row['rpki-invalid'], row['rpki-valid']))\
.map(toTSV)
k.saveAsTextFile(savePath)
try:
shutil.rmtree(localPath)
except:
pass
try:
os.makedirs(localPath)
except:
pass
hdfs.get(savePath, localPath)
mergeAndSort(localPath)
def runSparkValidationUniquePrefix(sc, dataset, ip_type):
readPath = "/spark-hdfs-path/rpki/results/rpki-enabled-unique-prefix-asn-%s/%s" % (
ip_type, dataset)
savePath = "/spark-hdfs-path/rpki/results/rpki-enabled-unique-prefix-asn-adv-%s/%s" % (
ip_type, dataset)
localPath = "/local-spark-result-path/research/rpki/src/spark/results/rpki-enabled-unique-prefix-asn-adv-%s/%s" % (
ip_type, dataset)
try:
hdfs.rmr(savePath)
except:
pass
k = sc.textFile(readPath)\
.map(lambda v: parseVerifyLineUniquePrefix(v))\
.filter(lambda v: v is not None)\
.filter(lambda v: notDataError(dataset, v))\
.filter(lambda v: isIPv4v6(v, ip_type))\
.filter(lambda v: ip_type == "ipv6" or not isLargerSlash24(v))\
.map(lambda j: ( (j['time'], classifyBGPAdvSparse(j)), 1))\
.reduceByKey(lambda a, b: a + b)\
.map(lambda ( (time, rpki_type), cnt): (time, rpki_type, cnt))
sqlContext = SQLContext(sc)
df = sqlContext.createDataFrame(k, ['timestamp', 'rpkiType', 'cnt'])
grouped = df.rdd\
.map(lambda row: (row.timestamp, (row.rpkiType, row.cnt)))\
.groupByKey()
def make_row(kv):
k, vs = kv # time: [(-1, cnt), (0, cnt), (1, cnt)] ...
tmp = dict(list(vs) + [("timestamp", k)])
return Row(
**{
k: tmp.get(k, 0)
for k in
["timestamp", "non-rpki", "rpki-invalid", "rpki-valid"]
})
reshaped = sqlContext.createDataFrame(grouped.map(make_row))
k = reshaped.rdd\
.map(lambda row: (row['timestamp'], row['non-rpki'], row['rpki-invalid'], row['rpki-valid']))\
.map(toTSV)
k.saveAsTextFile(savePath)
try:
shutil.rmtree(localPath)
except:
pass
try:
os.makedirs(localPath)
except:
pass
hdfs.get(savePath, localPath)
mergeAndSort(localPath)
df_y = df['label'] == 3
df_X = df[['x' + str(i)
for i in range(1, 49)] + ['y' + str(j) for j in range(1, 49)]]
X_train, X_test, y_train, y_test = train_test_split(df_X,
df_y,
test_size=0.2,
random_state=15)
# Feature Scaling
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
detector = dlib.get_frontal_face_detector()
hdfs.get("/drunkdetection/shape_predictor_68_face_landmarks.dat",
"tmp/shape_predictor_68_face_landmarks.dat")
predictor = dlib.shape_predictor("tmp/shape_predictor_68_face_landmarks.dat")
fa = FaceAligner(predictor, desiredFaceWidth=300)
hdfs.get("/drunkdetection/drunk3.jpg", "/tmp/drunk3.jpg")
img = cv2.imread("/tmp/drunk3.jpg")
print(img)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = detector(gray, 1)
dic = {}
x_values = [[] for _ in range(48)]
y_values = [[] for _ in range(48)]
for face in faces:
(x, y, w, h) = rect_to_bb(face)
faceOrig = imutils.resize(img[y:y + h, x:x + w], width=300)
import sys
import pydoop.hdfs as hdfs
#create package
#date = 'data/'+str(sys.argv[1])[2:]+'/'
st = '['
for x in hdfs.ls("data/18-02-21/"):
#for x in hdfs.ls("date"):
st = st + hdfs.load(x)
st = st.replace("\n", ",")
st = st[:-1]
st = st + ']'
#string4 = '{"input":"'+str(sys.argv[1])+'"}'
hdfs.dump(st, "test/hello.txt")
hdfs.get("test/hello.txt", "/tmp/tmp.txt")
def runSparkClassifyHijackingUniquePrefix(sc, dataset, ip_type):
readPath = "/spark-hdfs-path/rpki/results/rpki-enabled-unique-prefix-asn-%s/%s" % (
ip_type, dataset)
savePath = "/spark-hdfs-path/rpki/results/rpki-enabled-unique-prefix-classify-hijack-%s/%s" % (
ip_type, dataset)
localPath = "/local-spark-result-path/research/rpki/src/spark/results/rpki-unique-prefix-classify-hijack-%s/%s" % (
ip_type, dataset)
try:
hdfs.rmr(savePath)
except:
pass
k = sc.textFile(readPath)\
.map(lambda v: parseVerifyLineUniquePrefix(v))\
.filter(lambda v: v is not None)\
.filter(lambda v: notDataError(dataset, v))\
.filter(lambda v: isIPv4v6(v, ip_type))\
.filter(lambda v: classifyBGPAdvSparse(v) == "rpki-invalid")\
.filter(lambda v: ip_type == "ipv6" or not isLargerSlash24(v))\
.filter(lambda v: onlyHijackAttempt(v))\
.map(lambda v: ((v['time'], classifyHijack(v)), 1))\
.reduceByKey(lambda a, b: a + b)\
.map(lambda ( (time, status), cnt): (time, status, cnt))
sqlContext = SQLContext(sc)
df = sqlContext.createDataFrame(k, ['timestamp', 'hijackType', 'cnt'])
grouped = df.rdd\
.map(lambda row: (row.timestamp, (row.hijackType, row.cnt)))\
.groupByKey()
def make_row(kv):
k, vs = kv # time: [(-1, cnt), (0, cnt), (1, cnt)] ...
tmp = dict(list(vs) + [("timestamp", k)])
return Row(
**{
k: tmp.get(k, 0)
for k in [
"timestamp", "sameISP", "provider", "customer", "peer",
"DDoS", "None"
]
})
reshaped = sqlContext.createDataFrame(grouped.map(make_row))
k = reshaped.rdd\
.map(lambda row: (row['timestamp'], row['sameISP'], row['provider'], row['customer'], row['peer'], row['DDoS'], row['None']))\
.map(toTSV)
k.saveAsTextFile(savePath)
try:
shutil.rmtree(localPath)
except:
pass
try:
os.makedirs(localPath)
except:
pass
hdfs.get(savePath, localPath)
mergeAndSort(localPath)
