def scenario_A(file_in, file_out=None):
print("Input file: {}".format(file_in))
reader = FileReader(file_in)
problem = reader.read()
solver = Pizza(problem)
print("Description:")
print(solver.description())
slices = solver.solve()
is_valid, result = problem.validate_solution(slices)
if is_valid:
print("Solution for problem {} is correct. Score is {}".format(
file_in, result))
solution = Solution(problem)
solution.load_slices(slices)
if DEBUG:
solution.print_solution()
if file_out:
writer = FileWriter(file_in + ".out")
writer.write(solution)
else:
print("Incorrect solution. Please check the error messages below")
for msg in result:
print(msg)
def scenario_A(file_in):
print("Input file: {}".format(file_in))
global DEBUG
reader = FileReader(file_in)
problem = reader.read()
solver = Solver(problem)
print("Description:")
print(solver.description())
solution = solver.initial_solution()
# print(solver.initial_solution().print_free())
# print("-------")
# print(solver.initial_solution().print_solution())
# print("here")
# print("\n".join(str(i) for i in solver.initial_solution().slices))
# print("here2")
print("Initial solution score: {}".format(solution.score()))
if DEBUG:
solution.print_solution()
optimized_solution = solver.search(solution, time_limit=60)
print("Optimized solution score: {}".format(optimized_solution.score()))
# print("\n".join(str(i) for i in optimized_solution.slices))
if DEBUG:
optimized_solution.print_solution()
trace = solver.get_search_trace()
if trace is not None:
print("Trace:")
print("\n".join([str(x) for x in trace]))
def __init__(self):
self.val = Validator()
self.db = DatabaseMaker()
self.reader = FileReader()
self.py = PyGal()
self.converted_file = None
self.file_count = 1
def initHeapUsageAndGCPause(self, gclogFile, gcpauseFile):
fileLines = FileReader.readLines(gclogFile)
for line in fileLines:
if (line.strip != ""):
heapUsage = self.parseUsage(line)
if (line.startswith("[Young]")):
self.youngGen.append(heapUsage)
elif (line.startswith("[Old]")):
self.oldGen.append(heapUsage)
elif (line.startswith("[Metaspace]")):
self.metaGen.append(heapUsage)
fileLines2 = FileReader.readLines(gcpauseFile)
for line in fileLines2:
if not line[0].isdigit():
continue
items = re.split(",", line)
gc_type = items[-1]
if gc_type == "NONE":
continue
if gc_type not in self.gcTotal.keys():
self.gcTotal[gc_type] = []
self.gcTotal[gc_type].append(float(items[3]))
if gc_type not in self.gcTimeLine.keys():
self.gcTimeLine[gc_type] = []
self.gcTimeLine[gc_type].append(float(items[0]))
def main():
"""Main function for commandline call
"""
# end user version for user_input
# args = user_input(sys.argv[1:])
# add your own args = user_input() for testing and debugging so that you
# don't have to call the script with full command line input
args = user_input(['Input/Task1/', '-o', 'Output/Task2/'])
# read files
reader = FileReader(args.path)
input_df = reader.read()
# perform statistical analysis
stat_ana = analysis.Statistical_Analysis(args.output)
stat_ana.correlation(input_df['npop.t'])
stat_ana.eucl_distance(input_df['table.dat'])
# perfomr numerical analysis
num_ana = analysis.Numerical_Analysis(args.output)
# return new df with the desired columns
df_efield_relevant = num_ana.remove_low_variance(input_df['efield.t'])
# fft with freq of the df
df_efield_fft = num_ana.fft_with_freq_analysis(df_efield_relevant, "y")
# disabled plot to not have it get on my nerves
num_ana.plot_and_save(df_efield_fft,
"freq",
"intensitys",
"efield_fft_analysis",
xlabel="Freq",
show_graph=False)
df_autocorr = num_ana.autocorrelation(input_df["nstate_i.t"], "time")
num_ana.plot_and_save(df_autocorr,
"time",
["autocorr_abs", "autocorr_real", "autocorr_imag"],
"nstate_autocorr_analysis",
xlabel="time",
show_graph=False)
df_autocorr_fft = num_ana.fft_with_freq_analysis(df_autocorr,
"autocorr",
type="complex")
# adding abs**2 to the dataframe
df_autocorr_fft["intensitys_squared"] = np.abs(
df_autocorr_fft["intensitys"].values)**2
num_ana.plot_and_save(df_autocorr_fft,
"freq", ["intensitys", "intensitys_squared"],
"nstate_autocorr_fft_analysis",
xlabel="Freq",
show_graph=True,
crop_edge=3)
def scenario_A(file_in, file_out):
global DEBUG
reader = FileReader(file_in)
problem = reader.read()
init_solver = InitSolverSilly()
solution = init_solver.run(problem)
print("Initial solution score: {}".format(solution.score()))
if DEBUG:
solution.print_solution()
writer = FileWriter(file_out)
writer.write(solution)
def scenario_C(file_in, file_out, file_par=None):
global DEBUG
reader = FileReader(file_in)
problem = reader.read()
init_solver = ParallelInitSolver(InitSolverSillyParameterized,
file_output=file_out)
solution = init_solver.run(problem, file_par)
print("Initial solution score: {}".format(solution.score()))
if DEBUG:
solution.print_solution()
writer = FileWriter(file_out)
writer.write(solution)
def analyzeMetrics(self, metrics):
"""
:param metrics: [app.duration, stage0.duration, s0.task.jvmGcTime, ...]
"""
statisticsFiles = os.listdir(self.statisticsDir)
metricsSet = set(metrics)
for file in statisticsFiles:
if file.endswith(
"stat.txt"
): # [RDDJoin-CMS-1-7G-stat.txt, RDDJoin-CMS-2-14G-stat.txt, ...]
# [app.duration] mean = 224339.20, stdVar = 8311.91, median = 225233.00, min = 211999.00, quantile25 = 216682.50, quantile75 = 231549.00, max = 233837.00
# -------------------------------------------------------------------[Stage 0]-------------------------------------------------------------------
# [stage0.duration] mean = 42360.60, stdVar = 4069.63, median = 41404.00, min = 37094.00, quantile25 = 38942.50, quantile75 = 46257.00, max = 47801.00
# [stage0.inputBytes] mean = 8588671743.00, stdVar = 0.00, median = 8588671743.00, min = 8588671743.00, quantile25 = 8588671743.00, quantile75 = 8588671743.00, max = 8588671743.00
# [stage0.inputRecords] mean = 66000000.00, stdVar = 0.00, median = 66000000.00, min = 66000000.00, quantile25 = 66000000.00, quantile75 = 66000000.00, max = 66000000.00
for line in FileReader.readLines(
os.path.join(self.statisticsDir, file)):
metricName = line[line.find('[') + 1:line.find(']')]
if metricName in metricsSet:
if self.metricsMap.has_key(metricName):
self.metricsMap[metricName].addStatistics(
line, file)
else:
statistics = st.Statistics()
statistics.addStatistics(line, file)
self.metricsMap[metricName] = statistics
def fillStatistics(self, metrics, statisticsDir, statisticsFiles, metricsMap, withMax):
metricsTupleDict = {} # ["app.duration", ("app.duration", "Time (s)", 1000)]
for tuple in metrics:
metricsTupleDict[tuple[0]] = tuple
for file in statisticsFiles:
if file.endswith("stat.txt"): # [RDDJoin-CMS-1-7G-stat.txt, RDDJoin-CMS-2-14G-stat.txt, ...]
# [app.duration] mean = 224339.20, stdVar = 8311.91, median = 225233.00, min = 211999.00, quantile25 = 216682.50, quantile75 = 231549.00, max = 233837.00
# -------------------------------------------------------------------[Stage 0]-------------------------------------------------------------------
# [stage0.duration] mean = 42360.60, stdVar = 4069.63, median = 41404.00, min = 37094.00, quantile25 = 38942.50, quantile75 = 46257.00, max = 47801.00
# [stage0.inputBytes] mean = 8588671743.00, stdVar = 0.00, median = 8588671743.00, min = 8588671743.00, quantile25 = 8588671743.00, quantile75 = 8588671743.00, max = 8588671743.00
# [stage0.inputRecords] mean = 66000000.00, stdVar = 0.00, median = 66000000.00, min = 66000000.00, quantile25 = 66000000.00, quantile75 = 66000000.00, max = 66000000.00
for line in FileReader.readLines(os.path.join(statisticsDir, file)):
metricName = line[line.find('[') + 1: line.find(']')]
if metricsTupleDict.has_key(metricName):
if metricsMap.has_key(metricName):
metricsMap[metricName].addStatistics(line, file, withMax)
else:
statistics = st.BoxPlotStatistics(metricsTupleDict[metricName])
statistics.addStatistics(line, file, withMax)
metricsMap[metricName] = statistics
# Fill the NaA values
for metricName, statistics in metricsMap.items():
statistics.checkAndFillNulls()
def initHeapUsage(self, gclogFile, timeOffset):
fileLines = FileReader.readLines(gclogFile)
for line in fileLines:
if (line.strip != ""):
heapUsage = self.parseUsage(line, timeOffset)
if(line.startswith("[Young]")):
self.youngGen.append(heapUsage)
elif(line.startswith("[Old]")):
self.oldGen.append(heapUsage)
def analyzeMetrics(self):
taskInfoFiles = os.listdir(self.taskInfoDir)
for file in taskInfoFiles:
if file.endswith(
"tasks.txt"
): # [GroupByRDD-CMS-1-7G-0.5-tasks.txt, GroupByRDD-CMS-2-14G-0.5-tasks.txt, ...]
# [appName = GroupByRDD-CMS-1-7G-0.5]
# [appId = app-20170721105922-0025]
# [stageId = 1]
#
# [0.task.index] 0
# [0.task.attemptId] 0
# [0.task.executorId] 29
# [0.task.duration] 4715
# [0.task.executorDeserializeTime] 60
# [0.task.executorDeserializeCpuTime] 43705029
# [0.task.executorRunTime] 4715
# [0.task.executorCpuTime] 2614115738
# [0.task.resultSize] 2865
# [0.task.jvmGcTime] 1305
GC = ""
executor = ""
if (file.find("CMS") != -1):
GC = "CMS"
elif (file.find("G1") != -1):
GC = "G1"
elif (file.find("Parallel") != -1):
GC = "Parallel"
if (file.find("1-7G") != -1):
executor = "E1"
elif (file.find("2-14G") != -1):
executor = "E2"
elif (file.find("4-28G") != -1):
executor = "E4"
for line in FileReader.readLines(
os.path.join(self.taskInfoDir, file)):
if line.startswith('[appName'):
appName = line[line.find('=') + 2:line.find(']')]
elif line.startswith('[appId'):
appId = line[line.find('=') + 2:line.find(']')]
elif line.startswith('[stageId'):
stageId = int(line[line.find('=') + 2:line.find(']')])
elif line.startswith('['):
metricName = line[line.find('task') + 5:line.find(']')]
metricValue = float(line[line.find(']') + 2:])
if (metricName == 'index'):
taskAttempt = ta.TaskAttempt(
appId, appName, stageId, int(metricValue), GC,
executor)
taskAttempt.set(metricName, metricValue)
self.taskList.append(taskAttempt)
def scenario_A(file_in):
print("Input file: {}".format(file_in))
global DEBUG
reader = FileReader(file_in)
problem = reader.read()
solver = Solver(problem)
print("Description:")
print(solver.description())
solution = solver.initial_solution()
print("Initial solution score: {}".format(solution.score()))
if DEBUG:
solution.print_solution()
optimized_solution = solver.search(solution, time_limit=60)
print("Optimized solution score: {}".format(optimized_solution.score()))
if DEBUG:
optimized_solution.print_solution()
trace = solver.get_search_trace()
def scenario_B(file_in, file_out):
global DEBUG
reader = FileReader(file_in)
problem = reader.read()
init_solver = InitSolverSilly()
solution = init_solver.run(problem)
print("Initial solution score: {}".format(solution.score()))
if DEBUG:
solution.print_solution()
optimizer = Tabu(problem,
solution,
Neighbourhood_ChangeFormats,
debug=True)
optimized_solution = optimizer.run(time_limit=100)
print("optimized solution score: {}".format(optimized_solution.score()))
if DEBUG:
optimized_solution.print_solution()
writer = FileWriter(file_out)
writer.write(optimized_solution)
def scenario_D(file_in, file_out, file_par=None):
global DEBUG
reader = FileReader(file_in)
problem = reader.read()
init_solver = ParallelInitSolver(InitSolverSillyParameterized,
file_output=file_out)
solution = init_solver.run(problem, file_par)
print("Initial solution score: {}".format(solution.score()))
if DEBUG:
solution.print_solution()
optimizer = ParallelTabu(problem,
solution,
Neighbourhood_ChangeFormats,
debug=True)
optimized_solution = optimizer.run(time_limit=1000)
print("optimized solution score: {}".format(optimized_solution.score()))
if DEBUG:
optimized_solution.print_solution()
writer = FileWriter(file_out)
writer.write(solution)
def plot_specific_heat_ratio_isobaric(filename, plot_cp=True):
data = FileReader(filename)
# Plot 1
fig, ax1 = plt.subplots()
if plot_cp:
color = 'b'
else:
color = 'k'
ax1.plot(data.temperature, data.gamma, color, label='Specific Heat Ratio')
ax1.set_xlabel(data.titles.temperature)
ax1.set_ylabel(data.titles.gamma)
if plot_cp:
# Plot 2
ax2 = ax1.twinx()
ax2.plot(data.temperature, data.cp, 'r', label='Cp')
ax2.set_ylabel(data.titles.cp)
fig.legend(loc="upper center", bbox_to_anchor=(0.5, 0.9))
plt.title('Specific heat ratio of water (g) at 6 bar')
fig.tight_layout()
plt.show()
import argparse
from analyzer import Analyzer
from chart import BarGraph
from reader import FileReader
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Лабораторная №5')
parser.add_argument('--path=', action="store", required=True,
dest="path", help="путь до файла")
args = parser.parse_args()
data = FileReader.read(args.path)
analyzer = Analyzer(data)
spammers = analyzer.spammer_list
if not spammers:
print('Спамеры не обнаружены')
else:
print('Список спамеров:')
for spammer in spammers:
print(spammer)
BarGraph.draw(analyzer.info)
def __init__(self):
self.data = FileReader().readFile("client.csv")
def __init__(self, f):
self.has_more = True
self.reader = FileReader(f)
self.line_no = 0
self.queue = []
self.pattern = re.compile(self.regex_pat, re.X)
def __init__(self):
self.data = FileReader().readFile("trans.csv")
def __init__(self):
self.data = FileReader().readFile("district.csv")
class Controller(object):
def __init__(self):
self.val = Validator()
self.db = DatabaseMaker()
self.reader = FileReader()
self.py = PyGal()
self.converted_file = None
self.file_count = 1
def read_file(self, filename):
try:
self.converted_file = self.reader.read(filename)
for dict in self.converted_file:
print("FILE_DATA:", self.file_count, "\n", dict)
self.file_count += 1
except Exception as err:
print("The exception is: ", err)
def valid(self, flag):
try:
if flag == '':
self.validate()
if flag == '-f':
file = input("Validate which file?:")
self.read_file(file)
self.validate()
if flag == '-v':
self.view_valid()
if flag == '-fv':
file = input("Validate which file?:")
self.read_file(file)
self.validate()
else:
raise Exception("Not a valid flag")
except Exception as err:
print("The exception is: ", err)
def validate(self):
try:
for dict in self.converted_file:
self.val.valid(dict)
return self.view_valid()
except Exception as err:
print("The exception is: No file specified")
def view_valid(self):
print("\n VALID DATA:")
for dict in self.val.get():
print(dict)
def db_table(self, flag):
try:
if flag == "-d":
self.db_drop_table()
elif flag == '-c':
self.db_create_table()
elif flag == '-dc':
self.db_drop_table()
self.db_create_table()
elif flag == '-i':
for i in self.db_insert():
print(i)
elif flag == '-v':
value = input("What column do you want to\
see from the employee table?")
self.db_view(value)
elif flag == '-if':
file = input("Input which file?:")
self.read_file(file)
self.validate()
self.db_insert()
elif flag == '':
self.db_view("*")
else:
raise Exception("Not a valid flag")
except Exception as err:
print("The exception is: ", err)
def db_drop_table(self):
self.db.drop_table("employee")
def db_create_table(self):
self.db.create_table()
def db_insert(self):
try:
list_of_dictionaries = self.val.get()
if list_of_dictionaries:
for dict in list_of_dictionaries:
self.db.insert(dict["id"], dict["gender"], dict["age"],
dict["sales"], dict["bmi"], dict["salary"],
dict["birthday"])
return self.db.get("*")
else:
raise Exception("Cannot add invalid data to the database")
except Exception as err:
print("The exception is: ", err)
def db_view(self, value):
for v in self.db.get(value):
print(v)
def database_close(self):
self.db.close()
def pygal(self, flag):
try:
if flag == '':
value = input("Which data do you want to see a bar graph of?")
self.py_view(value)
if flag == '-d':
value = input("Input first bar graph comparison:")
value2 = input("Input second bar graph comparison:")
self.py_view(value, value2)
return
else:
raise Exception("Not a valid flag")
except Exception as err:
print("The exception is: ", err)
def py_view(self, value, value2=None):
try:
data = self.db.bar_get(value)
id = self.db.bar_get('id')
if value2 is not None:
data2 = self.db.bar_get(value2)
self.py.bar_char(id, value, data, value2, data2)
else:
self.py.bar_char(id, value, data)
except Exception as err:
print("The exception is: ", err)
def pickled(self, flag):
try:
if flag == '':
name = input("Name of pickle file?:")
if name == '':
raise Exception("Not a valid filename")
else:
self.pickled_write(name)
elif flag == '-r':
name = input("Name of pickle file?:")
if name == '':
raise Exception("Not a valid filename")
else:
self.pickled_read(name)
else:
raise Exception("Not a valid flag")
except Exception as err:
print("The exception is: ", err)
def pickled_write(self, name):
import pickle
with open('{name}.pickle'.format(name=name), 'wb') as f:
pickle.dump(self.converted_file, f)
def pickled_read(self, name):
import pickle
with open('{name}.pickle'.format(name=name), 'rb') as f:
data = pickle.load(f)
print(data)
def __init__(self, filename):
self.filename = FileReader(filename)
def __init__(self, f):
self.reader = FileReader(f)
self.state = 1
self.queue = []
self.word_buffer = ""
self.tmp_char = ""
from reader import FileReader, UrlReader
from sma import SMA, SMAPlt
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Лабораторная №4')
parser.add_argument('--paths=',
help='Список адресов через запятую',
required=True,
dest="paths")
parser.add_argument('--frame=',
help='Размер окна',
required=True,
dest="frame")
args = parser.parse_args()
paths = [item for item in args.paths.split(',')]
if len(paths) < 25:
print('Кол-во данных меньше 25')
exit()
sma = SMA()
print('Загрузка данных...')
for path in paths:
reader = FileReader() if FileReader.path_is_file(path) else UrlReader()
sma.read_data(reader, path)
sma.frame = args.frame
SMAPlt.draw(*sma.calculate_sma())
def __init__(self):
self.data = FileReader().readFile("loan.csv")
#plt.tight_layout()
plt.show()
#fig = plt.gcf()
#plt.show()
#fig.savefig(outputFile, dpi=300, bbox_inches='tight')
if __name__ == '__main__':
gcLogPath = "/Users/xulijie/Documents/GCResearch/Android/logs/"
appName = "Alexnet GC Metrics"
gcFileName = "cnndroid_alexnet_scale1_image_time_logcat.txt"
gcLines = FileReader.readLines(gcLogPath + gcFileName)
gcActivities = GCActivities(gcLines)
plotHeapUsage(appName, appName, gcActivities)
appName = "Nin GC Metrics"
gcFileName = "cnndroid_nin_scale1_image_time_logcat.txt"
gcLines = FileReader.readLines(gcLogPath + gcFileName)
gcActivities = GCActivities(gcLines)
plotHeapUsage(appName, appName, gcActivities)
def __init__(self):
self.data = FileReader().readFile("account.csv")
class ReLexer(Lexer):
regex_pat = r'''
\s*( #空白
(//.*)| #注释
(\d+)| #数字类型
("[^"]*")| #字符串类型
[A-Z_a-z][A-z_a-z0-9]*| #标识符
==|<=|>=|&&|\|\|| #符号
[\+\-\*/\{\}\=\|\&\[\]\(\)\<\>\;\%] #符号
)
'''
def __init__(self, f):
self.has_more = True
self.reader = FileReader(f)
self.line_no = 0
self.queue = []
self.pattern = re.compile(self.regex_pat, re.X)
def read(self):
if self.fill_queue(0):
return self.queue.pop(0)
return Token.EOF
def peek(self, i):
if self.fill_queue(i):
return self.queue[i]
return Token.EOF
def fill_queue(self, i):
while i >= len(self.queue):
if self.has_more:
self.read_line()
else:
return False
return True
def read_line(self):
line = self.reader.read_line()
if not line:
self.has_more = False
return
line = line[:-1]
self.line_no = self.reader.line_no
start = 0
end = len(line)
while start < end:
matcher = self.pattern.search(line[start:end])
start += matcher.end()
if matcher:
self.add_token(matcher)
self.queue.append(IdToken(self.line_no, Token.EOL))
def add_token(self, matcher):
m = matcher.group(1)
if m is not None: # not a space line
if matcher.group(2) == None: # not a comment
if matcher.group(3) != None:
token = NumToken(self.line_no, int(m))
elif matcher.group(4) != None:
token = StrToken(self.line_no, m)
else:
token = IdToken(self.line_no, m)
self.queue.append(token)
def plotResourceUsage(topMetricsFile, slowestTasksDir, appName):
fileLines = FileReader.readLines(topMetricsFile)
isExecutorMetric = False
isSlaveMetric = False
executorTime = []
executorCPU = []
executorMemory = []
slaveTime = []
slaveCPU = []
slaveMemory = []
for line in fileLines:
if (line.startswith("[Top Metrics][Executor")):
isExecutorMetric = True
elif (line.startswith("[Top Metrics][aliSlave")):
isSlaveMetric = True
isExecutorMetric = False
elif (isExecutorMetric == True and line.strip() != ""):
time = line[line.find('[') + 1:line.find(']')]
cpu = line[line.find('=') + 2:line.find(',')]
memory = line[line.find('Memory') + 9:]
executorTime.append(datetime.strptime(time, '%H:%M:%S'))
executorCPU.append(float(cpu))
executorMemory.append(float(memory))
elif (isSlaveMetric == True and line.strip() != ""):
time = line[line.find('[') + 1:line.find(']')]
cpu = line[line.find('=') + 2:line.find(',')]
memory = line[line.find('Memory') + 9:]
slaveTime.append(datetime.strptime(time, '%H:%M:%S'))
slaveCPU.append(float(cpu))
slaveMemory.append(float(memory))
fig, axes = plt.subplots(nrows=2, ncols=1, sharey=False, sharex=True)
# locator = mpl.dates.MinuteLocator()
xfmt = mdates.DateFormatter('%H:%M:%S')
#ax.xaxis.set_major_locator(locator)
axes[0].xaxis.set_major_formatter(xfmt)
axes[1].xaxis.set_major_formatter(xfmt)
axes[0].set_ylabel("Executor CPU (%)", color='r')
axes[0].tick_params('y', colors='r')
axes[1].set_ylabel("Worker CPU (%)", color='r')
axes[1].tick_params('y', colors='r')
axes[0].set_ylim(0, 840) # The ceil
axes[1].set_ylim(0, 105) # The ceil
# plt.ylim(0, statistics.max) # The ceil
# plt.legend()
fig.autofmt_xdate()
axes[0].plot_date(executorTime, executorCPU, '-r', label='CPU')
axes[1].plot_date(slaveTime, slaveCPU, '-r', label='CPU')
ax12 = axes[0].twinx()
ax12.plot_date(executorTime, executorMemory, '-b', label='Memory')
ax12.set_ylabel('Executor Memory (GB)', color='b')
ax12.tick_params('y', colors='b')
ax12.set_ylim(0, 32) # The ceil
# ax12.tick_params('y', colors='r')
ax22 = axes[1].twinx()
ax22.plot_date(slaveTime, slaveMemory, '-b', label='Memory')
ax22.set_ylabel('Worker Memory (GB)', color='b')
ax22.tick_params('y', colors='b')
ax22.set_ylim(0, 32) # The ceil
plt.suptitle(appName)
outputDir = os.path.join(slowestTasksDir, "topMetricsFigures")
if not os.path.exists(outputDir):
os.mkdir(outputDir)
file = os.path.join(outputDir, appName + ".pdf")
plt.show()
import math
import time
from flask import Flask, request, jsonify
from flask_cors import CORS
from reader import FileReader, Bolder
app = Flask(__name__)
CORS(app)
start_reading = time.time()
reader = FileReader('./data/IR-F19-Project01-Input.xlsx', True,
'./matches.json', './exceptions.txt')
# reader = FileReader('./data/IR-F19-Project01-Input-2k.xlsx', True, './matches.json', './exceptions.txt')
# reader = FileReader('./data/IR-F19-Project02-14k.csv', False, './matches.json', './exceptions.txt')
end_reading = time.time()
print("Reading Files:", end_reading - start_reading, "secs")
bolder = Bolder()
@app.route('/')
def query():
q = request.args.get('q')
start_searching = time.time()
print("Query:", q)
docs, query_tokens = reader.search(q, 50)
docs, query_tokens = list(docs), list(query_tokens)
print("Results:", len(docs))
items = int(request.args.get('items', 10))
print("Items:", items)
pages = math.ceil(len(docs) / items)
print("Pages:", pages)
page = int(request.args.get('page', 0))
class Manager(object):
'''Program manager'''
def __init__(self, filename):
self.filename = FileReader(filename)
def build_autonomous_systems(self):
print('Processing data:', end='')
data = self.filename.get_data()
for line in data:
if Prefix.get_count() % 10000 == 0:
print('.', end='')
sys.stdout.flush()
prefix = self.create_prefix(line)
asn = prefix.origin_asn
next_hop = prefix.next_hop_asn
if asn not in AutonomousSystem.dict_of_all:
self.create_new_asn(asn).ipv4_prefixes.append(prefix)
else:
self.find_asn(asn).ipv4_prefixes.append(prefix)
if next_hop:
if next_hop not in AutonomousSystem.dict_of_all:
self.create_new_asn(next_hop).ipv4_next_hop_prefixes.append(prefix)
else:
self.find_asn(next_hop).ipv4_next_hop_prefixes.append(prefix)
else:
pass
def create_new_asn(self, asn):
return AutonomousSystem(asn)
def find_asn(self, asn):
return AutonomousSystem.dict_of_all.get(asn)
def create_prefix(self, line):
return Prefix(*line)
def list_of_peers(self):
next_hops = []
for k, v in AutonomousSystem.dict_of_all.items():
for route in v.ipv4_prefixes:
next_hops.append(route.next_hop_asn)
return set(next_hops)
def find_prefixes(self, asn):
prefixes = self.find_asn(asn).ipv4_prefixes
count = len(prefixes)
return prefixes, count
def _sorted_ip_list(self, ip_list):
mylist = list()
for line in ip_list:
mylist.append(line.prefix)
return(mylist)
#return sorted(list({line.prefix for line in ip_list}), key=lambda x: socket.inet_aton(x.split("/")[0]))
def find_next_hop_prefixes(self, asn):
prefixes = self.find_asn(asn).ipv4_next_hop_prefixes
count = len(prefixes)
return prefixes, count
def _next_hop_asns(self, asn):
next_hops = []
for prefix in self.find_prefixes(asn)[0]:
next_hops.append(prefix.next_hop_asn)
return list(set(next_hops))
def _next_hop_ips(self, prefix_list):
next_hop_ips = []
for prefix in prefix_list:
next_hop_ips.append(prefix.next_hop_ip)
c = Counter(next_hop_ips)
return c, len(c)
def _dns_query(self, asn):
query = 'AS' + asn + '.asn.cymru.com'
answers = dns.resolver.query(query, 'TXT')
for rdata in answers:
for txt_string in rdata.strings:
return txt_string.split('|')[-1].split(",", 2)[0].strip()
def print_stats(self):
print()
print('IPv4 Routing Table Size:', Prefix.get_count())
print('Unique ASNs:', len(AutonomousSystem.dict_of_all))
print('Peer Networks:', len(self.list_of_peers()))
def print_details(self):
results = OrderedDict()
results['peers'] = OrderedDict()
print()
for peer in sorted(list(filter(None.__ne__, self.list_of_peers())), key=lambda x: int(x)):
if (peer and (int(peer) < 64512 or int(peer) > 65534)):
results['peers'][peer] = OrderedDict()
results['peers'][peer]['name'] = self._dns_query(peer)
results['peers'][peer]['prefixes originated'] = self.find_prefixes(peer)[1]
results['peers'][peer]['routes selected'] = self.find_next_hop_prefixes(peer)[1]
all_next_hop_prefixes, count_next_hop_prefixes = self.find_next_hop_prefixes(peer)
results['peers'][peer]['peering connections'] = self._next_hop_ips(all_next_hop_prefixes)[1]
results['peers'][peer]['peering connection ip list route count'] = self._next_hop_ips(all_next_hop_prefixes)[0]
else:
pass
print(json.dumps(results, indent=4))
def print_asn_details(self, asn, show_routes):
if self.find_asn(asn):
results = OrderedDict()
print()
if asn in self.list_of_peers():
name = self._dns_query(asn)
results['asn'] = asn
results['name'] = name
results['peer'] = True
prefixes, count = self.find_prefixes(asn)
all_next_hop_prefixes, count_next_hop_prefixes = self.find_next_hop_prefixes(asn)
received_prefixes_peering = []
received_prefixes_other = []
results['prefixes originated'] = count
for prefix in prefixes:
if prefix.as_path[0] == asn:
received_prefixes_peering.append(prefix)
else:
received_prefixes_other.append(prefix)
results['peering connections'] = self._next_hop_ips(all_next_hop_prefixes)[1]
results['peering connection ip list route count'] = self._next_hop_ips(all_next_hop_prefixes)[0]
results['prefixes originated by this asn that use the direct peering as a next hop'] = len(received_prefixes_peering)
results['prefixes originated by this asn that do use the direct peering'] = len(received_prefixes_other)
results['all prefixes using this asn as next hop'] = count_next_hop_prefixes
if show_routes:
results['direct peering prefixes'] = self._sorted_ip_list(received_prefixes_peering)
results['non-peering prefixes'] = self._sorted_ip_list(received_prefixes_other)
results['all prefixes using this asn as transit'] = self._sorted_ip_list(all_next_hop_prefixes)
results['next hop asn list'] = self._next_hop_asns(asn)
print(json.dumps(results, indent=4))
else:
name = self._dns_query(asn)
results['asn'] = asn
results['name'] = name
results['peer'] = False
prefixes, count = self.find_prefixes(asn)
received_prefixes_other = []
results['prefixes originated'] = count
for prefix in prefixes:
received_prefixes_other.append(prefix)
if show_routes:
results['prefix list'] = self._sorted_ip_list(received_prefixes_other)
results['next hop asn list'] = self._next_hop_asns(asn)
print(json.dumps(results, indent=4))
else:
print()
print("ASN %s not found" % asn)
print("Ignored Lines:")
for line in self.filename.get_ignored_lines():
print(line)
class AutomatonLexer(Lexer):
def __init__(self, f):
self.reader = FileReader(f)
self.state = 1
self.queue = []
self.word_buffer = ""
self.tmp_char = ""
def read(self):
if self.fill_queue(0):
return self.queue.pop(0)
return Token.EOF
def peek(self, i):
if self.fill_queue(i):
return self.queue[i]
return Token.EOF
def fill_queue(self, i):
while i >= len(self.queue):
token = self.add_token()
if token != Token.EOF:
self.queue.append(token)
else:
return False
return True
def get_char(self):
if not self.tmp_char:
char = self.reader.read_char()
return char
else:
char = self.tmp_char
self.tmp_char = ""
return char
def unget_char(self, char):
self.tmp_char = char
def ret_token(self, word):
if self.state in (2,):
token = NumToken(self.reader.line_no, word)
elif self.state in (12,):
token = StrToken(self.reader.line_no, word)
elif self.state in (3, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16,):
token = IdToken(self.reader.line_no, word)
self.state = 1
return token
def add_token(self):
self.word_buffer = ""
while True:
char = self.get_char()
if char == "":
if self.word_buffer:
word = self.word_buffer.strip(" ").strip("\t")
return self.ret_token(word)
return Token.EOF
ret = getattr(self, "state%s" % self.state)(char)
if ret:
token = None
word = self.word_buffer.strip(" ").strip("\t")
if word:
self.unget_char(char)
return self.ret_token(word)
def state1(self, char):
if char in [" ", "\t"]:
self.state = 1
elif re.match('\d', char):
self.state = 2
elif re.match('[A-Za-z]', char):
self.state = 3
elif char == "=":
self.state = 4
elif char == "<":
self.state = 6
elif char == ">":
self.state = 7
elif char in list("+-*/{}|&[]()<>\n%"):
self.state = 10
elif char == '"':
self.state = 11
elif char == "&":
self.state = 13
else:
raise Exception("%s行词法分析失败" % self.reader.line_no)
self.word_buffer += char
def state2(self, char):
if re.match('\d', char):
self.state = 2
self.word_buffer += char
else:
return True
def state3(self, char):
if re.match('\w', char):
self.state = 3
self.word_buffer += char
else:
return True
def state4(self, char):
if char == '=':
self.state = 5
self.word_buffer += char
else:
return True
def state5(self, char):
return True
def state6(self, char):
if char == "=":
self.state = 8
self.word_buffer += char
else:
return True
def state7(self, char):
if char == "=":
self.state = 9
self.word_buffer += char
else:
return True
def state8(self, char):
return True
def state9(self, char):
return True
def state10(self, char):
return True
def state11(self, char):
if char == '"':
self.state = 12
self.word_buffer += char
else:
self.state = 11
self.word_buffer += char
def state12(self, char):
return True
def state13(self, char):
if char == "&":
self.state = 14
self.word_buffer += char
else:
return True
def state14(self, char):
return True
def state15(self, char):
if char == "|":
self.state = 16
self.word_buffer += char
def state16(self, char):
return True
