def scenario_A(file_in, file_out=None): print("Input file: {}".format(file_in)) reader = FileReader(file_in) problem = reader.read() solver = Pizza(problem) print("Description:") print(solver.description()) slices = solver.solve() is_valid, result = problem.validate_solution(slices) if is_valid: print("Solution for problem {} is correct. Score is {}".format( file_in, result)) solution = Solution(problem) solution.load_slices(slices) if DEBUG: solution.print_solution() if file_out: writer = FileWriter(file_in + ".out") writer.write(solution) else: print("Incorrect solution. Please check the error messages below") for msg in result: print(msg)
def scenario_A(file_in): print("Input file: {}".format(file_in)) global DEBUG reader = FileReader(file_in) problem = reader.read() solver = Solver(problem) print("Description:") print(solver.description()) solution = solver.initial_solution() # print(solver.initial_solution().print_free()) # print("-------") # print(solver.initial_solution().print_solution()) # print("here") # print("\n".join(str(i) for i in solver.initial_solution().slices)) # print("here2") print("Initial solution score: {}".format(solution.score())) if DEBUG: solution.print_solution() optimized_solution = solver.search(solution, time_limit=60) print("Optimized solution score: {}".format(optimized_solution.score())) # print("\n".join(str(i) for i in optimized_solution.slices)) if DEBUG: optimized_solution.print_solution() trace = solver.get_search_trace() if trace is not None: print("Trace:") print("\n".join([str(x) for x in trace]))
def __init__(self): self.val = Validator() self.db = DatabaseMaker() self.reader = FileReader() self.py = PyGal() self.converted_file = None self.file_count = 1
def initHeapUsageAndGCPause(self, gclogFile, gcpauseFile): fileLines = FileReader.readLines(gclogFile) for line in fileLines: if (line.strip != ""): heapUsage = self.parseUsage(line) if (line.startswith("[Young]")): self.youngGen.append(heapUsage) elif (line.startswith("[Old]")): self.oldGen.append(heapUsage) elif (line.startswith("[Metaspace]")): self.metaGen.append(heapUsage) fileLines2 = FileReader.readLines(gcpauseFile) for line in fileLines2: if not line[0].isdigit(): continue items = re.split(",", line) gc_type = items[-1] if gc_type == "NONE": continue if gc_type not in self.gcTotal.keys(): self.gcTotal[gc_type] = [] self.gcTotal[gc_type].append(float(items[3])) if gc_type not in self.gcTimeLine.keys(): self.gcTimeLine[gc_type] = [] self.gcTimeLine[gc_type].append(float(items[0]))
def main(): """Main function for commandline call """ # end user version for user_input # args = user_input(sys.argv[1:]) # add your own args = user_input() for testing and debugging so that you # don't have to call the script with full command line input args = user_input(['Input/Task1/', '-o', 'Output/Task2/']) # read files reader = FileReader(args.path) input_df = reader.read() # perform statistical analysis stat_ana = analysis.Statistical_Analysis(args.output) stat_ana.correlation(input_df['npop.t']) stat_ana.eucl_distance(input_df['table.dat']) # perfomr numerical analysis num_ana = analysis.Numerical_Analysis(args.output) # return new df with the desired columns df_efield_relevant = num_ana.remove_low_variance(input_df['efield.t']) # fft with freq of the df df_efield_fft = num_ana.fft_with_freq_analysis(df_efield_relevant, "y") # disabled plot to not have it get on my nerves num_ana.plot_and_save(df_efield_fft, "freq", "intensitys", "efield_fft_analysis", xlabel="Freq", show_graph=False) df_autocorr = num_ana.autocorrelation(input_df["nstate_i.t"], "time") num_ana.plot_and_save(df_autocorr, "time", ["autocorr_abs", "autocorr_real", "autocorr_imag"], "nstate_autocorr_analysis", xlabel="time", show_graph=False) df_autocorr_fft = num_ana.fft_with_freq_analysis(df_autocorr, "autocorr", type="complex") # adding abs**2 to the dataframe df_autocorr_fft["intensitys_squared"] = np.abs( df_autocorr_fft["intensitys"].values)**2 num_ana.plot_and_save(df_autocorr_fft, "freq", ["intensitys", "intensitys_squared"], "nstate_autocorr_fft_analysis", xlabel="Freq", show_graph=True, crop_edge=3)
def scenario_A(file_in, file_out): global DEBUG reader = FileReader(file_in) problem = reader.read() init_solver = InitSolverSilly() solution = init_solver.run(problem) print("Initial solution score: {}".format(solution.score())) if DEBUG: solution.print_solution() writer = FileWriter(file_out) writer.write(solution)
def scenario_C(file_in, file_out, file_par=None): global DEBUG reader = FileReader(file_in) problem = reader.read() init_solver = ParallelInitSolver(InitSolverSillyParameterized, file_output=file_out) solution = init_solver.run(problem, file_par) print("Initial solution score: {}".format(solution.score())) if DEBUG: solution.print_solution() writer = FileWriter(file_out) writer.write(solution)
def analyzeMetrics(self, metrics): """ :param metrics: [app.duration, stage0.duration, s0.task.jvmGcTime, ...] """ statisticsFiles = os.listdir(self.statisticsDir) metricsSet = set(metrics) for file in statisticsFiles: if file.endswith( "stat.txt" ): # [RDDJoin-CMS-1-7G-stat.txt, RDDJoin-CMS-2-14G-stat.txt, ...] # [app.duration] mean = 224339.20, stdVar = 8311.91, median = 225233.00, min = 211999.00, quantile25 = 216682.50, quantile75 = 231549.00, max = 233837.00 # -------------------------------------------------------------------[Stage 0]------------------------------------------------------------------- # [stage0.duration] mean = 42360.60, stdVar = 4069.63, median = 41404.00, min = 37094.00, quantile25 = 38942.50, quantile75 = 46257.00, max = 47801.00 # [stage0.inputBytes] mean = 8588671743.00, stdVar = 0.00, median = 8588671743.00, min = 8588671743.00, quantile25 = 8588671743.00, quantile75 = 8588671743.00, max = 8588671743.00 # [stage0.inputRecords] mean = 66000000.00, stdVar = 0.00, median = 66000000.00, min = 66000000.00, quantile25 = 66000000.00, quantile75 = 66000000.00, max = 66000000.00 for line in FileReader.readLines( os.path.join(self.statisticsDir, file)): metricName = line[line.find('[') + 1:line.find(']')] if metricName in metricsSet: if self.metricsMap.has_key(metricName): self.metricsMap[metricName].addStatistics( line, file) else: statistics = st.Statistics() statistics.addStatistics(line, file) self.metricsMap[metricName] = statistics
def fillStatistics(self, metrics, statisticsDir, statisticsFiles, metricsMap, withMax): metricsTupleDict = {} # ["app.duration", ("app.duration", "Time (s)", 1000)] for tuple in metrics: metricsTupleDict[tuple[0]] = tuple for file in statisticsFiles: if file.endswith("stat.txt"): # [RDDJoin-CMS-1-7G-stat.txt, RDDJoin-CMS-2-14G-stat.txt, ...] # [app.duration] mean = 224339.20, stdVar = 8311.91, median = 225233.00, min = 211999.00, quantile25 = 216682.50, quantile75 = 231549.00, max = 233837.00 # -------------------------------------------------------------------[Stage 0]------------------------------------------------------------------- # [stage0.duration] mean = 42360.60, stdVar = 4069.63, median = 41404.00, min = 37094.00, quantile25 = 38942.50, quantile75 = 46257.00, max = 47801.00 # [stage0.inputBytes] mean = 8588671743.00, stdVar = 0.00, median = 8588671743.00, min = 8588671743.00, quantile25 = 8588671743.00, quantile75 = 8588671743.00, max = 8588671743.00 # [stage0.inputRecords] mean = 66000000.00, stdVar = 0.00, median = 66000000.00, min = 66000000.00, quantile25 = 66000000.00, quantile75 = 66000000.00, max = 66000000.00 for line in FileReader.readLines(os.path.join(statisticsDir, file)): metricName = line[line.find('[') + 1: line.find(']')] if metricsTupleDict.has_key(metricName): if metricsMap.has_key(metricName): metricsMap[metricName].addStatistics(line, file, withMax) else: statistics = st.BoxPlotStatistics(metricsTupleDict[metricName]) statistics.addStatistics(line, file, withMax) metricsMap[metricName] = statistics # Fill the NaA values for metricName, statistics in metricsMap.items(): statistics.checkAndFillNulls()
def initHeapUsage(self, gclogFile, timeOffset): fileLines = FileReader.readLines(gclogFile) for line in fileLines: if (line.strip != ""): heapUsage = self.parseUsage(line, timeOffset) if(line.startswith("[Young]")): self.youngGen.append(heapUsage) elif(line.startswith("[Old]")): self.oldGen.append(heapUsage)
def analyzeMetrics(self): taskInfoFiles = os.listdir(self.taskInfoDir) for file in taskInfoFiles: if file.endswith( "tasks.txt" ): # [GroupByRDD-CMS-1-7G-0.5-tasks.txt, GroupByRDD-CMS-2-14G-0.5-tasks.txt, ...] # [appName = GroupByRDD-CMS-1-7G-0.5] # [appId = app-20170721105922-0025] # [stageId = 1] # # [0.task.index] 0 # [0.task.attemptId] 0 # [0.task.executorId] 29 # [0.task.duration] 4715 # [0.task.executorDeserializeTime] 60 # [0.task.executorDeserializeCpuTime] 43705029 # [0.task.executorRunTime] 4715 # [0.task.executorCpuTime] 2614115738 # [0.task.resultSize] 2865 # [0.task.jvmGcTime] 1305 GC = "" executor = "" if (file.find("CMS") != -1): GC = "CMS" elif (file.find("G1") != -1): GC = "G1" elif (file.find("Parallel") != -1): GC = "Parallel" if (file.find("1-7G") != -1): executor = "E1" elif (file.find("2-14G") != -1): executor = "E2" elif (file.find("4-28G") != -1): executor = "E4" for line in FileReader.readLines( os.path.join(self.taskInfoDir, file)): if line.startswith('[appName'): appName = line[line.find('=') + 2:line.find(']')] elif line.startswith('[appId'): appId = line[line.find('=') + 2:line.find(']')] elif line.startswith('[stageId'): stageId = int(line[line.find('=') + 2:line.find(']')]) elif line.startswith('['): metricName = line[line.find('task') + 5:line.find(']')] metricValue = float(line[line.find(']') + 2:]) if (metricName == 'index'): taskAttempt = ta.TaskAttempt( appId, appName, stageId, int(metricValue), GC, executor) taskAttempt.set(metricName, metricValue) self.taskList.append(taskAttempt)
def scenario_A(file_in): print("Input file: {}".format(file_in)) global DEBUG reader = FileReader(file_in) problem = reader.read() solver = Solver(problem) print("Description:") print(solver.description()) solution = solver.initial_solution() print("Initial solution score: {}".format(solution.score())) if DEBUG: solution.print_solution() optimized_solution = solver.search(solution, time_limit=60) print("Optimized solution score: {}".format(optimized_solution.score())) if DEBUG: optimized_solution.print_solution() trace = solver.get_search_trace()
def scenario_B(file_in, file_out): global DEBUG reader = FileReader(file_in) problem = reader.read() init_solver = InitSolverSilly() solution = init_solver.run(problem) print("Initial solution score: {}".format(solution.score())) if DEBUG: solution.print_solution() optimizer = Tabu(problem, solution, Neighbourhood_ChangeFormats, debug=True) optimized_solution = optimizer.run(time_limit=100) print("optimized solution score: {}".format(optimized_solution.score())) if DEBUG: optimized_solution.print_solution() writer = FileWriter(file_out) writer.write(optimized_solution)
def scenario_D(file_in, file_out, file_par=None): global DEBUG reader = FileReader(file_in) problem = reader.read() init_solver = ParallelInitSolver(InitSolverSillyParameterized, file_output=file_out) solution = init_solver.run(problem, file_par) print("Initial solution score: {}".format(solution.score())) if DEBUG: solution.print_solution() optimizer = ParallelTabu(problem, solution, Neighbourhood_ChangeFormats, debug=True) optimized_solution = optimizer.run(time_limit=1000) print("optimized solution score: {}".format(optimized_solution.score())) if DEBUG: optimized_solution.print_solution() writer = FileWriter(file_out) writer.write(solution)
def plot_specific_heat_ratio_isobaric(filename, plot_cp=True): data = FileReader(filename) # Plot 1 fig, ax1 = plt.subplots() if plot_cp: color = 'b' else: color = 'k' ax1.plot(data.temperature, data.gamma, color, label='Specific Heat Ratio') ax1.set_xlabel(data.titles.temperature) ax1.set_ylabel(data.titles.gamma) if plot_cp: # Plot 2 ax2 = ax1.twinx() ax2.plot(data.temperature, data.cp, 'r', label='Cp') ax2.set_ylabel(data.titles.cp) fig.legend(loc="upper center", bbox_to_anchor=(0.5, 0.9)) plt.title('Specific heat ratio of water (g) at 6 bar') fig.tight_layout() plt.show()
import argparse from analyzer import Analyzer from chart import BarGraph from reader import FileReader if __name__ == "__main__": parser = argparse.ArgumentParser(description='Лабораторная №5') parser.add_argument('--path=', action="store", required=True, dest="path", help="путь до файла") args = parser.parse_args() data = FileReader.read(args.path) analyzer = Analyzer(data) spammers = analyzer.spammer_list if not spammers: print('Спамеры не обнаружены') else: print('Список спамеров:') for spammer in spammers: print(spammer) BarGraph.draw(analyzer.info)
def __init__(self): self.data = FileReader().readFile("client.csv")
def __init__(self, f): self.has_more = True self.reader = FileReader(f) self.line_no = 0 self.queue = [] self.pattern = re.compile(self.regex_pat, re.X)
def __init__(self): self.data = FileReader().readFile("trans.csv")
def __init__(self): self.data = FileReader().readFile("district.csv")
class Controller(object): def __init__(self): self.val = Validator() self.db = DatabaseMaker() self.reader = FileReader() self.py = PyGal() self.converted_file = None self.file_count = 1 def read_file(self, filename): try: self.converted_file = self.reader.read(filename) for dict in self.converted_file: print("FILE_DATA:", self.file_count, "\n", dict) self.file_count += 1 except Exception as err: print("The exception is: ", err) def valid(self, flag): try: if flag == '': self.validate() if flag == '-f': file = input("Validate which file?:") self.read_file(file) self.validate() if flag == '-v': self.view_valid() if flag == '-fv': file = input("Validate which file?:") self.read_file(file) self.validate() else: raise Exception("Not a valid flag") except Exception as err: print("The exception is: ", err) def validate(self): try: for dict in self.converted_file: self.val.valid(dict) return self.view_valid() except Exception as err: print("The exception is: No file specified") def view_valid(self): print("\n VALID DATA:") for dict in self.val.get(): print(dict) def db_table(self, flag): try: if flag == "-d": self.db_drop_table() elif flag == '-c': self.db_create_table() elif flag == '-dc': self.db_drop_table() self.db_create_table() elif flag == '-i': for i in self.db_insert(): print(i) elif flag == '-v': value = input("What column do you want to\ see from the employee table?") self.db_view(value) elif flag == '-if': file = input("Input which file?:") self.read_file(file) self.validate() self.db_insert() elif flag == '': self.db_view("*") else: raise Exception("Not a valid flag") except Exception as err: print("The exception is: ", err) def db_drop_table(self): self.db.drop_table("employee") def db_create_table(self): self.db.create_table() def db_insert(self): try: list_of_dictionaries = self.val.get() if list_of_dictionaries: for dict in list_of_dictionaries: self.db.insert(dict["id"], dict["gender"], dict["age"], dict["sales"], dict["bmi"], dict["salary"], dict["birthday"]) return self.db.get("*") else: raise Exception("Cannot add invalid data to the database") except Exception as err: print("The exception is: ", err) def db_view(self, value): for v in self.db.get(value): print(v) def database_close(self): self.db.close() def pygal(self, flag): try: if flag == '': value = input("Which data do you want to see a bar graph of?") self.py_view(value) if flag == '-d': value = input("Input first bar graph comparison:") value2 = input("Input second bar graph comparison:") self.py_view(value, value2) return else: raise Exception("Not a valid flag") except Exception as err: print("The exception is: ", err) def py_view(self, value, value2=None): try: data = self.db.bar_get(value) id = self.db.bar_get('id') if value2 is not None: data2 = self.db.bar_get(value2) self.py.bar_char(id, value, data, value2, data2) else: self.py.bar_char(id, value, data) except Exception as err: print("The exception is: ", err) def pickled(self, flag): try: if flag == '': name = input("Name of pickle file?:") if name == '': raise Exception("Not a valid filename") else: self.pickled_write(name) elif flag == '-r': name = input("Name of pickle file?:") if name == '': raise Exception("Not a valid filename") else: self.pickled_read(name) else: raise Exception("Not a valid flag") except Exception as err: print("The exception is: ", err) def pickled_write(self, name): import pickle with open('{name}.pickle'.format(name=name), 'wb') as f: pickle.dump(self.converted_file, f) def pickled_read(self, name): import pickle with open('{name}.pickle'.format(name=name), 'rb') as f: data = pickle.load(f) print(data)
def __init__(self, filename): self.filename = FileReader(filename)
def __init__(self, f): self.reader = FileReader(f) self.state = 1 self.queue = [] self.word_buffer = "" self.tmp_char = ""
from reader import FileReader, UrlReader from sma import SMA, SMAPlt if __name__ == "__main__": parser = argparse.ArgumentParser(description='Лабораторная №4') parser.add_argument('--paths=', help='Список адресов через запятую', required=True, dest="paths") parser.add_argument('--frame=', help='Размер окна', required=True, dest="frame") args = parser.parse_args() paths = [item for item in args.paths.split(',')] if len(paths) < 25: print('Кол-во данных меньше 25') exit() sma = SMA() print('Загрузка данных...') for path in paths: reader = FileReader() if FileReader.path_is_file(path) else UrlReader() sma.read_data(reader, path) sma.frame = args.frame SMAPlt.draw(*sma.calculate_sma())
def __init__(self): self.data = FileReader().readFile("loan.csv")
#plt.tight_layout() plt.show() #fig = plt.gcf() #plt.show() #fig.savefig(outputFile, dpi=300, bbox_inches='tight') if __name__ == '__main__': gcLogPath = "/Users/xulijie/Documents/GCResearch/Android/logs/" appName = "Alexnet GC Metrics" gcFileName = "cnndroid_alexnet_scale1_image_time_logcat.txt" gcLines = FileReader.readLines(gcLogPath + gcFileName) gcActivities = GCActivities(gcLines) plotHeapUsage(appName, appName, gcActivities) appName = "Nin GC Metrics" gcFileName = "cnndroid_nin_scale1_image_time_logcat.txt" gcLines = FileReader.readLines(gcLogPath + gcFileName) gcActivities = GCActivities(gcLines) plotHeapUsage(appName, appName, gcActivities)
def __init__(self): self.data = FileReader().readFile("account.csv")
class ReLexer(Lexer): regex_pat = r''' \s*( #空白 (//.*)| #注释 (\d+)| #数字类型 ("[^"]*")| #字符串类型 [A-Z_a-z][A-z_a-z0-9]*| #标识符 ==|<=|>=|&&|\|\|| #符号 [\+\-\*/\{\}\=\|\&\[\]\(\)\<\>\;\%] #符号 ) ''' def __init__(self, f): self.has_more = True self.reader = FileReader(f) self.line_no = 0 self.queue = [] self.pattern = re.compile(self.regex_pat, re.X) def read(self): if self.fill_queue(0): return self.queue.pop(0) return Token.EOF def peek(self, i): if self.fill_queue(i): return self.queue[i] return Token.EOF def fill_queue(self, i): while i >= len(self.queue): if self.has_more: self.read_line() else: return False return True def read_line(self): line = self.reader.read_line() if not line: self.has_more = False return line = line[:-1] self.line_no = self.reader.line_no start = 0 end = len(line) while start < end: matcher = self.pattern.search(line[start:end]) start += matcher.end() if matcher: self.add_token(matcher) self.queue.append(IdToken(self.line_no, Token.EOL)) def add_token(self, matcher): m = matcher.group(1) if m is not None: # not a space line if matcher.group(2) == None: # not a comment if matcher.group(3) != None: token = NumToken(self.line_no, int(m)) elif matcher.group(4) != None: token = StrToken(self.line_no, m) else: token = IdToken(self.line_no, m) self.queue.append(token)
def plotResourceUsage(topMetricsFile, slowestTasksDir, appName): fileLines = FileReader.readLines(topMetricsFile) isExecutorMetric = False isSlaveMetric = False executorTime = [] executorCPU = [] executorMemory = [] slaveTime = [] slaveCPU = [] slaveMemory = [] for line in fileLines: if (line.startswith("[Top Metrics][Executor")): isExecutorMetric = True elif (line.startswith("[Top Metrics][aliSlave")): isSlaveMetric = True isExecutorMetric = False elif (isExecutorMetric == True and line.strip() != ""): time = line[line.find('[') + 1:line.find(']')] cpu = line[line.find('=') + 2:line.find(',')] memory = line[line.find('Memory') + 9:] executorTime.append(datetime.strptime(time, '%H:%M:%S')) executorCPU.append(float(cpu)) executorMemory.append(float(memory)) elif (isSlaveMetric == True and line.strip() != ""): time = line[line.find('[') + 1:line.find(']')] cpu = line[line.find('=') + 2:line.find(',')] memory = line[line.find('Memory') + 9:] slaveTime.append(datetime.strptime(time, '%H:%M:%S')) slaveCPU.append(float(cpu)) slaveMemory.append(float(memory)) fig, axes = plt.subplots(nrows=2, ncols=1, sharey=False, sharex=True) # locator = mpl.dates.MinuteLocator() xfmt = mdates.DateFormatter('%H:%M:%S') #ax.xaxis.set_major_locator(locator) axes[0].xaxis.set_major_formatter(xfmt) axes[1].xaxis.set_major_formatter(xfmt) axes[0].set_ylabel("Executor CPU (%)", color='r') axes[0].tick_params('y', colors='r') axes[1].set_ylabel("Worker CPU (%)", color='r') axes[1].tick_params('y', colors='r') axes[0].set_ylim(0, 840) # The ceil axes[1].set_ylim(0, 105) # The ceil # plt.ylim(0, statistics.max) # The ceil # plt.legend() fig.autofmt_xdate() axes[0].plot_date(executorTime, executorCPU, '-r', label='CPU') axes[1].plot_date(slaveTime, slaveCPU, '-r', label='CPU') ax12 = axes[0].twinx() ax12.plot_date(executorTime, executorMemory, '-b', label='Memory') ax12.set_ylabel('Executor Memory (GB)', color='b') ax12.tick_params('y', colors='b') ax12.set_ylim(0, 32) # The ceil # ax12.tick_params('y', colors='r') ax22 = axes[1].twinx() ax22.plot_date(slaveTime, slaveMemory, '-b', label='Memory') ax22.set_ylabel('Worker Memory (GB)', color='b') ax22.tick_params('y', colors='b') ax22.set_ylim(0, 32) # The ceil plt.suptitle(appName) outputDir = os.path.join(slowestTasksDir, "topMetricsFigures") if not os.path.exists(outputDir): os.mkdir(outputDir) file = os.path.join(outputDir, appName + ".pdf") plt.show()
import math import time from flask import Flask, request, jsonify from flask_cors import CORS from reader import FileReader, Bolder app = Flask(__name__) CORS(app) start_reading = time.time() reader = FileReader('./data/IR-F19-Project01-Input.xlsx', True, './matches.json', './exceptions.txt') # reader = FileReader('./data/IR-F19-Project01-Input-2k.xlsx', True, './matches.json', './exceptions.txt') # reader = FileReader('./data/IR-F19-Project02-14k.csv', False, './matches.json', './exceptions.txt') end_reading = time.time() print("Reading Files:", end_reading - start_reading, "secs") bolder = Bolder() @app.route('/') def query(): q = request.args.get('q') start_searching = time.time() print("Query:", q) docs, query_tokens = reader.search(q, 50) docs, query_tokens = list(docs), list(query_tokens) print("Results:", len(docs)) items = int(request.args.get('items', 10)) print("Items:", items) pages = math.ceil(len(docs) / items) print("Pages:", pages) page = int(request.args.get('page', 0))
class Manager(object): '''Program manager''' def __init__(self, filename): self.filename = FileReader(filename) def build_autonomous_systems(self): print('Processing data:', end='') data = self.filename.get_data() for line in data: if Prefix.get_count() % 10000 == 0: print('.', end='') sys.stdout.flush() prefix = self.create_prefix(line) asn = prefix.origin_asn next_hop = prefix.next_hop_asn if asn not in AutonomousSystem.dict_of_all: self.create_new_asn(asn).ipv4_prefixes.append(prefix) else: self.find_asn(asn).ipv4_prefixes.append(prefix) if next_hop: if next_hop not in AutonomousSystem.dict_of_all: self.create_new_asn(next_hop).ipv4_next_hop_prefixes.append(prefix) else: self.find_asn(next_hop).ipv4_next_hop_prefixes.append(prefix) else: pass def create_new_asn(self, asn): return AutonomousSystem(asn) def find_asn(self, asn): return AutonomousSystem.dict_of_all.get(asn) def create_prefix(self, line): return Prefix(*line) def list_of_peers(self): next_hops = [] for k, v in AutonomousSystem.dict_of_all.items(): for route in v.ipv4_prefixes: next_hops.append(route.next_hop_asn) return set(next_hops) def find_prefixes(self, asn): prefixes = self.find_asn(asn).ipv4_prefixes count = len(prefixes) return prefixes, count def _sorted_ip_list(self, ip_list): mylist = list() for line in ip_list: mylist.append(line.prefix) return(mylist) #return sorted(list({line.prefix for line in ip_list}), key=lambda x: socket.inet_aton(x.split("/")[0])) def find_next_hop_prefixes(self, asn): prefixes = self.find_asn(asn).ipv4_next_hop_prefixes count = len(prefixes) return prefixes, count def _next_hop_asns(self, asn): next_hops = [] for prefix in self.find_prefixes(asn)[0]: next_hops.append(prefix.next_hop_asn) return list(set(next_hops)) def _next_hop_ips(self, prefix_list): next_hop_ips = [] for prefix in prefix_list: next_hop_ips.append(prefix.next_hop_ip) c = Counter(next_hop_ips) return c, len(c) def _dns_query(self, asn): query = 'AS' + asn + '.asn.cymru.com' answers = dns.resolver.query(query, 'TXT') for rdata in answers: for txt_string in rdata.strings: return txt_string.split('|')[-1].split(",", 2)[0].strip() def print_stats(self): print() print('IPv4 Routing Table Size:', Prefix.get_count()) print('Unique ASNs:', len(AutonomousSystem.dict_of_all)) print('Peer Networks:', len(self.list_of_peers())) def print_details(self): results = OrderedDict() results['peers'] = OrderedDict() print() for peer in sorted(list(filter(None.__ne__, self.list_of_peers())), key=lambda x: int(x)): if (peer and (int(peer) < 64512 or int(peer) > 65534)): results['peers'][peer] = OrderedDict() results['peers'][peer]['name'] = self._dns_query(peer) results['peers'][peer]['prefixes originated'] = self.find_prefixes(peer)[1] results['peers'][peer]['routes selected'] = self.find_next_hop_prefixes(peer)[1] all_next_hop_prefixes, count_next_hop_prefixes = self.find_next_hop_prefixes(peer) results['peers'][peer]['peering connections'] = self._next_hop_ips(all_next_hop_prefixes)[1] results['peers'][peer]['peering connection ip list route count'] = self._next_hop_ips(all_next_hop_prefixes)[0] else: pass print(json.dumps(results, indent=4)) def print_asn_details(self, asn, show_routes): if self.find_asn(asn): results = OrderedDict() print() if asn in self.list_of_peers(): name = self._dns_query(asn) results['asn'] = asn results['name'] = name results['peer'] = True prefixes, count = self.find_prefixes(asn) all_next_hop_prefixes, count_next_hop_prefixes = self.find_next_hop_prefixes(asn) received_prefixes_peering = [] received_prefixes_other = [] results['prefixes originated'] = count for prefix in prefixes: if prefix.as_path[0] == asn: received_prefixes_peering.append(prefix) else: received_prefixes_other.append(prefix) results['peering connections'] = self._next_hop_ips(all_next_hop_prefixes)[1] results['peering connection ip list route count'] = self._next_hop_ips(all_next_hop_prefixes)[0] results['prefixes originated by this asn that use the direct peering as a next hop'] = len(received_prefixes_peering) results['prefixes originated by this asn that do use the direct peering'] = len(received_prefixes_other) results['all prefixes using this asn as next hop'] = count_next_hop_prefixes if show_routes: results['direct peering prefixes'] = self._sorted_ip_list(received_prefixes_peering) results['non-peering prefixes'] = self._sorted_ip_list(received_prefixes_other) results['all prefixes using this asn as transit'] = self._sorted_ip_list(all_next_hop_prefixes) results['next hop asn list'] = self._next_hop_asns(asn) print(json.dumps(results, indent=4)) else: name = self._dns_query(asn) results['asn'] = asn results['name'] = name results['peer'] = False prefixes, count = self.find_prefixes(asn) received_prefixes_other = [] results['prefixes originated'] = count for prefix in prefixes: received_prefixes_other.append(prefix) if show_routes: results['prefix list'] = self._sorted_ip_list(received_prefixes_other) results['next hop asn list'] = self._next_hop_asns(asn) print(json.dumps(results, indent=4)) else: print() print("ASN %s not found" % asn) print("Ignored Lines:") for line in self.filename.get_ignored_lines(): print(line)
class AutomatonLexer(Lexer): def __init__(self, f): self.reader = FileReader(f) self.state = 1 self.queue = [] self.word_buffer = "" self.tmp_char = "" def read(self): if self.fill_queue(0): return self.queue.pop(0) return Token.EOF def peek(self, i): if self.fill_queue(i): return self.queue[i] return Token.EOF def fill_queue(self, i): while i >= len(self.queue): token = self.add_token() if token != Token.EOF: self.queue.append(token) else: return False return True def get_char(self): if not self.tmp_char: char = self.reader.read_char() return char else: char = self.tmp_char self.tmp_char = "" return char def unget_char(self, char): self.tmp_char = char def ret_token(self, word): if self.state in (2,): token = NumToken(self.reader.line_no, word) elif self.state in (12,): token = StrToken(self.reader.line_no, word) elif self.state in (3, 4, 5, 6, 7, 8, 9, 10, 13, 14, 15, 16,): token = IdToken(self.reader.line_no, word) self.state = 1 return token def add_token(self): self.word_buffer = "" while True: char = self.get_char() if char == "": if self.word_buffer: word = self.word_buffer.strip(" ").strip("\t") return self.ret_token(word) return Token.EOF ret = getattr(self, "state%s" % self.state)(char) if ret: token = None word = self.word_buffer.strip(" ").strip("\t") if word: self.unget_char(char) return self.ret_token(word) def state1(self, char): if char in [" ", "\t"]: self.state = 1 elif re.match('\d', char): self.state = 2 elif re.match('[A-Za-z]', char): self.state = 3 elif char == "=": self.state = 4 elif char == "<": self.state = 6 elif char == ">": self.state = 7 elif char in list("+-*/{}|&[]()<>\n%"): self.state = 10 elif char == '"': self.state = 11 elif char == "&": self.state = 13 else: raise Exception("%s行词法分析失败" % self.reader.line_no) self.word_buffer += char def state2(self, char): if re.match('\d', char): self.state = 2 self.word_buffer += char else: return True def state3(self, char): if re.match('\w', char): self.state = 3 self.word_buffer += char else: return True def state4(self, char): if char == '=': self.state = 5 self.word_buffer += char else: return True def state5(self, char): return True def state6(self, char): if char == "=": self.state = 8 self.word_buffer += char else: return True def state7(self, char): if char == "=": self.state = 9 self.word_buffer += char else: return True def state8(self, char): return True def state9(self, char): return True def state10(self, char): return True def state11(self, char): if char == '"': self.state = 12 self.word_buffer += char else: self.state = 11 self.word_buffer += char def state12(self, char): return True def state13(self, char): if char == "&": self.state = 14 self.word_buffer += char else: return True def state14(self, char): return True def state15(self, char): if char == "|": self.state = 16 self.word_buffer += char def state16(self, char): return True