import glob import time import sys import tarfile import shutil import cStringIO import platform from SCons.Script.SConscript import SConsEnvironment import SCons.Action import SCons.Util import SCons.Builder import SCons.Subst import SCons.Tool import bcolors bc = bcolors.bcolors() import btools VERSION = btools.VERSION VERSION_RELEASE_CYCLE = btools.VERSION_RELEASE_CYCLE Split = SCons.Util.Split Action = SCons.Action.Action Builder = SCons.Builder.Builder GetBuildPath = SConsEnvironment.GetBuildPath # a few globals root_build_dir = '' doc_build_dir = '' quickie = None # Anything else than None if BF_QUICK has been passed quicklist = [] # The list of libraries/programs to compile during a quickie program_list = [
import glob import time import sys import tarfile import shutil import cStringIO import platform from SCons.Script.SConscript import SConsEnvironment import SCons.Action import SCons.Util import SCons.Builder import SCons.Subst import SCons.Tool import bcolors bc = bcolors.bcolors() import btools VERSION = btools.VERSION VERSION_RELEASE_CYCLE = btools.VERSION_RELEASE_CYCLE Split = SCons.Util.Split Action = SCons.Action.Action Builder = SCons.Builder.Builder GetBuildPath = SConsEnvironment.GetBuildPath # a few globals root_build_dir = '' doc_build_dir = '' quickie = None # Anything else than None if BF_QUICK has been passed quicklist = [] # The list of libraries/programs to compile during a quickie program_list = [] # A list holding Nodes to final binaries, used to create installs
def main() : parser = OptionParser(usage="usage: run-simulator.py [-v] [-t <clock cycles>] filename", version="1.0") parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="Print cycle by cycle debug information to simulation log file") parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False, help="Supress terminal output. Be vewwy quiet (I'm hunting wabbits...)") parser.add_option("-t", "--timeout", action="store", type="int", dest="timeout", help="Terminal looping program. Specify number of clock cycles TIMEOUT") (options, args) = parser.parse_args() inputFile = None # For automated testing output B = bcolors("mathstuff") # Open the input file try: inputFile = open(args[0], "r"); except IOError: print "There was an error opening the input file: ", args[0] sys.exit() defaultSimASMFile = "simasm.sim" defaultDataMemFile = "datamem.sim" defaultPreProcLogFile = "preprocLog.sim" defaultSimRunFile = "simrun.sim" oldstdout = sys.stdout # Initialize parsers iparser = InstructionParser.InstructionParser() eparser = elf32parser.elf32parser() # Convert elf32-bigmips to simulator friendly format SimAsmFileName = args[2] if len(args) >= 3 else defaultSimASMFile SimAsmFile = open(SimAsmFileName, 'w') sys.stdout = SimAsmFile DataMemFileName = args[4] if len(args) >= 5 else defaultDataMemFile if(not options.quiet): oldstdout.write("> Starting Parser...\n") eparser.convertToSimASM(args[0], SimAsmFileName, DataMemFileName) lines = eparser.getLines() datamem = eparser.getDataMem() mainAddr = eparser.getMainAddr() # TODO: change this mainAdder = 0x48 from first line of "0: jal 0x48" # Parse in lines and check for dependencies PPLogFileName = args[3] if len(args) >= 4 else defaultPreProcLogFile PPLogFile = open(PPLogFileName, 'w') sys.stdout = PPLogFile if(not options.quiet): oldstdout.write("> Starting Assembler...\n") # Get line by line lines = iparser.parseLines(lines) pipelinesim = PipelineSimulator.PipelineSimulator(lines, datamem, mainAddr, oldstdout, options.verbose, options.quiet, options.timeout) if(not options.quiet): print "> Starting Simulation..." startTime = time.clock() # Set logfile simulationFileName = args[1] if len(args) >= 2 else defaultSimRunFile simulationFile = open(simulationFileName, 'w') sys.stdout = simulationFile # Run simulation pipelinesim.run() # Print out line by line #print lines elapsedTime = (time.clock() - startTime) if(not options.quiet): oldstdout.write("\n> Simulation Completed in ") oldstdout.write(str(elapsedTime)) oldstdout.write(" s") simulationFile.close() PPLogFile.close() sys.stdout = oldstdout checker = Checker.Checker(simulationFileName, options.quiet) success = False if(not options.quiet): checker.runCheck() else: success = checker.runCheck() if(success): B.printPass(args[0], checker.getCycles()) else: B.printFail(args[0], "")
def content_parsing(self, context): bc = bcolors.bcolors() r = BeautifulSoup(context, 'lxml') sys.stdout.write("\033[K") sys.stdout.write('[*] Parsing Contents...\r') sys.stdout.flush() try: self.title = r.title.string title = r.find("meta", attrs={"property": "og:title"}) url = r.find("meta", attrs={"property": "og:url"}) description = r.find("meta", attrs={"property": "og:description"}) image = r.find("meta", attrs={"property": "og:image"}) self.property_og_title = title["content"] if title else "" self.property_og_url = url[ "content"] if url else self.property_og_url self.property_og_description = description[ "content"] if description else self.property_og_description self.property_og_image = image[ "content"] if image else self.property_og_image name = r.find("meta", attrs={"itemprop": "name"}) description = r.find("meta", attrs={"itemprop": "description"}) image = r.find("meta", attrs={"itemprop": "og:image"}) self.itemprop_name = name["content"] if name else self.itemprop_name self.property_og_description = description[ "content"] if description else self.property_og_description self.property_og_image = image[ "content"] if image else self.property_og_image author = r.find("meta", attrs={"name": "author"}) description = r.find("meta", attrs={"name": "description"}) image = r.find("meta", attrs={"name": "image"}) self.name_author = author["content"] if author else self.name_author self.property_og_description = description[ "content"] if description else self.property_og_description self.property_og_image = image[ "content"] if image else self.property_og_image except: print('%s[!] Content Parsing Error - Unknown Elements%s\r' % (bc.OKBLUE, bc.ENDC)) self.title = '' for script in r( ["script", "style", "nav", "form", "footer", "noscript", "header"]): script.decompose() # rip it out # for code in r() self.body = r.get_text() # break into lines and remove leading and trailing space on each self.body = (line.strip() for line in self.body.splitlines()) # break multi-headlines into a line each self.body = (phrase.strip() for line in self.body for phrase in line.split(" ")) # drop blank lines self.body = ' '.join(chunk for chunk in self.body if chunk) return { 'title': self.title, 'property_og_title': self.property_og_title, 'property_og_url': self.property_og_url, 'property_og_description': self.property_og_description, 'property_og_image': self.property_og_image, 'itemprop_name': self.itemprop_name, 'name_author': self.name_author, 'body': self.body }
def main() : # Hard-coded table holding clock frequency information for HW frequencyTable = { 5: 193.723, 6: 257.931, 7: 266.241, 8: 311.818, 9: 405.844 } # Create option parser parser = OptionParser(usage="usage: run-simulator.py [-hvcqmpfdews] filename", version="1.0") # Add options (self-documenting) parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="log cycle by cycle debug information") parser.add_option("-c", "--core", action="store_true", dest="core", default=False, help="show only core cycles") parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False, help="less terminal output") parser.add_option("-m", "--mute", action="store_true", dest="mute", default=False, help="no summary output") parser.add_option("-p",# "--pipeline", type="int", dest="pipeline", default=5, help="set number of pipeline stages [default 5]") parser.add_option("-f",# "--IFStages", type="int", dest="ifcycles", default=-1, help="set number of Fetch (IF) Stage cycles") parser.add_option("-d",# "--IDStages", type="int", dest="idcycles", default=-1, help="set number of Decode (ID) Stage cycles") parser.add_option("-e",# "--EXStages", type="int", dest="excycles", default=-1, help="set number of Execution (EX) Stage cycles") parser.add_option("-w",# "--WBStages", type="int", dest="wbcycles", default=-1, help="set number of Writeback (WB) Stage cycles") parser.add_option("-s",# "--WBStages", type="int", dest="startAddr", default=0x0, help="set execution start address") (options, args) = parser.parse_args() # For automated coloured testing output B = bcolors() # Integer conversion options.pipeline = int(options.pipeline) options.ifcycles = int(options.ifcycles) options.idcycles = int(options.idcycles) options.excycles = int(options.excycles) options.wbcycles = int(options.wbcycles) pipelineConfigError = False # Pipeline checking if(options.pipeline < 4): pipelineConfigError = True # Use 1 ID and WB Cycle by default if(options.idcycles == -1): options.idcycles = 1 if(options.wbcycles == -1): options.wbcycles = 1 # Use maximum number of EX/MEM Cycles by default (ugly code...) if(options.excycles == -1 and options.pipeline >= 5): if(options.pipeline == 5): options.excycles = 2 elif(options.pipeline == 6): options.excycles = 3 else: options.excycles = 4 # The rest of the instructions will be WB if(options.ifcycles == -1): remCycles = options.pipeline-options.idcycles-options.excycles-options.wbcycles if(remCycles < 1): pipelineConfigError = True options.ifcycles = remCycles # Double check for correct pipeline configuration pipelineSum = options.ifcycles + options.idcycles + options.excycles + options.wbcycles if(pipelineSum != options.pipeline): pipelineConfigError = True # Give error if something is wrong if(pipelineConfigError): B.printError("Error: Incorrect pipeline configuration") sys.exit() inputFile = None # Open the input file try: inputFile = open(args[0], "r"); except IOError: B.printError("Error: Could not open input file\t" + args[0]) sys.exit() # Default values defaultSimASMFile = "simasm.sim" defaultDataMemFile = "datamem.sim" defaultPreProcLogFile = "preprocLog.sim" defaultSimRunFile = "simrun.sim" oldstdout = sys.stdout # Initialize parsers iparser = InstructionParser.InstructionParser() eparser = elf32parser.elf32parser() # If custom simulation assembly file output is set SimAsmFileName = args[2] if len(args) >= 3 else defaultSimASMFile SimAsmFile = open(SimAsmFileName, 'w') sys.stdout = SimAsmFile # If custom data memory file output is set DataMemFileName = args[4] if len(args) >= 5 else defaultDataMemFile if(not options.quiet): oldstdout.write("> Starting Parser...\n") # Convert elf32-bigmips to simulator friendly format eparser.convertToSimASM(args[0], SimAsmFileName, DataMemFileName) # Extract statistics lines = eparser.getLines() datamem = eparser.getDataMem() mainAddr = eparser.getMainAddr() coreInstr = eparser.getCCoreInstr() # IF custom preprocessing log file name is set PPLogFileName = args[3] if len(args) >= 4 else defaultPreProcLogFile PPLogFile = open(PPLogFileName, 'w') sys.stdout = PPLogFile if(not options.quiet): oldstdout.write("> Starting Assembler...\n") # Parse in lines, do preprocessing and check for dependencies lines = iparser.parseLines(lines, (options.pipeline-options.ifcycles), options.ifcycles, coreInstr) pipelineInfo = [options.pipeline, options.ifcycles, options.idcycles, options.excycles, options.wbcycles] # Initialize simulator pipelinesim = PipelineSimulator.PipelineSimulator(lines, datamem, options.startAddr, oldstdout, options.verbose, options.quiet, pipelineInfo) if(not options.quiet): print "> Starting Simulation..." startTime = time.clock() # Set logfile simulationFileName = args[1] if len(args) >= 2 else defaultSimRunFile simulationFile = open(simulationFileName, 'w') sys.stdout = simulationFile # Run simulation pipelinesim.run() elapsedTime = (time.clock() - startTime) if(not options.quiet): oldstdout.write("\n> Simulation Completed in ") oldstdout.write(str(elapsedTime)) oldstdout.write(" s") # Close output files simulationFile.close() PPLogFile.close() sys.stdout = oldstdout checker = Checker.Checker(simulationFileName, options.quiet) success = False # Give output according to the settings # Normal terminal-based single run output if(not options.quiet): checker.runCheck() # Only summary and statistics output (for use in automated scripts) elif(not options.mute): success = checker.runCheck() if(success): if(options.core): B.printCoreOnly(checker.getCoreCycles()) else: # Compile statistics pNOPs = str(round(float(str(float(checker.getCoreNops())/(float(checker.getCoreCycles()))))*100, 1)) c = checker.getCycles() n = checker.getNOPs() cpi = checker.getCPI() cc = checker.getCoreCycles() cn = checker.getCoreNops() ex = str(round(float(int(c) * float(1/frequencyTable.get(options.pipeline))), 8)) + "us" cex = str(round(float(int(cc) * float(1/frequencyTable.get(options.pipeline))), 8)) + "us" # Print successful test B.printPass(args[0], [c, n, cpi, cc, cn, pNOPs, ex, cex]) else: # Indicate failure B.printFail(args[0], "")
# Rivki Kanterovich # ID: 212030761 # Geula Shoshan # ID: 11826658 import math import csv import bcolors from collections import Counter from pathlib import Path from prettytable import PrettyTable bcolors = bcolors.bcolors() #Class like enum of the heuristics distance avilable class Heuristic: EUCLIDEAN = 0 MANHATTAN = 1 HAMMING = 2 #class with ctor that has distance and tag properties class distClass: def __init__(self, dist=-1, tag='-'): self.dist = dist # distance of current point from test point self.tag = tag # tag of current point #calculate the distance between 2 vectors by selected heuristic def calculate_distance(instance1, instance2, heuristic: Heuristic): if (heuristic == Heuristic.EUCLIDEAN):
# pip install validators beautifulsoup4 lxml # Python version: Python 3.6.3 :: Anaconda, Inc. from bs4 import BeautifulSoup, SoupStrainer import urllib.request import traceback import json import re import urllib.parse import validators from bcolors import bcolors _print = bcolors() class Spider: def __init__(self, root_url, max_links): self.root_url = root_url self.crawl_result = {} # a dictionary to store the crawl output as key, val pairs. self.crawl_set = set() self.bucket = [] self.MAX_CRAWL = max_links self.link_count = 0 self.default_scheme = 'http://' self.scheme = self.default_scheme def fetch_url(self, url): """
def main(): # Hard-coded table holding clock frequency information for HW frequencyTable = { 5: 193.723, 6: 257.931, 7: 266.241, 8: 311.818, 9: 405.844 } # Create option parser parser = OptionParser( usage="usage: run-simulator.py [-hvcqmpfdews] filename", version="1.0") # Add options (self-documenting) parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="log cycle by cycle debug information") parser.add_option("-c", "--core", action="store_true", dest="core", default=False, help="show only core cycles") parser.add_option("-q", "--quiet", action="store_true", dest="quiet", default=False, help="less terminal output") parser.add_option("-m", "--mute", action="store_true", dest="mute", default=False, help="no summary output") parser.add_option( "-p", # "--pipeline", type="int", dest="pipeline", default=5, help="set number of pipeline stages [default 5]") parser.add_option( "-f", # "--IFStages", type="int", dest="ifcycles", default=-1, help="set number of Fetch (IF) Stage cycles") parser.add_option( "-d", # "--IDStages", type="int", dest="idcycles", default=-1, help="set number of Decode (ID) Stage cycles") parser.add_option( "-e", # "--EXStages", type="int", dest="excycles", default=-1, help="set number of Execution (EX) Stage cycles") parser.add_option( "-w", # "--WBStages", type="int", dest="wbcycles", default=-1, help="set number of Writeback (WB) Stage cycles") parser.add_option( "-s", # "--WBStages", type="int", dest="startAddr", default=0x0, help="set execution start address") (options, args) = parser.parse_args() # For automated coloured testing output B = bcolors() # Integer conversion options.pipeline = int(options.pipeline) options.ifcycles = int(options.ifcycles) options.idcycles = int(options.idcycles) options.excycles = int(options.excycles) options.wbcycles = int(options.wbcycles) pipelineConfigError = False # Pipeline checking if (options.pipeline < 4): pipelineConfigError = True # Use 1 ID and WB Cycle by default if (options.idcycles == -1): options.idcycles = 1 if (options.wbcycles == -1): options.wbcycles = 1 # Use maximum number of EX/MEM Cycles by default (ugly code...) if (options.excycles == -1 and options.pipeline >= 5): if (options.pipeline == 5): options.excycles = 2 elif (options.pipeline == 6): options.excycles = 3 else: options.excycles = 4 # The rest of the instructions will be WB if (options.ifcycles == -1): remCycles = options.pipeline - options.idcycles - options.excycles - options.wbcycles if (remCycles < 1): pipelineConfigError = True options.ifcycles = remCycles # Double check for correct pipeline configuration pipelineSum = options.ifcycles + options.idcycles + options.excycles + options.wbcycles if (pipelineSum != options.pipeline): pipelineConfigError = True # Give error if something is wrong if (pipelineConfigError): B.printError("Error: Incorrect pipeline configuration") sys.exit() inputFile = None # Open the input file try: inputFile = open(args[0], "r") except IOError: B.printError("Error: Could not open input file\t" + args[0]) sys.exit() # Default values defaultSimASMFile = "simasm.sim" defaultDataMemFile = "datamem.sim" defaultPreProcLogFile = "preprocLog.sim" defaultSimRunFile = "simrun.sim" oldstdout = sys.stdout # Initialize parsers iparser = InstructionParser.InstructionParser() eparser = elf32parser.elf32parser() # If custom simulation assembly file output is set SimAsmFileName = args[2] if len(args) >= 3 else defaultSimASMFile SimAsmFile = open(SimAsmFileName, 'w') sys.stdout = SimAsmFile # If custom data memory file output is set DataMemFileName = args[4] if len(args) >= 5 else defaultDataMemFile if (not options.quiet): oldstdout.write("> Starting Parser...\n") # Convert elf32-bigmips to simulator friendly format eparser.convertToSimASM(args[0], SimAsmFileName, DataMemFileName) # Extract statistics lines = eparser.getLines() datamem = eparser.getDataMem() mainAddr = eparser.getMainAddr() coreInstr = eparser.getCCoreInstr() # IF custom preprocessing log file name is set PPLogFileName = args[3] if len(args) >= 4 else defaultPreProcLogFile PPLogFile = open(PPLogFileName, 'w') sys.stdout = PPLogFile if (not options.quiet): oldstdout.write("> Starting Assembler...\n") # Parse in lines, do preprocessing and check for dependencies lines = iparser.parseLines(lines, (options.pipeline - options.ifcycles), options.ifcycles, coreInstr) pipelineInfo = [ options.pipeline, options.ifcycles, options.idcycles, options.excycles, options.wbcycles ] # Initialize simulator pipelinesim = PipelineSimulator.PipelineSimulator( lines, datamem, options.startAddr, oldstdout, options.verbose, options.quiet, pipelineInfo) if (not options.quiet): print "> Starting Simulation..." startTime = time.clock() # Set logfile simulationFileName = args[1] if len(args) >= 2 else defaultSimRunFile simulationFile = open(simulationFileName, 'w') sys.stdout = simulationFile # Run simulation pipelinesim.run() elapsedTime = (time.clock() - startTime) if (not options.quiet): oldstdout.write("\n> Simulation Completed in ") oldstdout.write(str(elapsedTime)) oldstdout.write(" s") # Close output files simulationFile.close() PPLogFile.close() sys.stdout = oldstdout checker = Checker.Checker(simulationFileName, options.quiet) success = False # Give output according to the settings # Normal terminal-based single run output if (not options.quiet): checker.runCheck() # Only summary and statistics output (for use in automated scripts) elif (not options.mute): success = checker.runCheck() if (success): if (options.core): B.printCoreOnly(checker.getCoreCycles()) else: # Compile statistics pNOPs = str( round( float( str( float(checker.getCoreNops()) / (float(checker.getCoreCycles())))) * 100, 1)) c = checker.getCycles() n = checker.getNOPs() cpi = checker.getCPI() cc = checker.getCoreCycles() cn = checker.getCoreNops() ex = str( round( float( int(c) * float(1 / frequencyTable.get(options.pipeline))), 8)) + "us" cex = str( round( float( int(cc) * float(1 / frequencyTable.get(options.pipeline))), 8)) + "us" # Print successful test B.printPass(args[0], [c, n, cpi, cc, cn, pNOPs, ex, cex]) else: # Indicate failure B.printFail(args[0], "")
''' This is the test case file for python_tester. The purpose of this file is to help create Python output value and dtype test cases for user programs. ''' from bcolors import bcolors bc = bcolors() # global colors object ''' This is the Test Case class. This holdes the test case's id, the function call, the input value it is bonded to, the expected value that the function outputs based on the input value, and the types of both data values. ''' class TestCase(object): def __init__(self, test_id, func, expected_value): self.id = test_id self.func = func self.input_value = None self.input_value_type = None self.output_value = None self.expected_value = expected_value self.expected_value_type = type(self.expected_value) def bond_value(self, bc, value): try: self.input_value = value self.input_value_type = type(self.input_value) bc.cprint( bc.OKBLUE, "Test Case {} successfully bonded to value at mem address {}".