import glob import time import sys import tarfile import shutil import cStringIO import platform from SCons.Script.SConscript import SConsEnvironment import SCons.Action import SCons.Util import SCons.Builder import SCons.Subst import SCons.Tool import bcolors bc = bcolors.bcolors() import btools VERSION = btools.VERSION VERSION_RELEASE_CYCLE = btools.VERSION_RELEASE_CYCLE Split = SCons.Util.Split Action = SCons.Action.Action Builder = SCons.Builder.Builder GetBuildPath = SConsEnvironment.GetBuildPath # a few globals root_build_dir = '' doc_build_dir = '' quickie = None # Anything else than None if BF_QUICK has been passed quicklist = [] # The list of libraries/programs to compile during a quickie program_list = [
import glob import time import sys import tarfile import shutil import cStringIO import platform from SCons.Script.SConscript import SConsEnvironment import SCons.Action import SCons.Util import SCons.Builder import SCons.Subst import SCons.Tool import bcolors bc = bcolors.bcolors() import btools VERSION = btools.VERSION VERSION_RELEASE_CYCLE = btools.VERSION_RELEASE_CYCLE Split = SCons.Util.Split Action = SCons.Action.Action Builder = SCons.Builder.Builder GetBuildPath = SConsEnvironment.GetBuildPath # a few globals root_build_dir = '' doc_build_dir = '' quickie = None # Anything else than None if BF_QUICK has been passed quicklist = [] # The list of libraries/programs to compile during a quickie program_list = [] # A list holding Nodes to final binaries, used to create installs
def main() :
parser = OptionParser(usage="usage: run-simulator.py [-v] [-t <clock cycles>] filename", version="1.0")
parser.add_option("-v", "--verbose",
action="store_true",
dest="verbose",
default=False,
help="Print cycle by cycle debug information to simulation log file")
parser.add_option("-q", "--quiet",
action="store_true",
dest="quiet",
default=False,
help="Supress terminal output. Be vewwy quiet (I'm hunting wabbits...)")
parser.add_option("-t", "--timeout",
action="store",
type="int",
dest="timeout",
help="Terminal looping program. Specify number of clock cycles TIMEOUT")
(options, args) = parser.parse_args()
inputFile = None
# For automated testing output
B = bcolors("mathstuff")
# Open the input file
try:
inputFile = open(args[0], "r");
except IOError:
print "There was an error opening the input file: ", args[0]
sys.exit()
defaultSimASMFile = "simasm.sim"
defaultDataMemFile = "datamem.sim"
defaultPreProcLogFile = "preprocLog.sim"
defaultSimRunFile = "simrun.sim"
oldstdout = sys.stdout
# Initialize parsers
iparser = InstructionParser.InstructionParser()
eparser = elf32parser.elf32parser()
# Convert elf32-bigmips to simulator friendly format
SimAsmFileName = args[2] if len(args) >= 3 else defaultSimASMFile
SimAsmFile = open(SimAsmFileName, 'w')
sys.stdout = SimAsmFile
DataMemFileName = args[4] if len(args) >= 5 else defaultDataMemFile
if(not options.quiet):
oldstdout.write("> Starting Parser...\n")
eparser.convertToSimASM(args[0], SimAsmFileName, DataMemFileName)
lines = eparser.getLines()
datamem = eparser.getDataMem()
mainAddr = eparser.getMainAddr() # TODO: change this mainAdder = 0x48 from first line of "0: jal 0x48"
# Parse in lines and check for dependencies
PPLogFileName = args[3] if len(args) >= 4 else defaultPreProcLogFile
PPLogFile = open(PPLogFileName, 'w')
sys.stdout = PPLogFile
if(not options.quiet):
oldstdout.write("> Starting Assembler...\n")
# Get line by line
lines = iparser.parseLines(lines)
pipelinesim = PipelineSimulator.PipelineSimulator(lines, datamem, mainAddr, oldstdout, options.verbose, options.quiet, options.timeout)
if(not options.quiet):
print "> Starting Simulation..."
startTime = time.clock()
# Set logfile
simulationFileName = args[1] if len(args) >= 2 else defaultSimRunFile
simulationFile = open(simulationFileName, 'w')
sys.stdout = simulationFile
# Run simulation
pipelinesim.run()
# Print out line by line
#print lines
elapsedTime = (time.clock() - startTime)
if(not options.quiet):
oldstdout.write("\n> Simulation Completed in ")
oldstdout.write(str(elapsedTime))
oldstdout.write(" s")
simulationFile.close()
PPLogFile.close()
sys.stdout = oldstdout
checker = Checker.Checker(simulationFileName, options.quiet)
success = False
if(not options.quiet):
checker.runCheck()
else:
success = checker.runCheck()
if(success):
B.printPass(args[0], checker.getCycles())
else:
B.printFail(args[0], "")
def content_parsing(self, context):
bc = bcolors.bcolors()
r = BeautifulSoup(context, 'lxml')
sys.stdout.write("\033[K")
sys.stdout.write('[*] Parsing Contents...\r')
sys.stdout.flush()
try:
self.title = r.title.string
title = r.find("meta", attrs={"property": "og:title"})
url = r.find("meta", attrs={"property": "og:url"})
description = r.find("meta", attrs={"property": "og:description"})
image = r.find("meta", attrs={"property": "og:image"})
self.property_og_title = title["content"] if title else ""
self.property_og_url = url[
"content"] if url else self.property_og_url
self.property_og_description = description[
"content"] if description else self.property_og_description
self.property_og_image = image[
"content"] if image else self.property_og_image
name = r.find("meta", attrs={"itemprop": "name"})
description = r.find("meta", attrs={"itemprop": "description"})
image = r.find("meta", attrs={"itemprop": "og:image"})
self.itemprop_name = name["content"] if name else self.itemprop_name
self.property_og_description = description[
"content"] if description else self.property_og_description
self.property_og_image = image[
"content"] if image else self.property_og_image
author = r.find("meta", attrs={"name": "author"})
description = r.find("meta", attrs={"name": "description"})
image = r.find("meta", attrs={"name": "image"})
self.name_author = author["content"] if author else self.name_author
self.property_og_description = description[
"content"] if description else self.property_og_description
self.property_og_image = image[
"content"] if image else self.property_og_image
except:
print('%s[!] Content Parsing Error - Unknown Elements%s\r' %
(bc.OKBLUE, bc.ENDC))
self.title = ''
for script in r(
["script", "style", "nav", "form", "footer", "noscript",
"header"]):
script.decompose() # rip it out
# for code in r()
self.body = r.get_text()
# break into lines and remove leading and trailing space on each
self.body = (line.strip() for line in self.body.splitlines())
# break multi-headlines into a line each
self.body = (phrase.strip() for line in self.body
for phrase in line.split(" "))
# drop blank lines
self.body = ' '.join(chunk for chunk in self.body if chunk)
return {
'title': self.title,
'property_og_title': self.property_og_title,
'property_og_url': self.property_og_url,
'property_og_description': self.property_og_description,
'property_og_image': self.property_og_image,
'itemprop_name': self.itemprop_name,
'name_author': self.name_author,
'body': self.body
}
def main() :
# Hard-coded table holding clock frequency information for HW
frequencyTable = {
5: 193.723,
6: 257.931,
7: 266.241,
8: 311.818,
9: 405.844
}
# Create option parser
parser = OptionParser(usage="usage: run-simulator.py [-hvcqmpfdews] filename", version="1.0")
# Add options (self-documenting)
parser.add_option("-v", "--verbose",
action="store_true",
dest="verbose",
default=False,
help="log cycle by cycle debug information")
parser.add_option("-c", "--core",
action="store_true",
dest="core",
default=False,
help="show only core cycles")
parser.add_option("-q", "--quiet",
action="store_true",
dest="quiet",
default=False,
help="less terminal output")
parser.add_option("-m", "--mute",
action="store_true",
dest="mute",
default=False,
help="no summary output")
parser.add_option("-p",# "--pipeline",
type="int",
dest="pipeline",
default=5,
help="set number of pipeline stages [default 5]")
parser.add_option("-f",# "--IFStages",
type="int",
dest="ifcycles",
default=-1,
help="set number of Fetch (IF) Stage cycles")
parser.add_option("-d",# "--IDStages",
type="int",
dest="idcycles",
default=-1,
help="set number of Decode (ID) Stage cycles")
parser.add_option("-e",# "--EXStages",
type="int",
dest="excycles",
default=-1,
help="set number of Execution (EX) Stage cycles")
parser.add_option("-w",# "--WBStages",
type="int",
dest="wbcycles",
default=-1,
help="set number of Writeback (WB) Stage cycles")
parser.add_option("-s",# "--WBStages",
type="int",
dest="startAddr",
default=0x0,
help="set execution start address")
(options, args) = parser.parse_args()
# For automated coloured testing output
B = bcolors()
# Integer conversion
options.pipeline = int(options.pipeline)
options.ifcycles = int(options.ifcycles)
options.idcycles = int(options.idcycles)
options.excycles = int(options.excycles)
options.wbcycles = int(options.wbcycles)
pipelineConfigError = False
# Pipeline checking
if(options.pipeline < 4):
pipelineConfigError = True
# Use 1 ID and WB Cycle by default
if(options.idcycles == -1):
options.idcycles = 1
if(options.wbcycles == -1):
options.wbcycles = 1
# Use maximum number of EX/MEM Cycles by default (ugly code...)
if(options.excycles == -1 and options.pipeline >= 5):
if(options.pipeline == 5):
options.excycles = 2
elif(options.pipeline == 6):
options.excycles = 3
else:
options.excycles = 4
# The rest of the instructions will be WB
if(options.ifcycles == -1):
remCycles = options.pipeline-options.idcycles-options.excycles-options.wbcycles
if(remCycles < 1):
pipelineConfigError = True
options.ifcycles = remCycles
# Double check for correct pipeline configuration
pipelineSum = options.ifcycles + options.idcycles + options.excycles + options.wbcycles
if(pipelineSum != options.pipeline):
pipelineConfigError = True
# Give error if something is wrong
if(pipelineConfigError):
B.printError("Error: Incorrect pipeline configuration")
sys.exit()
inputFile = None
# Open the input file
try:
inputFile = open(args[0], "r");
except IOError:
B.printError("Error: Could not open input file\t" + args[0])
sys.exit()
# Default values
defaultSimASMFile = "simasm.sim"
defaultDataMemFile = "datamem.sim"
defaultPreProcLogFile = "preprocLog.sim"
defaultSimRunFile = "simrun.sim"
oldstdout = sys.stdout
# Initialize parsers
iparser = InstructionParser.InstructionParser()
eparser = elf32parser.elf32parser()
# If custom simulation assembly file output is set
SimAsmFileName = args[2] if len(args) >= 3 else defaultSimASMFile
SimAsmFile = open(SimAsmFileName, 'w')
sys.stdout = SimAsmFile
# If custom data memory file output is set
DataMemFileName = args[4] if len(args) >= 5 else defaultDataMemFile
if(not options.quiet):
oldstdout.write("> Starting Parser...\n")
# Convert elf32-bigmips to simulator friendly format
eparser.convertToSimASM(args[0], SimAsmFileName, DataMemFileName)
# Extract statistics
lines = eparser.getLines()
datamem = eparser.getDataMem()
mainAddr = eparser.getMainAddr()
coreInstr = eparser.getCCoreInstr()
# IF custom preprocessing log file name is set
PPLogFileName = args[3] if len(args) >= 4 else defaultPreProcLogFile
PPLogFile = open(PPLogFileName, 'w')
sys.stdout = PPLogFile
if(not options.quiet):
oldstdout.write("> Starting Assembler...\n")
# Parse in lines, do preprocessing and check for dependencies
lines = iparser.parseLines(lines, (options.pipeline-options.ifcycles), options.ifcycles, coreInstr)
pipelineInfo = [options.pipeline, options.ifcycles, options.idcycles, options.excycles, options.wbcycles]
# Initialize simulator
pipelinesim = PipelineSimulator.PipelineSimulator(lines, datamem, options.startAddr, oldstdout, options.verbose, options.quiet, pipelineInfo)
if(not options.quiet):
print "> Starting Simulation..."
startTime = time.clock()
# Set logfile
simulationFileName = args[1] if len(args) >= 2 else defaultSimRunFile
simulationFile = open(simulationFileName, 'w')
sys.stdout = simulationFile
# Run simulation
pipelinesim.run()
elapsedTime = (time.clock() - startTime)
if(not options.quiet):
oldstdout.write("\n> Simulation Completed in ")
oldstdout.write(str(elapsedTime))
oldstdout.write(" s")
# Close output files
simulationFile.close()
PPLogFile.close()
sys.stdout = oldstdout
checker = Checker.Checker(simulationFileName, options.quiet)
success = False
# Give output according to the settings
# Normal terminal-based single run output
if(not options.quiet):
checker.runCheck()
# Only summary and statistics output (for use in automated scripts)
elif(not options.mute):
success = checker.runCheck()
if(success):
if(options.core):
B.printCoreOnly(checker.getCoreCycles())
else:
# Compile statistics
pNOPs = str(round(float(str(float(checker.getCoreNops())/(float(checker.getCoreCycles()))))*100, 1))
c = checker.getCycles()
n = checker.getNOPs()
cpi = checker.getCPI()
cc = checker.getCoreCycles()
cn = checker.getCoreNops()
ex = str(round(float(int(c) * float(1/frequencyTable.get(options.pipeline))), 8)) + "us"
cex = str(round(float(int(cc) * float(1/frequencyTable.get(options.pipeline))), 8)) + "us"
# Print successful test
B.printPass(args[0], [c, n, cpi, cc, cn, pNOPs, ex, cex])
else:
# Indicate failure
B.printFail(args[0], "")
# Rivki Kanterovich
# ID: 212030761
# Geula Shoshan
# ID: 11826658
import math
import csv
import bcolors
from collections import Counter
from pathlib import Path
from prettytable import PrettyTable
bcolors = bcolors.bcolors()
#Class like enum of the heuristics distance avilable
class Heuristic:
EUCLIDEAN = 0
MANHATTAN = 1
HAMMING = 2
#class with ctor that has distance and tag properties
class distClass:
def __init__(self, dist=-1, tag='-'):
self.dist = dist # distance of current point from test point
self.tag = tag # tag of current point
#calculate the distance between 2 vectors by selected heuristic
def calculate_distance(instance1, instance2, heuristic: Heuristic):
if (heuristic == Heuristic.EUCLIDEAN):
# pip install validators beautifulsoup4 lxml
# Python version: Python 3.6.3 :: Anaconda, Inc.
from bs4 import BeautifulSoup, SoupStrainer
import urllib.request
import traceback
import json
import re
import urllib.parse
import validators
from bcolors import bcolors
_print = bcolors()
class Spider:
def __init__(self, root_url, max_links):
self.root_url = root_url
self.crawl_result = {} # a dictionary to store the crawl output as key, val pairs.
self.crawl_set = set()
self.bucket = []
self.MAX_CRAWL = max_links
self.link_count = 0
self.default_scheme = 'http://'
self.scheme = self.default_scheme
def fetch_url(self, url):
"""
def main():
# Hard-coded table holding clock frequency information for HW
frequencyTable = {
5: 193.723,
6: 257.931,
7: 266.241,
8: 311.818,
9: 405.844
}
# Create option parser
parser = OptionParser(
usage="usage: run-simulator.py [-hvcqmpfdews] filename", version="1.0")
# Add options (self-documenting)
parser.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="log cycle by cycle debug information")
parser.add_option("-c",
"--core",
action="store_true",
dest="core",
default=False,
help="show only core cycles")
parser.add_option("-q",
"--quiet",
action="store_true",
dest="quiet",
default=False,
help="less terminal output")
parser.add_option("-m",
"--mute",
action="store_true",
dest="mute",
default=False,
help="no summary output")
parser.add_option(
"-p", # "--pipeline",
type="int",
dest="pipeline",
default=5,
help="set number of pipeline stages [default 5]")
parser.add_option(
"-f", # "--IFStages",
type="int",
dest="ifcycles",
default=-1,
help="set number of Fetch (IF) Stage cycles")
parser.add_option(
"-d", # "--IDStages",
type="int",
dest="idcycles",
default=-1,
help="set number of Decode (ID) Stage cycles")
parser.add_option(
"-e", # "--EXStages",
type="int",
dest="excycles",
default=-1,
help="set number of Execution (EX) Stage cycles")
parser.add_option(
"-w", # "--WBStages",
type="int",
dest="wbcycles",
default=-1,
help="set number of Writeback (WB) Stage cycles")
parser.add_option(
"-s", # "--WBStages",
type="int",
dest="startAddr",
default=0x0,
help="set execution start address")
(options, args) = parser.parse_args()
# For automated coloured testing output
B = bcolors()
# Integer conversion
options.pipeline = int(options.pipeline)
options.ifcycles = int(options.ifcycles)
options.idcycles = int(options.idcycles)
options.excycles = int(options.excycles)
options.wbcycles = int(options.wbcycles)
pipelineConfigError = False
# Pipeline checking
if (options.pipeline < 4):
pipelineConfigError = True
# Use 1 ID and WB Cycle by default
if (options.idcycles == -1):
options.idcycles = 1
if (options.wbcycles == -1):
options.wbcycles = 1
# Use maximum number of EX/MEM Cycles by default (ugly code...)
if (options.excycles == -1 and options.pipeline >= 5):
if (options.pipeline == 5):
options.excycles = 2
elif (options.pipeline == 6):
options.excycles = 3
else:
options.excycles = 4
# The rest of the instructions will be WB
if (options.ifcycles == -1):
remCycles = options.pipeline - options.idcycles - options.excycles - options.wbcycles
if (remCycles < 1):
pipelineConfigError = True
options.ifcycles = remCycles
# Double check for correct pipeline configuration
pipelineSum = options.ifcycles + options.idcycles + options.excycles + options.wbcycles
if (pipelineSum != options.pipeline):
pipelineConfigError = True
# Give error if something is wrong
if (pipelineConfigError):
B.printError("Error: Incorrect pipeline configuration")
sys.exit()
inputFile = None
# Open the input file
try:
inputFile = open(args[0], "r")
except IOError:
B.printError("Error: Could not open input file\t" + args[0])
sys.exit()
# Default values
defaultSimASMFile = "simasm.sim"
defaultDataMemFile = "datamem.sim"
defaultPreProcLogFile = "preprocLog.sim"
defaultSimRunFile = "simrun.sim"
oldstdout = sys.stdout
# Initialize parsers
iparser = InstructionParser.InstructionParser()
eparser = elf32parser.elf32parser()
# If custom simulation assembly file output is set
SimAsmFileName = args[2] if len(args) >= 3 else defaultSimASMFile
SimAsmFile = open(SimAsmFileName, 'w')
sys.stdout = SimAsmFile
# If custom data memory file output is set
DataMemFileName = args[4] if len(args) >= 5 else defaultDataMemFile
if (not options.quiet):
oldstdout.write("> Starting Parser...\n")
# Convert elf32-bigmips to simulator friendly format
eparser.convertToSimASM(args[0], SimAsmFileName, DataMemFileName)
# Extract statistics
lines = eparser.getLines()
datamem = eparser.getDataMem()
mainAddr = eparser.getMainAddr()
coreInstr = eparser.getCCoreInstr()
# IF custom preprocessing log file name is set
PPLogFileName = args[3] if len(args) >= 4 else defaultPreProcLogFile
PPLogFile = open(PPLogFileName, 'w')
sys.stdout = PPLogFile
if (not options.quiet):
oldstdout.write("> Starting Assembler...\n")
# Parse in lines, do preprocessing and check for dependencies
lines = iparser.parseLines(lines, (options.pipeline - options.ifcycles),
options.ifcycles, coreInstr)
pipelineInfo = [
options.pipeline, options.ifcycles, options.idcycles, options.excycles,
options.wbcycles
]
# Initialize simulator
pipelinesim = PipelineSimulator.PipelineSimulator(
lines, datamem, options.startAddr, oldstdout, options.verbose,
options.quiet, pipelineInfo)
if (not options.quiet):
print "> Starting Simulation..."
startTime = time.clock()
# Set logfile
simulationFileName = args[1] if len(args) >= 2 else defaultSimRunFile
simulationFile = open(simulationFileName, 'w')
sys.stdout = simulationFile
# Run simulation
pipelinesim.run()
elapsedTime = (time.clock() - startTime)
if (not options.quiet):
oldstdout.write("\n> Simulation Completed in ")
oldstdout.write(str(elapsedTime))
oldstdout.write(" s")
# Close output files
simulationFile.close()
PPLogFile.close()
sys.stdout = oldstdout
checker = Checker.Checker(simulationFileName, options.quiet)
success = False
# Give output according to the settings
# Normal terminal-based single run output
if (not options.quiet):
checker.runCheck()
# Only summary and statistics output (for use in automated scripts)
elif (not options.mute):
success = checker.runCheck()
if (success):
if (options.core):
B.printCoreOnly(checker.getCoreCycles())
else:
# Compile statistics
pNOPs = str(
round(
float(
str(
float(checker.getCoreNops()) /
(float(checker.getCoreCycles())))) * 100, 1))
c = checker.getCycles()
n = checker.getNOPs()
cpi = checker.getCPI()
cc = checker.getCoreCycles()
cn = checker.getCoreNops()
ex = str(
round(
float(
int(c) *
float(1 / frequencyTable.get(options.pipeline))),
8)) + "us"
cex = str(
round(
float(
int(cc) *
float(1 / frequencyTable.get(options.pipeline))),
8)) + "us"
# Print successful test
B.printPass(args[0], [c, n, cpi, cc, cn, pNOPs, ex, cex])
else:
# Indicate failure
B.printFail(args[0], "")
'''
This is the test case file for python_tester. The purpose of this file is to help
create Python output value and dtype test cases for user programs.
'''
from bcolors import bcolors
bc = bcolors() # global colors object
'''
This is the Test Case class. This holdes the test case's id, the function call, the
input value it is bonded to, the expected value that the function outputs based on
the input value, and the types of both data values.
'''
class TestCase(object):
def __init__(self, test_id, func, expected_value):
self.id = test_id
self.func = func
self.input_value = None
self.input_value_type = None
self.output_value = None
self.expected_value = expected_value
self.expected_value_type = type(self.expected_value)
def bond_value(self, bc, value):
try:
self.input_value = value
self.input_value_type = type(self.input_value)
bc.cprint(
bc.OKBLUE,
"Test Case {} successfully bonded to value at mem address {}".
