def main(search):
'''
executes serach for car using make-model-year and cars.com
params: search is the reddit message/comment body that should
contain the make model year information that we would like to
look up
'''
search = search.lower()
#clean up info to create query
output = ''
sanitizer = Sanitizer()
query = sanitizer.sanitize_input(search)
#verify that we received data back, and make a request
if query:
print('making request with', query)
req = requests.get('http://www.cars.com/research/' + query)
else:
return output
#parse html response using beautiful soup https://www.crummy.com/software/BeautifulSoup/bs4/doc/
soup = BeautifulSoup(req.text,
'html.parser') #get the entire html of the site
specs = soup.findAll(
'div', {'class': 'mmy-spec'}) #find all list items in the list
other_trims = soup.findAll('div',
{'class': 'trim_listing'}) #find other trims
#print info
if bool(specs) or bool(other_trims):
output = output + print_information(specs)
output = output + '\n\n---\n\n '\
'in order to not be annoying I am not printing all trims!'
# output = output + print_trims(other_trims, output)
return output
def add_sanitizer(self,
method: Method,
sink: Sink,
sink_method_idx: int,
sanitizer: dict,
level: int = 0):
"""Add a new sanitizer to a sink.
New sanitizers are only relevant if a method's parameters are used in a sanitizer.
Parameters
----------
method : Method
The method where the sanitizer was added
sink : Sink
The sink to add the sanitizer to
sink_method_idx : int
The index of the method for which to add the new sanitizer
sanitizer : dict
A dictionary definition of the sanitizer as defined in the ruleset
level : int, optional
The depth of the nesting before the sanitizer is reached (sanitizers defined in the
rules get a level of 0, sanitizers that call those get a level of 1, sanitizers that
call those get a level of 2 and so on)
"""
new_sanitizer = Sanitizer(sanitizer, level)
duplicate = False
for existing_sanitizer in sink.methods[sink_method_idx]['Sanitizers']:
if existing_sanitizer.object_name != new_sanitizer.object_name:
continue
if new_sanitizer.methods == existing_sanitizer.methods:
duplicate = True
break
if not duplicate:
sink.methods[sink_method_idx]['Sanitizers'].append(new_sanitizer)
self.notify_observers(method, changed_sanitizer=True)
def prep_sanits(self):
#show how many bottles left
self.sanits = Group()
for sanit_number in range(self.stats.sanits_left):
sanit = Sanitizer(self.s,self.screen)
sanit.rect.x = 10 + sanit_number*sanit.rect.width
sanit.rect.y =10
self.sanits.add(sanit)
def get_identifier_flow(self, identifier):
if identifier in self.variable_flows:
# get existing flow
flow = self.variable_flows[identifier]
else:
# new variable: check if source/sink/sanitizer
flows = []
flows.append(Source(identifier, self.is_source(identifier)))
flows.append(Sink(identifier, self.is_sink(identifier)))
flows.append(Sanitizer(identifier, self.is_sanitizer(identifier)))
flow = Flow(flows)
return flow
def rungame():
#initializin pygame,settings,and scr objs
pygame.init()
s = Settings()
screen = pygame.display.set_mode((s.screen_width,s.screen_height))
pygame.display.set_caption("CORONA INVASION!")
#make the play button
play_button = Button(s,screen,"Play")
#create instance to store game statistics and create a scoreboard
stats = GameStats(s)
sb = Scoreboard(s,screen,stats)
#time to make a ship
sanit = Sanitizer(s , screen)
#make a virus
#coro = Coro(s,screen) ####optional for now
#making a group to store bullets in
bubbles = Group()
coros = Group()
#create fleet of viruses
f.create_fleet(s,screen,sanit,coros)
#main loop for the game
while True:
f.check_events(s,screen,stats,sb,play_button,sanit,coros,bubbles)
bubbles.update()
if stats.game_active:
sanit.update()
f.update_bubbles(s,screen,stats,sb,sanit,coros,bubbles)
f.update_coros(s,screen,stats,sb,sanit,coros,bubbles)
f.update_screen(s,screen,stats,sb,sanit,coros,bubbles,play_button)
def __init__(self, definition: dict):
"""Constructor for class `Sink`. Load definitions.
Parameters
----------
definition : dict
Definitions for the object name, methods and sanitizers
"""
self.object_name = next(iter(definition))
self.methods = definition[self.object_name]['Methods']
for idx, method in enumerate(self.methods):
original_sanitizers = copy.deepcopy(self.methods[idx].get(
'Sanitizers', []))
method['Sanitizers'] = list()
for sanitizer in original_sanitizers:
method['Sanitizers'].append(Sanitizer(sanitizer))
# Make sure that the definition has a valid format
assert all(['Methodname' in method for method in self.methods])
assert all(['Parameters' in method for method in self.methods])
assert all(['Comment' in method for method in self.methods])
def test_sanitizer(self):
sanitizer = Sanitizer()
file_removed = sanitizer.clean(
'/Users/cv/.m2/repository/ant/ant/1.6.5/ant-1.6.5.jar')
print(file_removed)
import nltk
import csv
from sanitizer import Sanitizer
from multiprocessing import Pool
csv.field_size_limit(2**30)
sanitizer = Sanitizer()
filename = 'dataset.csv'
def generate_training_data(row):
document_id, title, content, date_posted, court = row
judgement = sanitizer.extract_judgement(content)
sentences = nltk.sent_tokenize(unicode(judgement, errors='ignore'))
return map(sanitizer.tokenize, sentences)
if __name__ == "__main__":
reader = csv.reader(open(filename, 'rb'))
reader.next()
pool = Pool()
results = pool.map(generate_training_data, reader)
sentences = []
for sent in results:
sentences.extend(sent)
with open('sentences.txt', 'w') as f:
for sentence in sentences:
f.write(' '.join(sentence) + '\n')
def makeAbove(arguments):
global template_path_g, output_path_g, script_dir_g
mySanitizer = Sanitizer(
) # default sanitizer for mAm. Clears JS and CSS, leaves html in.
# for every argument, check if set and handle accordingly
with open(template_path_g, 'r') as templateFile:
template = Template(templateFile.read())
# set title if there should be one
if arguments['--title'] is not None:
title = arguments['--title']
else:
title = ""
# clean title
title = mySanitizer.cleanHTML(title)
image = ""
if arguments['--image'] is not None:
image = arguments['--image']
image = mySanitizer.cleanHTML(image)
# create all tags and store them in one long string
global TAG_HTML_TEMPLATE_STRING
alltags = ""
for tag in arguments['--tag']:
alltags += TAG_HTML_TEMPLATE_STRING.substitute(
{'tagtext': mySanitizer.cleanHTML(tag)})
# for the line with points and comments
global COMMENTLINE_TEMPLATE_STRING
if arguments['-C'] is not None:
argsC = mySanitizer.cleanHTML(arguments['-C'])
commentline = '<a href="" class="C">{0}</a>'.format(argsC)
elif (arguments['--comments'] is None) and (arguments['--points'] is None):
commentline = ""
else:
comments = 0 if arguments['--comments'] is None else arguments[
'--comments']
points = 0 if arguments['--points'] is None else arguments['--points']
comments = mySanitizer.cleanHTML(comments)
points = mySanitizer.cleanHTML(points)
subC = "{0} comments".format(comments)
subP = "{0} points".format(points)
commentline = COMMENTLINE_TEMPLATE_STRING.substitute({
'points': subP,
'comments': subC
})
# set text if there should be
global TXT_TEMPLATE_STRING
text = ''
if arguments['--text'] is not None:
text = arguments['--text']
text = mySanitizer.cleanHTML(text)
text = TXT_TEMPLATE_STRING.substitute(
{'text': text}) # write text into html-string
substDir = {
'title': title,
'image': image,
'tags': alltags,
'commentline': commentline,
'text': text
}
tempStr = template.substitute(substDir)
# write result to temp file
(fd, filename) = tempfile.mkstemp(
suffix='.html', dir=script_dir_g
) # create the tempfile in the script-containing directory
try:
tfile = os.fdopen(fd, "w")
tfile.write(tempStr)
tfile.close()
if not arguments['-X']:
webkitres = subprocess.check_output([
"webkit2png", filename, "-o", output_path_g, "-x", "70", "1000"
])
else:
webkitres = subprocess.check_output(
["webkit2png", filename, "-o", output_path_g])
print("Called webkit2png with filename {0} and output path {1}".format(
filename, output_path_g))
except subprocess.CalledProcessError as e:
print("webkit2png failed. DO SOMETHING."
) # handle error of webkit2png? I don't know how, so not my job
exit(2)
finally:
os.remove(filename)
]
def __init__(self):
pass
def __call__(self, arg):
return News(title='abc',
body=m_news.bodies[arg],
clean_body=m_news.bodies[arg],
url="http://dfsd.fd.com",
date=int(time()))
from newsgroup import NewsGroup
from sanitizer import Sanitizer
s = Sanitizer()
ng = NewsGroup()
m_news = m_news()
nr = [ng.quantity_reduce(m_news(i)) for i in range(0, len(m_news.bodies))]
print s.cleanup_news(dirty_news)
for i in range(0, len(m_news.bodies)):
for j in range(i, len(m_news.bodies)):
print "(" + str(i) + "," + str(j) + ")" + str(
ng.cosine_distance(nr[i], nr[j]))
print("jaccard")
nr = [ng.binary_reduce(m_news(i)) for i in range(0, len(m_news.bodies))]
for i in range(0, len(m_news.bodies)):
for j in range(i, len(m_news.bodies)):
print "(" + str(i) + "," + str(j) + ")" + str(
def __init__(self, output_directory):
self.output_directory = output_directory
self.sanitizer = Sanitizer()