def loadMf(self, filePath):
"""
Extracts a Toontown Beta multifile.
"""
extractor = Extractor.Extractor()
extractor.extract(Filename(filePath), Filename(filePath[:-3]))
def download_single(self, url, extra):
download_url, source_url = self.find_download_link(url)
hidden_url = self.find_hidden_url(url)
if self.resolution == '480':
download_url = download_url[0][1]
else:
try:
download_url = source_url[1][1]
except Exception:
download_url = download_url[0][1]
show_info = self.info_extractor(extra)
output = self.check_output(show_info[0])
Extractor(logger=self.logger,
download_url=download_url,
backup_url=source_url,
hidden_url=hidden_url,
output=output,
header=self.header,
user_agent=self.user_agent,
show_info=show_info,
settings=self.settings,
quiet=self.quiet)
def saisirUrl(self):
url = input("veuillez entrez une url ")
page = PageCheck(url)
if (page.urlChek() != " "):
extract = Extractor(url)
extract.extraction()
print("le nombre de tableau est {} ".format(
extract.countTable(url)))
else:
print("l'url n\' est pas valide")
class PostHandler(BaseHTTPRequestHandler): ex = Extractor.Extractor() def processed(self, data): #PARSE DATA #UPDATE MODEL #RETURN NEW MODEL DATA JSON #Sidd will implement self.model.add_vote(user_id, question_id, vote) out = self.model.get_graph_data() # Parse output into JSON of right form return out
def lister(self):
f = open("urls.txt", "r")
fichier_entier = f.read()
files = fichier_entier.split("\n")
for file in files:
page = PageCheck(file)
url = page.urlChek()
if (url != " "):
extract = Extractor(url)
extract.extraction()
print("le nombre de tableau est {} ".format(
extract.countTable(url)))
else:
print("l'url n\' est pas valide")
def extractFile(self, filePath):
"""Extract the downloaded file.
Args:
filePath (str): The file path to be extract.
Return:
str: The folder where the file was extracted.
"""
self.__extractor = Extractor.Extractor(self.__fileExt)
uniqueName = str(uuid.uuid4())
extractedPath = os.path.join(tempfile.gettempdir(), uniqueName)
self.__extractor.getFunction()(self.__fileDownloaded, extractedPath)
return extractedPath
def main():
#---------Extraccion de Informacion---------
extractor = Extractor()
OS_Data = extractor.get_os_information()
Server_Data = extractor.get_server_data()
Processor_Data = extractor.get_processor_information()
Processes_data = extractor.get_processes_information()
Users_Data = extractor.get_users_information()
#-------------------------------------------
#------Envio de Informacion a la API--------
dictonary_set = {
'OS': OS_Data,
'Proccesor': Processor_Data,
'Server': Server_Data,
'Users': Users_Data,
'Processes': Processes_data
}
post_api(config.URL_API, dictonary_set)
def genetic_alg(next_gen,
num_generations=200,
max_size_gen=1000,
size_final_gen=100,
mutations_per_solution_max=50,
name=""):
print('start solution:' + str(next_gen[0].get_score()))
for times in xrange(num_generations):
next_gen = next_generation(next_gen, max_size_gen, size_final_gen,
mutations_per_solution_max)
print('iter :' + str(times) + ', name: ' + name)
print('best solution:' + str(next_gen[0].get_score()))
print('worst solution:' +
str(next_gen[size_final_gen - 1].get_score()))
print('start solution:' + str(next_gen[0].get_score()))
file_write = open(name + '.obj', 'w')
pickle.dump(next_gen[0], file_write)
ex = Extractor.Extractor(next_gen[0].cars, name)
ex.write()
return next_gen[0]
from Extractor import *
from Wrapper import *
from Sender import *
if __name__ == '__main__':
extractor = Extractor()
wrapper = Wrapper()
sender = Sender()
rawData = extractor.get_site()
dataModel = wrapper.packData()
print(rawData)
import Extractor as ex
import question_generator as gen
# To speed up script, start servers:
##bash runStanfordParserServer.sh
##bash runSSTServer.sh
#Dish sample
#direct_path = "/Users/brandon/Documents/Northwestern Courses/Winter 2019/CS+Law Innovation Lab/Orrick, Harrington, & Sutcliffe/Documents/Dish_Sample.txt"
#Apple Brief
direct_path = '/Users/brandon/Documents/Northwestern Courses/Winter 2019/CS+Law Innovation Lab/Orrick, Harrington, & Sutcliffe/Documents/Test_Text.txt'
with open(direct_path, 'r') as file:
brief = file.read()
test = ex.Extractor(brief)
qGen = gen.QuestionGenerator()
test.fix_pronouns(silence=1)
sentences = test.get_sentences()
print(sentences)
for sentence in sentences:
flashcard = qGen.generate_question(sentence)
if flashcard:
#print(type(flashcard), type(flashcard[0]))
print("Question: {}\n\nAnswer: {}'\n-------------".format(
flashcard[0]['Q'], flashcard[0]['A']))
from Extractor import *
from Threads import *
from Curator import *
from gcamp_extractor import *
arguments = {
'root': '/Users/stevenban/Documents/Data/20190917/binned',
'numz': 20,
'frames': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
'offset': 0,
't': 999,
'gaussian': (25, 4, 3, 1),
'quantile': 0.99,
'reg_peak_dist': 40,
'anisotropy': (6, 1, 1),
'blob_merge_dist_thresh': 7,
'mip_movie': True,
'marker_movie': True,
'infill': True,
'save_threads': True,
'save_timeseries': True,
'suppress_output': False,
'regen': False,
}
e = Extractor(**arguments)
e.calc_blob_threads()
e.quantify()
c = Curator(e)
'''
@author: Yang
@time: 17-11-14 下午2:57
'''
'''
extractor text lines from a text file
acceleration dataset:
training data: 2000 | test data: 100
underfitting dataset:
training data: 500 | test data: 1000
'''
import Extractor
extractor = Extractor.Extractor(filename='text.txt')
extractor.load_data(trainingNum=2000, testNum=100)
extractor.save_data(target='dataset/text/acceleration/')
extractor.load_data(trainingNum=500, testNum=1000)
extractor.save_data(target='dataset/text/underfitting/')
extractor.charset()
import os
'''
imageGen.py
generate image for OCR
'''
os.system('python imageGen.py')
def download_show(self, url):
page = requests.get(url)
soup = BeautifulSoup(page.content, 'html.parser')
ep_range = self.ep_range
links = []
for link in soup.findAll('a', {'class': 'sonra'}):
if link['href'] not in links:
links.append(link['href'])
if self.exclude is not None:
excluded = [
i for e in self.exclude for i in links if re.search(e, i)
]
links = [item for item in links if item not in excluded]
season = "season-" + self.season
if self.update == True:
links = links[0:1]
if len(ep_range) == 1:
ep_range = '{0}-{0}'.format(ep_range)
if ep_range == 'l5' or ep_range == 'L5': # L5 (Last five)
links = links[:5]
ep_range = 'All'
season = 'season-All'
if self.newest:
links = links[0:1]
ep_range = 'All'
season = 'season-All'
if season != "season-All" and ep_range != "All":
episodes = [
"episode-{0}".format(n) for n in range(int(ep_range[0]),
int(ep_range[1]) + 1)
]
if season == 'season-1':
matching = [
s for s in links if 'season' not in s or season in s
]
else:
matching = [s for s in links if season in s]
matching = [
s for s in matching for i in episodes
if i == re.search(r'episode-[0-9]+', s).group(0)
]
elif season != "season-All":
if season == 'season-1':
matching = [
s for s in links if 'season' not in s or season in s
]
else:
matching = [s for s in links if season in s]
elif ep_range != 'All':
episodes = [
"episode-{0}".format(n) for n in range(int(ep_range[0]),
int(ep_range[1]) + 1)
]
matching = [
s for s in links for i in episodes
if re.search("{0}-".format(i), s)
]
else:
matching = links
if len(matching) < 1:
matching.reverse()
if (self.threads != None and self.threads != 0):
if (len(matching) == 1):
for item in matching:
source_url, backup_url = self.find_download_link(item)
hidden_url = self.find_hidden_url(item)
if self.resolution == '480' or len(source_url[0]) > 2:
download_url = source_url[0][1]
else:
try:
download_url = source_url[1][1]
except Exception:
download_url = source_url[0][1]
show_info = self.info_extractor(item)
output = self.check_output(show_info[0])
Extractor(logger=self.logger,
download_url=download_url,
backup_url=backup_url,
hidden_url=hidden_url,
output=output,
header=self.header,
user_agent=self.user_agent,
show_info=show_info,
settings=self.settings,
quiet=self.quiet)
else:
count = 0
while (True):
processes_count = 0
processes = []
processes_url = []
processes_extra = []
if (int(self.threads) > len(matching)):
self.threads = 3
procs = ProcessParallel(print('', end='\n\n'))
for x in range(int(self.threads)):
try:
item = matching[count]
_, extra = self.is_valid(item)
processes.append(self.download_single)
processes_url.append(item)
processes_extra.append(extra)
count += 1
except Exception as e:
if self.logger == 'True':
print('Error: {0}'.format(e))
pass
for x in processes:
procs.append_process(
x,
url=processes_url[processes_count],
extra=processes_extra[processes_count])
processes_count += 1
if ('' in processes_extra):
self.threads = None
self.download_show(url)
break
procs.fork_processes()
procs.start_all()
procs.join_all()
processes_url.clear()
processes_extra.clear()
processes.clear()
self.threads = self.original_thread
if (count >= len(matching)): break
else:
for item in matching:
source_url, backup_url = self.find_download_link(item)
hidden_url = self.find_hidden_url(item)
if self.resolution == '480' or len(source_url[0]) > 2:
download_url = source_url[0][1]
else:
try:
download_url = source_url[1][1]
except Exception:
download_url = source_url[0][1]
show_info = self.info_extractor(item)
output = self.check_output(show_info[0])
Extractor(logger=self.logger,
download_url=download_url,
backup_url=backup_url,
hidden_url=hidden_url,
output=output,
header=self.header,
user_agent=self.user_agent,
show_info=show_info,
settings=self.settings,
quiet=self.quiet)
if (self.original_thread != None and self.original_thread != 0):
self.threads = self.original_thread
# pylint: disable=no-member
import time
import cv2
import numpy as np
import Extractor
W = 1920 // 2
H = 1080 // 2
F = 1
extrac = Extractor.Extractor(F, H, W)
class Process():
def process_frame(self, img):
self.img = cv2.resize(img, (W, H))
#matches es un array-2D con los puntos normalizados y filtrados
matches = extrac.extract(self.img)
print("%d matches" % (len(matches)))
for pt1, pt2 in matches:
#Se desnormalizan las coordenadas de pt1 y pt2, provenientes del filtrado, para poder mostrarse
u1, v1 = extrac.denormalize(pt1)
u2, v2 = extrac.denormalize(pt2)
#Dibuja un circulo verde por cada keypoint
solution = Solution.Solution(cars, rule_out_rides, bonus, steps)
if genetic:
solutions = solutions + [solution]
print('genetic algorithm:')
solution = Solution.genetic_alg([solution], num_generations=1000, max_size_gen=500, size_final_gen=50, mutations_per_solution_max=50, name=file)
score_new = solution.get_score()
suma_total = suma_total + score_new
print('solution score ' + str(score_new))
if score_new > score:
print('NUEVA MEJORA DE PUNTUACION EN EL FICHERO: ' + file)
best_scores[index_file]=score_new
ex = Extractor.Extractor(solution.cars, file)
ex.write()
veces_mejorado = veces_mejorado + 1
index_file = index_file + 1
print('score: ' + str(suma_total/1000000.0) + ' M')
if best_global < suma_total:
print('Has mejorado el algoritmo!')
print('llevas mejoradas veces: ' + str(veces_mejorado) + '/' + str(index_total))
print('mejora: ' + str(sum(best_scores)-sum(old_best_scores)))
suma_total=0
index_total=index_total+1
def process(url): e = Extractor.Extractor() return e.get_out_data(url)
def __init__(self): self.extractor = Extractor() self.rootDirectory = '' self.movieDb = None
filemode='a')
def buildSequence(frameList):
sequence = []
for image in frameList:
features = model.extract(image)
sequence.append(features)
return np.array(sequence)
proc_csv = pd.read_pickle('dataset_with_file_list.pkl')
logging.info("Starting routine for features extraction with InceptionV3")
model = Extractor()
proc_csv.set_index('palavra', inplace=True)
folder = '/home/fabiana/Desktop/projeto-final-src/Classifier/InceptionV3_Features'
for n in ['5', '10', '15']:
print('Number of keyframes: ' + n)
for palavra, frameList in tqdm(proc_csv[f'files_list_{n}'].iteritems(),
total=len(proc_csv)):
if (len(frameList) < int(n)):
# print(f"Word {palavra} got less than {n} key frames")
logging.warning(f"Word {palavra} got less than {n} key frames")
seq = buildSequence(frameList)
np.save(f'{folder}/{n}/{palavra}', seq)
