def run(self): self.onStart.emit("Loading credits ..") data = get_data(self.year, self.month, self.day) self.onScrapping.emit(data) self.onFinish.emit('')
def run(self): self.onStart.emit("Loading credits ..") credit = get_data(self.amount, self.maturity, self.product_type) self.onScrapping.emit(credit) self.onFinish.emit('')
def index(): order_param = request.args.get("order_by") db_check = db.get(order_param) button = """<div><b>Popular</b></div><div><a href="/?order_by=new">New</a></div>""" if order_param: # query args가 있다면 if order_param == "new": button = """<div><a href="/?order_by=popular">Popular</a></div> <div><b>New</b></div>""" if db_check: # db에 저장되어 있다면 => db 데이터 사용 result = db_check else: # db에 저장 x => 수집 및 db 저장 if order_param == "popular": result = get_data(popular) else: result = get_data(new) db[order_param] = result else: # query args가 없다면 => popular 기준 수집 및 db 저장 result = get_data(popular) db[order_param] = result return render_template("index.html", result=result, button=button)
def load_data(): url = 'https://www.kvmeter.ru/information/homes_series/' raw_data = get_data(url, get_links, get_content) data = {} for key in raw_data: values = raw_data[key] for value in values: if value['key'] == 'Регионы строительства': if value['values'][0].find("Волгоград") > -1: data[key] = values break return data
def consulta_nota(browser,diccionario): dic={} cenacad = r"https://cenacad.espol.edu.ec/index.php/module/Report/action/Profesores" for materia in list(diccionario.keys()): dic[materia]={} for profesor in get_teachers_list(diccionario[materia]): browser.get(cenacad) browser=busc_profesor(browser,profesor) page = browser.page_source link=sc.find_name(page,profesor) notas=sc.get_data(link,materia) dic[materia][profesor]=notas return dic
def data_getter(): data = [] for i in [ 'https://www.opensecrets.org/outsidespending/summ.php?cycle=2018&disp=D&type=V&superonly=N', 'https://www.opensecrets.org/outsidespending/summ.php?cycle=2016&disp=D&type=V&superonly=N', 'https://www.opensecrets.org/outsidespending/summ.php?cycle=2014&disp=D&type=V&superonly=N', 'https://www.opensecrets.org/outsidespending/summ.php?cycle=2012&disp=D&type=V&superonly=N', 'https://www.opensecrets.org/outsidespending/summ.php?cycle=2010&disp=D&type=V&superonly=N', 'https://www.opensecrets.org/outsidespending/summ.php?cycle=2008&disp=D&type=V&superonly=N', 'https://www.opensecrets.org/outsidespending/summ.php?cycle=2006&disp=D&type=V&superonly=N', 'https://www.opensecrets.org/outsidespending/summ.php?cycle=2004&disp=D&type=V&superonly=N' ]: data.append(scrapper.get_data(i)) return data
def read(): result = [] sub_arr = [] for subreddit in subreddits: if request.args.get(subreddit): sub_arr.append(subreddit) if db.get(subreddit) == None: url = f"https://www.reddit.com/r/{subreddit}/top/?t=month" db[subreddit] = get_data(url, subreddit) result.extend(db.get(subreddit)) result = sorted(result, key=(lambda x: x['bote_count']), reverse=True) return render_template("read.html", reddit_data={ 'data': result, 'sub_arr': sub_arr })
def train(model_type): data = get_data(model_type) encoder = Encoder(data) model = None test_cases = None iterations = 20 if model_type == "sequences": iterations = 100 test_cases = SEQUENCES_TEST_CASES model = SequenceModel(model_type, encoder) else: test_cases = IDENTIFIERS_TEST_CASES model = IdentifiersModel(model_type, encoder) try: model.train(test_cases=test_cases, iterations=iterations) except KeyboardInterrupt: pass print("saving") model.save()
def index_order_by(select): result = get_data(select) return render_template("index.html", result=result)
from selenium import webdriver from navigation import busc_profesor import scrapper as sc path=r"D:\OneDrive - Escuela Superior Politécnica del Litoral\ESPOL\Programacion\Python\Proyectos\Horarios\Scrapper\chromedriver.exe"#editar path del chromedriver selenium browser = webdriver.Chrome(path) cenacad = r"https://cenacad.espol.edu.ec/index.php/module/Report/action/Profesores" browser.get(cenacad) profesor="ESCALA BENITES FRANCESCA ELIZABETH" materia="QUIG1001" browser=busc_profesor(browser,profesor) page = browser.page_source link=sc.find_name(page,profesor) notas=sc.get_data(link,materia) print(notas)
def load_data(): url = 'https://www.reformagkh.ru/myhouse?tid=2227388&limit=60' data = get_data(url, get_links, get_content) return data