def something():
products = []
if request.method == "POST":
query = request.form['query']
products = scrapper.scrap(query)
return render_template('index.html', products=products)
def scrap_into_database():
print("Scrapping into database")
crawlers = db.session.query(Crawler).all()
for c in crawlers:
data = scrap(c.url)
for d in data:
create_data(c.name, d["name"], d["price"], d["date"], d["link"])
def admin_scrap_and_cache_data(update: Update, _: CallbackContext) -> None:
audio_ids = {}
update.message.reply_text(
"Initiating scrapping and caching\n" +
"This can take several hours\n" +
"(!)Turn off notifications to this bot, there will be a lot of spam")
for tf2class, l in scrapper.scrap():
audio_ids[tf2class] = []
for e in l:
retry = True
while retry:
try:
retry = False
x = update.message.reply_audio(
BytesIO(e['data']),
performer=tf2class,
title=e['text'],
)
audio_ids[tf2class].append(
{'file_id': x.audio.file_id, 'text': e['text']})
except Exception as error:
logger.warning('triggering telegram anti-spam filter')
retry = True
sleep(3)
update.message.reply_text('Data cached successfully')
query_handler.save_ids(audio_ids)
file = json.dumps(query_handler.audio_ids).encode()
try:
update.message.reply_document(file, filename='audio_ids.json')
update.message.reply_text("use this if you want to move bot")
except NetworkError:
update.message.reply_text(
f'Seems like this file is too large: {len(file)}. Maximum file size is 1.5MB')
def startCrawling(date1, date2):
agents = UserAgent()
print("Crawling started for date range ", date1.strftime('%d-%m-%Y'), ' to ', date2.strftime('%d-%m-%Y'))
if date1 > date2:
temp = date1
date1 = date2
date2 = temp
pagenumber = 1
request_number = 0
agent = agents.random
while date1 <= date2:
request_number += 1
currenturl = 'https://www.prothomalo.com/archive/'+date1.strftime('%Y-%m-%d')+'?page='+str(pagenumber)
print("Currently fetching address: ", currenturl)
nextpage = scrapper.scrap(currenturl, date1.strftime('%Y-%m-%d'), agent)
if nextpage:
pagenumber += 1
else:
pagenumber = 1
agent = agents.random
date1 = date1 + timedelta(days=1)
sleep(0.3)
if request_number == 50:
sleep(10)
request_number = 0
print("ended")
def scrap_into_database(urls):
print("Scrapping into database")
for category, url in urls.items():
print(category, url)
data = scrap(url)
data = filter_data(data, exception_words, exception_words, price_range)
for d in data:
create_data(d["name"], d["price"], d["date"], d["link"], category)
def run(self):
r = scrapper.scrap(self.accounts, self.N, self.config,
self.output_folder)
self.res.append(r)
# on lance un événement <<thread_fini>> à la fenêtre principale
# pour lui dire que le thread est fini, l'événement est ensuite
# géré par la boucle principale de messages
# on peut transmettre également le résultat lors de l'envoi du message
# en utilisant un attribut de la classe Event pour son propre compte
self.win.event_generate("<<thread_fini>>")
def main():
# parse args
args = docopt(__doc__.format(self_filename=Path(__file__).name))
# Parse config file
with open(args['<config_file>'] or DEFAULT_CONFIG_FILE) as f:
config_yaml = yaml.load(f.read(), Loader=yaml.BaseLoader)
config = config_model.Config(**config_yaml)
# Init logging
log = logging.getLogger(__name__)
logging.basicConfig(
format="%(levelname)s: %(message)s",
level=(logging.DEBUG if args['--debug'] else logging.INFO))
# Parse already_seen file
if not SEEN_FILEPATH.exists():
log.error(
f'init "{SEEN_FILEPATH}" file before use. ("echo \'[]\' > {SEEN_FILEPATH})"'
)
sys.exit(-1)
already_seen_set = set()
if SEEN_FILEPATH.stat().st_size != 0:
with open(SEEN_FILEPATH) as f:
already_seen_set = set(ensure_list(json.load(f)))
new_ids_set = links = None
try:
_, new_ids_set, links = scrapper.scrap(
config,
log,
already_seen_set=already_seen_set,
send_sms=not args["--no-sms"])
except:
if not args["--test"] and not args["--no-sms"]:
scrapper.send_sms("EXCEPTION", config)
raise
finally:
# Write new found ids to seen file
if new_ids_set:
print(f'-> update {SEEN_FILEPATH!r}')
write_json_file(SEEN_FILEPATH,
list(already_seen_set | new_ids_set))
if args['--clipboard'] and links:
# Copy urls links to clipboard
import clipboard
try:
clipboard.copy("\n".join(links))
except Exception as e:
log.error(f"Error while copying to clipboard:\n{e}")
traceback.print_tb(e.__traceback__)
else:
log.info("URLs copied to clipboard")
async def scrappin():
while True:
a, b = scrapper.scrap()
print(a)
print(b)
if (a == "69"):
print("Changing Gate")
stateIO.stateChange("gate")
if (b == "69"):
print("Changing Garage")
stateIO.stateChange("garage")
await asyncio.sleep(1)
def fetch_menu(bot=None, job=None, callback=None):
print("Fetching menu...")
global latest_menu
latest_menu = None
try:
latest_menu = scrap()
except:
print("Couldn't fetch menu: ", sys.exc_info()[0])
# updater.job_queue.run_once(fetch_menu, 5)
return
try:
pickle.dump(
latest_menu,
open("menus/" + time.strftime("%H%M-%m-%d-%Y") + ".pickle", "wb"),
)
except:
print("Couldn't pickle menu to disk!")
pass # this is not important enough to crash the bot
if callback is not None:
callback()
print("Database is initialized/updated")
raise SystemExit(0)
CHECK_INPUT_DATA = checkdata.check_input_data(INPUT_DATA, IATA_LIST)
if CHECK_INPUT_DATA != 0:
print(CHECK_INPUT_DATA)
raise SystemExit(1)
CHECK_DATES = checkdata.check_dates(INPUT_DATA)
if CHECK_DATES != 0:
print(CHECK_DATES)
raise SystemExit(1)
TRIP = INPUT_DATA[1:]
CHECK_SCHEDULE = check_schedule(TRIP, DB_FILE)
if len(CHECK_SCHEDULE[0]) == 0:
print("No flights for " + str(TRIP[2]) + " and 2 next days")
raise SystemExit(0)
URL = "https://www.airblue.com/bookings/flight_selection.aspx"
ROUND_TRIP = True
if len(TRIP) == 3:
ROUND_TRIP = False
RESERVATION = scrapper.scrap(TRIP, CHECK_SCHEDULE, ROUND_TRIP)
print("**********************")
print(RESERVATION[0])
for flight in RESERVATION[1]:
print(flight)
print("**********************")
import base
import scrapper
import msvcrt as m
base.clear()
print("First step")
print("Create connection to database")
print("Make sure u have mysql database, user with have privileges to it and he using plugin mysql_native_password")
print("")
base.raw_print("Any key to continue ")
m.getch()
database = base.connect()
if base.create_schema(database):
print("Operation end successfully")
print("")
print("Start webscraping")
xx = "https://www.zalando.pl/odziez-damska"
xy = "https://www.zalando.pl/odziez-meska/"
scrapper.scrap(xx, "kobiety", database)
scrapper.scrap(xy, "mezczyzni", database)
else:
print("Error occur during creating schema")
def main():
a, b = scrapper.scrap()
print(a)
print(b)
import scrapper
from scrapper_mysql_db_manager import ScrapperMysqlDbManager
from config_manager import ConfigManager
pages_details = scrapper.scrap()
succeeded = True
try:
db_manager = ScrapperMysqlDbManager(ConfigManager())
db_manager.open_connection()
db_manager.insert_pages_details(pages_details)
db_manager.close_connection()
except Exception:
succeeded = False
scrapper.print_result(succeeded)
from helpers import *
import pickle
import sys
import scrapper
result = scrapper.scrap()
sys.exit()
menu = pickle.load(open("menus/07-01-2018.pickle", "rb"))["Menu"]
print(menu)
print(pickle.load(open("menus/07-01-2018.pickle", "rb"))["FullMenu"])
for i in menu:
print(i.allergens())