def load():
    try:
        with open('restaurants.pkl') as file:
            file.close()
            pass
    except IOError as e:
        main()
    restaurants = pickle.load(open('restaurants.pkl', 'rb'))
    getItems.restaurants = restaurants
    getRestaurants.restaurants = restaurants
    getMenu.restaurants = restaurants
Exemple #2
0
def init_get_data():
    global DATA
    data = {}
    # try:
    scraper.main()
    # except:
    #     print('data not updated')
    try:
        with open('./series.json', 'r') as f:
            text = f.read()
            data = json.loads(text)['data']
            f.close()
    except:
        print('No data found.')
    DATA = data
Exemple #3
0
def main():
    scraper.main()

    try:
        file = open("output.csv")
        reader = csv.reader(file)

        if len(list(reader)) > 1:
            send_results.main()
            print("Sent results")
        else:
            print("No results to send")

    except:
        pass
Exemple #4
0
def scrape_service_update(context):
    ids = scraper.main()
    print(ids)

    notification.send_notifications(context, ids["resolved"], "resolved")
    notification.send_notifications(context, ids["new"], "new")
    notification.send_notifications(context, ids["updated"], "updated")
Exemple #5
0
def addCourse(url, session=None, major_id=None):
    if session is None:
        Base.metadata.bind = engine
        DBSession = sessionmaker(bind=engine)
        session = DBSession()
    if len(url) == 0:
        return

    course = scraper.main([url])
    if len(course.lectures) == 0:
        return

    # Create course
    print "Creating course: " + course.title
    c = Course(name=course.title, description=course.description)
    session.add(c)
    session.commit()

    # Add to major
    if major_id is not None:
        rel = MajorCourse(major_id=major_id, course_id=c.course_id)
        session.add(rel)

    # Add Lectures
    for lecture in course.lectures:
        task = Task(course_id=c.course_id,
            name=lecture[0],
            url=lecture[1]
        )
        session.add(task)

    session.commit()
Exemple #6
0
def home(request):
    title = "Title Goes Here"
    Search = SearchBoxForm(request.POST or None)
    score = None
    polarity = None
    subjectivity = None
    buzzwords = None
    queried = False
    numberOfComments = 0

    if Search.is_valid():
        print "CALCULATE SUBREDDIT SCORE"
        title = Search.cleaned_data.get('Peek')
        score, buzzwords, polarity, subjectivity, numberOfComments = scraper.main(
            title)
        title = "/r/" + title
        print score
        queried = True
        Search.save()

    context = {
        "title": title,
        "searchBar": Search,
        "results": score,
        "polarity": polarity,
        "subjectivity": subjectivity,
        "queried": queried,
        "buzzwords": buzzwords,
        "numberOfComments": numberOfComments
    }

    return render(request, "home.html", context)
Exemple #7
0
def checkAvtivatie(versleutelBestand, ww, timer, tijd, sleutel):
    #deze variabel slaat de inhoud van de tweet op de aangegeven tijdstip in de csv betand
    bericht = tweet.main(gbr, ww, gebruiker, path)
    berichten = bericht
    # deze loop controleerd of er een tweet is gevond op de account door te kijken in de csv bestand
    if not berichten:  #als er geen bericht is gevonden ...
        timer = timer - 30
        sleep(
            20
        )  # er word een pauze ingelast van 20 seconde zodat twitter het account niet blockeerd
        if timer == 0:  #... en als de timer is afgegaan , start de versleuteling
            versleutel(versleutelBestand, ww)
        else:  #zo niet check net zolanf to de timer is gestopt of er een tweet met de key is verstuurd
            print(
                "Er zijn geen tweets gevonden. Heb gedult er moet nog gecheck worden ..."
            )
            checkAvtivatie(versleutelBestand, ww, timer)
    else:  # als er wel een tweet is met de key als inhoud stop het verleutelingsprogramma.
        print(berichten[tijd])
        if berichten[tijd] == sleutel:
            print(
                "Het versleutelmeganisme is uitgeschakeld - overgeschakeld op exit mode"
            )
            exit()
        else:
            checkAvtivatie(versleutelBestand, ww, timer, tijd, sleutel)
            print("okey")
Exemple #8
0
def home(request):
	title = "Title Goes Here"
	Search = SearchBoxForm(request.POST or None)
	score = None
	polarity = None
	subjectivity = None
	buzzwords = None
	queried = False
	numberOfComments = 0
	
	if Search.is_valid():
		print "CALCULATE SUBREDDIT SCORE"
		title = Search.cleaned_data.get('Peek')
		score, buzzwords, polarity, subjectivity, numberOfComments = scraper.main(title)
		title = "/r/" + title
		print score
		queried = True
		Search.save()

	context = {
		"title": title,
		"searchBar": Search,
		"results": score,
		"polarity": polarity,
		"subjectivity": subjectivity,
		"queried": queried,
		"buzzwords": buzzwords,
		"numberOfComments": numberOfComments
	}

	return render(request, "home.html", context)
Exemple #9
0
def addCourses(page, session=None):
    if session is None:
        Base.metadata.bind = engine
        DBSession = sessionmaker(bind=engine)
        session = DBSession()

    print "Locating course pages"
    coursesToAdd = scraper.getAllCoursePages(page)

    print "Locating course lecture videos"
    #major_id = majors.index("Computer Science") + 1
    course_id = 1
    for url in coursesToAdd:
        course = scraper.main([url])
        if len(course.lectures) == 0:
            continue
        print "Creating course: " + course.title
        c = Course(name=course.title, description=course.description)
        session.add(c)
        #rel = MajorCourse(major_id=2, course_id=course_id)
        #session.add(rel)
        for lecture in course.lectures:
            task = Task(course_id=course_id,
                name=lecture[0],
                url=lecture[1]
            )
            session.add(task)
        course_id += 1

        session.commit()
Exemple #10
0
def main(key=None, refresh=False):
    scrap_dict = scraper.main(refresh=refresh)
    # print(result_list)
    model = WhiteFishModel(scrap_dict)
    model.commit_to_SQL()
    # print("committed to SQL")
    # model.get_recent_SQL("example.sqlite")
    return model
Exemple #11
0
def main():
    print('To skip a step, press enter without typing "yes".')
    answer = input('Enter yes to overwrite or create the database.')
    if 'yes' == answer:
        create_yaml_lists.main()
    else:
        print('We will skip this and continue with the next step:')

    answer = input('Enter yes to start building a web scraper for images.')
    if 'yes' == answer:
        scraper.main()
    else:
        print('We will skip this and continue with the next step:')

    answer = input('Enter yes to check the new product names.')
    if 'yes' == answer:
        check_model_names.main()
    else:
        print('We will skip this and continue with the next step:')

    answer = input('Enter yes to check for annotations without metadata.')
    if 'yes' == answer:
        check_annotations.main()
    else:
        print('We will skip this and continue with the next step:')

    answer = input('Enter yes to remove images with duplicates in image URLs.')
    if 'yes' == answer:
        remove_images_with_URL_duplicates.main()
    else:
        print('We will skip this and continue with the next step:')

    answer = input(
        'Enter yes to remove images with annotations and URL specified by filename of the image.'
    )
    if 'yes' == answer:
        remove_annotated_images.main()
    else:
        print('We will skip this and continue with the next step:')

    answer = input('Enter yes to build tfrecord with data augmentation.')
    if 'yes' == answer:
        tfrecord_pipeline.main()
    else:
        print('We skipped this.')
Exemple #12
0
def main():
    pp1, score_date2, user_name1 = scraper.main()

    pp1, score_date2 = zip(*sorted(zip(score_date2, pp1)))  # sorts dates and pp

    # print(pp1)
    # print(score_date2)

    set_sizes()
    plot_graph(pp1, score_date2, user_name1)
Exemple #13
0
def create():
	objects = main()
	db.create_all()
	for i in objects:
		try:
			newProj = Project(i.title, i.des, i.link, i.image, i.source)
			db.session.add(newProj)
			db.session.commit()
		except IntegrityError:
			db.session.rollback()
Exemple #14
0
def main(topteams=False, stop_short='99999999', period=7):
    '''
	By default, this updates the data by scraping games through yesterday and returns an elo simulation run on the latest data
	It includes several options:
	- output the x 'topteams' by elo rating along with each teams projected point spread over the next team and their change in elo in the last 'period' days
	- 'stop_short' of simulating through the entire dataset by specifying a day to simulate through instead
	'''
    filepath = utils.get_latest_data_filepath()
    yesterday = (datetime.date.today() -
                 datetime.timedelta(days=1)).strftime('%Y%m%d')
    if filepath[len(DATA_FOLDER):][-12:-4] != yesterday:
        print('updating data...')
        scrape_start = utils.shift_dstring(filepath[len(DATA_FOLDER):][-12:-4],
                                           1)
        scraper.main(filepath[len(DATA_FOLDER):][0:8], scrape_start, yesterday,
                     filepath)
        filepath = utils.get_latest_data_filepath()

    data = utils.read_csv(filepath)
    this_sim = sim(data, K_FACTOR, SEASON_CARRY, HOME_ADVANTAGE, stop_short,
                   period)

    if topteams != False:
        output = pd.DataFrame(
            this_sim.get_top(int(topteams)),
            columns=['Team', 'Elo Rating',
                     '%i Day Change' % period])
        output['Point Spread vs. Next Rank'] = [
            "{0:+.1f}".format(
                ((output['Elo Rating'][i] - output['Elo Rating'][i + 1]) /
                 ELO_TO_POINTS_FACTOR)) for i in range(topteams - 1)
        ] + ['']
        output['Rank'] = [i for i in range(1, topteams + 1)]
        utils.table_output(
            output,
            'Ratings through ' + this_sim.date + ' - Top ' + str(topteams), [
                'Rank', 'Team', 'Elo Rating', 'Point Spread vs. Next Rank',
                '%i Day Change' % period
            ])

    return this_sim
Exemple #15
0
def handle_message(uid, message):
    cache = r.get('CACHE:' + uid)
    if cache is not None:
        app.logger.info('cache hit for uid:{}'.format(uid))
        send_message(uid, 'Cache Hit' + str(cache))
    else:
        details = r.get(uid)
        details = json.loads(details)
        app.logger.info('scraping for uid:{}'.format(uid))
        resp = scraper.main(details['regno'], details['password'])
        app.logger.info('setting cache for uid:{}'.format(uid))
        r.setex('CACHE:' + uid, json.dumps(resp), 600)

        send_message(uid, 'Ran Scraper' + str(resp))
Exemple #16
0
def get_results():
    library = request.args.get("library")
    search_keywords = request.args.get("search_keywords")
    page = request.args.get("page")

    if library == "" or library is None:
        results = {"error": "cannot execute search without library input"}
        return jsonify(results)

    if search_keywords == "" or search_keywords is None:
        results = {"error": "cannot execute search without search keywords"}
        return jsonify(results)

    try:
        if page is None:
            results = main(library, search_keywords)
        else:
            results = main(library, search_keywords, page)

    except Exception as e:
        results = {"error": str(e)}

    return jsonify(results)
Exemple #17
0
def home():
    infoList = ['', '']
    j = 0
    for i in session.pop('mylist', []):
        infoList[j] = i 
        j += 1

    JSON, dining_transactions, schiller_transactions = scraper.main(infoList[0], infoList[1])

    if not JSON:
	return redirect(url_for('login'))

    parseJSON = json.loads(JSON)

    dining_dollars = parseJSON["dining_dollars"]
    schillers = parseJSON["schillers"]
    guest_swipes = parseJSON["guest_meals"]
    if 'meals_week' in parseJSON:
	meals_left = parseJSON["meals_week"]
    else:
 	meals_left = ''
    json_obj = JSON

    temp = 0
    
    if int(guest_swipes) > 3:
        temp = meals_left
        meals_left = guest_swipes
        guest_swipes = temp

    if 'spending' in parseJSON:
    	spending = parseJSON["spending"]
    else:
	spending = ''

	dining_transactions = "[{day: '%s', balance: %s}, {day: '%s', balance: %s}]" % (datetime.datetime.strftime(datetime.datetime.now() + datetime.timedelta(days=-1), '%Y-%m-%d %H:%M'), dining_dollars, time.strftime('%Y-%m-%d %H:%M'), dining_dollars)

	schiller_transactions = "[{day: '%s', balance: %s}, {day: '%s', balance: %s}]" % (datetime.datetime.strftime(datetime.datetime.now() + datetime.timedelta(days=-1), '%Y-%m-%d %H:%M'), schillers, time.strftime('%Y-%m-%d %H:%M'), schillers)
	 
 
    if 'swipes' in parseJSON:
    	swipes = parseJSON["swipes"]
    else:
	swipes = ''

    week, diningDollarBudget, dailyDiningBudget, laundryLeft, LDCCardSwipes, burtonCardSwipes = getASI(parseJSON)

    return render_template('index.html', dining= "$" + dining_dollars, meals=meals_left, schill="$" + schillers, guest=guest_swipes, spending=spending, swipes=swipes, laundry=laundryLeft, diningBudget=diningDollarBudget, dailyDiningBudget=dailyDiningBudget, ldc=LDCCardSwipes, burton=burtonCardSwipes, dining_transactions=dining_transactions, schiller_transactions=schiller_transactions)
def loadData():
    eventsAndHorses = main()

    newDict = {}

    for i in eventsAndHorses:

        horseSents = {}
        for j in eventsAndHorses[i]:
            sents = mainAnalysis(j)
            horseSents[j] = sents

        print(horseSents)
        newDict[i] = horseSents

    print(newDict)

    with open(FILE, 'wb') as handle:
        pickle.dump(newDict, handle)
Exemple #19
0
def main():
    args = parse_input()
    log.info("INPUT: %s" % os.path.basename(args.PATH))

    media_files = find_all_files(args.PATH, True)  # True for subtitles, FIX
    for media_file in media_files:
        media_info = scraper.main(media_file)
        if 'episode' not in media_info:
            # Grab movie file and directory
            media_file = findfile(media_file['file'])
            # Replace defined Format with real values
            dst = rename(args.set, media_file, media_info)
        elif 'episode' in media_info:
            media_file = findfile(media_file['file'])
            # Replace defined Format with real values
            dst = rename(args.set, media_file, media_info)
        else:
            log.error("ERROR FINDING FILE.")

        # Update destination
        dst = os.path.join(args.output, dst)
        #print(media_file)
        # Execute changes
        exe_changes(media_file, dst, args.action, args.conflict)
    """Make tuple containing date from string (from google event json)

    Param str_: 'yyyy-mm-ddT.......'
    Return: tuple ({year}, {month}, {day})"""
    str_ = str_[:str_.index('T')]
    date = [int(a) for a in str_.split('-')]
    return tuple(date)


if __name__ == '__main__':
    if not os.path.exists('token.pkl'):
        make_token()

    username = input('username: '******'password: '******'token.pkl', 'rb') as file:
        credentials = pickle.load(file)
    with open('schedule.json', 'r') as file:
        mydict = json.load(file)
        schedule = Schedule.from_dict(mydict)
    days_to_clear = schedule.days_updated

    service = build('calendar', 'v3', credentials=credentials)
    calendar_id = get_cal_id(service, 'schedule_migrater')
    event_ids = get_event_ids_by_dts(service, calendar_id, days_to_clear)

    if event_ids:
        for event_id in track(
                event_ids, description='deleting events from google calendar'):
Exemple #21
0
def home():
    return json.dumps(scraper.main(), indent=2)
Exemple #22
0
import re

import scraper


class BatotoScraper(scraper.Scraper):
    feed_url = 'http://www.batoto.net/recent_rss'
    title_re = re.compile(r'(?P<series>.+?) - (?P<language>\w+) - '
    r'(:?Vol.(?P<volume>\d+) )?Ch.(?P<chapter>\w+?):?(?P<chapter_title>.+)')
    result_groups = ('volume', 'series', 'chapter', 'chapter_title', 'language')

    def read_item(self, raw_item):
        item = super(BatotoScraper, self).read_item(raw_item)
        results = self.title_re.match(item['title'])
        for group in self.result_groups:
            item[group] = results.group(group)
        return item


if __name__ == '__main__':
    scraper.main(None, scraper_class=BatotoScraper)
Exemple #23
0
import finder
import scraper
import reader

with open('log.txt', 'w', newline = '', encoding = 'UTF-8') as writer:
    writer.write("Master: START--START--START--START--START\n")
    print("Master: START--START--START--START--START")
"""
    * Starting point, give it a real player ID
    * zsda123 is only a placeholder!!!
    * If it happens to be a real ID or your ID, I apologize for that!
"""
    finder.main(writer, "zsda123")

    with open('userIdList.txt', 'r', newline = '', encoding = 'UTF-8') as f:
        userIdListStr = f.read()

    userIdList = userIdListStr.split()

    scraper.main(writer, userIdList)

    reader.main(writer)

    writer.write("Master: DONE--DONE--DONE--DONE--DONE\n")
    print("Master: DONE--DONE--DONE--DONE--DONE")
Exemple #24
0
 def refresh(self, sender):
     scraper.main()
     self.reload_data(None)
Exemple #25
0
def main(event, context):
    scraper.main()
Exemple #26
0
 def test_parses_kanjipedia_EI(self):
     output = scraper.main()
     print(output)
     # assert kanji_string in output
     # self.assertEquals(output, "")
     self.assertIn("エイ", output)
def returnCalendar():
    list = request.get_json(force=True)
    m = scraper.main(list)
    print(list)
    return m
Exemple #28
0
from multiprocessing import freeze_support

from scraper import main

if __name__ == '__main__':
    freeze_support()
    main()
Exemple #29
0
import scraper
from requests import get
from bs4 import BeautifulSoup as bs4
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
from urllib.parse import urlparse
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException
import time
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
import re

info = scraper.main()
url = 'https://zoom.us/join'
scraper.driver.get(info["url"])
time.sleep(5)

try:
    WebDriverWait(scraper.driver, 3).until(
        EC.alert_is_present(),
        'Timed out waiting for PA creation ' + 'confirmation popup to appear.')
    scraper.driver.switch_to.alert().accept()
except NoAlertPresentException as e:
    print("no alert")
Exemple #30
0
import scraper
import gsheets_writer

if __name__ == "__main__":
    scraper.main()
    gsheets_writer.main()
 def setUp(self):
     self.n_tags, self.top_five = scraper.main(verbose=False)
     self.top_five_tags = [x[0] for x in self.top_five]
     self.top_five_counts = [x[1] for x in self.top_five]
IDEntry = Entry(root)
IDEntry.grid(row=4, column=2, sticky=E)

# Remove ID
removeButton = Button(root,
                      text="Remove",
                      fg="purple",
                      command=lambda: removeID())
removeButton.grid(row=4, column=3)

# Scrape
scrapeButton = Button(root,
                      text="Scrape",
                      fg="orange",
                      command=lambda: scrap.main())
scrapeButton.grid(row=5, column=1)

# Refresh CSV File
refreshButton = Button(root,
                       text="Refresh",
                       fg="green",
                       command=lambda: c.resetIDs())
refreshButton.grid(row=5, column=2)

# Quit
quitButton = Button(root, text="Quit", command=root.quit)
quitButton.grid(row=5, column=3)

root.mainloop()
Exemple #33
0
def retrieve_data():
    text = request.form["selectcountries"]
    return render_template("postform.html", text=text, data=scraper.main(text))
Exemple #34
0
def hello_world():
   link,ver = scraper.main()
   return jsonify(link=link,
                   version=ver)
def main():
    scraper.main()
Exemple #36
0
        )
        session.add(newMajor)

    # Add users
    admin = User(name="John Sutton", email="*****@*****.**", picture="https://lh4.googleusercontent.com/-C6cSzCA5-Bw/AAAAAAAAAAI/AAAAAAAACVo/OrC0MgMptnI/photo.jpg", isAdmin=True)
    session.add(admin)

    # Add courses
    print "Locating course pages"
    coursesToAdd = scraper.getAllCoursePages()

    print "Locating course lecture videos"
    major_id = majors.index("Computer Science") + 1
    course_id = 1
    for url in coursesToAdd:
        course = scraper.main([url])
        if len(course.lectures) == 0:
            continue
        print "Creating course: " + course.title
        c = Course(name=course.title, description=course.description)
        session.add(c)
        #rel = MajorCourse(major_id=2, course_id=course_id)
        #session.add(rel)
        for lecture in course.lectures:
            task = Task(course_id=course_id,
                name=lecture[0],
                url=lecture[1]
            )
            session.add(task)
        course_id += 1
Exemple #37
0
def scrape():
    import scraper
    return scraper.main()
Exemple #38
0
teamDict = mi.read('teamDict.json')

newDict = {}


for i in teamDict: 
    listOfIDs = []
    try: 
        for j in rosterDict[i]['2017']: 
            listOfIDs.append(j)
        newDict[teamDict[i]] = listOfIDs
    except: 
        pass

#mi.write('teamAndPlayerIDS.json', newDict)
#pdb.set_trace()
playerStatisticsDict = {}
for i in newDict: 
    #pdb.set_trace()
    urls = generateURLs(i, newDict[i])
    start = time.time()
    elements = scraper.main(urls, newDict[i])
    end = time.time()
    print(end - start)
    playerStatisticsDict[str(i)] = elements

mi.write('finalPlayerStatsDict.json', playerStatisticsDict)



from datetime import date
from dateutil.rrule import rrule, DAILY
import scraper
import parser

a = date(2013, 3, 4)
b = date(2013, 3, 5)

if __name__ == '__main__':
	for dt in rrule(DAILY, dtstart=a, until=b):
    		current_date = dt.strftime("%Y/%m/%d")
		print 'Scraping ' + current_date
		scraper.main(dt.strftime("%d/%m/%Y"))
		print 'Parsing ' + current_date
		parser.main(dt.strftime("%Y/%m/%d"))