def main(): args = list(sys.argv) if len(args) == 2: user_timestamp = int(args[1]) scrapperObj = scrapper.Scrapper() print "Running scrapper..." # scrapperObj.cleanDB() """ Second way I tried to do the refresh. This time I tried sending all parameters, also didn't work with multiple parameters at once. SetInterval(10, make_call, reddit, db_conn, "python") try: sleep(35) finally: thread.stop() thread2 = set_interval(make_call(reddit, db_conn, "python"), 15) Finally, I gave up and did a very stupid - though working - way of handling this task. """ while True: for sub in scrapperObj.subreddits: scrapperObj.make_call(sub, int(user_timestamp)) time.sleep(120) else: print "Usage: python main.py TIMESTAMP"
def test_make_call(self): test_json = [{ "id": "1234123", "title": "This is a test title", "selftext": "Test is a test for testing tests.", "num_comments": 45, "created": 1495490048, "subreddit": "test", "comments": [ "this is the first comment", "this is the second commment", "this is the third comment" ] }] scrapper_call = scrapper.Scrapper() mock_subreddit = mock.MagicMock() # mocking the submissions part of the subreddits call. mock_subreddit.submissions.return_value = test_json scrapper_call.reddit.subreddit = mock.MagicMock( ).side_effect = mock_subreddit result = scrapper_call.make_call("python", 1495490048) self.assertEqual(result, True)
def clickedButton(self): print("Entered") handle = self.name_input.text() print(handle) try: s = scrapper.Scrapper(handle) list = s.problemsolved() for items in list: self.cb.addItem(items) self.rating.setText("Codechef Rating :: " + s.overAllRating()) self.countryRank.setText("Country Rank :: " + s.countryRank()) self.globalRank.setText("Global Rank :: " + s.globalRank()) self.ratingStars.setText("Rating Stars :: " + s.ratingStars()) self.highestRating.setText("Highest Rating :: " + s.highestRating()) self.contestAttended.setText("Contest Attended :: " + str(s.contestAttended())) except: pri = "NULL" self.button.setText(":( Invalid Input") self.rating.setText("Codechef Rating :: " + pri) self.countryRank.setText("Country Rank :: " + pri) self.globalRank.setText("Global Rank :: " + pri) self.ratingStars.setText("Rating Stars :: " + pri) self.highestRating.setText("Highest Rating :: " + pri) self.contestAttended.setText("Contest Attended :: " + pri)
def scrape(url, n=0): meetup = scr.Scrapper(url) meetup.event_urls() meetup.event_info(n) print('Events data load') meetup.attendees_info() print('Attendees data load') meetup.members_info() print('Members data load') meetup.organisers_info() print('Organisers data load') return meetup
def main(): url = 'https://www.meetup.com/find/?keywords=data%20science' meetup = scrapper.Scrapper(url) meetup.event_urls() meetup.event_info() meetup.attendees_info() meetup.members_info() meetup.organisers_info() data = Database() data.populate_tables_organizers(meetup.organisers_df) data.populate_tables_members(meetup.members_df) data.populate_tables_events(meetup.event_df) data.populate_tables_attendee(meetup.attendees_df)
def getResultsAndAddToTable(self): scrap = scrapper.Scrapper(self.kt, self.ktu, self.v, self.f, self.s) for i in scrap.getTable(): for c, j in enumerate(range(0, len(i))): self.gbSonuclar.setVisible(True) self.lblTaksit.setText(_translate("Form", "Taksit Tutarı : {}".format(i[j]['Taksit']), None)) self.lblFaiz.setText(_translate("Form", "Faiz Oranı : {}".format(str(self.txtFaiz.text())), None)) self.tableResult.setRowCount(len(i)) self.tableResult.setItem(c, 0, QtGui.QTableWidgetItem(str(i[j]['Sira']))) self.tableResult.setItem(c, 1, QtGui.QTableWidgetItem(str(i[j]['Taksit']))) self.tableResult.setItem(c, 2, QtGui.QTableWidgetItem(str(i[j]['Anapara']))) self.tableResult.setItem(c, 3, QtGui.QTableWidgetItem(str(i[j]['Faiz']))) self.tableResult.setItem(c, 4, QtGui.QTableWidgetItem(str(i[j]['Bakiye']))) self.tableResult.setItem(c, 5, QtGui.QTableWidgetItem(str(i[j]['KKDF']))) self.tableResult.setItem(c, 6, QtGui.QTableWidgetItem(str(i[j]['BSMV'])))
def test_stage_no_kwd(self): submissions_json = [{ "id": "1234123", "title": "This is a test title", "selftext": "Test is a test for testing tests.", "num_comments": 45, "created": 1495490048, "subreddit": "test", "comments": [ "this is the first comment", "this is the second commment", "this is the third comment" ] }] comments_json = [{ "id": "1234123", "body": "Test", "sub_id": "Test", "subreddit": "python", "created": "123" }] scrapper_call = scrapper.Scrapper() mock_comments = mock.MagicMock() mock_comments.find.return_value = comments_json mock_submissions = mock.MagicMock() mock_submissions.find.return_value = submissions_json scrapper_call.db_conn.Submissions = mock.MagicMock( ).side_effect = mock_submissions scrapper_call.db_conn.Comments = mock.MagicMock( ).side_effect = mock_comments result = scrapper_call.stage_one("subreddit", 1245522425, 1244546225) self.assertTrue(result)
def __init__(self): self.scrapper = scrapper.Scrapper() self.version = self.scrapper.get_version() self.con = sqlite3.connect("./DataTFT") self.cur = self.con.cursor()
def schedule(): team = request.form.get('team_select') sc = scrapper.Scrapper() items = sc.scrape(team) return render_template('schedule.html', content=items)
import scrapper as sc import predictions as pre import statistics as st default = True s = sc.Scrapper() p = pre.Prediction() stats = st.Statistics() def commands(): coms = "/createLinks --> Creating links for drivers standings and constructor standings\n" \ "/editLinks --> Editing the links for driver standings and constructor standings\n" \ "/viewStandings --> View the current standings for driver standings and constructor standings\n" \ "/makeNewPrediction --> Add a new prediction of how the drivers and teams will rank\n" \ "/addAnotherPrediction --> Adding another prediction of how the drivers and teams will rank\n" \ "/viewPrediction --> Viewing a prediction\n" \ "/deletePrediction --> Delete a prediction\n" \ "/showZeros --> Show the number of teams/drivers who have not gained points\n" \ "/calculateScore --> Calculate the inaccuracy score for the predictions\n" \ "/end --> Ends the program" return coms while default: response = input("What would you like to access: ") if response == "/createLinks": s.create_links() print("Links created") elif response == "/commands": print(commands())
def Command1(self): i=self.input1.get() scrapper.Scrapper(i)
def __init__(self): self.config_parser = config_parser.Configuration_Parser() self.scrapper = scrapper.Scrapper(self.config_parser)
''' Main Module for Project ''' import sys import scrapper import deployment import report_builder if __name__ == "__main__": if len(sys.argv) > 1: if sys.argv[1] == '-d': deployment.deploy() scrapper = scrapper.Scrapper() report_builder = report_builder.ReportBuilder() report_builder.print_reports(scrapper.scrape())
import scrapper import driver_config def addNewLine(string): return "{}\n".format(string) if __name__ == '__main__': url = "https://moat.com/" obj = scrapper.Scrapper(driver_config.DRIVER_PATH) with open(driver_config.OUTPUT_PATH, "w") as file_write: # 1. Verify the search bar autocomplete drop down text. obj.open_website(url) autocomplete_result = obj.verify_autocomplete("Saturn") file_write.write(addNewLine(autocomplete_result)) # 2. Verify the creative count on the search results page is # correct for these 3 search terms: Saturn, Saturday’s Market, # and Krux. products_list = ["Saturn", "Saturday’s Market", "Krux"] for product in products_list: obj.open_website(url) creative_check_result = obj.verify_creative(product) file_write.write(addNewLine(creative_check_result)) # 3. Verify the “Random Brand” link on the search results # page is random.
import requests import random import time import scrapper if __name__ == '__main__': path_to_key = 'secret.txt' scrapper = scrapper.Scrapper(path_to_key) scrapper.request_match_list('11121212122')
def setUp(self): self.scrapper = scrapper.Scrapper(interval=settings.interval)
def test_stage_one_false(self): scrapper_call = scrapper.Scrapper() self.assertFalse( scrapper_call.stage_one("subreddit", "string", 123442.123123)) self.assertFalse( scrapper_call.stage_one("subr¤©«±¶µeddit", 1245522425, 1244546225))
def test_make_call_non_ascii(self): scrapperObj = scrapper.Scrapper() result = scrapperObj.make_call("¤©«±¶µ", 1495490048) self.assertEqual(result, False)