def tryScrape(date, chatlist, id):
try:
return scraper.Scraper(
"http://www.osu.ru/pages/schedule/?who=1&what=1&filial=1&group=" +
str(chatlist[id][0]) + "&mode=full", date)
except Exception as e:
print('error', e)
def updateTrailers(self):
ms = scraper.Scraper()
if self.recentIsDue():
util.DEBUG_LOG(' - Fetching recent trailers')
return [Trailer(t) for t in ms.get_most_recent_movies(None)]
return []
def __init__(self, scraper_definition_file, logger=logging.getLogger()):
"""
"""
f = open(scraper_definition_file, 'r')
xml = f.read()
f.close()
self.scraper = scraper.Scraper(xml, logger)
def run(self):
# parse command line arguments
parser, arguments = self.parse_arguments()
if len(sys.argv) == 2 and (sys.argv[1] == '-h' or sys.argv[1] == '--help'):
parser.print_help()
return
elif len(sys.argv) < 3:
raise RuntimeError("ERROR: URL and HTML_TAG are required")
config = self.config(arguments)
url, tag = sys.argv[1], sys.argv[2]
print("Web Scraping with url={} tag={} next={} max={}"\
.format(url, tag, config.next, config.max if config.max > 0 else 'infinite'))
# tokenize HTML_TAG
t = tokenizer.Tokenizer()
tag_info = t.tokenize(tag)
# scrape HTML_TAG in URL
s = scraper.Scraper()
crawled_data = s.crawl(url, tag_info, config.next, config.max)
# print out data
for data in crawled_data:
print(data)
def main():
params = parse_args(sys.argv)
print "##", sys.argv, params
mode = params.get("path", "menu")
print "$$", mode
sc = scraper.Scraper(folders, play, record, notify)
getattr(sc, mode)(params)
def main():
sc = scraper.Scraper()
number_of_posts_num = 15
print("Subreddit to WordCloud Generator")
while True:
sub_fail = 0
subreddit_name = input("What subreddit would you like a WordCloud of? (Do not include /r/): ")
number_of_posts = input("How many top posts do you want to use? Max of 1000: ")
try:
number_of_posts_num = int(number_of_posts)
except ValueError:
print("Not a valid number")
if number_of_posts_num <= 0:
print('Number is less than zero')
try:
sc.check_if_subreddit_exist(subreddit_name)
except:
print('Not a valid sub!')
sub_fail = 1
if sub_fail == 0:
sc.top_submission_to_word_cloud(subreddit_name, number_of_posts_num)
run_again = input('Would you like to run the generator again? type 1 to run again')
if run_again == str(1):
print('----------------------------------------')
else:
break
def getPlayableURL(ID, res=None, url=None):
res = ItunesTrailerScraper.RES.get(res, 'hd720p')
ts = scraper.Scraper()
id_location = ID.split('.', 1)
all_ = [t for t in ts.get_trailers(id_location[-1], id_location[0]) if t]
if not all_:
return None
trailers = [t for t in all_ if t['title'] == 'Trailer']
url = None
if trailers:
try:
version = [v for v in trailers if any(res in u for u in v['streams'])][0]
if version:
url = [u for u in version['streams'] if res in u][0]
except:
import traceback
traceback.print_exc()
if not url:
try:
streams = all_[0]['streams']
url = streams.get(res, streams.get('hd720', streams.get('sd')))
except:
import traceback
traceback.print_exc()
url = re.sub(r'(480p|720p|1080p)', r'h\1', url)
return url
def scrape_with_threads():
print("Scraping with threads...")
products_df = scraper.Filemanager.get_products_data()
# Create instances of class "Scraper"
products = [
scraper.Scraper(category, url)
for category, url in zip(products_df["category"], products_df["url"])
]
# Create threads
threads = [
threading.Thread(target=product.scrape_info) for product in products
]
# Start scraping on all threads
for thread in threads:
time.sleep(scraper.REQUEST_DELAY)
thread.start()
# Wait for all threads to finish
for thread in threads:
thread.join()
# Save scraped data for each product (sequentially)
for product in products:
product.save_info()
def getTrailers(self):
ms = scraper.Scraper()
if self.allIsDue():
util.DEBUG_LOG(' - Fetching all trailers')
return [Trailer(t) for t in ms.get_all_movies(None)]
return []
def get(self):
sehir = self.request.get('sehir')
if memcache.get(sehir) == None:
s = scraper.Scraper()
res = s.hava_sehir(sehir)
logging.debug(res)
memcache.set(key=sehir, value=res, time=3600)
self.response.out.write(simplejson.dumps(memcache.get(sehir)))
def run(option):
qdspath = "N:/Common/Joe Hardy/Quality/Quality Documents"
outputpath = "C:/Users/User/SyncedFolder/Quality Share/ASTM Mapping/Data/links.csv"
s = scraper.Scraper(qdspath, option)
s.walk()
s.scrape()
s.organize_results()
s.output_results(outputpath)
def __init__(self, *args, **kwargs):
super(MainFrame, self).__init__(*args, **kwargs)
self.webpage = scraper.Scraper('http://feeds.huffingtonpost.com/huffingtonpost/LatestNews')
self.webpage.scrape() #Scrape the feed for titles, links, and paragraphs
self.InitUI()
self.Centre()
self.Show()
def find_links(urls: List[str]) -> List[Dict[str, str]]:
"""Find all links recursive"""
scr = scraper.Scraper()
links = []
for url in urls:
links.extend(scr.find_urls(url))
scr.close()
return links
def test_get_unsaved_reviews():
scrape = scraper.Scraper(datetime(2018, 1, 3))
file = open('correct/scraper1.txt')
correct_results = file.read()
file.close()
test_results = ''
for review in scrape.get_unsaved_reviews():
test_results += (str(review.artists) + '\n')
test_results += (review.album_title + '\n')
test_results += (review.score + '\n')
assert(test_results == correct_results)
def readPlayers(fileName):
file_players = []
f = open(fileName, 'r')
s = scraper.Scraper()
for line in f:
playerID = line[:-1]
print(playerID)
p = player.Player(playerID)
s.scrape(p)
file_players.append(p)
f.close()
return file_players
def get_result(keywords: str,
search: str,
webdriver: str,
qty: int = 0) -> List[Dict[str, str]]:
"""Get main seach result"""
scr = scraper.Scraper(webdriver)
qty = qty or 0
result = scr.get_urls(' '.join(keywords), qty=qty, search=search)
scr.close()
return result
def run_scraper(debug=False):
date = util.get_current_date()
# create the csv file
csv_name = '{}_data.csv'.format(date)
log_file = '{}_log.txt'.format(date)
util.create_csv(csv_name)
for date_pair in util.get_date_pairs():
worker = scraper.Scraper(date_pair[0], date_pair[1], debug)
data = worker.crawl(log_file)
if not data:
data_row = [date_pair[0], date_pair[1], None, None, None, None]
util.append_row(csv_name, data_row)
else:
for room_type in sorted(data):
room_data = data[room_type]
# Merge the accomodation data
for accom_type in sorted(room_data['price_bar']):
m_price = merge(room_data, accom_type)
data_row = [date_pair[0], date_pair[1],
room_type,
accom_type,
m_price['p_bar'],
m_price['p_sum'],
m_price['p_adv']
]
util.append_row(csv_name, data_row)
for accom_type in sorted(room_data['price_summer']):
m_price = merge(room_data, accom_type)
data_row = [date_pair[0], date_pair[1],
room_type,
accom_type,
m_price['p_bar'],
m_price['p_sum'],
m_price['p_adv']
]
util.append_row(csv_name, data_row)
for accom_type in sorted(room_data['price_adv']):
m_price = merge(room_data, accom_type)
data_row = [date_pair[0], date_pair[1],
room_type,
accom_type,
m_price['p_bar'],
m_price['p_sum'],
m_price['p_adv']
]
util.append_row(csv_name, data_row)
worker.clean()
def OnRefresh(self, e): #Reloads the scraper module and mimics previous processes, replaces the old articles in the list with the new articles
gauge = LoadingGauge(None, title='', size=(300,200)) #Create an instance of the loading screen
threading.Thread(target=gauge.InitUI).start() #Show the loading screen while the main program is refreshing (But it doesn't...)
reload(scraper)
self.webpage = scraper.Scraper('http://feeds.huffingtonpost.com/huffingtonpost/LatestNews')
self.statusBar.SetStatusText('Refreshing feed...')
self.webpage.scrape()
self.statusBar.SetStatusText('')
self.listCtrl.DeleteAllItems()
for i in range(0, 15):
self.listCtrl.InsertStringItem(i, self.webpage.titles[i])
self.browser.LoadURL(self.webpage.links[0])
self.Layout()
def __init__(self, company):
Thread.__init__(self)
self._scraper = scraper.Scraper((company[0], company[1]))
self._parser = article_parser.Parser(self._scraper.company_name)
self._company = company
self._scraper_results = ()
self.report = {}
self.report['articles'] = {}
self.report['good-count'] = 0
self.report['bad-count'] = 0
self.report['op-count'] = 0
self.report['pe-count'] = 0
self.driver_done = False
def scrape(sku, prod_name, source):
"""
Run the scraper until the queue is empty.
"""
thread_count = config.get("scraper").get("thread_count")
url_objs = list(q_db.find({"sku": sku}).sort('timestamp', pymongo.ASCENDING))
if url_objs: random.shuffle(url_objs)
with ThreadPoolExecutor(max_workers=thread_count) as executor:
sc = scraper.Scraper(sku, prod_name, source)
for url_obj in url_objs:
url = url_obj.get("url")
# sc.get_request(url, init=False) # Enable when debugging
executor.submit(sc.get_request, url, init=False) # Disable when debugging
def leaderboard(state, key):
s = scraper.Scraper()
tid = 0 # Scrape all teams for now, add individual teams later if needed
year = datetime.date.today().strftime('%Y')
url = """http://www.fangraphs.com/leaders.aspx?pos=all&stats={0}\
&lg=all&qual=0&type=8&season={1}\
&month=0&season1={1}\
&ind=0&team={2}&page=1_1000"""\
.format(state, year, tid)\
.replace(' ', '')
s.scrape(url, key, 'fangraphs', 'fangraphs', 24 * 60 * 60)
def post_tip(self):
"""
DevTo or Medium API sends a post daily here
:return:
"""
print("Posting tip to slack now...")
message = "***TIPS FOR TODAY*** \n\n\n\n"
scraper_obj = scraper.Scraper()
for article in scraper_obj.get_articles():
message += f"{article['title']} \n {article['link']} \n\n"
message += "\n\n\n\n\n\n\n <!channel> :heart::slightly_smiling_face: *I am Dele, a simple rule" \
" based bot, my code lives here: * https://github.com/olayemii/algorismbot " \
":heart::slightly_smiling_face: "
self._post_message(message)
def setUp(self):
self.scraper_config = {
'scraper': {
'frequency': 1,
'run_once': True
},
'request': {
'timeout': 10
},
'http://127.0.0.1:81': {
'pattern': "foo.*"
}
}
self.s = scraper.Scraper(tests.mocks.MockOutput(), self.scraper_config)
pass
def automation():
d = str(date.today())
t = datetime.now()
#Create a massive dataframe for processing
x = processor.Processor()
df = x.mergeLabels()
y = scraper.Scraper()
#Seperate dataframes for pre today and today
df_test = y.getNewHeadlines().drop_duplicates(keep='first')
df1 = df[df['date'] == d].drop_duplicates(ignore_index=True)
df = df[~df['date'].str.contains(d)]
#Vectorizes your phrases and creates your classifier
counter = CountVectorizer(ngram_range=(2, 3))
classifier = MultinomialNB()
counter.fit(df['title'] + df['description'])
training_counts = counter.transform(df['title'])
labels = df['label']
#The vectorized counts of the headlines in df_test
headline_counts = counter.transform(df_test['title'])
headline_counts_ticker = counter.transform(df1['title'])
#Training the model
classifier.fit(training_counts, labels)
prediction = classifier.predict(headline_counts)
prediction1 = classifier.predict(headline_counts_ticker)
chance = 100 * sum(prediction) / len(prediction)
chanceticker = 100 * sum(prediction1) / len(prediction1)
totalChance = (chance + chanceticker) / 2
print('Chances of market going up tomorrow: {0:.2f}%'.format(totalChance))
print('New Headline Chances: {0:.2f}%'.format(chance))
print('Ticker Headline Chances: {0:.2f}%'.format(chanceticker))
print('Prediction New Headline Length: {}'.format(
np.size(classifier.predict_proba(headline_counts), 0)))
print('Prediction Ticker Headline Length: {}'.format(
np.size(classifier.predict_proba(headline_counts_ticker), 0)))
with open('predictions/predictionsForTomorrow.csv', 'a',
newline='') as currentCSV:
writer = csv.writer(currentCSV)
writer.writerow([d, t, totalChance])
def run():
#Scraper
titles_path = "N:/Common/ASTMs & Standards/ASTM/2015 Standards"
output_path = "C:/Users/User/SyncedFolder/Quality Share/ASTM Mapping/Data/titles.csv"
scraper_ = scraper.Scraper(titles_path)
scraper_.scrape()
scraper_.output_results(output_path)
#Clusterer
cat_output_path = "C:/Users/User/SyncedFolder/Quality Share/ASTM Mapping/Data/ASTMCategorization.csv"
uncat_output_path = "C:/Users/User/SyncedFolder/Quality Share/ASTM Mapping/Data/ASTMUncategorized.csv"
clusterer_ = clusterer.Clusterer(output_path)
clusterer_.categorize_data()
clusterer_.output_categorization(cat_output_path, uncat_output_path)
def scrape():
print("Scraping...")
products_df = scraper.Filemanager.get_products_data()
# Create instances of class "Scraper"
products = [
scraper.Scraper(category, url)
for category, url in zip(products_df["category"], products_df["url"])
]
# Scrape and save scraped data for each product (sequentially)
for product in products:
print(product.url)
time.sleep(scraper.REQUEST_DELAY)
product.scrape_info()
product.save_info()
def modifyUI(self):
""" Method to modify UIs for the tabs after scraping.
First, the required web page is loaded on the webpage tab.
Second, the python script is generated and stored in script member variable
Third, scraper instance is created and scraping starts on a separate thread.
As soon as scraping finishes, the method addScriptAndData() is called.
"""
url = self.urlInput.text()
selectors = self.selectorInput.text()
self.web.load(QUrl(url))
print("Webpage Loaded \n")
self.script = ScriptGenerator(url, selectors).generate()
self.scraper_ = scraper.Scraper(str(url), str(selectors))
self.scraper_.threadChange.connect(self.addScriptAndData)
self.scraper_.start()
def readPlayers(new_players_file, known_players_file):
new_players = openFile(new_players_file)
known_players = openFile(known_players_file)
player_list = []
s = scraper.Scraper()
for playerID in new_players:
if len(playerID) > 1:
p = player.Player(playerID)
player_info = indexIntoLine(p.getID(), known_players)
if len(player_info) > 1:
print("Loading " + p.getID() + " from known players")
p.setSR(int(player_info.split(',')[1]))
p.setRole(player_info.split(',')[2])
else:
print(p.getID() + " not known. Fetching info")
s.scrape(p)
player_list.append(p)
return player_list
def main(argv):
config = None
try:
file = open(CONFIG, "r")
config = file.read()
file.close()
config = json.loads(config)
config = config["scraper"]
except:
print("Config syntax error!")
return 1
app = scraper.Scraper(config, argv)
return app.run()
def __init__(self, access_token=None, db=None):
self.access_token = access_token
self.db = db
self.scraper = scraper.Scraper(self, self.db)
self.scraper.start()
self.rep = {
"/": "-",
"\\": "-",
":": "-",
";": "-",
",": "-",
".": "-"
}
# define desired replacements here.
# Used in set reminder to get a standard format to work with
self.rep = dict((re.escape(k), v) for k, v in self.rep.items())
self.pattern = re.compile("|".join(self.rep.keys()))
self.delete_conf = {}
self.user_reminders = {}
