def first_page_links():
data = scraper.get_data("http://codingbat.com/java", 'temp.txt')
soup = BeautifulSoup(data, 'lxml')
categories = soup.find_all('div', attrs={'class': 'summ'})
rel_links = ['http://codingbat.com'+item.a['href'] for item in categories]
return rel_links
def displayData(subredditName):
postsData = get_data(subredditName)
result = {
"Developer": "Parag Jyoti Pal",
"subreddit":
subredditName + ", This api has been developed by Parag Jyoti Pal",
"total_members": postsData[0],
"active_users": postsData[1],
"posts": postsData[2]
}
return jsonify(result)
def show_products(update: Update, context: CallbackContext) -> int:
if context.user_data.get('links') is None:
update.message.reply_text('You have not inserted links yet',
reply_markup=markup)
else:
for link in context.user_data.get('links'):
df = get_data(link)
text = print_data_ordered(df, link)
update.message.reply_text(text)
make_choise(update, context)
return CHOOSING
def find_links():
for link in first_page_links():
data = scraper.get_data(link, 'test.txt')
soup = BeautifulSoup(data, 'lxml')
td = soup.find("div", class_="indent").table.find_all('td')
for item in td:
temp = item.find_all('a')
for link in temp:
all_links.append(('http://codingbat.com' + link['href']))
def main(path=''):
"""Handles all routes for TiledHN app"""
color_values = colors.randomize_colors()
data, has_next = scraper.get_data('https://news.ycombinator.com/' + path.replace(':', '?'))
# Only one attribute to show
if 'jobs' in path:
flag = 1
else:
flag = None
return render_template('index.html', zipped=zip(data, color_values), jobs=flag, next=has_next)
def generate_response(msg):
"""
indices_list = []
try:
indices_list = QueryResults.objects.get(keyname = sno)
except OperationalError:
pass
"""
if 'search' in msg:
q = ''.join([ix for ix in msg.split('search', 1)[1]])
#result = get_data(q) #display the data resulting from a query search
return get_data(q)
elif int(msg) in range(1,100): #fetch the link of that video
link = get_video(int(msg))
return link
else:
return "Use search 'search-term' to look for the video you want"
def write_output():
output = open(output_file, "a+")
for link in all_links:
main_string = []
data = scraper.get_data(link, 'temp.txt')
soup = BeautifulSoup(data, 'lxml')
main = soup.find("div", class_="minh")
main_string.append(str(main.string) + "\n")
for br_tag in main.parent.find_all('br'):
main_string.append(str(br_tag.next_sibling))
del main_string[-1]
for item in main_string:
output.write(item)
output.write("\n")
output.write("\n\n")
output.close()
def on_chat_message(msg):
content_type, chat_type, chat_id = telepot.glance(msg)
# print(msg)
# print(chat_id)
if content_type == 'text':
name = msg["from"]["first_name"]
txt = msg['text']
if txt == '/list':
for index, url in enumerate(urls):
text = get_data(url)
bot.sendMessage(chat_id, text)
elif '/add' in txt:
urls.append(re.findall("(?P<url>https?://[^\s]+)", txt)[0])
print(urls)
elif txt == '/remove':
bot.sendMessage(chat_id, 'bravo')
else:
bot.sendMessage(chat_id, 'This command does not exist')
bot.sendMessage(chat_id, hello_msg(name))
# Import necessary packages, OS, TWEEPY, and JSON
import os
import tweepy as tw
import pandas as pd
import inspect
import re
import scraper
practitioner = ['KirkDBorne', 'gp_pulipaka', 'MrDataScience', 'hadleywickham']
influencer = ['MikeQuindazzi', 'Fisher85M', 'HaroldSinnott', 'jblefevre60']
organization = ['analyticbridge', 'Talenter_io', 'rstudio', '_100DaysOfCode']
technologies = ['pytorch', 'rstats', 'javascript', 'reactjs', 'angular']
filename = 'practitioner.csv'
alldf = []
for user in practitioner:
print('user: '******'a') as f:
dff.to_csv(f, header=False)
def api_return():
return jsonify(get_data())
import argparse
import scraper
parser = argparse.ArgumentParser()
parser.add_argument("url",
help="The url for the Amazon wishlist to scrape.", type=str)
args = parser.parse_args()
url = args.url
print(scraper.get_data(url))
no INT,
ticket VARCHAR(10),
company VARCHAR(255),
sector VARCHAR(255),
industry VARCHAR(255),
country VARCHAR(255),
market_cap FLOAT(4),
pe FLOAT(4),
price FLOAT(4),
change FLOAT(4),
volume INT
);
""".format(table_name))
DB.commit()
data = get_data()
from pprint import pprint
pprint(data)
for row in data:
cursor.execute("""
INSERT INTO {}(no, ticket, company, sector, industry, country, market_cap, pe, price, change, volume)
VALUES({}, '{}', '{}', '{}', '{}', '{}', {}, {}, {}, {}, {})
""".format(
table_name,
row['No.'],
row['Ticker'],
row['Company'],
row['Sector'],
row['Industry'],
def updateDb():
with app.app_context():
results = get_data()
db.table('table1').update({'data': results}, where('id') == 1)
print('Updated DB')
def get_user(username):
return IGUser(
scraper.get_data(IG_URL.format(username))['entry_data']['ProfilePage']
[0]['graphql']['user'])
no INT,
ticket VARCHAR(10),
company VARCHAR(255),
sector VARCHAR(255),
industry VARCHAR(255),
country VARCHAR(255),
market_cap FLOAT(4),
pe FLOAT(4),
price FLOAT(4),
change FLOAT(4),
volume INT
);
""".format(table_name))
DB.commit()
data = get_data()
from pprint import pprint
pprint(data)
for row in data:
cursor.execute("""
INSERT INTO {}(no, ticket, company, sector, industry, country, market_cap, pe, price, change, volume)
VALUES({}, '{}', '{}', '{}', '{}', '{}', {}, {}, {}, {}, {})
""".format(
table_name,
row['No.'],
row['Ticker'],
row['Company'],
row['Sector'],
row['Industry'],
# -*- coding: utf-8 -*-
import dash
import dash_core_components as dcc
import dash_html_components as html
import scraper as scrap
import colors as colors
colors = colors.get()
# getting data from scraping module
data = scrap.get_data()
data['percentage D/C'] = data['Total Deaths']/data['Total Cases']*100
data_europe_est = data[data['Continent'] == "Europe de l'est"][:10]
data_europe_ouest = data[data['Continent'] == "Europe de l'ouest"][:10]
data_amerique = data[data['Continent'] == 'Amerique'][:10]
data_oceanie = data[data['Continent'] == 'Oceanie']
data_afrique = data[data['Continent'] == 'Afrique'][:10]
data_asie = data[data['Continent'] == 'Asie'][:10]
data_asie_occ = data[data['Continent'] == 'Asie occidentale'][:10]
data_autre = data[data['Continent'] == 'Autres']
def get_content():
return html.Div([
html.Div([
html.Div([
# Total Cases europe de l'est
dcc.Graph(
id='total-cases-europe-est',
figure={
'data': [
def main():
scraper.get_data()
