def scraping():
mars_info = mongo.db.mars_info
#scrape_news()
#scrape_image()
#scrape_tweets()
#scrape_facts()
#scrape_hemispheres()
news = scrape_mars.scrape_news()
image = scrape_mars.scrape_image()
tweets = scrape_mars.scrape_tweets()
facts = scrape_mars.scrape_facts()
hemispheres = scrape_mars.scrape_hemispheres()
update_data = {
'news': news,
'image': image,
'tweets': tweets,
'facts': facts,
'hemispheres': hemispheres
}
# listings.update({}, listings_data, upsert=True)
mars_info.update({}, update_data, upsert=True)
return redirect("/", code=302)
def scrape():
result_dict = {}
db = client.mars_db
db.news_collection.drop()
db.mars_fact.drop()
db.mars_weather.drop()
db.mars_featured_images.drop()
db.scrape_hemispheres_images.drop()
db.news_collection.insert_many(scrape_news())
db.mars_fact.insert_one(scrape_facts())
db.mars_featured_images.insert_many(scrape_featured_images())
db.scrape_hemispheres_images.insert_many(scrape_hemispheres_images())
db.mars_weather.insert_many(scrape_weather())
result_dict['facts'] = list(db.mars_fact.find({}, {"_id": 0}))
result_dict['hemispheres'] = list(
db.scrape_hemispheres_images.find({}, {"_id": 0}))
result_dict['news'] = list(
db.news_collection.find({}, {
"_id": 0
}).limit(2))
result_dict['weather'] = list(
db.mars_weather.find({}, {
"_id": 0
}).limit(2))
result_dict['featured'] = list(
db.mars_featured_images.find({}, {
"_id": 0
}).limit(2))
return render_template("index.html", result_dict=result_dict)
def scrape():
# Run scraped functions
news = scrape_mars.scrape_news()
jpl = scrape_mars.scrape_jpl()
weather = scrape_mars.scrape_weather()
facts = scrape_mars.scrape_facts()
hemi = scrape_mars.scrape_hemisphere()
# Store results into a dictionary
mars_data = {
"news_title": news["news_title"],
"news_p": news["news_p"],
"featured_image_url": jpl["featured_image_url"],
"mars_weather": weather["mars_weather"],
"mars_facts": facts["mars_facts"],
"mars_hemisphere": hemi["mars_hemisphere"],
}
# Remove old record
mongo.db.collection.remove({})
# Insert forecast into database
mongo.db.collection.insert_one(mars_data)
# Redirect back to home page
return redirect("/", code=302)
def scraper():
# listings = mongo.db.listings
# listings_data = scrape_craigslist.scrape()
mars = mongo.db.mars
mars_web = scrape_mars.scrape_news()
mars_web = scrape_mars.scrape_marsImg()
mars_web = scrape_mars.scrape_marsTwitter()
mars_web = scrape_mars.scrape_marsFacts()
mars_web = scrape_mars.scrape_marsHemi1()
mars_web = scrape_mars.scrape_marsHemi2()
mars_web = scrape_mars.scrape_marsHemi3()
mars_web = scrape_mars.scrape_marsHemi4()
mars.update({}, mars_web, upsert=True)
return redirect("/", code=302)
def scrape():
# Scrape for mars news data
news_data = mongo.db.news_data
news_scrape_data = scrape_mars.scrape_news()
news_data.update({}, news_scrape_data, upsert=True)
# Scrape for mars images data
images_data = mongo.db.images_data
images_scrape_data = scrape_mars.scrape_images()
images_data.update({}, images_scrape_data, upsert=True)
# Scrape for mars weather data
weather_data = mongo.db.weather_data
weather_scrape_data = scrape_mars.scrape_weather()
weather_data.update({}, weather_scrape_data, upsert=True)
# Scrape for mars facts data
facts_data = mongo.db.facts_data
facts_scrape_data = scrape_mars.scrape_facts()
facts_data.update({}, facts_scrape_data, upsert=True)
# Scrape for mars hemisphere data
hemi_data = mongo.db.hemi_data
hemi_scrape_data = scrape_mars.scrape_hemi()
hemi_data.update({}, hemi_scrape_data, upsert=True)
return redirect("/", code=302)
def scrape():
mars_information = mongo.db.mars_information
mars_data = scrape_mars.scrape_news()
mars_data = scrape_mars.featured_image()
mars_data = scrape_mars.weather()
mars_data = scrape_mars.mars_facts()
mars_data = scrape_mars.mars_hemisphere()
mars_information.update({}, mars_data, upsert=True)
return redirect("/", code=302)
def scraper():
mars_data = mongo.db.mars_data
mars_info = scrape_mars.scrape_news()
mars_info = scrape_mars.scrape_image()
mars_info = scrape_mars.scrape_weather()
mars_info = scrape_mars.scrape_facts()
mars_info = scrape_mars.scrape_hemispheres()
mars_data.update({}, mars_info, upsert=True)
return redirect("/", code=302)
def scrape():
#run scrape function
mars_data = mongo.db.mars_data
mars_information = scrape_mars.scrape_news()
mars_information = scrape_mars.scrape_image()
mars_information = scrape_mars.scrape_facts()
mars_information = scrape_mars.scrape_hem()
mars_data.update({}, mars_information, upsert=True)
return redirect("/", code=302)
def scrape():
# Run the scrape functions
mars_data = scrape_mars.scrape_news()
mars_data = scrape_mars.scrape_image()
mars_data = scrape_mars.scrape_facts()
mars_data = scrape_mars.scrape_weather()
mars_data = scrape_mars.scrape_hemis()
# Update the Mongo database using update and upsert=True
mongo.db.collection.update({}, mars_data, upsert=True)
return redirect("/", code=302)
def scrape():
mars_dict = mongo.db.mars_dict
mars_data = scrape_mars.scrape_news()
mars_data = scrape_mars.scrape_image()
mars_data = scrape_mars.scrape_facts()
mars_data = scrape_mars.scrape_hemispheres()
# Update the Mongo Database using update and upsert=True
mars_dict.update({}, mars_data, upsert=True)
# Redirect back to home page
return redirect("/", code=302)
def scrape():
mars = mongo.db.mars
mars_web = scrape_mars.scrape_news()
mars_web = scrape_mars.scrape_marsImage()
mars_web = scrape_mars.scrape_marsWeather()
mars_web = scrape_mars.scrape_marsFacts()
mars_web = scrape_mars.scrape_mars_hemispheres()
mars.update({}, mars_web, upsert=True)
return redirect("/", code=302)
def scrape():
mars_db=mongo.db.mars_db
mars_data=scrape_mars.scrape_news()
mars_data=scrape_mars.scrape_img()
mars_data=scrape_mars.scrape_weather()
mars_data=scrape_mars.scrape_facts()
mars_data=scrape_mars.scrape_hemispheres()
mars_db.update({}, mars_data, upsert=True)
return redirect("/",code=302)
def scrape():
mars = mongo.db.mars
mars_web = scrape_mars.scrape_news()
mars_web = scrape_mars.scrape_marsImg()
mars_web = scrape_mars.scrape_marsFacts()
mars_web = scrape_mars.scrape_marsHemi1()
mars_web = scrape_mars.scrape_marsHemi2()
mars_web = scrape_mars.scrape_marsHemi3()
mars_web = scrape_mars.scrape_marsHemi4()
mars.update({}, mars_web, upsert=True)
return redirect("/", code=302)
def scrape():
# Run scrapped functions
mars_info = mongo.db.mars_info
mars_data = scrape_mars.scrape_news()
mars_data = scrape_mars.scrape_image()
mars_f = scrape_mars.scrape_facts()
mars_w = scrape_mars.scrape_weather()
mars_data = scrape_mars.scrape_hemisphere()
mars_info.update({}, mars_data, upsert=True)
return redirect("/", code=302)
def scrape():
all_mars_info = mongo.db.all_mars_info
new_data = scrape_mars.scrape_news()
new_data = scrape_mars.scrape_image()
new_data = scrape_mars.scrape_facts()
new_data = scrape_mars.scrape_hemispheres()
new_data = scrape_mars.scrape_weather()
all_mars_info.update({}, new_data, upsert=True)
return redirect("/", code=302)
def scrape():
# Run the scrape function
mars_data = scrape_mars.scrape_news()
title_paragraph = scrape_mars.scrape_news()
featured_image_url = scrape_mars.scrape_featured_image()
mars_facts_table = scrape_mars.scrape_facts()
mars_hemi = scrape_mars.scrape_hemi_img()
mars_news_data = {
'news_title_p': title_paragraph,
'featured_image_url': featured_image_url,
'table':mars_facts_table,
'hemisphere_image_urls': mars_hemi,
}
# update database
mongo.db.collection.update({}, mars_news_data, upsert=True)
mars_news_data = mongo.db.collection.find_one()
return render_template("scrape.html", mars_news_data = mars_news_data)
def scrape():
# Run the scrape function
mars_data = scrape_mars.scrape_news()
mars_data = scrape_mars.scrape_images()
mars_data = scrape_mars.scrape_facts()
mars_data = scrape_mars.scrape_hemisphere()
# Update the Mongo database using update and upsert=True
mongo.db.collection.update({}, mars_data, upsert=True)
# Redirect back to home page
return redirect("/")
def scrape():
# Run the scrape function
mars_info = mongo.db.mars_info
mars_data = scrape_mars.scrape_news()
mars_data = scrape_mars.scrape_feature()
mars_data = scrape_mars.scrape_facts()
mars_data = scrape_mars.scrape_weather()
# mars_data = scrape_mars.scrape_hemispheres()
mars_info.update({}, mars_data, upsert=True)
# Redirect back to home page
return redirect("/", code=302)
def scrape():
# Run scraped functions
mars_info = mongo.db.mars_info
mars_data = scrape_mars.scrape_news()
mars_data = scrape_mars.scrape_image()
mars_data = scrape_mars.scrape_weather()
mars_data = scrape_mars.scrape_facts()
mars_data = scrape_mars.scrape_hemi_1()
mars_data = scrape_mars.scrape_hemi_2()
mars_data = scrape_mars.scrape_hemi_3()
mars_data = scrape_mars.scrape_hemi_4()
mars_info.update({}, mars_data, upsert=True)
return redirect("/")
def scrape():
# Run the scrape function
# mars = client.db.mars
mars = mongo.db.mars
mars_web = scrape_mars.scrape_news()
mars_web = scrape_mars.scrape_marsImage()
mars_web = scrape_mars.scrape_marsTwitter()
mars_web = scrape_mars.scrape_marsFacts()
mars_web = scrape_mars.scrape_marsH1Cerberus()
mars_web = scrape_mars.scrape_marsH2Schiaparelli()
mars_web = scrape_mars.scrape_marsH3SyrtisMajor()
mars_web = scrape_mars.scrape_marsH4VallesMarineris()
mars.update({}, mars_web, upsert=True)
return redirect("/", code=302)
def scrape():
# Run the scrape function
# mars = client.db.mars
mars = mongo.db.mars
mars_web = scrape_mars.scrape_news()
mars_web = scrape_mars.scrape_marsImage()
mars_web = scrape_mars.scrape_marsFacts()
mars_web = scrape_mars.scrape_marsH1Cerberus()
mars_web = scrape_mars.scrape_marsH2Schiaparelli()
mars_web = scrape_mars.scrape_marsH3SyrtisMajor()
mars_web = scrape_mars.scrape_marsH4VallesMarineris()
# Update the Mongo Database using update and upsert=True
mars.update({}, mars_web, upsert=True)
# Redirect back to home page
return redirect("/", code=302)
def scrape():
# Run the scrape function
# mars = client.db.mars
mars = db.mars
mars_news = scrape_mars.scrape_news()
mars_news = scrape_mars.scrape_img()
mars_news = scrape_mars.scrape_marsfact()
mars_news = scrape_mars.scrape_cerhemi()
mars_news = scrape_mars.scrape_schhemi()
mars_news = scrape_mars.scrape_smhemi()
mars_news = scrape_mars.scrape_vmhemi()
# Update the Mongo Database using update and upsert=True
mars.update({}, mars_news, upsert=True)
# Redirect back to home page
return redirect("/", code=302)
def scraper():
news = mongo.db.news
news_data = scrape_mars.scrape_news()
news.update({}, news_data, upsert=True)
image = mongo.db.image
image_data = scrape_mars.scrape_image()
image.update({}, image_data, upsert=True)
weather = mongo.db.weather
weather_data = scrape_mars.scrape_weather()
weather.update({}, weather_data, upsert=True)
table = mongo.db.table
table_data = scrape_mars.scrape_facts()
table.update({}, table_data, upsert=True)
return redirect("/", code=302)
def scrape():
# Run scraped functions
news = scrape_mars.scrape_news()
featured = scrape_mars.scrape_featured()
weather = scrape_mars.scrape_weather()
table = scrape_mars.scrape_table()
images = scrape_mars.scrape_images()
mars_data = {
"news_title": news["news_title"],
"news_p": news["news_p"],
"featured_image_url": featured["featured_image_url"],
"weather": weather["mars_weather"],
"table": table["mars_table"],
"images": images["hemisphere_image_url"]
}
# Insert forecast into database
mongo.db.collection.insert_one(mars_data)
# Redirect back to home page
return redirect("http://localhost:5000/", code=302)
def scrape():
# Run scraped functions
news = scrape_mars.scrape_news()
weather = scrape_mars.scrape_weather()
facts = scrape_mars.scrape_facts()
hemispere = scrape_mars.scrape_hemisperes()
# Store results into a dictionary
listings = {
"news_headline": news["news_headline"],
"news_discription": news["news_discription"],
"news_weather": weather["news_weather"],
"mars_facts": facts["mars_facts"],
"hemisperes": hemispere["hemisperes"]
}
# Insert forecast into database
mongo.db.collection.insert_one(listings)
# Redirect back to home page
return redirect("http://localhost:5000/", code=302)
def scrape():
# run the news scrape function
news_data = scrape_mars.scrape_news()
# run the hemisphere scrape function
hemisphere_image_urls = scrape_mars.scrape_hemispheres()
# update dictionary for hemisphere scrape
for i in range(len(hemisphere_image_urls)):
news_data.update(hemisphere_image_urls[i])
# run the mars facts scrape function
mars_facts = scrape_mars.scrape_facts()
# update dictionary for mars facts scrape
news_data["mars_facts"] = mars_facts
# update the Mongo database using update and upsert=True
mongo.db.collection.drop()
mongo.db.collection.update({}, news_data, upsert=True)
# redirect back to home page
return redirect("/")
