def get(self, url_name):
company = Company.all().filter('urlname = ', url_name).fetch(1)
company = company[0]
reviews = company.GetReviews();
template_values = {
'reviews': reviews,
'company_name': company.name
}
self.Render(GetTemplate('company_reviews'), template_values)
def get(self, url_name):
company = Company.all().filter('urlname = ', url_name).fetch(1)
company = company[0]
reviews = db.GqlQuery("SELECT * FROM Review ORDER BY date DESC")
template_values = {
'reviews': reviews,
'company_name': company.name
}
self.Render(GetTemplate('review'), template_values)
def companies():
q = Company.all()
q.order("datetime")
companies = q.fetch(1000)
duplicates = []
for company in companies:
if company.name in duplicates:
db.delete(company)
else:
duplicates.append(company.name)
def get(self, company_name):
url_name = Company.UrlName(company_name)
company = Company.all().filter("urlname = ", url_name).fetch(1)
if company:
company = company[0]
else:
self.CompanyNotFound(company_name)
return
template_data = {
"company": company,
"reviews": [(r.text, r.rating.WeightedAverage()) for r in company.GetReviews(order="-date") if r.rating],
}
self.Render(GetTemplate("company_profile"), template_data)
def get(self):
companies = Company.all()
keys_names = []
for company in companies:
keys_names.append([company.key().id(),company.name])
template_values = {
'keys_names' : keys_names,
'companies' : companies
}
template = jinja_environment.get_template('companies.html')
self.response.out.write(template.render(template_values))
def count_all_stats():
q = Company.all()
companies = q.fetch(10000)
for company in companies:
company.pos_ctr = 0
company.neg_ctr = 0
articles = company.articles
for article in articles:
if article.sentiment == "positive":
company.pos_ctr += 1
elif article.sentiment == "negative":
company.neg_ctr += 1
company.put()
def post(self, url_name):
company = Company.all().filter('urlname = ', url_name).fetch(1)
company = company[0]
review = Review()
review.company = company
review.text = self.request.get('content')
rating = Rating()
rating.overall = self.GetIntOrDefault('overall')
rating.benefits = self.GetIntOrDefault('benefits')
rating.salary = self.GetIntOrDefault('salary')
rating.environment = self.GetIntOrDefault('environment')
rating.peers = self.GetIntOrDefault('peers')
rating.location = self.GetIntOrDefault('location')
rating.growth = self.GetIntOrDefault('growth')
rating.put()
review.rating = rating
review.put()
self.redirect('/companies/view/' + company.urlname)
def companies():
q = Company.all(keys_only=True)
# q.order("datetime") no datetime on keys
company_keys = q.fetch(1000)
duplicates = []
checked = []
delete_ctr = 0
for key in company_keys:
company = Company.get_by_id(key.id())
# logging.debug("company key id: %s", key.id())
# logging.debug("company key name: %s", key.name())
# logging.debug("company name: %s", company.name())
# if company.name == "General Electric Company":
# company.exchange = "NYSE"
# company.put()
# # db.delete(company)
#dev only:
# #comp_list = ["GOOG","FB","AAPL","GE","TSLA"]
# if company.exchange == "NYSE" and company.ticker != "GE":
# if company.ticker == "IBM":
# db.delete(company)
if company.name in duplicates:
db.delete(company)
delete_ctr += 1
else:
duplicates.append(company.name)
checked.append(company)
duplicates = []
for company in checked:
if company.ticker in duplicates:
db.delete(company)
delete_ctr += 1
else:
duplicates.append(company.ticker)
logging.debug("deleted %s duplicate companies", delete_ctr)
def get(self):
path = GetTemplate("company_list")
companies = Company.all().order("name")
autocomplete_list = ",".join(['"%s"' % c.name for c in companies])
template_data = {"companies": companies, "autocomplete_list": autocomplete_list}
self.Render(path, template_data)
def getContributionCEO(contribution):
company = Company.all().filter("name =", contribution.get("company")).get()
ceo = {}
ceo["to_email"] = company.ceo_email
ceo["to_name"] = company.ceo_name
return ceo
def scrape():
# q = Company.all()
# companies = q.fetch(10000)
# for company in companies:
# process_links(company)
# company = Company()
# company.name = "Microsoft Corporation"
# company.name_lower = "microsoft corporation"
# company.ticker = "MSFT"
# company.ticker_lower = "msft"
# company.exchange = "NASDAQ"
# company.put()
q = Company.all() # her resettes det til start, ja.
q.order("datetime") # nyeste sist, slik at de er dukker opp etter cursor
# company_cursor = memcache.get("company_cursor")
# if company_cursor:
# q.with_cursor(start_cursor = company_cursor)
# chunk_size = 5
companies = q.fetch(10000) # should be optimized by storing companies in memcache
# duplicate_check = memcache.get("duplicate_check")
scrape_ctr = 0
# bongo = "bongo"
# bingi = "bingi"
# memcache.add("bongo bongo", bongo)
# memcache.set("bingi bingi", bingi)
for company in companies:
# logging.debug("company: %s", company.name)
article_keys = memcache.get("article_keys")
# logging.debug("one more company")
new_article_keys = process_links(company)
logging.debug("scraped %s links for %s", len(new_article_keys), company.name)
scrape_ctr += len(new_article_keys)
# c.append(company.name)
if new_article_keys:
if article_keys:
# logging.debug("article keys exist in scrape")
article_keys = new_article_keys + article_keys
# memcache.delete("article_keys")
memcache.set("article_keys", article_keys, 7200) # clean does not update, so store until analyze.
# if duplicate_check:
# memcache.set("duplicate_check",article_keys, 11000)
# else:
# memcache.add("duplicate_check",article_keys, 11000)
else:
# logging.debug("article keys do NOT exist in scrape")
# memcache.delete("article_keys") # used when soup line in clean gets error.
memcache.set("article_keys",new_article_keys, 7200) # SET works. ADD does not.
# logging.debug("new article keys added to memcache")
# if duplicate_check:
# memcache.set("duplicate_check",new_article_keys, 11000) # tvilsom.
# # pass
# else:
# memcache.add("duplicate_check",new_article_keys, 11000)
else: # no new articles
# logging.debug("NO new articles")
if article_keys:
memcache.set("article_keys",article_keys)
# a = memcache.get("bongo bongo")
# if a:
# logging.debug("bongo bongo")
# b = memcache.get("bingi bingi")
# if b:
# logging.debug("bingi bingi")
# ak = memcache.get("article_keys")
# if ak:
# logging.debug("ak exists in scrape")
# else:
# logging.debug("ak does NOT exist in scrape")
logging.debug("scraped %s articles", scrape_ctr)
