def fill_data():
countries = ['Portugal', 'Germany', 'Spain', 'France', 'USA', 'China', 'Russia', 'Japan']
for country in countries:
c = Country(name=country)
try:
db.session.add(c)
db.session.commit()
except Exception as e:
log.error("Update ViewMenu error: {0}".format(str(e)))
db.session.rollback()
try:
data = db.session.query(CountryStats).all()
if len(data) == 0:
for x in range(1, 40):
cs = CountryStats()
cs.population = random.randint(1, 100)
cs.unemployed = random.randint(1, 100)
cs.college = random.randint(1, 100)
year = random.choice(range(1900, 2012))
month = random.choice(range(1, 12))
day = random.choice(range(1, 28))
cs.stat_date = datetime.datetime(year, month, day)
cs.country_id = random.randint(1, len(countries))
db.session.add(cs)
db.session.commit()
except Exception as e:
log.error("Update Data error: {0}".format(str(e)))
db.session.rollback()
def get(self):
allCountries = CountryStats.gql('order by count desc').fetch(5000)
for c in allCountries:
memcache_key='city_queue'+'_'+c.countryCode+str(datetime.datetime.now().date())
if memcache.get(memcache_key):
logging.info('processed country %s skipping' % c.countryCode)
continue
logging.info('adding to queue %s' % c.countryCode)
taskqueue.add(queue_name='redundant',url='/stats/country/redundant/task', params={'country':c.countryCode})
memcache.set(memcache_key, 1)
def get(self):
'''
The class serving the page for showing statistics about countries.
'''
# Init selectable countries
selectable_countries = ['India', 'Pakistan', 'Bangladesh', 'United States', 'Vietnam', 'Romania',
'United Kingdom', 'Philippines', 'China', 'Sri Lanka', 'Ukraine', 'Canada', 'Egypt', 'Australia',
'Indonesia', 'Serbia and Montenegro', 'Bulgaria', 'Nepal', 'Poland', 'Russian Federation',
'Argentina', 'Italy', 'Spain', 'Brazil', 'Portugal', 'Germany', 'Turkey', 'Malaysia', 'Kenya',
'United Arab Emirates', 'Florida', 'Mexico', 'Nigeria', 'France', 'Greece', 'Morocco', 'Bolivia',
'Moldova', 'Croatia', 'South Africa', 'Sweden', 'Iran', 'Bosnia and Herzegovina', 'Macedonia',
'Singapore', 'Belarus', 'Hungary', 'Tunisia', 'Algeria', 'Peru', 'Netherlands', 'Hong Kong',
'Saudi Arabia', 'Israel', 'New Zealand', 'Estonia', 'Colombia', 'Denmark', 'Lithuania', 'Thailand',
'Armenia', 'Switzerland', 'Austria', 'Uruguay', 'Japan', 'Cyprus', 'Venezuela', 'Czech Republic',
'Kazakhstan', 'Slovak Republic', 'Albania', 'Dominican Republic', 'Lebanon', 'Slovenia', 'Chile',
'Mauritius', 'Ireland', 'Belgium', 'Latvia', 'Palestine', 'Jamaica', 'Georgia', 'Cameroon', 'Gambia',
'Puerto Rico', 'Qatar', 'Jordan', 'Costa Rica', 'Finland', 'El Salvador', 'Panama', 'Ethiopia', 'Norway'];
selectable_countries.sort();
# Get the selected countries
countries = self.request.get_all("country")
countries = [j for j in countries if j in selectable_countries]
log.info("Stats for countries: " + ",".join(countries))
#Get the stats from the database
stats = CountryStats.all()
stats.filter("country IN", countries)
expanded_stats = []
for s in stats:
s.expand()
log.info(s);
expanded_stats.append(s)
countries = [c.country for c in stats]
expanded_stats.sort(key=lambda x: x.country)
#Get the outbound bids from the database
bids = OutBidderBehavior.all()
bids.filter("country IN", countries)
expanded_bids = []
for b in bids:
b.expand()
log.info(b);
expanded_bids.append(b)
expanded_bids.sort(key=lambda x: x.country)
#Generate the page
template_values = { 'selectable_countries': selectable_countries, 'countries':countries,
'stats': expanded_stats, 'bids': expanded_bids }
template = jinja_environment.get_template('templates/country_stats.html')
self.response.out.write(template.render(template_values))
def countryFeed(self, reqId):
logging.info('Country stats feed')
countryStats= CountryStats.gql('WHERE count > 10 ORDER BY count desc ').fetch(40)
if countryStats is None:
logging.info('Not enough data for graph')
self.repsonse.out.write('Not enough data for graph')
return
countryStats = [ x for x in countryStats if x.countryCode != 'XX' and x.countryCode != 'EU' ]
logging.info('retrieved %s stats' % len(countryStats))
description = {"countryCode": ("string", "Country Code"),
"count":("number", "Count")}
columnnames = [ "countryCode", "count" ]
data_table = gviz_api.DataTable(description)
cntrCnt = []
for countryCnt in countryStats:
entry = {"countryCode": countryCnt.countryCode, "count":math.log(countryCnt.count)}
cntrCnt.append(entry)
data_table.LoadData(cntrCnt)
self.response.headers['Content-Type'] = 'text/plain'
self.response.out.write(data_table.ToJSonResponse(columns_order=(columnnames) , req_id=reqId))
def post(self):
countryCode=self.request.get('country',None)
memcache_key='reduncant_country'+'_'+countryCode+'_'+str(datetime.datetime.now().date())
if memcache.get(memcache_key):
logging.info('already processed %s %s' %countryCode)
return
redundant=CountryStats.gql('WHERE countryCode = :1', countryCode).fetch(1000)
if len(redundant) == 1:
logging.info('no duplicates for %s ' % countryCode)
return
logging.info('country: %s has duplidates: %s' %(countryCode, len(redundant)))
pivot = max(redundant, key=redundant.count)
logging.info('max stats:%s' % pivot.count)
redundant.remove(pivot)
for c in redundant:
pivot.count += c.count
if c.dateUpdated is not None and (pivot.dateUpdated is None or c.dateUpdated > pivot.dateUpdated):
pivot.dateUpdated = c.dateUpdated
c.delete()
pivot.put()
memcache.set(memcache_key, '1')
logging.info('new count %s' % pivot.count)
def fill_data():
countries = ['Portugal', 'Germany', 'Spain', 'France', 'USA', 'China', 'Russia', 'Japan']
politicals = ['Democratic', 'Authorative']
for country in countries:
c = Country(name=country)
try:
db.session.add(c)
db.session.commit()
except Exception as e:
log.error("Update ViewMenu error: {0}".format(str(e)))
db.session.rollback()
for political in politicals:
c = PoliticalType(name=political)
try:
db.session.add(c)
db.session.commit()
except Exception as e:
log.error("Update ViewMenu error: {0}".format(str(e)))
db.session.rollback()
try:
for x in range(1, 20):
cs = CountryStats()
cs.population = random.randint(1, 100)
cs.unemployed = random.randint(1, 100)
cs.college = random.randint(1, 100)
year = random.choice(range(1900, 2012))
month = random.choice(range(1, 12))
day = random.choice(range(1, 28))
cs.stat_date = datetime.datetime(year, month, day)
cs.country_id = random.randint(1, len(countries))
cs.political_type_id = random.randint(1, len(politicals))
db.session.add(cs)
db.session.commit()
except Exception as e:
log.error("Update ViewMenu error: {0}".format(str(e)))
db.session.rollback()
def aggregateData(self, currentSession, upper_limit_date):
logging.info("agregate data for %s" % currentSession.date)
locationData = RequestUtils.ipResolverAPI(currentSession.ip)
if len(locationData) == 2:
logging.info("updating location data")
city = locationData[0]
countryCode = locationData[1]
# self.response.out.write('location api response:<BR> city : %s; country: %s ' % (locationData[1], locationData[3]))
logging.info("location api response: %s " % locationData)
userLocation = UserLocationModel()
userLocation.user = currentSession.instaright_account
userLocation.city = city
userLocation.countryCode = countryCode
userLocation.date = currentSession.date
userLocation.put()
# update country stats and city stats
logging.info("country update")
existingCountryStat = CountryStats.gql("WHERE countryCode = :1 ", countryCode).get()
if existingCountryStat:
# hack to avoid exception
if existingCountryStat.count is None:
existingCountryStat.count = 1
else:
existingCountryStat.count += 1
logging.info("updating count %s" % existingCountryStat.count)
existingCountryStat.dateUpdated = upper_limit_date
existingCountryStat.put()
else:
logging.info("new country")
countryStat = CountryStats()
countryStat.countryCode = countryCode
countryStat.count = 1
countryStat.count = upper_limit_date
countryStat.put()
logging.info("city update")
existingCityStat = CityStats.gql("WHERE city = :1 and countryCode = :2", city, countryCode).get()
if existingCityStat:
# hack to avoid exception
if existingCityStat.count is None:
existingCityStat.count = 1
else:
existingCityStat.count += 1
existingCityStat.dateUpdated = upper_limit_date
logging.info("updating count %s" % existingCityStat.count)
existingCityStat.put()
else:
logging.info("new city")
cityStat = CityStats()
cityStat.countryCode = countryCode
cityStat.city = city
cityStat.count = 1
cityStat.updateDate = upper_limit_date
cityStat.put()
existingLinkStat = LinkStats.gql("WHERE link = :1", currentSession.url).get()
logging.info("link stats update")
if existingLinkStat:
logging.info("new link %s" % currentSession.url)
existingLinkStat.count = existingLinkStat.count + 1
existingLinkStat.countUpdated = currentSession.date
existingLinkStat.lastUpdatedBy = currentSession.instaright_account
existingLinkStat.put()
else:
logging.info("updating link stats: %s" % currentSession.url)
linkStat = LinkStats()
linkStat.link = currentSession.url
linkStat.count = 1
linkStat.countUpdated = currentSession.date
linkStat.lastUpdatedBy = currentSession.instaright_account
linkStat.put()
# domain update
shouldUpdateSession = 0
mode = ""
if currentSession.domain is None or currentSession.domain == "":
currentSession.domain = RequestUtils.getDomain(currentSession.url)
shouldUpdateSession = 1
mode = "domain change: %s" % currentSession.domain
if currentSession.date is None or currentSession.date == "":
date = datetime.datetime.strptime("2009-11-15", "%Y-%m-%d").date()
currentSession.date = date
shouldUpdateSession = 2
mode = "date change: %s" % date
if shouldUpdateSession > 0:
logging.info("updating session mode: %s" % mode)
currentSession.put()
else:
logging.info("unchanged session")
logging.info("done data aggregation")