def figs(username, password):
"""Generates figures of trends for pairs of trends.
Plots one from list "goods" with one from list "celebs". Each pair creates plot.
"""
goods = ('martin luther king', 'muhammad yunus', 'mother teresa', 'the madonna', 'martti ahtisaari', 'gandhi', 'nelson mandela')
celebs = ('kanye west', 'paris hilton', 'madonna', 'kim kardashian', 'jessica simpson', 'hannah montana', 'brad pitt')
pygt = pyGTrends.pyGTrends(username, password)
pygt.download_report(goods)
date, rep1ar = csvarray(pygt)
# goods = pygt.keywords
xaxis1 = range(len(date))
pygt = pyGTrends.pyGTrends(username, password)
pygt.download_report(celebs)
# celebs = pygt.keywords
date, rep2ar = csvarray(pygt)
xaxis2 = range(len(date))
skip = 30
for i in range(len(rep1ar)):
if len(rep1ar[i]) == 0: continue
for j in range(len(rep2ar)):
if len(rep2ar[j]) == 0: continue
pylab.plot(xaxis1, rep1ar[i], label=goods[i])
pylab.plot(xaxis2, rep2ar[j], label=celebs[j])
pylab.xticks(range(0, len(date), skip), (date[::skip]), rotation=60, size='x-small')
pylab.ylabel('Popularity')
pylab.legend(loc=2)
pylab.title('Relative popularity plot # %i' % (random.randint(1,1000)))
pylab.savefig('compare%i.png' % (j + i*len(goods)))
pylab.clf()
def getGTData(self, search_query = "debt", date="all", geo="all", scale="1", position = "end" ):
print search_query
dict = {}
endlist = []
countlist = []
connector = pyGTrends( "larotondeapi", "dstegroup")
connector.download_report( ( search_query )
, date = date
, geo = geo
, scale = scale )
#connector.writer()
# print connector.getData()
d = csv.DictReader( connector.getData().split('\n'))
x = connector.csv( section='Main', as_list=True )
for item in x:
date = item[0]
if " " and "-" in date:
enddate = date.split(" ")[2]
if (len(item[1])>0):
try:
count = int(item[1])
dict[enddate] = count
endlist.append(enddate[5:])
countlist.append(count)
except:
break
print dict
print endlist[-62:]
print countlist[-62:]
def getGTData(search_query, date="all", geo="all", scale="1", position="end"):
connector = pyGTrends(google_username, google_password)
connector.download_report(search_query, date=date, geo=geo, scale=scale)
data = connector.csv(section='Main').split('\n')
csv_reader = csv.reader(data)
# remove all whitespaces
search_query = search_query.strip()
search_query = " ".join(search_query.split())
search_query = search_query.replace(" ", "")
with open( search_query + '_google_report.csv', 'w') as csv_out:
positionInWeek = { "start" : 0, "end" : 1 }
separator = " - "
csv_writer = csv.writer( csv_out )
#print "LOOPING ON ENTRIES"
for count, row in enumerate( csv_reader ):
if separator not in row[0] :
csv_writer.writerow( row )
continue
date = row[0].split( separator )[ positionInWeek[ position ] ]
# we want to remove any whitespaces from the value entry since we are not interested in blank data points
val = re.sub(r'\s+', '', row[1] )
if len(val) < 1 :
#print "We will skip Null value at date %s" % str(date)
continue
#row = str(date)+','+str(val)
if count == 0:
csv_writer.writerow( row )
else:
csv_writer.writerow( [ str(date) ] + [ str(val) ])
print "File saved: %s " % ( search_query + '_google_report.csv' )
def getGTData( search_query , date, geo, cmpt, cid , export, reqId ) :
#output file name
fnametag = search_query + "_suburban"
if cid == "GEO_MAP_ANIMATION_0_2":
fnametag= search_query + "_metro"
q = search_query
print "search_query ",search_query
print "date ",date
print "geo ",geo
print "q ",q
print "cmpt ",cmpt
print "cid ",cid
print "export ",export
print "reqId ",reqId
connector = pyGTrends( google_username, google_password )
connector.download_report( ( search_query )
, date = date
, geo = geo
,q = q, cmpt = cmpt, cid = cid, export = export, reqId = reqId
)
rsp=connector.getData()
print rsp
#extract the json
regex= 'animationResponse\((.+?)\);'
pattern=re.compile(regex)
json_rsp=re.findall(pattern,rsp)
for j in json_rsp:
json_rsp1 = j
#print json_rsp1, "xxx\n"
rsp = json.loads(json_rsp1)
print rsp['data'][0], "xxx\n"
print "first: ", rsp['data'][0]["frameData"], "xxx\n"
print "second: ", rsp["data"][0]["timeStr"], "xxx\n"
with open( fnametag + '_google_report.csv', 'w') as csv_out:
csv_writer = csv.writer( csv_out )
print len(rsp['data'])
#outer loop
for index in range(len(rsp['data'])):
print rsp['data'][index]
col4=rsp["data"][index]["timeStr"]
for innerindex in range(len(rsp['data'][index]['frameData'])):
col1=rsp['data'][index]['frameData'][innerindex][0]
col2=rsp['data'][index]['frameData'][innerindex][1]
col3=rsp['data'][index]['frameData'][innerindex][2]
#print col1,col2,col3,col4
csv_writer.writerow( [ col1 ] + [ col2 ] + [ col3 ] + [ col4 ])
print "File saved: %s " % ( fnametag + '_google_report.csv' )
def getGTData(search_query="debt",
date="all",
geo="all",
scale="1",
position="end"):
connector = pyGTrends(google_username, google_password)
connector.download_report((search_query), date=date, geo=geo, scale=scale)
#connector.writer()
#print connector.getData()
#d = DictReader( connector.getData().split('\n'))
#print csv.DictReader( connector.csv().split('\n'))
#print connector.csv( section='Main', as_list=True )
data = connector.csv(section='Main').split('\n')
csv_reader = csv.reader(data)
# example using City to cache search terms
#csv_reader = csv.reader( connector.csv( section='City' ).split('\n') )
#for row in csv_reader:
# print row
#data1 = read_csv_data( data )
#print data1
csv_reader = csv.reader(data)
#fields = csv_reader.next() # Read the column names
# remove all whitespaces
search_query = search_query.strip()
search_query = " ".join(search_query.split())
search_query = search_query.replace(" ", "")
with open(search_query + '_google_report.csv', 'w') as csv_out:
positionInWeek = {"start": 0, "end": 1}
separator = " - "
csv_writer = csv.writer(csv_out)
#print "LOOPING ON ENTRIES"
for count, row in enumerate(csv_reader):
if separator not in row[0]:
csv_writer.writerow(row)
continue
date = row[0].split(separator)[positionInWeek[position]]
# we want to remove any whitespaces from the value entry since we are not interested in blank data points
val = re.sub(r'\s+', '', row[1])
if len(val) < 1:
#print "We will skip Null value at date %s" % str(date)
continue
#row = str(date)+','+str(val)
if count == 0:
csv_writer.writerow(row)
else:
csv_writer.writerow([str(date)] + [str(val)])
print "File saved: %s " % (search_query + '_google_report.csv')
def getGTData( search_query = "debt", date="all", geo="all", scale="1", position = "end" ) :
connector = pyGTrends( google_username, google_password )
connector.download_report( ( search_query )
, date = date
, geo = geo
, scale = scale )
#connector.writer()
#print connector.getData()
#d = DictReader( connector.getData().split('\n'))
#print csv.DictReader( connector.csv().split('\n'))
#print connector.csv( section='Main', as_list=True )
data = connector.csv( section='Main' ).split('\n')
csv_reader = csv.reader( data )
# example using City to cache search SEARCHTERMs
#csv_reader = csv.reader( connector.csv( section='City' ).split('\n') )
#for row in csv_reader:
# print row
#data1 = read_csv_data( data )
#print data1
csv_reader = csv.reader( data )
#fields = csv_reader.next() # Read the column names
# remove all whitespaces
search_query = search_query.strip()
search_query = " ".join( search_query.split() )
search_query = search_query.replace(" ", "")
with open( search_query + '_google_report.csv', 'w') as csv_out:
positionInWeek = { "start" : 0, "end" : 1 }
separator = " - "
csv_writer = csv.writer( csv_out )
#print "LOOPING ON ENTRIES"
for count, row in enumerate( csv_reader ):
if separator not in row[0] :
csv_writer.writerow( row )
continue
date = row[0].split( separator )[ positionInWeek[ position ] ]
# we want to remove any whitespaces from the value entry since we are not interested in blank data points
val = re.sub(r'\s+', '', row[1] )
if len(val) < 1 :
#print "We will skip Null value at date %s" % str(date)
continue
#row = str(date)+','+str(val)
if count == 0:
csv_writer.writerow( row )
else:
csv_writer.writerow( [ str(date) ] + [ str(val) ])
print "File saved: %s " % ( search_query + '_google_report.csv' )
def download_data(region, device):
while True:
try:
connector = pyGTrends(google_username, google_password)
except Exception, e:
if str(e).startswith('Cannot parse GALX'):
continue
else:
raise
else:
break
def download_data(region, device):
while True:
try:
connector = pyGTrends(google_username, google_password)
except Exception, e:
if str(e).startswith('Cannot parse GALX'):
continue
else:
raise
else:
break
def getGTData( search_query = "debt", date="all", geo="all", scale="1", position = "end" ) :
connector = pyGTrends( google_username, google_password )
connector.download_report( ( search_query )
, date = date
, geo = geo
, scale = scale )
connector.writer()
# print connector.getData()
# d = DictReader( connector.getData().split('\n'))
# print csv.DictReader( connector.csv().split('\n'))
# print connector.csv( section='Main', as_list=True )
# data = connector.csv( section='Main' ).split('\n')
# csv_reader = csv.reader( data )
return connector.getData()
def getGTData(search_query="debt",
date="all",
geo="all",
scale="1",
position="end"):
connector = pyGTrends(google_username, google_password)
connector.download_report((search_query), date=date, geo=geo, scale=scale)
connector.writer()
# print connector.getData()
# d = DictReader( connector.getData().split('\n'))
# print csv.DictReader( connector.csv().split('\n'))
# print connector.csv( section='Main', as_list=True )
# data = connector.csv( section='Main' ).split('\n')
# csv_reader = csv.reader( data )
return connector.getData()
def index():
global connection
global popList
popList = popularList()
if request.method == "GET":
if connection == None:
return render_template("index.html", logged=False)
else:
return render_template("index.html", logged=True, popular=popList)
else:
google_username = request.form['username']
google_password = request.form['password']
try:
connection = pyGTrends(google_username, google_password)
except:
return render_template("index.html", err="Incorrect Username or Password", logged=False)
return render_template("index.html", logged=True, popular=popList)
return render_template("index.html", logged=False)
def googletrends(keywords, start_month, start_year, duration=12, geo="US"):
# Quta limit reached for 16/05 Don't call
connector = pyGTrends.pyGTrends("*****@*****.**", "RB$Hackathon")
querydate = str(start_month) + "/" + str(start_year) + " " + str(
duration) + "m"
#print querydate
connector.download_report(keywords, geo=geo, date=querydate)
result = []
try:
result = connector.csv().split("\n")
except Exception as e:
if "Could not find requested section" in e.__str__():
with open("data/googletrends.csv", 'rb') as f:
reader = csv.reader(f)
for row in reader:
result.append(row[0] + "," + row[1])
#print result
del result[0]
response = []
previousnumber = 100
for r in result:
s = r.split(" ")
date = datetime.strptime(s[0], '%Y-%m-%d')
number = s[2].split(",")[1]
diff = int(number) - previousnumber
text = ""
if diff >= 0:
text = "The search for " + keywords + " has increased " + str(
diff) + "%" + " on Google this week"
else:
text = "The search for " + keywords + " has decreased " + str(
diff) + "%" + " on Google this week"
response += [{
'timestamp': date.isoformat(' '),
'description': text,
'source': 'google',
'author': 'zhuangy',
'sentiment': NaiveBayesAnalyzerParser(text)
}]
previousnumber = int(number)
#print len(response)
return response
def index():
global connection
global popList
popList = popularList()
if request.method == "GET":
if connection == None:
return render_template("index.html", logged=False)
else:
return render_template("index.html", logged=True, popular=popList)
else:
google_username = request.form['username']
google_password = request.form['password']
try:
connection = pyGTrends(google_username, google_password)
except:
return render_template("index.html",
err="Incorrect Username or Password",
logged=False)
return render_template("index.html", logged=True, popular=popList)
return render_template("index.html", logged=False)
def googletrends(keywords, start_month, start_year, duration=12, geo="US"):
# Quta limit reached for 16/05 Don't call
connector = pyGTrends.pyGTrends("*****@*****.**", "RB$Hackathon" )
querydate=str(start_month)+"/"+str(start_year)+" "+str(duration)+"m"
#print querydate
connector.download_report(keywords,geo=geo,date=querydate)
result = []
try:
result = connector.csv().split("\n")
except Exception as e:
if "Could not find requested section" in e.__str__():
with open("data/googletrends.csv", 'rb') as f:
reader = csv.reader(f)
for row in reader:
result.append(row[0]+","+row[1])
#print result
del result[0]
response = []
previousnumber = 100
for r in result:
s = r.split(" ")
date = datetime.strptime(s[0], '%Y-%m-%d')
number = s[2].split(",")[1]
diff = int(number) - previousnumber
text=""
if diff >= 0:
text = "The search for " + keywords+ " has increased "+ str(diff) + "%" +" on Google this week"
else:
text = "The search for " + keywords+ " has decreased "+ str(diff) + "%" +" on Google this week"
response += [{
'timestamp': date.isoformat(' '),
'description': text,
'source':'google',
'author':'zhuangy',
'sentiment': NaiveBayesAnalyzerParser(text)
}]
previousnumber = int(number)
#print len(response)
return response
__author__ = 'fang'
from pyGTrends import pyGTrends
connector = pyGTrends('*****@*****.**', 'amy8426ainos')
connector.download_report(('apple', 'microsoft'))
print connector.csv()
from pyGTrends import pyGTrends
from pyGparse import pyGparse
import httplib
import urllib
import urllib2
connector = pyGTrends('edmund.m.hart','3dmundh4rt')
connector.download_report(('jaguar', 'bmw'),)
3:])
while timestamp < timestamp2:
update_report(candidat, value, timestamp, regionName)
timestamp = timestamp + 60 * 60 * 24 * 1000
if in_region:
region = regions[row[0]]['name']
value = int(row[1])
print(region)
candidat['geoReport'][region] = value
elif len(row) > 0 and row[0] == "Sous-région":
in_region = True
i = i + 1
db.candidat.update({"_id": candidat['_id']}, candidat)
connector = pyGTrends('avricot.team', 'table87table')
connection = Connection('serv.avricot.com', 27017)
db = connection.prediction
for candidat in db.candidat.find({}):
if candidat['candidatName'] == "SARKOZY":
sarkozy_query = build_query(candidat)
if candidat['candidatName'] == "HOLLANDE":
hollande_query = build_query(candidat)
j = 0
for candidat in db.candidat.find({}):
for region in regions:
if j > 1:
#break
time.sleep(15)
j = j + 1
release_date = [ "2012-07-15"] # , "2014-02-10", "2015-03-17", "2014-11-10", "2014-08-18"]
release_index = [241] #, 362, 382, 362, 350]
gprop_list = ["youtube"] #, ""]
country_list = []
country_map = {}
infi = open("codes.csv")
tx = infi.readlines()
for line in tx:
cut = line.split(',')
country_list.append(cut[0])
country_map[cut[0]] = cut[1].strip()
path = "/home/calvin/Data/"
# comment/uncomment below to download
con = pyGTrends(google_username, google_password)
for k in keyword_list:
for g in gprop_list:
for c in country_list:
con.request_report(k, gprop=g, geo=c)
time.sleep(randint(4, 9))
file_name = k + "_" + g + "_" + c
con.save_csv(path, file_name)
# let's parse these into a JSON file
j_m = {}
index = -1
return self._stream.close()
if __name__=="__main__":
# read GOOGLE ID/PW from local control file
f=open(GIDPWFile, 'r')
f.readline() #skip the file instruction line
google_username = f.readline()
google_password = f.readline()
print "user name = " + google_username
print "user pw = " + google_password
# read Search_Terms_Control file to get list of search terms
i=0
f=codecs.open(Search_Terms_Control_File,'r', encoding='utf-16')
#list_of_Search_Terms = f.readlines()
list_of_Search_Terms = list(f)
for Search_Term in list_of_Search_Terms:
i+=1
print "(" + str(i) + ") " + Search_Term
# sign on to google
connector = pyGTrends( google_username, google_password )
#Get Trend data for each line in Search Terms control File - iterating over the GEOs control file lines
for Search_Term in list_of_Search_Terms :
#if getGoogleTrendData( search_query = Search_Term, geo = Geo_Dictionary.keys()) :
if getGoogleTrendData( search_query = Search_Term, geo = ['xx']) :
print "Google Trend Data processing complete"
def getGTData(search_query, date, geo, cmpt, cid, export, reqId):
#output file name
fnametag = search_query + "_suburban"
if cid == "GEO_MAP_ANIMATION_0_2":
fnametag = search_query + "_metro"
q = search_query
print "search_query ", search_query
print "date ", date
print "geo ", geo
print "q ", q
print "cmpt ", cmpt
print "cid ", cid
print "export ", export
print "reqId ", reqId
connector = pyGTrends(google_username, google_password)
connector.download_report((search_query),
date=date,
geo=geo,
q=q,
cmpt=cmpt,
cid=cid,
export=export,
reqId=reqId)
rsp = connector.getData()
print rsp
#extract the json
regex = 'animationResponse\((.+?)\);'
pattern = re.compile(regex)
json_rsp = re.findall(pattern, rsp)
for j in json_rsp:
json_rsp1 = j
#print json_rsp1, "xxx\n"
rsp = json.loads(json_rsp1)
print rsp['data'][0], "xxx\n"
print "first: ", rsp['data'][0]["frameData"], "xxx\n"
print "second: ", rsp["data"][0]["timeStr"], "xxx\n"
with open(fnametag + '_google_report.csv', 'w') as csv_out:
csv_writer = csv.writer(csv_out)
print len(rsp['data'])
#outer loop
for index in range(len(rsp['data'])):
print rsp['data'][index]
col4 = rsp["data"][index]["timeStr"]
for innerindex in range(len(rsp['data'][index]['frameData'])):
col1 = rsp['data'][index]['frameData'][innerindex][0]
col2 = rsp['data'][index]['frameData'][innerindex][1]
col3 = rsp['data'][index]['frameData'][innerindex][2]
#print col1,col2,col3,col4
csv_writer.writerow([col1] + [col2] + [col3] + [col4])
print "File saved: %s " % (fnametag + '_google_report.csv')
def trend_csv_download():
r = pyGTrends('my email', 'passwd')
r.download_report(('spain', 'wine'))### read it from csv file
print DictReader(r.csv().split('\n'))
# recieve search:
# /search/'STRING 1,STRING 2'
# 1 sender:
# /player1/progress/
# /player1/reset/
google_username = '******'
google_password = '******'
csv_path = 'csv/'
if not os.path.exists(csv_path):
os.makedirs(csv_path)
#connect to Google
connector = pyGTrends(google_username, google_password)
players = [
{
'id': 1,
'ip': '10.0.1.11',
# 'ip': '192.168.1.73',
'port': 5001,
},
{
'id': 2,
'ip': '192.168.1.70',
'port': 5001,
}
]
while timestamp<timestamp2:
update_report(candidat, value, timestamp, regionName)
timestamp = timestamp + 60*60*24*1000
if in_region:
region = regions[row[0]]['name']
value = int(row[1])
print(region)
candidat['geoReport'][region] = value
elif len(row)>0 and row[0] == "Sous-région":
in_region = True
i=i+1
db.candidat.update({"_id": candidat['_id']}, candidat)
connector = pyGTrends('avricot.team','table87table')
connection = Connection('serv.avricot.com', 27017)
db = connection.prediction
for candidat in db.candidat.find({}):
if candidat['candidatName'] == "SARKOZY":
sarkozy_query = build_query(candidat)
if candidat['candidatName'] == "HOLLANDE":
hollande_query = build_query(candidat)
j=0
for candidat in db.candidat.find({}):
for region in regions:
if j>1:
#break
time.sleep(15)
j=j+1
from pyGTrends import pyGTrends
connector = pyGTrends('*****@*****.**', 'tilaivytila77')
connector.download_report(('drought', 'aids', 'obama'))
csv = connector.csv()
o = open( 'report.csv', "wb" )
o.write( csv )
def __init__(self,username,password):
self._r = ptrends.pyGTrends(constants.Username, constants.Password)
import urllib
from pyGTrends import pyGTrends
import ystockquote
from datetime import date
#my module to convert dates into a numerical string in the YMD format
from date_converter import date_converter
#this will need to be changed as it currently uses my information
connector = pyGTrends("USERNAME","PASSWORD")
today = str(date.today())
today = today.replace('-','')
#prompts the user for a stock and converts it to upper case
#prompt = ">>> "
#print "Stock Symbol?"
#stock_symbol = raw_input(prompt)
#stock_symbol = stock_symbol.upper()
#print "Data type?"
#metric_type = raw_input(prompt)
class ychart_data(object):
#sets up the initial varibles
#defines the chart type, and sets up the URL
def __init__(self, symbol, data_type):
self.url = 'http://ycharts.com/companies/%s/%s' % (symbol, data_type)
self.raw_url = urllib.urlopen(self.url).read()
self.raw_url = self.raw_url.split('\n')
from pyGTrends import pyGTrends
from csv import DictReader
connector = pyGTrends('google username', 'google password')
connector.download_report(('occupy wall street', 'Õ¼Á컪¶û½Ö'))
# date='2011-10',
# geo='AT',
# scale=1)
print connector.csv()
# print connector.csv(section='Language')
# print DictReader(connector.csv().split('\n'))
