def figs(username, password): """Generates figures of trends for pairs of trends. Plots one from list "goods" with one from list "celebs". Each pair creates plot. """ goods = ('martin luther king', 'muhammad yunus', 'mother teresa', 'the madonna', 'martti ahtisaari', 'gandhi', 'nelson mandela') celebs = ('kanye west', 'paris hilton', 'madonna', 'kim kardashian', 'jessica simpson', 'hannah montana', 'brad pitt') pygt = pyGTrends.pyGTrends(username, password) pygt.download_report(goods) date, rep1ar = csvarray(pygt) # goods = pygt.keywords xaxis1 = range(len(date)) pygt = pyGTrends.pyGTrends(username, password) pygt.download_report(celebs) # celebs = pygt.keywords date, rep2ar = csvarray(pygt) xaxis2 = range(len(date)) skip = 30 for i in range(len(rep1ar)): if len(rep1ar[i]) == 0: continue for j in range(len(rep2ar)): if len(rep2ar[j]) == 0: continue pylab.plot(xaxis1, rep1ar[i], label=goods[i]) pylab.plot(xaxis2, rep2ar[j], label=celebs[j]) pylab.xticks(range(0, len(date), skip), (date[::skip]), rotation=60, size='x-small') pylab.ylabel('Popularity') pylab.legend(loc=2) pylab.title('Relative popularity plot # %i' % (random.randint(1,1000))) pylab.savefig('compare%i.png' % (j + i*len(goods))) pylab.clf()
def getGTData(self, search_query = "debt", date="all", geo="all", scale="1", position = "end" ): print search_query dict = {} endlist = [] countlist = [] connector = pyGTrends( "larotondeapi", "dstegroup") connector.download_report( ( search_query ) , date = date , geo = geo , scale = scale ) #connector.writer() # print connector.getData() d = csv.DictReader( connector.getData().split('\n')) x = connector.csv( section='Main', as_list=True ) for item in x: date = item[0] if " " and "-" in date: enddate = date.split(" ")[2] if (len(item[1])>0): try: count = int(item[1]) dict[enddate] = count endlist.append(enddate[5:]) countlist.append(count) except: break print dict print endlist[-62:] print countlist[-62:]
def getGTData(search_query, date="all", geo="all", scale="1", position="end"): connector = pyGTrends(google_username, google_password) connector.download_report(search_query, date=date, geo=geo, scale=scale) data = connector.csv(section='Main').split('\n') csv_reader = csv.reader(data) # remove all whitespaces search_query = search_query.strip() search_query = " ".join(search_query.split()) search_query = search_query.replace(" ", "") with open( search_query + '_google_report.csv', 'w') as csv_out: positionInWeek = { "start" : 0, "end" : 1 } separator = " - " csv_writer = csv.writer( csv_out ) #print "LOOPING ON ENTRIES" for count, row in enumerate( csv_reader ): if separator not in row[0] : csv_writer.writerow( row ) continue date = row[0].split( separator )[ positionInWeek[ position ] ] # we want to remove any whitespaces from the value entry since we are not interested in blank data points val = re.sub(r'\s+', '', row[1] ) if len(val) < 1 : #print "We will skip Null value at date %s" % str(date) continue #row = str(date)+','+str(val) if count == 0: csv_writer.writerow( row ) else: csv_writer.writerow( [ str(date) ] + [ str(val) ]) print "File saved: %s " % ( search_query + '_google_report.csv' )
def getGTData( search_query , date, geo, cmpt, cid , export, reqId ) : #output file name fnametag = search_query + "_suburban" if cid == "GEO_MAP_ANIMATION_0_2": fnametag= search_query + "_metro" q = search_query print "search_query ",search_query print "date ",date print "geo ",geo print "q ",q print "cmpt ",cmpt print "cid ",cid print "export ",export print "reqId ",reqId connector = pyGTrends( google_username, google_password ) connector.download_report( ( search_query ) , date = date , geo = geo ,q = q, cmpt = cmpt, cid = cid, export = export, reqId = reqId ) rsp=connector.getData() print rsp #extract the json regex= 'animationResponse\((.+?)\);' pattern=re.compile(regex) json_rsp=re.findall(pattern,rsp) for j in json_rsp: json_rsp1 = j #print json_rsp1, "xxx\n" rsp = json.loads(json_rsp1) print rsp['data'][0], "xxx\n" print "first: ", rsp['data'][0]["frameData"], "xxx\n" print "second: ", rsp["data"][0]["timeStr"], "xxx\n" with open( fnametag + '_google_report.csv', 'w') as csv_out: csv_writer = csv.writer( csv_out ) print len(rsp['data']) #outer loop for index in range(len(rsp['data'])): print rsp['data'][index] col4=rsp["data"][index]["timeStr"] for innerindex in range(len(rsp['data'][index]['frameData'])): col1=rsp['data'][index]['frameData'][innerindex][0] col2=rsp['data'][index]['frameData'][innerindex][1] col3=rsp['data'][index]['frameData'][innerindex][2] #print col1,col2,col3,col4 csv_writer.writerow( [ col1 ] + [ col2 ] + [ col3 ] + [ col4 ]) print "File saved: %s " % ( fnametag + '_google_report.csv' )
def getGTData(search_query="debt", date="all", geo="all", scale="1", position="end"): connector = pyGTrends(google_username, google_password) connector.download_report((search_query), date=date, geo=geo, scale=scale) #connector.writer() #print connector.getData() #d = DictReader( connector.getData().split('\n')) #print csv.DictReader( connector.csv().split('\n')) #print connector.csv( section='Main', as_list=True ) data = connector.csv(section='Main').split('\n') csv_reader = csv.reader(data) # example using City to cache search terms #csv_reader = csv.reader( connector.csv( section='City' ).split('\n') ) #for row in csv_reader: # print row #data1 = read_csv_data( data ) #print data1 csv_reader = csv.reader(data) #fields = csv_reader.next() # Read the column names # remove all whitespaces search_query = search_query.strip() search_query = " ".join(search_query.split()) search_query = search_query.replace(" ", "") with open(search_query + '_google_report.csv', 'w') as csv_out: positionInWeek = {"start": 0, "end": 1} separator = " - " csv_writer = csv.writer(csv_out) #print "LOOPING ON ENTRIES" for count, row in enumerate(csv_reader): if separator not in row[0]: csv_writer.writerow(row) continue date = row[0].split(separator)[positionInWeek[position]] # we want to remove any whitespaces from the value entry since we are not interested in blank data points val = re.sub(r'\s+', '', row[1]) if len(val) < 1: #print "We will skip Null value at date %s" % str(date) continue #row = str(date)+','+str(val) if count == 0: csv_writer.writerow(row) else: csv_writer.writerow([str(date)] + [str(val)]) print "File saved: %s " % (search_query + '_google_report.csv')
def getGTData( search_query = "debt", date="all", geo="all", scale="1", position = "end" ) : connector = pyGTrends( google_username, google_password ) connector.download_report( ( search_query ) , date = date , geo = geo , scale = scale ) #connector.writer() #print connector.getData() #d = DictReader( connector.getData().split('\n')) #print csv.DictReader( connector.csv().split('\n')) #print connector.csv( section='Main', as_list=True ) data = connector.csv( section='Main' ).split('\n') csv_reader = csv.reader( data ) # example using City to cache search SEARCHTERMs #csv_reader = csv.reader( connector.csv( section='City' ).split('\n') ) #for row in csv_reader: # print row #data1 = read_csv_data( data ) #print data1 csv_reader = csv.reader( data ) #fields = csv_reader.next() # Read the column names # remove all whitespaces search_query = search_query.strip() search_query = " ".join( search_query.split() ) search_query = search_query.replace(" ", "") with open( search_query + '_google_report.csv', 'w') as csv_out: positionInWeek = { "start" : 0, "end" : 1 } separator = " - " csv_writer = csv.writer( csv_out ) #print "LOOPING ON ENTRIES" for count, row in enumerate( csv_reader ): if separator not in row[0] : csv_writer.writerow( row ) continue date = row[0].split( separator )[ positionInWeek[ position ] ] # we want to remove any whitespaces from the value entry since we are not interested in blank data points val = re.sub(r'\s+', '', row[1] ) if len(val) < 1 : #print "We will skip Null value at date %s" % str(date) continue #row = str(date)+','+str(val) if count == 0: csv_writer.writerow( row ) else: csv_writer.writerow( [ str(date) ] + [ str(val) ]) print "File saved: %s " % ( search_query + '_google_report.csv' )
def download_data(region, device): while True: try: connector = pyGTrends(google_username, google_password) except Exception, e: if str(e).startswith('Cannot parse GALX'): continue else: raise else: break
def getGTData( search_query = "debt", date="all", geo="all", scale="1", position = "end" ) : connector = pyGTrends( google_username, google_password ) connector.download_report( ( search_query ) , date = date , geo = geo , scale = scale ) connector.writer() # print connector.getData() # d = DictReader( connector.getData().split('\n')) # print csv.DictReader( connector.csv().split('\n')) # print connector.csv( section='Main', as_list=True ) # data = connector.csv( section='Main' ).split('\n') # csv_reader = csv.reader( data ) return connector.getData()
def getGTData(search_query="debt", date="all", geo="all", scale="1", position="end"): connector = pyGTrends(google_username, google_password) connector.download_report((search_query), date=date, geo=geo, scale=scale) connector.writer() # print connector.getData() # d = DictReader( connector.getData().split('\n')) # print csv.DictReader( connector.csv().split('\n')) # print connector.csv( section='Main', as_list=True ) # data = connector.csv( section='Main' ).split('\n') # csv_reader = csv.reader( data ) return connector.getData()
def index(): global connection global popList popList = popularList() if request.method == "GET": if connection == None: return render_template("index.html", logged=False) else: return render_template("index.html", logged=True, popular=popList) else: google_username = request.form['username'] google_password = request.form['password'] try: connection = pyGTrends(google_username, google_password) except: return render_template("index.html", err="Incorrect Username or Password", logged=False) return render_template("index.html", logged=True, popular=popList) return render_template("index.html", logged=False)
def googletrends(keywords, start_month, start_year, duration=12, geo="US"): # Quta limit reached for 16/05 Don't call connector = pyGTrends.pyGTrends("*****@*****.**", "RB$Hackathon") querydate = str(start_month) + "/" + str(start_year) + " " + str( duration) + "m" #print querydate connector.download_report(keywords, geo=geo, date=querydate) result = [] try: result = connector.csv().split("\n") except Exception as e: if "Could not find requested section" in e.__str__(): with open("data/googletrends.csv", 'rb') as f: reader = csv.reader(f) for row in reader: result.append(row[0] + "," + row[1]) #print result del result[0] response = [] previousnumber = 100 for r in result: s = r.split(" ") date = datetime.strptime(s[0], '%Y-%m-%d') number = s[2].split(",")[1] diff = int(number) - previousnumber text = "" if diff >= 0: text = "The search for " + keywords + " has increased " + str( diff) + "%" + " on Google this week" else: text = "The search for " + keywords + " has decreased " + str( diff) + "%" + " on Google this week" response += [{ 'timestamp': date.isoformat(' '), 'description': text, 'source': 'google', 'author': 'zhuangy', 'sentiment': NaiveBayesAnalyzerParser(text) }] previousnumber = int(number) #print len(response) return response
def googletrends(keywords, start_month, start_year, duration=12, geo="US"): # Quta limit reached for 16/05 Don't call connector = pyGTrends.pyGTrends("*****@*****.**", "RB$Hackathon" ) querydate=str(start_month)+"/"+str(start_year)+" "+str(duration)+"m" #print querydate connector.download_report(keywords,geo=geo,date=querydate) result = [] try: result = connector.csv().split("\n") except Exception as e: if "Could not find requested section" in e.__str__(): with open("data/googletrends.csv", 'rb') as f: reader = csv.reader(f) for row in reader: result.append(row[0]+","+row[1]) #print result del result[0] response = [] previousnumber = 100 for r in result: s = r.split(" ") date = datetime.strptime(s[0], '%Y-%m-%d') number = s[2].split(",")[1] diff = int(number) - previousnumber text="" if diff >= 0: text = "The search for " + keywords+ " has increased "+ str(diff) + "%" +" on Google this week" else: text = "The search for " + keywords+ " has decreased "+ str(diff) + "%" +" on Google this week" response += [{ 'timestamp': date.isoformat(' '), 'description': text, 'source':'google', 'author':'zhuangy', 'sentiment': NaiveBayesAnalyzerParser(text) }] previousnumber = int(number) #print len(response) return response
__author__ = 'fang' from pyGTrends import pyGTrends connector = pyGTrends('*****@*****.**', 'amy8426ainos') connector.download_report(('apple', 'microsoft')) print connector.csv()
from pyGTrends import pyGTrends from pyGparse import pyGparse import httplib import urllib import urllib2 connector = pyGTrends('edmund.m.hart','3dmundh4rt') connector.download_report(('jaguar', 'bmw'),)
3:]) while timestamp < timestamp2: update_report(candidat, value, timestamp, regionName) timestamp = timestamp + 60 * 60 * 24 * 1000 if in_region: region = regions[row[0]]['name'] value = int(row[1]) print(region) candidat['geoReport'][region] = value elif len(row) > 0 and row[0] == "Sous-région": in_region = True i = i + 1 db.candidat.update({"_id": candidat['_id']}, candidat) connector = pyGTrends('avricot.team', 'table87table') connection = Connection('serv.avricot.com', 27017) db = connection.prediction for candidat in db.candidat.find({}): if candidat['candidatName'] == "SARKOZY": sarkozy_query = build_query(candidat) if candidat['candidatName'] == "HOLLANDE": hollande_query = build_query(candidat) j = 0 for candidat in db.candidat.find({}): for region in regions: if j > 1: #break time.sleep(15) j = j + 1
release_date = [ "2012-07-15"] # , "2014-02-10", "2015-03-17", "2014-11-10", "2014-08-18"] release_index = [241] #, 362, 382, 362, 350] gprop_list = ["youtube"] #, ""] country_list = [] country_map = {} infi = open("codes.csv") tx = infi.readlines() for line in tx: cut = line.split(',') country_list.append(cut[0]) country_map[cut[0]] = cut[1].strip() path = "/home/calvin/Data/" # comment/uncomment below to download con = pyGTrends(google_username, google_password) for k in keyword_list: for g in gprop_list: for c in country_list: con.request_report(k, gprop=g, geo=c) time.sleep(randint(4, 9)) file_name = k + "_" + g + "_" + c con.save_csv(path, file_name) # let's parse these into a JSON file j_m = {} index = -1
return self._stream.close() if __name__=="__main__": # read GOOGLE ID/PW from local control file f=open(GIDPWFile, 'r') f.readline() #skip the file instruction line google_username = f.readline() google_password = f.readline() print "user name = " + google_username print "user pw = " + google_password # read Search_Terms_Control file to get list of search terms i=0 f=codecs.open(Search_Terms_Control_File,'r', encoding='utf-16') #list_of_Search_Terms = f.readlines() list_of_Search_Terms = list(f) for Search_Term in list_of_Search_Terms: i+=1 print "(" + str(i) + ") " + Search_Term # sign on to google connector = pyGTrends( google_username, google_password ) #Get Trend data for each line in Search Terms control File - iterating over the GEOs control file lines for Search_Term in list_of_Search_Terms : #if getGoogleTrendData( search_query = Search_Term, geo = Geo_Dictionary.keys()) : if getGoogleTrendData( search_query = Search_Term, geo = ['xx']) : print "Google Trend Data processing complete"
def getGTData(search_query, date, geo, cmpt, cid, export, reqId): #output file name fnametag = search_query + "_suburban" if cid == "GEO_MAP_ANIMATION_0_2": fnametag = search_query + "_metro" q = search_query print "search_query ", search_query print "date ", date print "geo ", geo print "q ", q print "cmpt ", cmpt print "cid ", cid print "export ", export print "reqId ", reqId connector = pyGTrends(google_username, google_password) connector.download_report((search_query), date=date, geo=geo, q=q, cmpt=cmpt, cid=cid, export=export, reqId=reqId) rsp = connector.getData() print rsp #extract the json regex = 'animationResponse\((.+?)\);' pattern = re.compile(regex) json_rsp = re.findall(pattern, rsp) for j in json_rsp: json_rsp1 = j #print json_rsp1, "xxx\n" rsp = json.loads(json_rsp1) print rsp['data'][0], "xxx\n" print "first: ", rsp['data'][0]["frameData"], "xxx\n" print "second: ", rsp["data"][0]["timeStr"], "xxx\n" with open(fnametag + '_google_report.csv', 'w') as csv_out: csv_writer = csv.writer(csv_out) print len(rsp['data']) #outer loop for index in range(len(rsp['data'])): print rsp['data'][index] col4 = rsp["data"][index]["timeStr"] for innerindex in range(len(rsp['data'][index]['frameData'])): col1 = rsp['data'][index]['frameData'][innerindex][0] col2 = rsp['data'][index]['frameData'][innerindex][1] col3 = rsp['data'][index]['frameData'][innerindex][2] #print col1,col2,col3,col4 csv_writer.writerow([col1] + [col2] + [col3] + [col4]) print "File saved: %s " % (fnametag + '_google_report.csv')
def trend_csv_download(): r = pyGTrends('my email', 'passwd') r.download_report(('spain', 'wine'))### read it from csv file print DictReader(r.csv().split('\n'))
# recieve search: # /search/'STRING 1,STRING 2' # 1 sender: # /player1/progress/ # /player1/reset/ google_username = '******' google_password = '******' csv_path = 'csv/' if not os.path.exists(csv_path): os.makedirs(csv_path) #connect to Google connector = pyGTrends(google_username, google_password) players = [ { 'id': 1, 'ip': '10.0.1.11', # 'ip': '192.168.1.73', 'port': 5001, }, { 'id': 2, 'ip': '192.168.1.70', 'port': 5001, } ]
while timestamp<timestamp2: update_report(candidat, value, timestamp, regionName) timestamp = timestamp + 60*60*24*1000 if in_region: region = regions[row[0]]['name'] value = int(row[1]) print(region) candidat['geoReport'][region] = value elif len(row)>0 and row[0] == "Sous-région": in_region = True i=i+1 db.candidat.update({"_id": candidat['_id']}, candidat) connector = pyGTrends('avricot.team','table87table') connection = Connection('serv.avricot.com', 27017) db = connection.prediction for candidat in db.candidat.find({}): if candidat['candidatName'] == "SARKOZY": sarkozy_query = build_query(candidat) if candidat['candidatName'] == "HOLLANDE": hollande_query = build_query(candidat) j=0 for candidat in db.candidat.find({}): for region in regions: if j>1: #break time.sleep(15) j=j+1
from pyGTrends import pyGTrends connector = pyGTrends('*****@*****.**', 'tilaivytila77') connector.download_report(('drought', 'aids', 'obama')) csv = connector.csv() o = open( 'report.csv', "wb" ) o.write( csv )
def __init__(self,username,password): self._r = ptrends.pyGTrends(constants.Username, constants.Password)
import urllib from pyGTrends import pyGTrends import ystockquote from datetime import date #my module to convert dates into a numerical string in the YMD format from date_converter import date_converter #this will need to be changed as it currently uses my information connector = pyGTrends("USERNAME","PASSWORD") today = str(date.today()) today = today.replace('-','') #prompts the user for a stock and converts it to upper case #prompt = ">>> " #print "Stock Symbol?" #stock_symbol = raw_input(prompt) #stock_symbol = stock_symbol.upper() #print "Data type?" #metric_type = raw_input(prompt) class ychart_data(object): #sets up the initial varibles #defines the chart type, and sets up the URL def __init__(self, symbol, data_type): self.url = 'http://ycharts.com/companies/%s/%s' % (symbol, data_type) self.raw_url = urllib.urlopen(self.url).read() self.raw_url = self.raw_url.split('\n')
from pyGTrends import pyGTrends from csv import DictReader connector = pyGTrends('google username', 'google password') connector.download_report(('occupy wall street', 'Õ¼Á컪¶û½Ö')) # date='2011-10', # geo='AT', # scale=1) print connector.csv() # print connector.csv(section='Language') # print DictReader(connector.csv().split('\n'))