def main(argv):
'''
Main Logic
1 - Read Config filesq
2 - Read queries list
3 - If first running, create empty schema
4 - Grab google trends results into csv file
5 - Parse csv file and export into sqlite db
'''
dbfile = "google_trends_db"
print "SQLite db file: ", dbfile
config_file = "config.txt"
queries_file = "query_args_lst.txt"
thisfile = 'pyGoogleTrendsCsvToSQLite.py'
pathToStore = "csv_results"
#parsing input args
try:
opts, args = getopt.getopt(argv,"hc:q:",["conffile=","queryfile="])
except getopt.GetoptError:
print thisfile + ' -c <conffile> -q <queryfile> -s <csvsubfolder>'
print thisfile + ' --conffile <conffile> --queryfile <queryfile> --csvsubfolder <csvsubfolder>'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print thisfile + ' -c <conffile> -q <queryfile> -s <csvsubfolder>'
print thisfile + ' --conffile <conffile> --queryfile <queryfile> --csvsubfolder <csvsubfolder>'
sys.exit()
elif opt in ("-c", "--conffile"):
config_file = arg
elif opt in ("-q", "--queryfile"):
queries_file = arg
elif opt in ("-s", "--csvsubfolder"):
pathToStore = arg
if not os.path.exists(pathToStore):
os.makedirs(pathToStore)
print 'conffile file is "' + config_file + '"'
print 'queryfile file is "' + queries_file + '"'
# 1 - Read Config file
if keyboard_config:
rtn_confinfo = getManualConfigInfo()
else:
rtn_confinfo = read_ConfInfo(config_file)
username = rtn_confinfo['username']
password = rtn_confinfo['password']
print "\t", "username : "******"\t",queries_lst
# 3 - If first running, create empty schema from sql file
db_filename = 'google_trends_db'
schema_filename = 'google_trends_db_schema.sql'
db_is_new = not os.path.exists(db_filename)
if db_is_new:
print 'Creating schema ...'
with lite.connect(db_filename) as conn:
with open(schema_filename, 'rt') as f:
schema = f.read()
conn.executescript(schema)
print 'Created a new db file -"' + db_filename + '"'
# 4 - Grab google trends results into csv file
#parseCsvAndExportToSQLite(1,
# "romney,obama",
# os.path.join(pathToStore,"trends_q-romney,obama.csv"),
# "http://www.google.com/trends/explore?q=romney%2Cphp#q=obama&cmpt=q",
# dbfile)
#sys.exit()
r = pyGoogleTrendsCsvDownloader(username, password)
query_id = 1
for query in queries_lst:
rtn = r.get_csv(pathToStore, q=query)
# 5 - Parse csv file and export into sqlite db
parseCsvAndExportToSQLite(query_id, query, os.path.join(pathToStore,rtn['csvfilename']), rtn['url'], dbfile)
query_id += 1
from pyGoogleTrendsCsvDownloader import pyGoogleTrendsCsvDownloader
import time
from random import randint
google_username = '******'
google_pass = '******'
# Create the csv downloader object
downloader = pyGoogleTrendsCsvDownloader(google_username, google_pass)
# Wait some time to avoid blocking by google
time.sleep(randint(0, 5))
# Attributes for the url
kwargs = {'hl':'en-US', 'q':'\"Pizza\"', 'cmpt':'q', 'content' :'1'}
downloader.get_csv(**kwargs)
def login():
login = raw_input("Please enter your email address: ")
pwd = getpass.getpass("Enter your password: ")
return pyGoogleTrendsCsvDownloader(login, pwd)
def login():
login = raw_input("Please enter your email address: ")
pwd = getpass.getpass("Enter your password: ")
return pyGoogleTrendsCsvDownloader(login, pwd)
'''
Created on Sep 22, 2013
@author: mayank
'''
from pyGoogleTrendsCsvDownloader import pyGoogleTrendsCsvDownloader
r = pyGoogleTrendsCsvDownloader('username', 'password')
r.get_csv(geo='US-NY',hl='en-US',q='obamacare,affordable')