def main(argv): ''' Main Logic 1 - Read Config filesq 2 - Read queries list 3 - If first running, create empty schema 4 - Grab google trends results into csv file 5 - Parse csv file and export into sqlite db ''' dbfile = "google_trends_db" print "SQLite db file: ", dbfile config_file = "config.txt" queries_file = "query_args_lst.txt" thisfile = 'pyGoogleTrendsCsvToSQLite.py' pathToStore = "csv_results" #parsing input args try: opts, args = getopt.getopt(argv,"hc:q:",["conffile=","queryfile="]) except getopt.GetoptError: print thisfile + ' -c <conffile> -q <queryfile> -s <csvsubfolder>' print thisfile + ' --conffile <conffile> --queryfile <queryfile> --csvsubfolder <csvsubfolder>' sys.exit(2) for opt, arg in opts: if opt == '-h': print thisfile + ' -c <conffile> -q <queryfile> -s <csvsubfolder>' print thisfile + ' --conffile <conffile> --queryfile <queryfile> --csvsubfolder <csvsubfolder>' sys.exit() elif opt in ("-c", "--conffile"): config_file = arg elif opt in ("-q", "--queryfile"): queries_file = arg elif opt in ("-s", "--csvsubfolder"): pathToStore = arg if not os.path.exists(pathToStore): os.makedirs(pathToStore) print 'conffile file is "' + config_file + '"' print 'queryfile file is "' + queries_file + '"' # 1 - Read Config file if keyboard_config: rtn_confinfo = getManualConfigInfo() else: rtn_confinfo = read_ConfInfo(config_file) username = rtn_confinfo['username'] password = rtn_confinfo['password'] print "\t", "username : "******"\t",queries_lst # 3 - If first running, create empty schema from sql file db_filename = 'google_trends_db' schema_filename = 'google_trends_db_schema.sql' db_is_new = not os.path.exists(db_filename) if db_is_new: print 'Creating schema ...' with lite.connect(db_filename) as conn: with open(schema_filename, 'rt') as f: schema = f.read() conn.executescript(schema) print 'Created a new db file -"' + db_filename + '"' # 4 - Grab google trends results into csv file #parseCsvAndExportToSQLite(1, # "romney,obama", # os.path.join(pathToStore,"trends_q-romney,obama.csv"), # "http://www.google.com/trends/explore?q=romney%2Cphp#q=obama&cmpt=q", # dbfile) #sys.exit() r = pyGoogleTrendsCsvDownloader(username, password) query_id = 1 for query in queries_lst: rtn = r.get_csv(pathToStore, q=query) # 5 - Parse csv file and export into sqlite db parseCsvAndExportToSQLite(query_id, query, os.path.join(pathToStore,rtn['csvfilename']), rtn['url'], dbfile) query_id += 1
from pyGoogleTrendsCsvDownloader import pyGoogleTrendsCsvDownloader import time from random import randint google_username = '******' google_pass = '******' # Create the csv downloader object downloader = pyGoogleTrendsCsvDownloader(google_username, google_pass) # Wait some time to avoid blocking by google time.sleep(randint(0, 5)) # Attributes for the url kwargs = {'hl':'en-US', 'q':'\"Pizza\"', 'cmpt':'q', 'content' :'1'} downloader.get_csv(**kwargs)
def login(): login = raw_input("Please enter your email address: ") pwd = getpass.getpass("Enter your password: ") return pyGoogleTrendsCsvDownloader(login, pwd)
''' Created on Sep 22, 2013 @author: mayank ''' from pyGoogleTrendsCsvDownloader import pyGoogleTrendsCsvDownloader r = pyGoogleTrendsCsvDownloader('username', 'password') r.get_csv(geo='US-NY',hl='en-US',q='obamacare,affordable')