def main(): if len(sys.argv) <= 2: doHelp() return elif len(sys.argv) >= 3: configFile = sys.argv[1] action = sys.argv[2].lower() args = sys.argv[3:] else: return if action == "configure": BigQuery_Configuration.configureBigquery(configFile, args) return defaults = BigQuery_Configuration.loadConfig(configFile) args, params = BigQuery_Configuration.loadParams(args) for key in defaults: if key not in params: params[key] = defaults[key] try: bigquery = Google_Service_Builder.buildBigQuery(**params) except Exception as e: print e print "failed to build bigquery" return if action == "list": try: BigQuery_Basics.listTables(bigquery, **params) except Exception as e: print e print "failed to list tables in: %s.%s" % (params["projectId"], params["datasetId"]) elif action == "create": try: with open(args[0]) as f: schema = json.load(f) BigQuery_Basics.createTable(bigquery, schema, **params) except Exception as e: print e print "failed to create table: %s" % params["tableId"] elif action == "status": BigQuery_Basics.getStatus(bigquery, params["projectId"], args[0]) elif action == "upload": if not DataUploader.upload(args[0], params["bucket"]): print "<DataLoader> Failed to upload file, exiting"
def main(): if len(sys.argv) == 1: helpFunc() return if sys.argv[1] == "help": helpFunc(sys.argv[2:]) return if len(sys.argv) >= 3: configFile = sys.argv[1] action = sys.argv[2].lower() args = sys.argv[3:] else: return if action == "configure": BigQuery_Configuration.configureBigquery(configFile, args) return if action != "load": helpFunc() return #get parameters defaults = BigQuery_Configuration.loadConfig(configFile) args, params = BigQuery_Configuration.loadParams(args) for key in defaults: if key not in params: params[key] = defaults[key] #check source source = args[0] try: if os.path.isdir(source): print "<DataLoader> Source is a directory" for tarball in [f for f in os.listdir(source) if \ os.path.isfile(os.path.join(source, f))]: print "<DataLoader> Loading file: %s" % tarball loadData(os.path.join(source, tarball), params) else: loadData(source, params) DataExtractor.cleanup() except Exception as e: print e