def loadData(tarball, params): sourceFile, formattedFile = extractFile(tarball) #open and format input file print "<DataLoader> formatting file: %s" % sourceFile try: inFile = open(sourceFile) except: print "<DataLoader> Could not retrieve data from file: %s" % sourceFile return DataFormatter.formatFile(inFile, formattedFile) inFile.close() print "<DataLoader> Successfully formatted file: %s" % sourceFile #attempt to upload file uploaded = DataUploader.upload(formattedFile, params["bucket"], **params) if not uploaded: print "<DataLoader> Failed to upload file, exiting" return #create bigquery object try: bigquery = Google_Service_Builder.buildBigQuery(**params) except Exception as e: print "<DataLoader>", e return source = "gs://"+params["bucket"]+"/"+formattedFile BigQuery_Append.appendData(bigquery, source, **params)
def loadData(tarball, params): sourceFile, formattedFile = extractFile(tarball) #open and format input file print "<DataLoader> formatting file: %s" % sourceFile try: inFile = open(sourceFile) except: print "<DataLoader> Could not retrieve data from file: %s" % sourceFile return DataFormatter.formatFile(inFile, formattedFile) inFile.close() print "<DataLoader> Successfully formatted file: %s" % sourceFile #attempt to upload file uploaded = DataUploader.upload(formattedFile, params["bucket"], **params) if not uploaded: print "<DataLoader> Failed to upload file, exiting" return #create bigquery object try: bigquery = Google_Service_Builder.buildBigQuery(**params) except Exception as e: print "<DataLoader>", e return source = "gs://" + params["bucket"] + "/" + formattedFile BigQuery_Append.appendData(bigquery, source, **params)
def main(): if len(sys.argv) <= 2: doHelp() return elif len(sys.argv) >= 3: configFile = sys.argv[1] action = sys.argv[2].lower() args = sys.argv[3:] else: return if action == "configure": BigQuery_Configuration.configureBigquery(configFile, args) return defaults = BigQuery_Configuration.loadConfig(configFile) args, params = BigQuery_Configuration.loadParams(args) for key in defaults: if key not in params: params[key] = defaults[key] try: bigquery = Google_Service_Builder.buildBigQuery(**params) except Exception as e: print e print "failed to build bigquery" return if action == "list": try: BigQuery_Basics.listTables(bigquery, **params) except Exception as e: print e print "failed to list tables in: %s.%s" % (params["projectId"], params["datasetId"]) elif action == "create": try: with open(args[0]) as f: schema = json.load(f) BigQuery_Basics.createTable(bigquery, schema, **params) except Exception as e: print e print "failed to create table: %s" % params["tableId"] elif action == "status": BigQuery_Basics.getStatus(bigquery, params["projectId"], args[0]) elif action == "upload": if not DataUploader.upload(args[0], params["bucket"]): print "<DataLoader> Failed to upload file, exiting"