def loadData(tarball, params):
sourceFile, formattedFile = extractFile(tarball)
#open and format input file
print "<DataLoader> formatting file: %s" % sourceFile
try:
inFile = open(sourceFile)
except:
print "<DataLoader> Could not retrieve data from file: %s" % sourceFile
return
DataFormatter.formatFile(inFile, formattedFile)
inFile.close()
print "<DataLoader> Successfully formatted file: %s" % sourceFile
#attempt to upload file
uploaded = DataUploader.upload(formattedFile, params["bucket"], **params)
if not uploaded:
print "<DataLoader> Failed to upload file, exiting"
return
#create bigquery object
try:
bigquery = Google_Service_Builder.buildBigQuery(**params)
except Exception as e:
print "<DataLoader>", e
return
source = "gs://"+params["bucket"]+"/"+formattedFile
BigQuery_Append.appendData(bigquery, source, **params)
def loadData(tarball, params):
sourceFile, formattedFile = extractFile(tarball)
#open and format input file
print "<DataLoader> formatting file: %s" % sourceFile
try:
inFile = open(sourceFile)
except:
print "<DataLoader> Could not retrieve data from file: %s" % sourceFile
return
DataFormatter.formatFile(inFile, formattedFile)
inFile.close()
print "<DataLoader> Successfully formatted file: %s" % sourceFile
#attempt to upload file
uploaded = DataUploader.upload(formattedFile, params["bucket"], **params)
if not uploaded:
print "<DataLoader> Failed to upload file, exiting"
return
#create bigquery object
try:
bigquery = Google_Service_Builder.buildBigQuery(**params)
except Exception as e:
print "<DataLoader>", e
return
source = "gs://" + params["bucket"] + "/" + formattedFile
BigQuery_Append.appendData(bigquery, source, **params)
def main():
if len(sys.argv) <= 2:
doHelp()
return
elif len(sys.argv) >= 3:
configFile = sys.argv[1]
action = sys.argv[2].lower()
args = sys.argv[3:]
else:
return
if action == "configure":
BigQuery_Configuration.configureBigquery(configFile, args)
return
defaults = BigQuery_Configuration.loadConfig(configFile)
args, params = BigQuery_Configuration.loadParams(args)
for key in defaults:
if key not in params:
params[key] = defaults[key]
try:
bigquery = Google_Service_Builder.buildBigQuery(**params)
except Exception as e:
print e
print "failed to build bigquery"
return
if action == "list":
try:
BigQuery_Basics.listTables(bigquery, **params)
except Exception as e:
print e
print "failed to list tables in: %s.%s" % (params["projectId"], params["datasetId"])
elif action == "create":
try:
with open(args[0]) as f:
schema = json.load(f)
BigQuery_Basics.createTable(bigquery, schema, **params)
except Exception as e:
print e
print "failed to create table: %s" % params["tableId"]
elif action == "status":
BigQuery_Basics.getStatus(bigquery, params["projectId"], args[0])
elif action == "upload":
if not DataUploader.upload(args[0], params["bucket"]):
print "<DataLoader> Failed to upload file, exiting"
