def run_job(self, line):
ip = get_ipython()
try:
args = self.parse_job_args(line)
except SystemExit:
return
self.code_list = []
self.push_code("# translated code")
if args.connection:
con = ip.ev(args.connection)
else:
con = self.context.connect()
# queue
if args.queue:
return self.submit_job(args.job_id, con, args)
# engine
job = con.client.job(args.job_id)
engine = self.build_engine(job.type, job.database, args)
# read_td_query
self.push_code("_d = td.read_td_job({}, _e)".format(args.job_id))
if args.dry_run:
return self.display_code_block()
d = td.read_td_job(args.job_id, engine)
# output
r = self.post_process(d, args)
if args.verbose:
self.display_code_block()
return r
def run_dynamic_query(parameters):
#0. Initialize our connection to Treasure Data
apikey = os.environ['MASTER_TD_API_KEY']
endpoint = 'https://api.treasuredata.com'
con = td.connect(apikey, endpoint)
#1. Connect to the query engine
con_engine = con.query_engine(database=parameters['db_name'],
type=parameters['query_engine'])
#2. Setup query limit string
if parameters['limit'] != '0':
limit_str = "LIMIT " + str(parameters['limit']) + ";"
else:
limit_str = ";"
#3. Compose Query String
if not 'min_time' in parameters.keys():
parameters['min_time'] = 'NULL'
if not 'max_time' in parameters.keys():
parameters['max_time'] = 'NULL'
if parameters['min_time'] == 'NULL' and parameters['max_time'] == 'NULL':
compose_query = "SELECT " + parameters['col_list'] + " " + \
"FROM " + parameters['table_name'] + " " + limit_str
else:
compose_query = "SELECT " + parameters['col_list'] + " " + \
"FROM " + parameters['table_name'] + " " + \
"WHERE " + "td_time_range(time," + parameters['min_time'] + "," + parameters['max_time'] + ") " + \
limit_str
print("Executing..." + compose_query)
#4. Run query as a job and wait for job to finish
#Assign result set to a data frame
with tdclient.Client(apikey) as client:
job = client.query(parameters['db_name'],
compose_query,
type=parameters['query_engine'])
job.wait()
try:
#Assign result set to a data frame
df = td.read_td_job(job.job_id, con_engine)
except RuntimeError:
print("Please review the column names and delimited by commas: " +
parameters['col_list'])
return
#5. Write the results to a csv or tabular format file
if parameters['format'] == 'csv':
print("Downloading results to " + job.job_id + ".csv" + " file")
df.to_csv(job.job_id + ".csv")
else:
#Write data into tabular grid format
print("Downloading results to " + job.job_id + ".txt" + " file")
filename = job.job_id + ".txt"
outfile = open(filename, "a")
outfile.write(tabulate(df, tablefmt="grid"))
outfile.close()
def run_dynamic_query(parameters):
#0. Initialize our connection to Treasure Data
apikey=os.environ['MASTER_TD_API_KEY']
endpoint='https://api.treasuredata.com'
con = td.connect(apikey, endpoint)
#1. Connect to the query engine
con_engine=con.query_engine(database=parameters['db_name'], type=parameters['query_engine'])
#2. Setup query limit string
if parameters['limit'] != '0':
limit_str = "LIMIT " + str(parameters['limit']) + ";"
else:
limit_str = ";"
#3. Compose Query String
if not 'min_time' in parameters.keys():
parameters['min_time'] = 'NULL'
if not 'max_time' in parameters.keys():
parameters['max_time'] = 'NULL'
if parameters['min_time'] == 'NULL' and parameters['max_time'] == 'NULL':
compose_query = "SELECT " + parameters['col_list'] + " " + \
"FROM " + parameters['table_name'] + " " + limit_str
else:
compose_query = "SELECT " + parameters['col_list'] + " " + \
"FROM " + parameters['table_name'] + " " + \
"WHERE " + "td_time_range(time," + parameters['min_time'] + "," + parameters['max_time'] + ") " + \
limit_str
print("Executing..." + compose_query)
#4. Run query as a job and wait for job to finish
#Assign result set to a data frame
with tdclient.Client(apikey) as client:
job = client.query(parameters['db_name'],compose_query,type=parameters['query_engine'])
job.wait()
try:
#Assign result set to a data frame
df = td.read_td_job(job.job_id, con_engine)
except RuntimeError:
print("Please review the column names and delimited by commas: " + parameters['col_list'])
return
#5. Write the results to a csv or tabular format file
if parameters['format'] == 'csv':
print("Downloading results to " + job.job_id + ".csv" + " file")
df.to_csv(job.job_id + ".csv")
else:
#Write data into tabular grid format
print("Downloading results to " + job.job_id + ".txt" + " file")
filename = job.job_id + ".txt"
outfile = open(filename,"a")
outfile.write(tabulate(df, tablefmt="grid"))
outfile.close()
def run_job(self, line):
ip = get_ipython()
try:
args = self.parse_job_args(line)
except SystemExit:
return
self.code_list = []
self.push_code("# translated code")
if args.connection:
con = ip.ev(args.connection)
else:
con = self.context.connect()
# queue
if args.queue:
return self.submit_job(args.job_id, con, args)
# engine
job = con.client.job(args.job_id)
if hasattr(job, 'database'):
database = job.database
else:
# NOTE: tdclient <= 0.3.2 is broken
database = 'sample_datasets'
engine = self.build_engine(job.type, engine, args)
# read_td_query
self.push_code("_d = td.read_td_job({}, _e)".format(args.job_id))
if args.dry_run:
return self.display_code_block()
d = td.read_td_job(args.job_id, engine)
# output
r = self.post_process(d, args)
if args.verbose:
self.display_code_block()
return r
def test_ok(self):
df = read_td_job(1, self.engine)
assert len(df) == len(self.job.result)
def test_ok(self):
df = read_td_job(1, self.engine)
eq_(len(df), len(self.job.result))
def test_ok(self):
df = read_td_job(1, self.engine)
eq_(len(df), len(self.job.result))
