def run_job(self, line): ip = get_ipython() try: args = self.parse_job_args(line) except SystemExit: return self.code_list = [] self.push_code("# translated code") if args.connection: con = ip.ev(args.connection) else: con = self.context.connect() # queue if args.queue: return self.submit_job(args.job_id, con, args) # engine job = con.client.job(args.job_id) engine = self.build_engine(job.type, job.database, args) # read_td_query self.push_code("_d = td.read_td_job({}, _e)".format(args.job_id)) if args.dry_run: return self.display_code_block() d = td.read_td_job(args.job_id, engine) # output r = self.post_process(d, args) if args.verbose: self.display_code_block() return r
def run_dynamic_query(parameters): #0. Initialize our connection to Treasure Data apikey = os.environ['MASTER_TD_API_KEY'] endpoint = 'https://api.treasuredata.com' con = td.connect(apikey, endpoint) #1. Connect to the query engine con_engine = con.query_engine(database=parameters['db_name'], type=parameters['query_engine']) #2. Setup query limit string if parameters['limit'] != '0': limit_str = "LIMIT " + str(parameters['limit']) + ";" else: limit_str = ";" #3. Compose Query String if not 'min_time' in parameters.keys(): parameters['min_time'] = 'NULL' if not 'max_time' in parameters.keys(): parameters['max_time'] = 'NULL' if parameters['min_time'] == 'NULL' and parameters['max_time'] == 'NULL': compose_query = "SELECT " + parameters['col_list'] + " " + \ "FROM " + parameters['table_name'] + " " + limit_str else: compose_query = "SELECT " + parameters['col_list'] + " " + \ "FROM " + parameters['table_name'] + " " + \ "WHERE " + "td_time_range(time," + parameters['min_time'] + "," + parameters['max_time'] + ") " + \ limit_str print("Executing..." + compose_query) #4. Run query as a job and wait for job to finish #Assign result set to a data frame with tdclient.Client(apikey) as client: job = client.query(parameters['db_name'], compose_query, type=parameters['query_engine']) job.wait() try: #Assign result set to a data frame df = td.read_td_job(job.job_id, con_engine) except RuntimeError: print("Please review the column names and delimited by commas: " + parameters['col_list']) return #5. Write the results to a csv or tabular format file if parameters['format'] == 'csv': print("Downloading results to " + job.job_id + ".csv" + " file") df.to_csv(job.job_id + ".csv") else: #Write data into tabular grid format print("Downloading results to " + job.job_id + ".txt" + " file") filename = job.job_id + ".txt" outfile = open(filename, "a") outfile.write(tabulate(df, tablefmt="grid")) outfile.close()
def run_dynamic_query(parameters): #0. Initialize our connection to Treasure Data apikey=os.environ['MASTER_TD_API_KEY'] endpoint='https://api.treasuredata.com' con = td.connect(apikey, endpoint) #1. Connect to the query engine con_engine=con.query_engine(database=parameters['db_name'], type=parameters['query_engine']) #2. Setup query limit string if parameters['limit'] != '0': limit_str = "LIMIT " + str(parameters['limit']) + ";" else: limit_str = ";" #3. Compose Query String if not 'min_time' in parameters.keys(): parameters['min_time'] = 'NULL' if not 'max_time' in parameters.keys(): parameters['max_time'] = 'NULL' if parameters['min_time'] == 'NULL' and parameters['max_time'] == 'NULL': compose_query = "SELECT " + parameters['col_list'] + " " + \ "FROM " + parameters['table_name'] + " " + limit_str else: compose_query = "SELECT " + parameters['col_list'] + " " + \ "FROM " + parameters['table_name'] + " " + \ "WHERE " + "td_time_range(time," + parameters['min_time'] + "," + parameters['max_time'] + ") " + \ limit_str print("Executing..." + compose_query) #4. Run query as a job and wait for job to finish #Assign result set to a data frame with tdclient.Client(apikey) as client: job = client.query(parameters['db_name'],compose_query,type=parameters['query_engine']) job.wait() try: #Assign result set to a data frame df = td.read_td_job(job.job_id, con_engine) except RuntimeError: print("Please review the column names and delimited by commas: " + parameters['col_list']) return #5. Write the results to a csv or tabular format file if parameters['format'] == 'csv': print("Downloading results to " + job.job_id + ".csv" + " file") df.to_csv(job.job_id + ".csv") else: #Write data into tabular grid format print("Downloading results to " + job.job_id + ".txt" + " file") filename = job.job_id + ".txt" outfile = open(filename,"a") outfile.write(tabulate(df, tablefmt="grid")) outfile.close()
def run_job(self, line): ip = get_ipython() try: args = self.parse_job_args(line) except SystemExit: return self.code_list = [] self.push_code("# translated code") if args.connection: con = ip.ev(args.connection) else: con = self.context.connect() # queue if args.queue: return self.submit_job(args.job_id, con, args) # engine job = con.client.job(args.job_id) if hasattr(job, 'database'): database = job.database else: # NOTE: tdclient <= 0.3.2 is broken database = 'sample_datasets' engine = self.build_engine(job.type, engine, args) # read_td_query self.push_code("_d = td.read_td_job({}, _e)".format(args.job_id)) if args.dry_run: return self.display_code_block() d = td.read_td_job(args.job_id, engine) # output r = self.post_process(d, args) if args.verbose: self.display_code_block() return r
def test_ok(self): df = read_td_job(1, self.engine) assert len(df) == len(self.job.result)
def test_ok(self): df = read_td_job(1, self.engine) eq_(len(df), len(self.job.result))