def testListInstancesInPage(self): test_table = tn('pyodps_t_tmp_list_instances_in_page') delay_udf = textwrap.dedent(""" from odps.udf import annotate import sys import time @annotate("bigint->bigint") class Delayer(object): def evaluate(self, arg0): print('Start Logging') sys.stdout.flush() time.sleep(45) print('End Logging') sys.stdout.flush() return arg0 """) resource_name = tn('test_delayer_function_resource') function_name = tn('test_delayer_function') if self.odps.exist_resource(resource_name + '.py'): self.odps.delete_resource(resource_name + '.py') res = self.odps.create_resource(resource_name + '.py', 'py', file_obj=delay_udf) if self.odps.exist_function(function_name): self.odps.delete_function(function_name) fun = self.odps.create_function(function_name, class_type=resource_name + '.Delayer', resources=[res, ]) data = [[random.randint(0, 1000)] for _ in compat.irange(100)] self.odps.delete_table(test_table, if_exists=True) t = self.odps.create_table(test_table, Schema.from_lists(['num'], ['bigint'])) self.odps.write_table(t, data) instance = self.odps.run_sql("select sum({0}(num)), 1 + '1' as warn_col from {1} group by num" .format(function_name, test_table)) try: self.assertEqual(instance.status, Instance.Status.RUNNING) self.assertIn(instance.id, [it.id for it in self.odps.get_project().instances.iterate( status=Instance.Status.RUNNING, from_time=datetime.now()-timedelta(days=2), end_time=datetime.now()+timedelta(days=1), max_items=20)]) self.waitContainerFilled(lambda: instance.tasks) task = instance.tasks[0] task.put_info('testInfo', 'TestInfo') self.assertIsNotNone(task.warnings) self.waitContainerFilled(lambda: task.workers, 30) self.assertIsNotNone(task.workers[0].get_log('stdout')) finally: try: instance.stop() except: pass res.drop() fun.drop() t.drop()
def testRawUploadDownloadGreenlet(self): block = bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)])) partition = self.odps.get_volume_partition(TEST_VOLUME_NAME, TEST_PARTITION_NAME) with partition.open_writer() as writer: writer.write(TEST_FILE_NAME, block) with partition.open_reader(TEST_FILE_NAME) as reader: assert reader.read() == block
def testRawUploadDownloadGreenlet(self): block = bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)])) partition = self.odps.get_volume_partition(TEST_VOLUME_NAME, TEST_PARTITION_NAME) with partition.open_writer() as writer: writer.write(TEST_FILE_NAME, block) with partition.open_reader(TEST_FILE_NAME) as reader: assert reader.read() == block
def testZLibUploadDownload(self): block = bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)])) comp_option = CompressOption(level=9) partition = self.odps.get_volume_partition(TEST_VOLUME_NAME, TEST_PARTITION_NAME) with partition.open_writer(compress_option=comp_option) as writer: writer.write(TEST_FILE_NAME, block, compress=True) with partition.open_reader(TEST_FILE_NAME, compress_option=comp_option) as reader: assert reader.read() == block
def testZLibUploadDownload(self): block = bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)])) comp_option = CompressOption(level=9) partition = self.odps.get_volume_partition(TEST_VOLUME_NAME, TEST_PARTITION_NAME) with partition.open_writer(compress_option=comp_option) as writer: writer.write(TEST_FILE_NAME, block, compress=True) with partition.open_reader(TEST_FILE_NAME, compress_option=comp_option) as reader: assert reader.read() == block
def testRawUploadDownloadThread(self): from odps.tunnel import io io._FORCE_THREAD = True block = bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)])) partition = self.odps.get_volume_partition(TEST_VOLUME_NAME, TEST_PARTITION_NAME) with partition.open_writer() as writer: writer.write(TEST_FILE_NAME, block) with partition.open_reader(TEST_FILE_NAME) as reader: assert reader.read() == block
def testRawUploadDownloadThread(self): from odps.tunnel import io io._FORCE_THREAD = True block = bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)])) partition = self.odps.get_volume_partition(TEST_VOLUME_NAME, TEST_PARTITION_NAME) with partition.open_writer() as writer: writer.write(TEST_FILE_NAME, block) with partition.open_reader(TEST_FILE_NAME) as reader: assert reader.read() == block
def testListInstancesInPage(self): test_table = tn('pyodps_t_tmp_list_instances_in_page') data = [[random.randint(0, 1000)] for _ in compat.irange(10000)] self.odps.delete_table(test_table, if_exists=True) t = self.odps.create_table(test_table, Schema.from_lists(['num'], ['bigint'])) self.odps.write_table(t, data) instance = self.odps.run_sql('select sum(num) from {0} group by num'.format(test_table)) try: self.assertEqual(instance.status, Instance.Status.RUNNING) self.assertIn(instance.id, [it.id for it in self.odps.get_project().instances.iterate( status=Instance.Status.RUNNING, from_time=datetime.now()-timedelta(days=2), end_time=datetime.now()+timedelta(days=1), max_items=20)]) finally: try: instance.stop() except: pass t.drop()
def testListInstancesInPage(self): test_table = tn('pyodps_t_tmp_list_instances_in_page') delay_udf = textwrap.dedent(""" from odps.udf import annotate import sys import time @annotate("bigint->bigint") class Delayer(object): def evaluate(self, arg0): print('Start Logging') sys.stdout.flush() time.sleep(45) print('End Logging') sys.stdout.flush() return arg0 """) resource_name = tn('test_delayer_function_resource') function_name = tn('test_delayer_function') if self.odps.exist_resource(resource_name + '.py'): self.odps.delete_resource(resource_name + '.py') res = self.odps.create_resource(resource_name + '.py', 'py', file_obj=delay_udf) if self.odps.exist_function(function_name): self.odps.delete_function(function_name) fun = self.odps.create_function(function_name, class_type=resource_name + '.Delayer', resources=[ res, ]) data = [[random.randint(0, 1000)] for _ in compat.irange(100)] self.odps.delete_table(test_table, if_exists=True) t = self.odps.create_table(test_table, Schema.from_lists(['num'], ['bigint'])) self.odps.write_table(t, data) instance = self.odps.run_sql( "select sum({0}(num)), 1 + '1' as warn_col from {1} group by num". format(function_name, test_table)) try: self.assertEqual(instance.status, Instance.Status.RUNNING) self.assertIn(instance.id, [ it.id for it in self.odps.get_project().instances.iterate( status=Instance.Status.RUNNING, start_time=datetime.now() - timedelta(days=2), end_time=datetime.now() + timedelta(days=1), max_items=20) ]) self.waitContainerFilled(lambda: instance.tasks) task = instance.tasks[0] task.put_info('testInfo', 'TestInfo') self.assertIsNotNone(task.warnings) self.waitContainerFilled(lambda: task.workers, 30) self.assertIsNotNone(task.workers[0].get_log('stdout')) finally: try: instance.stop() except: pass res.drop() fun.drop() t.drop()
def _gen_byte_block(): return bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)]))
def _gen_byte_block(): return bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)]))