def flush(self): if self._curr_cursor > 0: self._buffer.flush() data = self._buffer.getvalue() self._buffer = compat.BytesIO() self._curr_cursor = 0 self._output_callback(data)
def __init__(self, output_callback, buffer_size=None, encoding='utf-8'): self._buffer = compat.BytesIO() self._output_callback = output_callback self._buffer_size = buffer_size or self.BUFFER_SIZE self._curr_cursor = 0 self._n_total = 0 self._encoding = encoding
def flush(self): self._buffer.flush() if self._curr_cursor > 0: self._queue.put(self._buffer.getvalue()) self._buffer = compat.BytesIO() self._curr_cursor = 0
def submit_job(param_pb, running_mode, priority=None, with_resource=False, session=None, running_cluster=None, task_name='cupid_task'): from odps import ODPS, compat from odps.models import CupidTask if logger.getEffectiveLevel() <= logging.DEBUG: param_pb_str = str(param_pb) if isinstance(param_pb, cupidtaskparam_pb2.CupidTaskParam): for conf in param_pb.jobconf.jobconfitem: if conf.key == 'odps.access.id': param_pb_str = param_pb_str.replace( conf.value, '** access-id **') elif conf.key == 'odps.access.key': param_pb_str = param_pb_str.replace( conf.value, '** access-key **') logger.debug('Job param proto: %s', param_pb_str) odps = session.odps if session else ODPS.from_global() plan_string = param_pb.SerializeToString() res_kw = dict(fileobj=compat.BytesIO(plan_string)) if not with_resource: res_kw['is_temp_resource'] = True res = odps.create_resource('cupid_plan_' + str(uuid.uuid4()), 'file', **res_kw) task_info = ','.join([res.name, odps.project, running_mode]) props = dict() if options.cupid.application_type: props[ 'odps.cupid.application.type'] = options.cupid.application_type props['odps.moye.runtime.type'] = options.cupid.application_type if options.biz_id: props['biz_id'] = options.biz_id if options.cupid.major_task_version: props['odps.task.major.version'] = options.cupid.major_task_version context_file = get_property('odps.exec.context.file') if context_file and os.path.exists(context_file): with open(context_file, 'r') as cf: file_settings = json.loads(cf.read()).get('settings', {}) props.update(file_settings) task = CupidTask(task_name, task_info, props) inst = odps.get_project().instances.create( task=task, priority=priority, running_cluster=running_cluster) inst = odps.get_instance(inst.id) return inst
def _get_buffer(cls, compress_option=None): if compress_option is None or \ compress_option == CompressOption.CompressAlgorithm.ODPS_RAW: return compat.BytesIO() elif compress_option.algorithm == \ CompressOption.CompressAlgorithm.ODPS_ZLIB: compress_obj = zlib.compressobj( compress_option.level, zlib.DEFLATED, zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL, compress_option.strategy) return ProtobufWriter.ZlibBuffer(compress_obj) elif compress_option.algorithm == \ CompressOption.CompressAlgorithm.ODPS_SNAPPY: return ProtobufWriter.SnappyBuffer() else: raise IOError('Invalid compression option.')
def CallMethod(self, method, controller, request, response_class, done): from .runtime import context context = context() try: sio = compat.BytesIO() sio.write(struct.pack('<I', method.index)) sio.write(request.SerializeToString()) logger.debug( 'SandboxRpcChannel sync_call service: %s, method id: %d request: %s', method.containing_service.full_name, method.index, request) res = context.channel_client.sync_call( method.containing_service.full_name, sio.getvalue()) resp = response_class() resp.ParseFromString(res) logger.debug('SandboxRpcChannel sync_call result: %s', resp) return resp except Exception as exc: logger.exception('SandboxRpcChannel CallMethod fail: %s', str(exc)) controller.SetFailed(str(exc))