def flush(self):
if self._curr_cursor > 0:
self._buffer.flush()
data = self._buffer.getvalue()
self._buffer = compat.BytesIO()
self._curr_cursor = 0
self._output_callback(data)
def __init__(self, output_callback, buffer_size=None, encoding='utf-8'):
self._buffer = compat.BytesIO()
self._output_callback = output_callback
self._buffer_size = buffer_size or self.BUFFER_SIZE
self._curr_cursor = 0
self._n_total = 0
self._encoding = encoding
def flush(self):
self._buffer.flush()
if self._curr_cursor > 0:
self._queue.put(self._buffer.getvalue())
self._buffer = compat.BytesIO()
self._curr_cursor = 0
def submit_job(param_pb,
running_mode,
priority=None,
with_resource=False,
session=None,
running_cluster=None,
task_name='cupid_task'):
from odps import ODPS, compat
from odps.models import CupidTask
if logger.getEffectiveLevel() <= logging.DEBUG:
param_pb_str = str(param_pb)
if isinstance(param_pb, cupidtaskparam_pb2.CupidTaskParam):
for conf in param_pb.jobconf.jobconfitem:
if conf.key == 'odps.access.id':
param_pb_str = param_pb_str.replace(
conf.value, '** access-id **')
elif conf.key == 'odps.access.key':
param_pb_str = param_pb_str.replace(
conf.value, '** access-key **')
logger.debug('Job param proto: %s', param_pb_str)
odps = session.odps if session else ODPS.from_global()
plan_string = param_pb.SerializeToString()
res_kw = dict(fileobj=compat.BytesIO(plan_string))
if not with_resource:
res_kw['is_temp_resource'] = True
res = odps.create_resource('cupid_plan_' + str(uuid.uuid4()), 'file',
**res_kw)
task_info = ','.join([res.name, odps.project, running_mode])
props = dict()
if options.cupid.application_type:
props[
'odps.cupid.application.type'] = options.cupid.application_type
props['odps.moye.runtime.type'] = options.cupid.application_type
if options.biz_id:
props['biz_id'] = options.biz_id
if options.cupid.major_task_version:
props['odps.task.major.version'] = options.cupid.major_task_version
context_file = get_property('odps.exec.context.file')
if context_file and os.path.exists(context_file):
with open(context_file, 'r') as cf:
file_settings = json.loads(cf.read()).get('settings', {})
props.update(file_settings)
task = CupidTask(task_name, task_info, props)
inst = odps.get_project().instances.create(
task=task, priority=priority, running_cluster=running_cluster)
inst = odps.get_instance(inst.id)
return inst
def _get_buffer(cls, compress_option=None):
if compress_option is None or \
compress_option == CompressOption.CompressAlgorithm.ODPS_RAW:
return compat.BytesIO()
elif compress_option.algorithm == \
CompressOption.CompressAlgorithm.ODPS_ZLIB:
compress_obj = zlib.compressobj(
compress_option.level, zlib.DEFLATED,
zlib.MAX_WBITS, zlib.DEF_MEM_LEVEL,
compress_option.strategy)
return ProtobufWriter.ZlibBuffer(compress_obj)
elif compress_option.algorithm == \
CompressOption.CompressAlgorithm.ODPS_SNAPPY:
return ProtobufWriter.SnappyBuffer()
else:
raise IOError('Invalid compression option.')
def CallMethod(self, method, controller, request, response_class, done):
from .runtime import context
context = context()
try:
sio = compat.BytesIO()
sio.write(struct.pack('<I', method.index))
sio.write(request.SerializeToString())
logger.debug(
'SandboxRpcChannel sync_call service: %s, method id: %d request: %s',
method.containing_service.full_name, method.index, request)
res = context.channel_client.sync_call(
method.containing_service.full_name, sio.getvalue())
resp = response_class()
resp.ParseFromString(res)
logger.debug('SandboxRpcChannel sync_call result: %s', resp)
return resp
except Exception as exc:
logger.exception('SandboxRpcChannel CallMethod fail: %s', str(exc))
controller.SetFailed(str(exc))
