def submit_job(param_pb,
running_mode,
priority=None,
with_resource=False,
session=None,
running_cluster=None,
task_name='cupid_task'):
from odps import ODPS, compat
from odps.models import CupidTask
if logger.getEffectiveLevel() <= logging.DEBUG:
param_pb_str = str(param_pb)
if isinstance(param_pb, cupidtaskparam_pb2.CupidTaskParam):
for conf in param_pb.jobconf.jobconfitem:
if conf.key == 'odps.access.id':
param_pb_str = param_pb_str.replace(
conf.value, '** access-id **')
elif conf.key == 'odps.access.key':
param_pb_str = param_pb_str.replace(
conf.value, '** access-key **')
logger.debug('Job param proto: %s', param_pb_str)
odps = session.odps if session else ODPS.from_global()
plan_string = param_pb.SerializeToString()
res_kw = dict(fileobj=compat.BytesIO(plan_string))
if not with_resource:
res_kw['is_temp_resource'] = True
res = odps.create_resource('cupid_plan_' + str(uuid.uuid4()), 'file',
**res_kw)
task_info = ','.join([res.name, odps.project, running_mode])
props = dict()
if options.cupid.application_type:
props[
'odps.cupid.application.type'] = options.cupid.application_type
props['odps.moye.runtime.type'] = options.cupid.application_type
if options.biz_id:
props['biz_id'] = options.biz_id
if options.cupid.major_task_version:
props['odps.task.major.version'] = options.cupid.major_task_version
context_file = get_property('odps.exec.context.file')
if context_file and os.path.exists(context_file):
with open(context_file, 'r') as cf:
file_settings = json.loads(cf.read()).get('settings', {})
props.update(file_settings)
task = CupidTask(task_name, task_info, props)
inst = odps.get_project().instances.create(
task=task, priority=priority, running_cluster=running_cluster)
inst = odps.get_instance(inst.id)
return inst
def copy_file_to_resource(local_file,
resource_path,
resource_type,
overwritten,
o=None):
"""Copy local file as odps resource."""
from odps import ODPS
if o is None:
o = ODPS.from_global()
if overwritten and o.exist_resource(resource_path):
o.delete_resource(resource_path)
if isinstance(local_file, str) or isinstance(local_file, Text):
fobj = open(local_file, 'rb')
else:
fobj = local_file
return o.create_resource(resource_path, resource_type, file_obj=fobj)
def __init__(self, odps=None, project=None):
from .runtime import context
from odps import ODPS
self._context = context()
if odps is not None:
self.odps = odps
else:
self.odps = ODPS.from_global()
self.project = project if project is not None else self.odps.project
self.lookup_name = get_environ('META_LOOKUP_NAME', '')
self.running = False
self.save_id = None
self.job_running_event = threading.Event()
self._kube_url = None
