def get_job_desc():
argv = {}
argv['-m'] = str(cfg.M)
argv['-s'] = str(cfg.SYNC)
argv_str = ''
for k, v in argv.iteritems():
argv_str += ' %s %s'%(k, v)
job_desc = JobDescription()
task = TaskDescription()
resource = ResourceDescription()
resource.Cpu = cfg.CPU_NUM*100
resource.Memory = 4000
# Create task.
task.ResourceDescription = resource
task.PackageUri = cfg.FULL_PACKAGE
task.ProgramArguments = argv_str
task.ProgramName = 'ud.py'
task.ProgramType = 'python'
task.ImageId = cfg.IMAGE_ID
#task.InstanceCount = cfg.COUNT_TASK_NUM
task.InstanceCount = cfg.INSTANCE_NUM
task.EnvironmentVariables = {}
task.StdoutRedirectPath = cfg.LOG_PATH
task.StderrRedirectPath = cfg.LOG_PATH
# Create task dag.
task_dag = TaskDag()
task_dag.add_task(task_name='upload', task=task)
job_desc.TaskDag = task_dag
job_desc.JobName = 'dtest'
job_desc.Priority = 1000
return job_desc
def get_job_desc():
job_desc = JobDescription()
task = TaskDescription()
resource = ResourceDescription()
resource.Cpu = cfg.CPU_NUM*100
resource.Memory = 16000
# Create task.
task.ResourceDescription = resource
task.PackageUri = cfg.FULL_PACKAGE
task.ProgramName = 'end.py'
task.ProgramType = 'python'
task.ImageId = cfg.IMAGE_ID
#task.InstanceCount = cfg.COUNT_TASK_NUM
task.InstanceCount = cfg.INSTANCE_NUM
task.EnvironmentVariables = {}
task.StdoutRedirectPath = cfg.LOG_PATH
task.StderrRedirectPath = cfg.LOG_PATH
task.OssMapping={cfg.FULL_DATAPATH:cfg.LOCAL_DATA}
task.OssMappingLock=cfg.LOCK
task.OssMappingLocale=cfg.LOCALE
# Create task dag.
task_dag = TaskDag()
task_dag.add_task(task_name='download', task=task)
job_desc.TaskDag = task_dag
job_desc.JobName = 'dtest'
job_desc.Priority = 1000
return job_desc
def get_job_desc():
job_desc = JobDescription()
task = TaskDescription()
resource = ResourceDescription()
resource.Cpu = cfg.CPU_NUM * 100
resource.Memory = 10000
# Create task.
task.ResourceDescription = resource
task.PackageUri = cfg.FULL_PACKAGE
task.ProgramName = 'test1.py'
task.ProgramType = 'python'
task.ImageId = cfg.IMAGE_ID
#task.InstanceCount = cfg.COUNT_TASK_NUM
task.InstanceCount = cfg.INSTANCE_NUM
task.EnvironmentVariables = {}
task.StdoutRedirectPath = cfg.LOG_PATH
task.StderrRedirectPath = cfg.LOG_PATH
# Create task dag.
task_dag = TaskDag()
task_dag.add_task(task_name='upload', task=task)
job_desc.TaskDag = task_dag
job_desc.JobName = 'bctest'
job_desc.Priority = 1000
return job_desc
def get_job_desc():
argv = {}
argv['-m'] = str(cfg.M)
argv['-s'] = str(cfg.SYNC)
argv_str = ''
for k, v in argv.iteritems():
argv_str += ' %s %s' % (k, v)
job_desc = JobDescription()
task = TaskDescription()
resource = ResourceDescription()
resource.Cpu = cfg.CPU_NUM * 100
resource.Memory = 4000
# Create task.
task.ResourceDescription = resource
task.PackageUri = cfg.FULL_PACKAGE
task.ProgramArguments = argv_str
task.ProgramName = 'ud.py'
task.ProgramType = 'python'
task.ImageId = cfg.IMAGE_ID
#task.InstanceCount = cfg.COUNT_TASK_NUM
task.InstanceCount = cfg.INSTANCE_NUM
task.EnvironmentVariables = {}
task.StdoutRedirectPath = cfg.LOG_PATH
task.StderrRedirectPath = cfg.LOG_PATH
# Create task dag.
task_dag = TaskDag()
task_dag.add_task(task_name='upload', task=task)
job_desc.TaskDag = task_dag
job_desc.JobName = 'dtest'
job_desc.Priority = 1000
return job_desc
def get_job_desc(package_path, verbose=True):
job_desc = JobDescription()
find_task = TaskDescription()
# Create find task.
find_task.PackageUri = package_path
find_task.ProgramName = 'find_prime_multi_instance_worker.py'
find_task.ProgramType = 'python'
find_task.ImageId = cfg.IMAGE_ID
#find_task.InstanceCount = cfg.COUNT_TASK_NUM
find_task.InstanceCount = 3
find_task.EnvironmentVariables = {}
find_task.StdoutRedirectPath = cfg.LOG_PATH
find_task.StderrRedirectPath = cfg.LOG_PATH
# Create count task.
count_task = TaskDescription(find_task)
count_task.InstanceCount = 1
# Create task dag.
task_dag = TaskDag()
task_dag.add_task(task_name='Find', task=find_task)
task_dag.add_task(task_name='Count', task=count_task)
task_dag.Dependencies = {'Find': ['Count']}
# count prime job description.
job_desc.TaskDag = task_dag
job_desc.JobName = 'PythonSDK2'
job_desc.Priority = 1
return job_desc
def _get_job_desc(self):
job_desc = JobDescription()
find_task = TaskDescription()
res_desc = ResourceDescription()
find_task.PackageUri = "oss://your-bucket/batch_python_sdk/worker.tar.gz"
find_task.ProgramName = 'find_prime_worker.py'
find_task.ProgramType = 'python'
find_task.ImageId = self.image_id
find_task.InstanceCount = 3
find_task.EnvironmentVariables = {}
find_task.StdoutRedirectPath = "oss://your-bucket/batch_python_sdk/logs/"
find_task.StderrRedirectPath = "oss://your-bucket/batch_python_sdk/logs/"
find_task.ResourceDescription = res_desc
# Create count task.
count_task = TaskDescription(find_task)
count_task['InstanceCount'] = 1
# Create task dag.
task_dag = TaskDag()
task_dag.add_task(task_name='Find', task=find_task)
task_dag.add_task(task_name='Count', task=count_task)
task_dag.Dependencies = {'Find': ['Count']}
# count prime job description.
job_desc.TaskDag = task_dag
job_desc.JobName = 'PythonSDK'
job_desc.Priority = 0
return job_desc
def get_job_desc(package_path, verbose=True):
job_desc = JobDescription()
find_task = TaskDescription()
# Create find task.
find_task.PackageUri = package_path
find_task.ProgramName = 'find_prime_multi_instance_worker.py'
find_task.ProgramType = 'python'
find_task.ImageId = cfg.IMAGE_ID
#find_task.InstanceCount = cfg.COUNT_TASK_NUM
find_task.InstanceCount = 3
find_task.EnvironmentVariables = {}
find_task.StdoutRedirectPath = cfg.LOG_PATH
find_task.StderrRedirectPath = cfg.LOG_PATH
# Create count task.
count_task = TaskDescription(find_task)
count_task.InstanceCount = 1
# Create task dag.
task_dag = TaskDag()
task_dag.add_task(task_name='Find', task=find_task)
task_dag.add_task(task_name='Count', task=count_task)
task_dag.Dependencies = {
'Find': ['Count']
}
# count prime job description.
job_desc.TaskDag = task_dag
job_desc.JobName = 'PythonSDK2'
job_desc.Priority = 1
return job_desc
def _get_job_desc(self):
job_desc = JobDescription()
find_task = TaskDescription()
res_desc = ResourceDescription()
find_task.PackageUri = "oss://your-bucket/batch_python_sdk/worker.tar.gz"
find_task.ProgramName = 'find_prime_worker.py'
find_task.ProgramType = 'python'
find_task.ImageId = self.image_id
find_task.InstanceCount = 3
find_task.EnvironmentVariables = {}
find_task.StdoutRedirectPath = "oss://your-bucket/batch_python_sdk/logs/"
find_task.StderrRedirectPath = "oss://your-bucket/batch_python_sdk/logs/"
find_task.ResourceDescription = res_desc
# Create count task.
count_task = TaskDescription(find_task)
count_task['InstanceCount'] = 1
# Create task dag.
task_dag = TaskDag()
task_dag.add_task(task_name='Find', task=find_task)
task_dag.add_task(task_name='Count', task=count_task)
task_dag.Dependencies = {
'Find': ['Count']
}
# count prime job description.
job_desc.TaskDag = task_dag
job_desc.JobName = 'PythonSDK'
job_desc.Priority = 0
return job_desc
def get_job_desc(package_path, verbose=True):
job_desc = JobDescription()
find_task= TaskDescription()
# find task description map.
find_task.PackageUri = package_path
find_task.ProgramName = 'find_prime_multi_instance_worker.py'
find_task.ProgramType = 'python'
find_task.ImageId = cfg.IMAGE_ID
find_task.InstanceCount = INSTANCE_COUNT
find_task.EnvironmentVariables = {}
find_task.StdoutRedirectPath = cfg.LOG_PATH
find_task.StderrRedirectPath = cfg.LOG_PATH
# Create task dag.
task_dag = TaskDag()
task_dag.add_task(task_name='Find', task=find_task)
# find prime job description.
job_desc.TaskDag = task_dag
job_desc.JobName = 'find-prime'
job_desc.Priority = 1
return job_desc
def get_job_desc(package_path, verbose=True):
# generate the worker command line string and encode it.
argv = {}
argv['-s'] = str(cfg.DATA_START)
argv['-e'] = str(cfg.DATA_END)
argv['-d'] = cfg.DATA_PATH
argv['-o'] = cfg.OUTPUT_PATH
argv['-b'] = cfg.OSS_BUCKET
argv['-c'] = str(cfg.COUNT_TASK_NUM)
argv['-l'] = cfg.LOCAL_DATA
argv_str = ''
for k, v in argv.iteritems():
argv_str += ' %s %s' % (k, v)
job_desc = JobDescription()
find_task = TaskDescription()
# Create find task.
find_task.PackageUri = package_path
find_task.ProgramName = 'count_prime_with_ossmapping_worker.py'
find_task.ProgramType = 'python'
find_task.ProgramArguments = argv_str
find_task.ImageId = cfg.IMAGE_ID
find_task.InstanceCount = cfg.COUNT_TASK_NUM
find_task.EnvironmentVariables = {}
find_task.StdoutRedirectPath = cfg.LOG_PATH
find_task.StderrRedirectPath = cfg.LOG_PATH
# Create count task.
count_task = TaskDescription(find_task)
count_task.InstanceCount = cfg.SUM_TASK_NUM
# This mapping will mount a oss path to a local path in VM.
count_task.OssMapping = {cfg.FULL_DATA: cfg.LOCAL_DATA}
# Create task dag.
task_dag = TaskDag()
task_dag.add_task(task_name='Find', task=find_task)
task_dag.add_task(task_name='Count', task=count_task)
task_dag.Dependencies = {'Find': ['Count']}
# count prime job description.
job_desc.TaskDag = task_dag
job_desc.JobName = 'find-prime'
job_desc.Priority = 100
return job_desc
def get_job_desc(package_path, verbose=True):
# generate the worker command line string and encode it.
argv = {}
argv['-s'] = str(cfg.DATA_START)
argv['-e'] = str(cfg.DATA_END)
argv['-d'] = cfg.DATA_PATH
argv['-o'] = cfg.OUTPUT_PATH
argv['-b'] = cfg.OSS_BUCKET
argv['-c'] = str(cfg.COUNT_TASK_NUM)
argv['-l'] = cfg.LOCAL_DATA
argv_str = ''
for k, v in argv.iteritems():
argv_str += ' %s %s'%(k, v)
job_desc = JobDescription()
find_task = TaskDescription()
# Create find task.
find_task.PackageUri = package_path
find_task.ProgramName = 'count_prime_with_ossmapping_worker.py'
find_task.ProgramType = 'python'
find_task.ProgramArguments = argv_str
find_task.ImageId = cfg.IMAGE_ID
find_task.InstanceCount = cfg.COUNT_TASK_NUM
find_task.EnvironmentVariables = {}
find_task.StdoutRedirectPath = cfg.LOG_PATH
find_task.StderrRedirectPath = cfg.LOG_PATH
# Create count task.
count_task = TaskDescription(find_task)
count_task.InstanceCount = cfg.SUM_TASK_NUM
# This mapping will mount a oss path to a local path in VM.
count_task.OssMapping = {
cfg.FULL_DATA: cfg.LOCAL_DATA
}
# Create task dag.
task_dag = TaskDag()
task_dag.add_task(task_name='Find', task=find_task)
task_dag.add_task(task_name='Count', task=count_task)
task_dag.Dependencies = {
'Find': ['Count']
}
# count prime job description.
job_desc.TaskDag = task_dag
job_desc.JobName = 'find-prime'
job_desc.Priority = 100
return job_desc