def main():
upload_worker(cfg.OSS_BUCKET, 'worker_package', cfg.PACKAGE_PATH)
# Submit job to batch compute.
clnt = Client(cfg.REGION, cfg.ID, cfg.KEY)
job_json = get_job_desc(cfg.FULL_PACKAGE)
job = clnt.create_job(job_json)
t = 10
print('Sleep %s second, please wait.'%t)
time.sleep(t)
# Wait for jobs terminated.
while(True):
s = clnt.get_job(job)
if s.State in ['Waiting', 'Running']:
print('Job %s is now %s'%(job, s.State))
time.sleep(3)
continue
else:
# 'Failed', 'Stopped', 'Finished'
print('Job %s is now %s'%(job, s.State))
if s.State == 'Finished':
oss_clnt = SimpleOss(cfg.OSS_HOST, cfg.ID, cfg.KEY)
result = oss_clnt.download_str(cfg.OSS_BUCKET, cfg.OUTPUT_PATH)
# Print out all prime numbers from 0 to 10000.
print result.splitlines()
break
clnt.delete_job(job)
def upload_worker(bucket, local_dir, oss_path):
'''
A function to help upload worker package to oss.
'''
oss_clnt = SimpleOss(cfg.OSS_HOST, cfg.ID, cfg.KEY)
local_tarfile = 'worker.tar.gz'
if os.path.exists(local_tarfile): os.remove(local_tarfile)
def do_tar(worker_dir, tar_file):
'''
A function to tar worker package.
'''
tar = tarfile.open(tar_file, 'w:gz')
cwd = os.getcwd()
os.chdir(worker_dir)
for root,dir,files in os.walk('.'):
for file in files:
tar.add(os.path.join(root, file))
os.chdir(cwd)
tar.close()
do_tar(local_dir, local_tarfile)
oss_clnt.upload(bucket, local_tarfile, oss_path)
import os
import time
import tarfile
from externals.simple_oss import SimpleOss
from batchcompute import (Client, JobDescription, TaskDag, TaskDescription,
ResourceDescription)
import config as cfg
oss_clnt = SimpleOss(cfg.OSS_HOST, cfg.ID, cfg.KEY)
def upload_worker(bucket, local_dir, oss_path):
'''
A function to help upload worker package to oss.
'''
local_tarfile = 'worker.tar.gz'
if os.path.exists(local_tarfile): os.remove(local_tarfile)
def do_tar(worker_dir, tar_file):
'''
A function to tar worker package.
'''
tar = tarfile.open(tar_file, 'w:gz')
cwd = os.getcwd()
os.chdir(worker_dir)
for root, dir, files in os.walk('.'):
for file in files:
tar.add(os.path.join(root, file))
os.chdir(cwd)
tar.close()
#!usr/bin/env python
#encoding: utf-8
import config as cfg
import os, sys
from multiprocessing import Pool
from time import time
from subprocess import call, PIPE, Popen
from externals.simple_oss import SimpleOss
__author__ = "luzhijun"
OSS_HOST = os.environ.get('ALI_DIKU_OSS_HOST')
oss_clnt = SimpleOss(OSS_HOST, cfg.ID, cfg.KEY, cfg.OSS_BUCKET)
str_list = []
rec = '%s %s\n'
def make_file1(size):
p = size * 1024 * 1024 * 1024 - 1
with open("test.iso", 'w') as f:
f.seek(p)
f.writelines('end')
f.write('\x00')
def make_file(size):
size *= 1024 * 1024 * 1024
fsize = 0
i = 0
datetime = "strftime('%a, %d %b %Y %H:%M:%S +0000', gmtime())"
with open("test.iso", 'w') as f:
while True: