def train(args):
ctx = Context(s3_path=args.context,
cache_dir=args.cache_dir,
workload_id=args.workload_id)
package.install_packages(ctx.python_packages, ctx.bucket)
model = ctx.models_id_map[args.model]
logger.info("Training")
with util.Tempdir(ctx.cache_dir) as temp_dir:
model_dir = os.path.join(temp_dir, "model_dir")
ctx.upload_resource_status_start(model)
try:
model_impl = ctx.get_model_impl(model["name"])
train_util.train(model["name"], model_impl, ctx, model_dir)
ctx.upload_resource_status_success(model)
logger.info("Caching")
logger.info("Caching model " + model["name"])
model_export_dir = os.path.join(model_dir, "export", "estimator")
model_zip_path = os.path.join(temp_dir, "model.zip")
util.zip_dir(model_export_dir, model_zip_path)
aws.upload_file_to_s3(local_path=model_zip_path,
key=model["key"],
bucket=ctx.bucket)
util.log_job_finished(ctx.workload_id)
except CortexException as e:
ctx.upload_resource_status_failed(model)
e.wrap("error")
logger.error(str(e))
logger.exception(
"An error occurred, see `cx logs model {}` for more details.".
format(model["name"]))
sys.exit(1)
except Exception as e:
ctx.upload_resource_status_failed(model)
logger.exception(
"An error occurred, see `cx logs model {}` for more details.".
format(model["name"]))
sys.exit(1)
def zip_and_upload(self, local_path, key):
util.zip_dir(local_path, "temp.zip")
self.s3.upload_file("temp.zip", self.bucket, key)
util.rm_file("temp.zip")
#!/usr/bin/env python2
# coding:utf-8
import sys
sys.path.append(r"../")
from lib.util import zip_dir
usage = '1 : output filename like xxx.zip,2: input_dir'
if len(sys.argv) != 3:
print usage
sys.exit(1)
filename = sys.argv[1]
input_dir = sys.argv[2]
zip_dir(input_dir, filename)
def compress_zip_and_upload(local_path, key, bucket, client_config={}):
s3 = s3_client(client_config)
util.zip_dir(local_path, "temp.zip")
s3.upload_file("temp.zip", bucket, key)
util.rm_file("temp.zip")
# convert txt to html
processCate(cate,doc_prefix,epub_path+'/OEBPS/Text')
# output toc.ncx
params['cate'] = cate
params['spine'] = sections[:]
params['files'] = sections[:]
env.trim_blocks = True
env.lstrip_blocks = True
toc_tpl = env.get_template('00_toc.ncx')
content_tpl = env.get_template('00_content.opf')
toc = toc_tpl.render(params)
cnt = content_tpl.render(params)
out_toc = open(epub_path+'/OEBPS/toc.ncx', 'w')
out_cnt = open(epub_path+'/OEBPS/content.opf', 'w')
# ouput toc
out_toc.write(toc)
out_toc.close()
# output content.opf
out_cnt.write(cnt)
out_cnt.close()
# buildEpub
zip_dir(epub_path,params['title']+'.epub')
