def train(args): ctx = Context(s3_path=args.context, cache_dir=args.cache_dir, workload_id=args.workload_id) package.install_packages(ctx.python_packages, ctx.bucket) model = ctx.models_id_map[args.model] logger.info("Training") with util.Tempdir(ctx.cache_dir) as temp_dir: model_dir = os.path.join(temp_dir, "model_dir") ctx.upload_resource_status_start(model) try: model_impl = ctx.get_model_impl(model["name"]) train_util.train(model["name"], model_impl, ctx, model_dir) ctx.upload_resource_status_success(model) logger.info("Caching") logger.info("Caching model " + model["name"]) model_export_dir = os.path.join(model_dir, "export", "estimator") model_zip_path = os.path.join(temp_dir, "model.zip") util.zip_dir(model_export_dir, model_zip_path) aws.upload_file_to_s3(local_path=model_zip_path, key=model["key"], bucket=ctx.bucket) util.log_job_finished(ctx.workload_id) except CortexException as e: ctx.upload_resource_status_failed(model) e.wrap("error") logger.error(str(e)) logger.exception( "An error occurred, see `cx logs model {}` for more details.". format(model["name"])) sys.exit(1) except Exception as e: ctx.upload_resource_status_failed(model) logger.exception( "An error occurred, see `cx logs model {}` for more details.". format(model["name"])) sys.exit(1)
def zip_and_upload(self, local_path, key): util.zip_dir(local_path, "temp.zip") self.s3.upload_file("temp.zip", self.bucket, key) util.rm_file("temp.zip")
#!/usr/bin/env python2 # coding:utf-8 import sys sys.path.append(r"../") from lib.util import zip_dir usage = '1 : output filename like xxx.zip,2: input_dir' if len(sys.argv) != 3: print usage sys.exit(1) filename = sys.argv[1] input_dir = sys.argv[2] zip_dir(input_dir, filename)
def compress_zip_and_upload(local_path, key, bucket, client_config={}): s3 = s3_client(client_config) util.zip_dir(local_path, "temp.zip") s3.upload_file("temp.zip", bucket, key) util.rm_file("temp.zip")
# convert txt to html processCate(cate,doc_prefix,epub_path+'/OEBPS/Text') # output toc.ncx params['cate'] = cate params['spine'] = sections[:] params['files'] = sections[:] env.trim_blocks = True env.lstrip_blocks = True toc_tpl = env.get_template('00_toc.ncx') content_tpl = env.get_template('00_content.opf') toc = toc_tpl.render(params) cnt = content_tpl.render(params) out_toc = open(epub_path+'/OEBPS/toc.ncx', 'w') out_cnt = open(epub_path+'/OEBPS/content.opf', 'w') # ouput toc out_toc.write(toc) out_toc.close() # output content.opf out_cnt.write(cnt) out_cnt.close() # buildEpub zip_dir(epub_path,params['title']+'.epub')