def create_workflow_dir(where, job): constructed_dir = '/user/hadoop/' constructed_dir = _add_postfix(constructed_dir) constructed_dir += '%s/%s' % (job.name, uuidutils.generate_uuid()) with remote.get_remote(where) as r: h.create_dir(r, constructed_dir) return constructed_dir
def create_workflow_dir(where, job, hdfs_user): constructed_dir = '/user/%s/' % hdfs_user constructed_dir = _add_postfix(constructed_dir) constructed_dir += '%s/%s' % (job.name, six.text_type(uuid.uuid4())) with remote.get_remote(where) as r: h.create_dir(r, constructed_dir, hdfs_user) return constructed_dir
def create_workflow_dir(where, job, hdfs_user): constructed_dir = '/user/hadoop/' constructed_dir = _add_postfix(constructed_dir) constructed_dir += '%s/%s' % (job.name, uuidutils.generate_uuid()) with remote.get_remote(where) as r: h.create_dir(r, constructed_dir, hdfs_user) return constructed_dir
def upload_job_files(where, job_dir, job, hdfs_user): mains = job.mains or [] libs = job.libs or [] uploaded_paths = [] with remote.get_remote(where) as r: for main in mains: raw_data = dispatch.get_raw_binary(main) h.put_file_to_hdfs(r, raw_data, main.name, job_dir, hdfs_user) uploaded_paths.append(job_dir + '/' + main.name) for lib in libs: raw_data = dispatch.get_raw_binary(lib) # HDFS 2.2.0 fails to put file if the lib dir does not exist h.create_dir(r, job_dir + "/lib", hdfs_user) h.put_file_to_hdfs(r, raw_data, lib.name, job_dir + "/lib", hdfs_user) uploaded_paths.append(job_dir + '/lib/' + lib.name) return uploaded_paths