def infer_topic(in_fn, model_fn, out_fn, conf): parser = SafeConfigParser() parser.read(conf) root = parser.get("basic", "root") hadoop_stream = parser.get('basic', 'hadoop_stream') topic_num = parser.getint('plda+', 'topic_num') alpha = 50.0 / topic_num task_id = uuid.uuid4() infer_in_path = "%s/%s" % (parser.get('plda+', 'infer_in_path'), task_id) infer_out_path = "%s/%s" % (parser.get('plda+', 'infer_out_path'), task_id) infer_burn_in_iter = parser.getint('plda+', 'infer_burn_in_iter') infer_total_iter = parser.getint('plda+', 'infer_total_iter') infer_reduce_tasks = parser.getint('plda+', 'infer_reduce_tasks') infer_reducer_mb = parser.getint('plda+', 'infer_reducer_mb') mapper = '%s/plda/infer_mapper' % root reducer = '%s/plda/infer_reducer' % root reducer_wrapper = '%s/data/temp/reducer_wrapper.sh' % root hdfs = luigi.contrib.hdfs.hadoopcli_clients.create_hadoopcli_client() hdfs.mkdir(infer_in_path) hdfs.put(in_fn, infer_in_path) with open(reducer_wrapper, 'w') as wrapper_fd: print >> wrapper_fd, "#!/bin/bash" print >> wrapper_fd, "./infer_reducer --alpha %f --beta 0.01 --model_file ./%s --burn_in_iterations %d --total_iterations %d -sparse true" % \ (alpha, os.path.basename(model_fn), infer_burn_in_iter, infer_total_iter) cmd = '''hadoop jar %s \ -D mapred.job.name="mr plda+ infer" \ -D mapred.job.map.memory.mb=32 \ -D mapred.job.reduce.memory.mb=%d \ -D io.compression.codecs=org.apache.hadoop.io.compress.DefaultCodec \ -input %s \ -output %s \ -file %s \ -file %s \ -file %s \ -file %s \ -mapper ./infer_mapper \ -reducer ./reducer_wrapper.sh \ -numReduceTasks %d ''' cmd = cmd % (hadoop_stream, infer_reducer_mb, infer_in_path, infer_out_path, model_fn, mapper, reducer, reducer_wrapper, infer_reduce_tasks) os.system(cmd) os.remove(reducer_wrapper) if check_mr_success(infer_out_path): with open(out_fn, 'w') as out_fd: get_mr_dir(infer_out_path, out_fd) hdfs.remove(infer_in_path) hdfs.remove(infer_out_path) else: hdfs.remove(infer_in_path) hdfs.remove(infer_out_path) raise Exception("failed to infer topic")
def run(self): if os.path.exists(self.external): os.remove(self.external) with open(self.external, "w") as external_fd: get_mr_dir(self.input()[0].path, external_fd) df = sf.SFrame.read_csv(self.external, column_type_hints=[str, str, str], delimiter='\t', header=False) cols = {} for i in xrange(len(self.schema)): cols["X%d" % (i + 1)] = self.schema[i] df.rename(cols) df.save(self.output().fn) os.remove(self.external)
def run(self): #copy user data if os.path.exists(self.external_user): os.remove(self.external_user) with open(self.external_user, "w") as ext_user: get_mr_dir(self.input()[0]['user'].path, ext_user) #copy user data version hdfs = luigi.contrib.hdfs.hadoopcli_clients.create_hadoopcli_client() hdfs.get(self.input()[0]['version'].path, self.output()['version'].fn) #transform to sframe ans save df = sf.SFrame.read_csv(self.external_user, delimiter="\t", column_type_hints=[str, str, list], header=False) cols = {} for i in xrange(len(self.schema)): cols["X%d" % (i + 1)] = self.schema[i] df.rename(cols) df.save(self.output()['user'].fn) os.remove(self.external_user)