예제 #1
0
파일: HdfsUtil.py 프로젝트: fmdeng/xtahcew
def restoreData(conf_str):
    conf = HexinConfiguration(conf_str)
    hdfs_dir = conf.getProperty("Configuration.hdfs_dir.value")
    local_data_dir = conf.getProperty("Configuration.local_data_dir.value")
    data_list = []
    node_list = conf.getNodeList("Configuration.restoreDataList")
    for node in node_list:
        data_list.append(node["value"])
    for data_file in data_list:
        getHdfsFile(hdfs_dir, data_file, local_data_dir + "/" + data_file)
예제 #2
0
def restoreData(conf_str):
    conf = HexinConfiguration(conf_str)
    hdfs_dir = conf.getProperty("Configuration.hdfs_dir.value")
    local_data_dir = conf.getProperty("Configuration.local_data_dir.value")
    data_list = []
    node_list = conf.getNodeList("Configuration.restoreDataList")
    for node in node_list:
        data_list.append(node["value"])
    for data_file in data_list:
        getHdfsFile(hdfs_dir,data_file,local_data_dir+"/"+data_file)
예제 #3
0
파일: HdfsUtil.py 프로젝트: fmdeng/xtahcew
    data_list = []
    node_list = conf.getNodeList("Configuration.restoreDataList")
    for node in node_list:
        data_list.append(node["value"])
    for data_file in data_list:
        getHdfsFile(hdfs_dir, data_file, local_data_dir + "/" + data_file)


# main
if __name__ == '__main__':
    if len(sys.argv) < 3:
        "Wrong usage: python HdfsUtil setting.xml save"
        sys.exit(0)
    conf_str = sys.argv[1]
    cmd = sys.argv[2]
    if cmd == "save":
        saveData(conf_str)
    elif cmd == "restore":
        restoreData(conf_str)

    conf = HexinConfiguration(conf_str)
    hdfs_dir = conf.getProperty("Configuration.hdfs_dir.value")
    print lsHdfsDir(hdfs_dir)
    '''mkHdfsDir(hdfs_dir)
    putHdfsFile(hdfs_dir,"neg_dict","data/neg_dict")
    putHdfsFile(hdfs_dir,"pos_dict","data/pos_dict")
    putHdfsFile(hdfs_dir,"stopwords.txt","data/stopwords.txt")
    putHdfsFile(hdfs_dir,"orig_word","data/orig_word")
    putHdfsFile(hdfs_dir,"myhexin.trade.kdd.txt","data/myhexin.trade.kdd.txt")
    '''
예제 #4
0
    data_list = []
    node_list = conf.getNodeList("Configuration.restoreDataList")
    for node in node_list:
        data_list.append(node["value"])
    for data_file in data_list:
        getHdfsFile(hdfs_dir,data_file,local_data_dir+"/"+data_file)
    
# main
if __name__ == '__main__':
    if len(sys.argv) < 3:
        "Wrong usage: python HdfsUtil setting.xml save"
        sys.exit(0)
    conf_str = sys.argv[1]
    cmd = sys.argv[2]
    if cmd == "save":
        saveData(conf_str)
    elif cmd == "restore":
        restoreData(conf_str)
        
    conf = HexinConfiguration(conf_str)
    hdfs_dir = conf.getProperty("Configuration.hdfs_dir.value")
    print lsHdfsDir(hdfs_dir)
    '''mkHdfsDir(hdfs_dir)
    putHdfsFile(hdfs_dir,"neg_dict","data/neg_dict")
    putHdfsFile(hdfs_dir,"pos_dict","data/pos_dict")
    putHdfsFile(hdfs_dir,"stopwords.txt","data/stopwords.txt")
    putHdfsFile(hdfs_dir,"orig_word","data/orig_word")
    putHdfsFile(hdfs_dir,"myhexin.trade.kdd.txt","data/myhexin.trade.kdd.txt")
    '''