コード例 #1
0
def do_mergetable(job_q,result_q,server_ip,user_name):
    print 'do_mergetable start ...'
    scp = paramiko.Transport((server_ip,22))
    scp.connect(username=user_name,password=common.g_server_password)
    sftp = paramiko.SFTPClient.from_transport(scp)
    
    while True:
        try:
            remote_input_table_list = job_q.get_nowait()
    
            for remote_input_table in remote_input_table_list:
                filename = remote_input_table[remote_input_table.rfind('/')+1:]
                local_input_table = MERGE_TABLE_CACHE_DIR + filename
                #sftp.get(remote_input_table,local_input_table)
                robust_sftp(sftp,remote_input_table,local_input_table,True)
        
            local_output_table = MergeTable_single.parallel_merge_tables(MERGE_TABLE_CACHE_DIR)
            print '-->local_output_table=',local_output_table
            output_filename = local_output_table[local_output_table.rfind('/')+1:]
            remote_output_table = remote_input_table[:remote_input_table.rfind('/')+1]+output_filename
            print 'remote_output_table=',remote_output_table
            #sftp.put(local_output_table,remote_output_table)
            if robust_sftp(sftp,remote_output_table,local_output_table,False):
                os.remove(local_output_table)
            result_q.put(1)
        except Exception,e:
            if job_q.empty():
                print 'Queue.Empty, will exit loop. pid=%d'%(os.getpid())
                break
            else:
                print 'Is not job_q.empty() exception, pid=%d'%(os.getpid())
                print e
                print traceback.format_exc()
コード例 #2
0
def do_remaintable(server_ip,user_name):
    print 'do_remaintable start ...'
    scp = paramiko.Transport((server_ip,22))
    scp.connect(username=user_name,password=common.g_server_password)
    sftp = paramiko.SFTPClient.from_transport(scp)

    try:
        local_output_table = MergeTable_single.parallel_merge_tables(MERGE_TABLE_CACHE_DIR)
        print '-->local_output_table=',local_output_table
        output_filename = local_output_table[local_output_table.rfind('/')+1:]
        remote_output_table = '/home/%s/%s'%(user_name,output_filename)
        print 'remote_output_table=',remote_output_table
        sftp.put(local_output_table,remote_output_table)
        os.remove(local_output_table)
    except Exception,e:
        print e
        print traceback.format_exc()