def do_mergetable(job_q,result_q,server_ip,user_name):
print 'do_mergetable start ...'
scp = paramiko.Transport((server_ip,22))
scp.connect(username=user_name,password=common.g_server_password)
sftp = paramiko.SFTPClient.from_transport(scp)
while True:
try:
remote_input_table_list = job_q.get_nowait()
for remote_input_table in remote_input_table_list:
filename = remote_input_table[remote_input_table.rfind('/')+1:]
local_input_table = MERGE_TABLE_CACHE_DIR + filename
#sftp.get(remote_input_table,local_input_table)
robust_sftp(sftp,remote_input_table,local_input_table,True)
local_output_table = MergeTable_single.parallel_merge_tables(MERGE_TABLE_CACHE_DIR)
print '-->local_output_table=',local_output_table
output_filename = local_output_table[local_output_table.rfind('/')+1:]
remote_output_table = remote_input_table[:remote_input_table.rfind('/')+1]+output_filename
print 'remote_output_table=',remote_output_table
#sftp.put(local_output_table,remote_output_table)
if robust_sftp(sftp,remote_output_table,local_output_table,False):
os.remove(local_output_table)
result_q.put(1)
except Exception,e:
if job_q.empty():
print 'Queue.Empty, will exit loop. pid=%d'%(os.getpid())
break
else:
print 'Is not job_q.empty() exception, pid=%d'%(os.getpid())
print e
print traceback.format_exc()
def do_remaintable(server_ip,user_name):
print 'do_remaintable start ...'
scp = paramiko.Transport((server_ip,22))
scp.connect(username=user_name,password=common.g_server_password)
sftp = paramiko.SFTPClient.from_transport(scp)
try:
local_output_table = MergeTable_single.parallel_merge_tables(MERGE_TABLE_CACHE_DIR)
print '-->local_output_table=',local_output_table
output_filename = local_output_table[local_output_table.rfind('/')+1:]
remote_output_table = '/home/%s/%s'%(user_name,output_filename)
print 'remote_output_table=',remote_output_table
sftp.put(local_output_table,remote_output_table)
os.remove(local_output_table)
except Exception,e:
print e
print traceback.format_exc()
