def join_bug_log_file(self, bugid, path, file_name, max_index):
local_file_dir = ''
local_file_dir = path + r'/' + str(bugid)
local_file_name = file_name
print_string = "join bug %s file %s log ..." % (str(bugid),
local_file_name)
print_info.print_info(print_info.PRINT_INFO, print_string)
if os.path.exists(local_file_dir + r'/' + local_file_name):
os.remove(local_file_dir + r'/' + local_file_name)
for x in range(max_index + 1):
try:
fread = open(
local_file_dir + r'/' + str(x) + '_' + local_file_name,
'r')
except Exception, e:
print_string = "join bug %s file error,open file %s" % (
str(bugid), e)
print_info.print_info(print_info.PRINT_ERROR, print_string)
else:
data = fread.read()
with open(local_file_dir + r'/' + local_file_name,
'ab') as lwrite:
lwrite.write(data)
fread.close()
os.remove(local_file_dir + r'/' + str(x) + '_' +
local_file_name)
def download_samba_file(self, sambafilepath, localfilepath, filename):
try:
time_start = time.time()
conn = SMBConnection(self.user_name, self.password, self.my_name, self.domain_name, use_ntlm_v2=True)
print_string = 'samba connection %s' % conn
print_info.print_info(print_info.PRINT_DEBUG, print_string)
assert conn.connect(self.remote_smb_ip, self.port)
print_string = "local file path : %s" % (localfilepath + "/" + filename)
print_info.print_info(print_info.PRINT_DEBUG, print_string)
file_fd = open(localfilepath + "/" + filename, 'wb')
sambafilepath = sambafilepath + "/" + filename
print_string = "samba file path : %s" % sambafilepath
print_info.print_info(print_info.PRINT_DEBUG, print_string)
conn.retrieveFile(self.dir, sambafilepath, file_fd)
time_use = time.time() - time_start
file_size = os.path.getsize(localfilepath + r'/' + filename)
samba_download_speed = (int)(file_size / time_use)
file_fd.close()
print_string = 'samba download file %s ok .\ndownload size %d use time is %0.2f s; speed is %d bit/s' % \
(filename, file_size, time_use, samba_download_speed)
print_info.print_info(print_info.PRINT_INFO, print_string)
return 1
return 1
except Exception, e :
print_string = "samba download file error %s" % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
return 0
def socket_server_send(self, sock, data):
try:
sock.send(data)
except Exception, e:
print_string = "send data to socket %s ,client error %s" % (sock,
e)
print_info.print_info(print_info.PRINT_ERROR, print_string)
return 0
def chang_work_dir(self, fd, remotefilepatch):
#change remote work dir
try:
#fd.set_pasv(0)
fd.cwd(remotefilepatch)
except Exception, e:
print_string = 'ftp change dir error %s' % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
return (0, 'ftp change dir error')
def socket_server_status(self, sock, addr):
print_string = 'Accept new connection from %s:%s...' % (addr[0],
str(addr[1]))
print_info.print_info(print_info.PRINT_DEBUG, print_string)
while True:
recv_data = self.socket_server_recv(sock)
if not self.handle_recv_data(sock, addr, recv_data):
break
print_string = "close connection from %s:%s..." % addr
print_info.print_info(print_info.PRINT_DEBUG, print_string)
def upload_samba_file(self, filename, upload_path):
try:
conn = SMBConnection(self.user_name, self.password, self.my_name, self.domain_name, use_ntlm_v2=True)
conn.connect(self.remote_smb_ip, self.port)
file_fd = open(upload_path + filename, 'rb')
conn.storeFile(self.dir, filename, file_fd)
file_fd.close()
print_info.print_info(print_info.PRINT_INFO, "samba upload file ok")
return 1
except Exception, e :
print_string = "samba upload file error %s" % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
return 0
def ssh_exec_scp(self, dst_file, src_file):
for x in range(3, 13):
try:
print_string = "scp file : %s to %s ... " % (src_file,
dst_file)
print_info.print_info(print_info.PRINT_DEBUG, print_string)
scp_command = pexpect.spawn('/usr/bin/scp -r ' + src_file +
' ' + dst_file,
timeout=x * 100)
expect_result = scp_command.expect([r'assword:', r'yes/no'],
timeout=300)
if expect_result == 0:
scp_command.sendline(ssh_password)
scp_command.read()
elif expect_result == 1:
print_info.print_info(print_info.PRINT_DEBUG,
'ssh scp :send yes')
scp_command.sendline('yes')
scp_command.expect('assword:', timeout=300)
scp_command.sendline(ssh_password)
scp_command.read()
except Exception, e:
print_string = "ssh scp file error %d:%s" % (x, e)
print_info.print_info(print_info.PRINT_ERROR, print_string)
else:
print_string = "ssh scp file %s to %s ok" % (src_file,
dst_file)
print_info.print_info(print_info.PRINT_INFO, print_string)
return 1
def connect_ftp_server(self, fd):
for n in range(3):
try:
# set 10s timeout
fd.connect(self.remoteip, self.remoteport, 10)
except Exception, e:
print_string = "conncet ftp server failed - %s" % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
else:
try:
fd.login(self.user, self.password)
except Exception, e:
print_string = 'login ftp server failed - %s' % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
else:
def dispense_work(work, sock_list):
client_speed_sum = 0
speed_min = ftp_speed_max
speed_max = 0
sock_ftp_speed_list = []
dispense_work_list = []
ftp_download_index = 0
ftp_file_size = 0
ftp_file_start = 0
dispense_size_per = 0
client_download_size = 0
try:
ftp_file_size = int(work[6])
except Exception, e:
print_string = "dispense work file size error %s" % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
return (0, "dispense work file size error")
def __init__(self, ip, port, user, password, localpath, remotepath):
global ssh_ip
global ssh_port
global ssh_user
global ssh_password
global local_path
global remote_path
ssh_ip = ip
ssh_port = port
ssh_user = user
ssh_password = password
local_path = localpath
remote_path = remotepath
# print_string = "ip: %s; port: %d; user: %s ; password: %s" % (ip, port, user, password)
# print_info.print_info(print_info.PRINT_DEBUG, print_string)
print_string = "local path : %s \nremote path : %s" % (local_path,
remote_path)
print_info.print_info(print_info.PRINT_DEBUG, print_string)
def get_ftp_filesize(self, fd, remotefilepatch, remotefilename, bugid):
ftp_file_list = []
fsize = 0
change_ret = fd.chang_work_dir(fd, remotefilepatch)
if change_ret[0]:
try:
dir_list = fd.nlst()
print_string = 'dir file list %s' % dir_list
print_info.print_info(print_info.PRINT_DEBUG, print_string)
except Exception, e:
print_string = 'get ftp dir error %s' % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
return (0, 'get ftp dir error')
for x in dir_list:
if remotefilename:
if remotefilename == x:
try:
fsize = fd.size(remotefilename)
except Exception, e:
print_string = 'ftp get file %s size error %s' % (
remotefilename, e)
print_info.print_info(print_info.PRINT_ERROR,
print_string)
return (0, 'ftp get file size error')
else:
ftp_file_list.append([remotefilename, fsize])
print_string = 'get file name %s size %d' % (
remotefilename, fsize)
print_info.print_info(print_info.PRINT_INFO,
print_string)
return (1, ftp_file_list)
elif str(bugid) in x:
try:
fsize = fd.size(x)
except Exception, e:
print_string = 'ftp get file %s size error %s' % (x, e)
print_info.print_info(print_info.PRINT_ERROR,
print_string)
return (0, 'ftp get file size error')
def client_link(self):
global socket_client_fd
while True:
# link socket server
while True:
try:
socket_client_fd = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
socket_client_fd.connect(
(self.socket_ip, self.socket_port))
except Exception, e:
print_string = "link socket server error %s" % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
socket_client_fd.close()
else:
break
time.sleep(30)
print_string = "connet socket server ok %s " % socket_client_fd
print_info.print_info(print_info.PRINT_INFO, print_string)
self.save_log_info_to_file(print_string + '\n')
#recv data socket
while True:
try:
data = self.client_recv(socket_client_fd)
if not self.handle_client_recvdata(data):
break
except Exception, e:
print_string = "socket client recv data error %s" % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
break
def filter_spikes_by_region_stimulus_across_images(multi_probe_expt_info,
region,
stimulus,
short_run=False):
spike_trains = {}
pre_stimulus_time = 0.1
for multi_probe_example in range(len(multi_probe_expt_info)):
multi_probe_filename = multi_probe_expt_info.iloc[multi_probe_example][
'nwb_filename']
# Specify full path to the .nwb file
nwb_file = os.path.join(drive_path, multi_probe_filename)
data_set = NWB_adapter(nwb_file)
for c_probe in np.unique(data_set.unit_df['probe']):
region_units = data_set.unit_df[
(data_set.unit_df['structure'] == region)
& (data_set.unit_df['probe'] == c_probe)]
all_units = data_set.spike_times[c_probe]
for index, region_unit in region_units.iterrows():
spike_train = all_units[region_unit['unit_id']]
for ind, stim_row in data_set.stim_tables[
'natural_scenes'].iterrows():
current_train = spike_train[
(spike_train > stim_row['start'] - pre_stimulus_time)
& (spike_train < stim_row['end'])] - stim_row['start']
train_id = multi_probe_filename + '_' + c_probe + '_' + region_unit[
'unit_id'] + '_' + str(region_unit['depth'])
if not spike_trains.has_key(train_id):
spike_trains[train_id] = []
spike_trains[train_id].append(current_train)
if short_run:
break
print_info(spike_trains)
return spike_trains
def window(start,
stop,
step,
width,
signal,
turn,
color='b',
plot_all=False,
chroma=1):
#if not plot_all:
#plt.figure(figsize=(10,10))
amp = []
frequency = []
while True:
naff_list = []
#windowing
for min_turn in range(start, stop, step):
max_turn = min_turn + width
#print "Turns: ",min_turn,max_turn
amplitude, counter, naff_list = naff_transformation(
width, signal[min_turn:max_turn], naff_list)
#print 'naff', naff_list
amp.append(amplitude[counter - 1])
#print amp
mean = print_info(naff_list)
#use again NAFF to find tune oscillation frequency
a, c, frequency = naff_transformation(len(naff_list),
np.array(naff_list), frequency)
for cnt in xrange(len(frequency)):
print 'Tune frequency', frequency[
cnt] #in this case, the prominent frequency is only one, so this array has only one element!
#plot the tunes over the number of turns to see tune oscillation
tune_plots(turn, start, stop, step, naff_list, mean, color, plot_all)
if max_turn >= len(turn) - width:
break
return mean, frequency[0]
def socket_server_thread(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#set SO_REUSEADDR is true
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# bind
s.bind((self.socket_server_ip, self.socket_server_port))
s.listen(self.socket_server_conn_num)
print_info.print_info(print_info.PRINT_INFO,
'Socket server waiting for connection...')
while True:
sock, addr = s.accept()
print_string = 'socket sock %s; addr %s' % (sock, addr)
print_info.print_info(print_info.PRINT_DEBUG, print_string)
t = threading.Thread(target=self.socket_server_status,
args=(sock, addr),
name="socket_master")
t.start()
self.socket_client_list_lock.acquire()
self.socket_client_list.append([sock, addr, '', 100000])
self.socket_server_send(sock, 'getusername*')
self.socket_server_send(sock, 'getftpspeed*')
self.socket_client_list_lock.release()
if self.update_code:
if addr[0] not in self.update_code_addr_list:
self.socket_server_send(sock, 'updatecode*')
self.update_code_addr_list_lock.acquire()
self.update_code_addr_list.append(addr[0])
self.update_code_addr_list_lock.release()
print_string = 'send updatecode to sock :%s' % sock
print_info.print_info(print_info.PRINT_DEBUG, print_string)
print_info.print_info(print_info.PRINT_DEBUG, 'socket sock server end')
def download_ftp_file_size(self, fd, remotefilepatch, remotefilename,
localfilepath, bugid, filename, start, size):
lsize = 0
cmpsize = 0
time_start = time.time()
if size == 0:
return (1, "remote file size is null")
if not os.path.exists(localfilepath + '/' + str(bugid)):
os.mkdir(localfilepath + '/' + str(bugid))
print_string = 'mkdir local file path %s' % (localfilepath + '/' +
str(bugid))
print_info.print_info(print_info.PRINT_DEBUG, print_string)
localfilename = localfilepath + '/' + str(bugid) + '/' + filename
if os.path.exists(localfilename):
lsize = os.stat(localfilename).st_size
if lsize >= size:
print_string = 'file %s is exist, download file ok' % filename
print_info.print_info(print_info.PRINT_INFO, print_string)
return (1, print_string)
change_ret = fd.chang_work_dir(fd, remotefilepatch)
if not change_ret[0]:
return change_ret
blocksize = 1024 * 8
cmpsize = lsize
fd.voidcmd('TYPE I')
try:
download_file_conn = fd.transfercmd('RETR ' + remotefilename,
start + lsize)
except Exception, e:
print_string = "ftp transfer cmd error %s" % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
return (0, 'time out-----ftp transfer cmd error')
def download_http(html_path, localfilepath, filename):
urlrequest = 0
fd = 0
time_start = time.time()
print_string = 'get file %s ...' % filename
print_info.print_info(print_info.PRINT_INFO, print_string)
try:
urlrequest = requests.get(html_path + '/' + filename, timeout=30)
except:
return (0, 'time out')
if int(urlrequest.status_code) > 400:
print_string = 'get file error %d ' % int(urlrequest.status_code)
print_info.print_info(print_info.PRINT_WARNING, print_string)
return (0, 'get file error')
print_string = 'get file status %d ' % int(urlrequest.status_code)
print_info.print_info(print_info.PRINT_DEBUG, print_string)
with open(localfilepath + r'/' + filename, "wb") as fd:
fd.write(urlrequest.content)
time_use = time.time() - time_start
file_size = os.path.getsize(localfilepath + r'/' + filename)
http_download_speed = (int)(file_size / time_use)
print_string = 'http download file %s end .\ndownload size %d use time is %0.2f s; speed is %d bit/s' % \
(filename, file_size, time_use, http_download_speed)
print_info.print_info(print_info.PRINT_INFO, print_string)
if urlrequest.headers.get('content-length', 0):
if os.path.getsize(localfilepath + r'/' + filename) >= int(
urlrequest.headers['content-length']):
print_info.print_info(print_info.PRINT_INFO, 'get file size ok')
return (1, 'get file size ok')
else:
print_string = 'local size %d html size %d ' % (
os.path.getsize(localfilepath + r'/' + filename),
int(urlrequest.headers['content-length']))
print_info.print_info(print_info.PRINT_ERROR, print_string)
return (0, 'get file size error')
print_string = 'get symbols %s ok' % filename
print_info.print_info(print_info.PRINT_INFO, print_string)
return (1, 'get symbols ok')
class socket_server():
socket_client_list = []
# socket_client_close_list = []
socket_client_list_lock = threading.Lock()
# socket_client_close_list_lock = threading.Lock()
dispense_work_list = []
redispense_work_list = []
dispense_work_list_lock = threading.Lock()
download_log_ok_list = []
download_log_ok_list_lock = threading.Lock()
bugid_max_index = []
bugid_max_index_lock = threading.Lock()
server_main_path = ''
socket_server_proc = 0
socket_recv_data_remnant = ''
update_code = 0
update_code_addr_list = []
update_code_addr_list_lock = threading.Lock()
def __init__(self, ip, port, recv_size, connect_num):
self.socket_server_ip = ip
self.socket_server_port = port
self.socket_server_size = recv_size
self.socket_server_conn_num = connect_num
if self.socket_server_proc == 0:
self.socket_server_proc = threading.Thread(
target=self.socket_server_thread, name='socket_server_proc')
self.socket_server_proc.start()
def join_bug_log_file(self, bugid, path, file_name, max_index):
local_file_dir = ''
local_file_dir = path + r'/' + str(bugid)
local_file_name = file_name
print_string = "join bug %s file %s log ..." % (str(bugid),
local_file_name)
print_info.print_info(print_info.PRINT_INFO, print_string)
if os.path.exists(local_file_dir + r'/' + local_file_name):
os.remove(local_file_dir + r'/' + local_file_name)
for x in range(max_index + 1):
try:
fread = open(
local_file_dir + r'/' + str(x) + '_' + local_file_name,
'r')
except Exception, e:
print_string = "join bug %s file error,open file %s" % (
str(bugid), e)
print_info.print_info(print_info.PRINT_ERROR, print_string)
else:
data = fread.read()
with open(local_file_dir + r'/' + local_file_name,
'ab') as lwrite:
lwrite.write(data)
fread.close()
os.remove(local_file_dir + r'/' + str(x) + '_' +
local_file_name)
self.download_log_ok_list_lock.acquire()
self.download_log_ok_list.append(bugid)
self.download_log_ok_list_lock.release()
print_string = "join bug %s file %s ok" % (str(bugid), local_file_name)
print_info.print_info(print_info.PRINT_INFO, print_string)
def ftp_close(self, fd):
try:
fd.quit()
except Exception, e:
print_string = 'ftp:%s quit error %s ' % (fd, e)
print_info.print_info(print_info.PRINT_ERROR, print_string)
def connect_ftp_server(self, fd):
for n in range(3):
try:
# set 10s timeout
fd.connect(self.remoteip, self.remoteport, 10)
except Exception, e:
print_string = "conncet ftp server failed - %s" % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
else:
try:
fd.login(self.user, self.password)
except Exception, e:
print_string = 'login ftp server failed - %s' % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
else:
print_info.print_info(print_info.PRINT_DEBUG,
'login ftp server success')
return (1, 'connect ftp server success')
print_info.print_info(print_info.PRINT_ERROR,
'connect ftp server failed')
return (0, 'connect ftp server failed')
def chang_work_dir(self, fd, remotefilepatch):
#change remote work dir
try:
#fd.set_pasv(0)
fd.cwd(remotefilepatch)
except Exception, e:
print_string = 'ftp change dir error %s' % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
return (0, 'ftp change dir error')
else:
def handle_client_recvdata(self, data):
global socket_recv_data_remnant
cut_data = []
print_string = "resv data len [%d] :%s " % (len(data), data)
print_info.print_info(print_info.PRINT_DEBUG, print_string)
if len(socket_recv_data_remnant):
data = socket_recv_data_remnant + data
socket_recv_data_remnant = ''
if '*' in data:
cut_data = data.strip('*').split('*')
if data[-1] != '*':
socket_recv_data_remnant = cut_data[-1]
cut_data.pop(-1)
else:
if not data or data.decode('utf-8') == 'exit':
print_string = "recv exit cmd or link close"
print_info.print_info(print_info.PRINT_DEBUG, print_string)
socket_recv_data_remnant += data
return 0
print_string = "client socket recv data %s " % cut_data
print_info.print_info(print_info.PRINT_DEBUG, print_string)
for recv_data_per in cut_data:
if 'getusername' in recv_data_per:
username = getpass.getuser()
print_string = "user name %s" % username
print_info.print_info(print_info.PRINT_DEBUG, print_string)
self.client_send(socket_client_fd,
"username:"******"ftp speed %s" % socket_ftp_download_file_speed
print_info.print_info(print_info.PRINT_DEBUG, print_string)
self.client_send(
socket_client_fd,
"ftpspeed:" + str(socket_ftp_download_file_speed) + '*')
elif "downloadlog:" in recv_data_per:
cut_data = recv_data_per[len('downloadlog:'):].split(';')
for x in cut_data:
if 'bugid:' in x:
bugid = x.split(':')[1]
elif 'ftpserver:' in x:
ftp_server = x.split(':')[1]
elif 'ftpport:' in x:
ftp_port = int(x.split(':')[1])
elif 'ftpuser:'******':')[1]
elif 'ftppw:' in x:
ftp_password = x.split(':')[1]
elif 'filepath:' in x:
file_path = x.split(':')[1]
elif 'filename:' in x:
file_name = x.split(':')[1]
elif 'start:' in x:
ftp_filestart = int(x.split(':')[1])
elif 'size:' in x:
ftp_filesize = int(x.split(':')[1])
elif 'index:' in x:
ftp_index = int(x.split(':')[1])
ftp_download_bug_log_proc = threading.Thread(
target=self.ftp_download_log,
args=(bugid, ftp_server, ftp_port, ftp_user, ftp_password,
file_path, file_name, ftp_filestart, ftp_filesize,
ftp_index),
name='ftp_download_log')
ftp_download_bug_log_proc.start()
elif "updatecode" in recv_data_per:
scp_fd = ssh_scp(self.scp_server_ip, 22, self.scp_user,
self.scp_password, self.client_main_path,
self.scp_dir)
if scp_fd.scp_file_dst_to_local(
os.path.split(os.path.split(self.code_path)[0])[1]):
#self.client_send(socket_client_fd, "updatecodeok:" + socket_client_fd + '*')
cur_pid = os.getpid()
print_string = "update code ok, old pid %d" % cur_pid
print_info.print_info(print_info.PRINT_DEBUG, print_string)
socket_client_fd.close()
os.system("/usr/bin/python " + self.code_path + " &")
os.system("sudo kill -9 " + str(cur_pid))
# elif len(recv_data_per) > 0:
# socket_recv_data_remnant = socket_recv_data_remnant + recv_data_per
return 1
def ftp_download_log(self, *params):
ftp_fd = 0
ret = []
download_file_num = 0
local_file_name = ''
scp_fd = 0
print_string = "download log params: %s" % str(params)
print_info.print_info(print_info.PRINT_DEBUG, print_string)
self.save_log_info_to_file(print_string + '\n')
if len(params) > 9:
local_file_name = str(params[9]) + '_' + params[6]
if os.path.exists(self.client_main_path + '/' + str(params[0]) +
'/' + local_file_name):
os.remove(self.client_main_path + '/' + str(params[0]) + '/' +
local_file_name)
print_string = "remove old file,path:%s" % (
self.client_main_path + '/' + str(params[0]) + '/' +
local_file_name)
print_info.print_info(print_info.PRINT_DEBUG, print_string)
ftp_fd = ftpdownloadfile.download_ftp(params[1], params[2],
params[3], params[4])
ret = ftp_fd.connect_ftp_server(ftp_fd)
if ret[0]:
while download_file_num < 10000:
ret = ftp_fd.download_ftp_file_size(
ftp_fd, params[5], params[6], self.client_main_path,
params[0], local_file_name, params[7], params[8])
if ret[0]:
#update ftp download speed
socket_ftp_download_file_speed = ftp_fd.ftp_download_speed
scp_fd = ssh_scp(
self.scp_server_ip, 22, self.scp_user,
self.scp_password,
self.client_main_path + '/' + str(params[0]),
self.scp_dir + '/' + str(params[0]))
if scp_fd.scp_file_to_dst(local_file_name):
self.client_send(
socket_client_fd, "downloadok:bugid:" +
str(params[0]) + ";filename:" + params[6] +
";index:" + str(params[9]) + '*')
os.remove(self.client_main_path + '/' +
str(params[0]) + '/' + local_file_name)
print_string = "scp file %s to server %s ok" % (
local_file_name, self.scp_server_ip)
print_info.print_info(print_info.PRINT_DEBUG,
print_string)
self.save_log_info_to_file(print_string + '\n')
print_string = "remove file name %s" % (
self.client_main_path + '/' + str(params[0]) +
'/' + local_file_name)
print_info.print_info(print_info.PRINT_DEBUG,
print_string)
print_string = "ftp speed %s" % socket_ftp_download_file_speed
print_info.print_info(print_info.PRINT_DEBUG,
print_string)
self.client_send(
socket_client_fd, "ftpspeed:" +
str(socket_ftp_download_file_speed) + '*')
else:
print_string = "scp file %s to server %s error" % (
local_file_name, self.scp_server_ip)
print_info.print_info(print_info.PRINT_DEBUG,
print_string)
break
ftp_fd.ftp_close(ftp_fd)
while download_file_num < 10000:
time.sleep(30)
ftp_fd = ftpdownloadfile.download_ftp(
params[1], params[2], params[3], params[4])
ret = ftp_fd.connect_ftp_server(ftp_fd)
if ret[0]:
break
download_file_num += 1
def handle_recv_data(self, sock, addr, data):
cut_data = []
max_index_num = 0
max_index = 0
download_ok_bugid = ''
download_ok_index = 0
download_ok_filename = ''
list_index = 0
print_string = "recv data %s " % data
print_info.print_info(print_info.PRINT_DEBUG, print_string)
if len(self.socket_recv_data_remnant):
data = self.socket_recv_data_remnant + data
self.socket_recv_data_remnant = ''
if '*' in data:
cut_data = data.strip('*').split('*')
if data[-1] != '*':
self.socket_recv_data_remnant = cut_data[-1]
cut_data.pop(-1)
else:
if not data or data.decode('utf-8') == 'exit':
sock.close()
self.socket_client_list_lock.acquire()
for x in self.socket_client_list:
if sock in x:
self.socket_client_list.pop(
self.socket_client_list.index(x))
break
self.socket_client_list_lock.release()
if len(self.dispense_work_list):
print_string = 'dispense_work_list %s' % self.dispense_work_list
print_info.print_info(print_info.PRINT_INFO, print_string)
self.dispense_work_list_lock.acquire()
for i in range(len(self.dispense_work_list) - 1, -1, -1):
if sock in self.dispense_work_list[i]:
self.redispense_work_list.append(
self.dispense_work_list[i])
self.dispense_work_list.pop(i)
self.dispense_work_list_lock.release()
self.socket_recv_data_remnant += data
return 0
print_string = "server socket recv data %s " % cut_data
print_info.print_info(print_info.PRINT_DEBUG, print_string)
for recv_data_per in cut_data:
if "username" in recv_data_per:
if ':' in recv_data_per:
cut_data_per = recv_data_per.split(':')
self.socket_client_list_lock.acquire()
for x in self.socket_client_list:
if sock in x and addr in x:
self.socket_client_list[
self.socket_client_list.index(
x)][2] = cut_data_per[1]
print_string = "client socket info(username) %s " % self.socket_client_list[
self.socket_client_list.index(x)]
print_info.print_info(print_info.PRINT_DEBUG,
print_string)
self.socket_client_list_lock.release()
elif "ftpspeed" in recv_data_per:
if ':' in recv_data_per:
cut_data_per = recv_data_per.split(':')
self.socket_client_list_lock.acquire()
for x in self.socket_client_list:
if sock in x:
self.socket_client_list[
self.socket_client_list.index(x)][3] = int(
cut_data_per[1])
self.socket_client_list.sort(reverse=True,
key=lambda x: x[3])
print_string = "client socket info(speed) %s " % self.socket_client_list[
self.socket_client_list.index(x)]
print_info.print_info(print_info.PRINT_DEBUG,
print_string)
break
self.socket_client_list_lock.release()
elif 'downloadok:' in recv_data_per:
cut_data_per = recv_data_per[len('downloadok:'):].split(';')
for x in cut_data_per:
if 'bugid:' in x:
download_ok_bugid = x.split(':')[1]
if 'index:' in x:
download_ok_index = int(x.split(':')[1])
if 'filename:' in x:
download_ok_filename = x.split(':')[1]
if len(self.bugid_max_index):
for x in self.bugid_max_index:
if download_ok_bugid in x:
if x[1] < download_ok_index:
self.bugid_max_index_lock.acquire()
self.bugid_max_index[
self.bugid_max_index.index(
x)][1] = download_ok_index
max_index = download_ok_index
self.bugid_max_index_lock.release()
else:
max_index = x[1]
break
else:
max_index_num += 1
if max_index_num == len(self.bugid_max_index):
self.bugid_max_index_lock.acquire()
self.bugid_max_index.append(
[download_ok_bugid, download_ok_index])
max_index = download_ok_index
self.bugid_max_index_lock.release()
max_index_num = 0
else:
self.bugid_max_index_lock.acquire()
self.bugid_max_index.append(
[download_ok_bugid, download_ok_index])
max_index = download_ok_index
self.bugid_max_index_lock.release()
self.dispense_work_list_lock.acquire()
for x in self.dispense_work_list:
if download_ok_bugid in x and download_ok_index in x:
self.dispense_work_list.pop(
self.dispense_work_list.index(x))
self.dispense_work_list_lock.release()
self.dispense_work_list_lock.acquire()
list_index = 0
for y in range(len(self.dispense_work_list) - 1, -1, -1):
if download_ok_bugid in self.dispense_work_list[y]:
list_index = y + 1
break
if list_index > 0:
self.dispense_work_list_lock.release()
continue
list_index = 0
for y in range(len(self.redispense_work_list) - 1, -1, -1):
if download_ok_bugid in self.redispense_work_list[y]:
list_index = y + 1
break
if list_index > 0:
self.dispense_work_list_lock.release()
continue
self.dispense_work_list_lock.release()
self.bugid_max_index_lock.acquire()
self.bugid_max_index.pop(
self.bugid_max_index.index([download_ok_bugid, max_index]))
self.bugid_max_index_lock.release()
join_bug_log_file_proc = threading.Thread(
target=self.join_bug_log_file,
args=(download_ok_bugid, self.server_main_path,
download_ok_filename, max_index),
name="join_bug_log")
join_bug_log_file_proc.start()
max_index = 0
elif 'updatecodeok:' in recv_data_per:
updatecodeok_socket = recv_data_per.split(':')[1]
self.update_code_addr_list_lock.acquire()
for x in self.socket_client_list:
if updatecodeok_socket in x:
if x[1][0] not in self.update_code_addr_list:
# self.update_code_addr_list_lock.acquire()
self.update_code_addr_list.append(x[1][0])
# self.update_code_addr_list_lock.release()
break
self.update_code_addr_list_lock.release()
print_string = "update code ok socket %s" % updatecodeok_socket
print_info.print_info(print_info.PRINT_DEBUG, print_string)
# elif len(recv_data_per) > 0:
# self.socket_recv_data_remnant = self.socket_recv_data_remnant + recv_data_per
return 1
import matplotlib.pyplot as plt
# Import NWB_adapter
import os
import sys
sys.path.append(basic_path + 'resources/swdb_2018_neuropixels')
from swdb_2018_neuropixels.ephys_nwb_adapter import NWB_adapter
from print_info import print_info
# Provide path to manifest file
manifest_file = os.path.join(drive_path, 'ephys_manifest.csv')
# Create a dataframe
expt_info_df = pd.read_csv(manifest_file)
#make new dataframe by selecting only multi-probe experiments
multi_probe_expt_info = expt_info_df[expt_info_df.experiment_type ==
'multi_probe']
multi_probe_example = 0
multi_probe_filename = multi_probe_expt_info.iloc[multi_probe_example][
'nwb_filename']
# Specify full path to the .nwb file
nwb_file = os.path.join(drive_path, multi_probe_filename)
data_set = NWB_adapter(nwb_file)
print_info(data_set.unit_df)
def client_send(self, sock, data):
try:
sock.send(data)
except Exception, e:
print_string = "send data %s to socket server error %s" % (data, e)
print_info.print_info(print_info.PRINT_ERROR, print_string)
f.close()
def read_line(path, file_name):
read_file = path + '/' + file_name
f = open(read_file, 'rb')
lines = f.readlines()
f.close()
return lines
if __name__ == '__main__':
print_info.init(print_info.PRINT_INFO)
save_write("/home/local/SPREADTRUM/lc.fan/flc/code/pybz", "test_file", "log_status:0" + '\n')
file_data = read_data("/home/local/SPREADTRUM/lc.fan/flc/code/pybz", "test_file")
print_info.print_info(print_info.PRINT_INFO, file_data)
save_append("/home/local/SPREADTRUM/lc.fan/flc/code/pybz", "test_file", "symbols_status:0:3" + '\n')
data_list = read_line("/home/local/SPREADTRUM/lc.fan/flc/code/pybz", "test_file")
for x in data_list:
if '\n' in x:
# delete '\n'
data_list[data_list.index(x)] = x.strip('\n')
for x in data_list:
print 'data line % s ' % x
print_info.print_info(print_info.PRINT_INFO, x)
if ':' in x:
cut_data = x.split(':')
if 'symbols_status' in cut_data[0]:
if int(cut_data[1]) == 0:
break
ftp_file_size = 0
ftp_file_start = 0
dispense_size_per = 0
client_download_size = 0
try:
ftp_file_size = int(work[6])
except Exception, e:
print_string = "dispense work file size error %s" % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
return (0, "dispense work file size error")
for x in sock_list:
print_string = "sock addr:%s ftp downdload speed %d bit/s" % (x[1],
x[3])
print_info.print_info(print_info.PRINT_DEBUG, print_string)
try:
if x[3] >= 0 and x[3] < ftp_speed_max:
client_speed_sum += x[3]
if x[3] < speed_min:
speed_min = x[3]
if x[3] > speed_max:
speed_max = x[3]
sock_ftp_speed_list.append([x[0], x[1], x[2], x[3]])
except Exception, e:
print_string = "dispense work get speed error %s" % e
print_info.print_info(print_info.PRINT_ERROR, print_string)
if not client_speed_sum:
return (0, 'client ftp speed is 0')
dispense_size_per = int(ftp_file_size / client_speed_sum)
