def multi_thread_copy_to_server(local_files, max_thread):
start_time = time.time()
try:
max_thread = int(max_thread)
except ValueError:
ConColorShow().error_show("Wrong max_thread specifid!")
return
files_num_each = len(local_files) // max_thread
reminder = len(local_files) % max_thread
start_pos = 0
end_pos = files_num_each
if files_num_each:
while True:
p = multiprocessing.Process(
target=copy_local_to_server,
args=(local_files[start_pos:end_pos], ))
p.start()
p.join()
start_pos = end_pos
end_pos += files_num_each
if end_pos > len(local_files):
break
if reminder:
p = multiprocessing.Process(target=copy_local_to_server,
args=(local_files[-reminder:], ))
p.start()
p.join()
pass
end_time = time.time()
ConColorShow().common_show("copy total time use %f" %
(end_time - start_time))
def copy_local_to_server(local_files, ssh_handle=None):
global G_server_ip
global G_username
global G_password
global G_server_base_path
global G_local_base_path
default_config_init()
if not ssh_handle:
ssh_handle = SSHHandle(server_ip=G_server_ip,
username=G_username,
password=G_password)
for local_file_full_path in local_files:
remote_path_tail = get_tail_path(G_local_base_path,
local_file_full_path)
remote_file_full_path = os.path.join(G_server_base_path,
remote_path_tail).replace(
'\\', '/')
try:
ssh_handle.copy_file_to_server(remote_file_full_path,
local_file_full_path)
except IOError:
e = sys.exc_info()[0]
print e
ConColorShow().error_show(
'Fail to copy from %s to %s' %
(local_file_full_path, remote_file_full_path))
s = raw_input("--continue?[y/n]").lower()
if s == 'y':
continue
ConColorShow().error_show('stop')
exit(1)
print 'copy to server done %s ' % local_file_full_path
pass
def main():
parser = default_config_init()
arg_parser = arg_parser_init()
scan_dir_cfg_list = get_scan_dir_list_tup()
G_scan_filter_tail.append('.a')
if not arg_parser.parse(sys.argv):
ConColorShow().error_show('please input valid args')
print arg_parser
exit(1)
pass
if arg_parser.check_option('-update'):
new_file_list = list()
for scan_dir_cfg in scan_dir_cfg_list:
tmp_file_list = scan_new_files_v2(scan_dir_cfg['dir_path'], 0,
scan_dir_cfg['depth'])
new_file_list.extend(tmp_file_list)
copy_local_to_server(new_file_list)
exit(0)
time_gap = None
if arg_parser.check_option('-t'):
time_gap = arg_parser.get_option_args('-t')[0]
else:
ConColorShow().error_show('please specific time')
exit(1)
ssh_handle = None
while True:
new_file_list = list()
for scan_dir_cfg in scan_dir_cfg_list:
tmp_file_list = scan_new_files_v2(scan_dir_cfg['dir_path'],
time_gap, scan_dir_cfg['depth'])
new_file_list.extend(tmp_file_list)
if arg_parser.check_option('-cp'):
# multi_thread_copy_to_server(new_file_list, 100)
start_time = time.time()
if not ssh_handle:
ssh_handle = SSHHandle(server_ip=G_server_ip,
username=G_username,
password=G_password)
copy_local_to_server(new_file_list, ssh_handle)
end_time = time.time()
ConColorShow().highlight_show('Done, copy time use %f' %
(end_time - start_time))
del new_file_list
ConColorShow().common_show('Enter to rescan or ctrl+c to quit')
try:
s = raw_input()
except KeyboardInterrupt:
ConColorShow().highlight_show('Quit')
exit(0)
pass
def clear_folder(folder_to_clean):
global G_store_folder_path
full_path_folder = folder_to_clean
ConColorShow().warning_show('do clean at %s' % full_path_folder)
if full_path_folder in ['/', '/home', '/root']:
ConColorShow().error_show('!!!are you mad to delete %s' % folder_to_clean)
exit(1)
s = raw_input(ConColor.Red + '---> input y to continue [y/n]' + ConColor.Reset)
if s.lower() == 'y':
os.system('rm -rvf %s' % full_path_folder)
else:
print 'nothing done'
pass
def scan_new_files_v2(scan_folder, time_gap, scan_depth=1000):
"""
time_gap:>0 scan file changed within time_gap(sec) from now
=0 will scan and return all files in scan_folder
"""
try:
time_min = int(time_gap)
except ValueError:
print 'ERROR:only number! %s' % time_gap
return None
limit_sec = time_min * 60
now_sec = time.time()
new_file_full_path_list = list()
scan_folder_list = list()
start_time = time.time()
if type(scan_folder) == list:
scan_folder_list = scan_folder
else:
scan_folder_list.append(scan_folder)
for scan_folder in scan_folder_list:
if not os.path.exists(scan_folder):
print 'folder [%s] not exist!' % scan_folder
return None
for (dirpath, dirnames, filenames) in os.walk(scan_folder):
# stat = os.stat(dirpath)
# if stat.st_mtime + limit_sec < now_sec:
# continue
dir_depth = get_dir_depth(dirpath) - get_dir_depth(scan_folder)
if dir_depth >= scan_depth:
# print ('skip dub dir of [%s]' % dirpath)
count = len(dirnames)
for i in range(count):
dirnames.pop()
for filename in filenames:
stat = os.stat(os.path.join(dirpath, filename))
if stat.st_mtime + limit_sec >= now_sec or limit_sec == 0:
new_file_full_path = os.path.join(dirpath, filename)
ConColorShow().color_show(
'%24s %12s' % (filename, time.ctime(stat.st_mtime)),
ConColorShow.Green)
new_file_full_path_list.append(new_file_full_path)
pass
print '### Dir %s changed count [%d]###' % (
scan_folder, len(new_file_full_path_list))
print 'time use %f' % (time.time() - start_time)
return new_file_full_path_list
pass
def scan_new_files(scan_folder, time_gap):
try:
time_min = int(time_gap)
except ValueError:
print 'ERROR:only number!'
return None
limit_sec = time_min * 60
now_sec = time.time()
new_file_full_path_list = list()
scan_folder_list = list()
if type(scan_folder) == list:
scan_folder_list = scan_folder
else:
scan_folder_list.append(scan_folder)
for scan_folder in scan_folder_list:
if not os.path.exists(scan_folder):
print 'folder [%s] not exist!' % scan_folder
return None
file_list = os.listdir(scan_folder)
changed_file_cnt = 0
for file_name in file_list:
full_file_path = os.path.join(scan_folder, file_name)
if os.path.isfile(full_file_path):
stat = os.stat(full_file_path)
if stat.st_mtime + limit_sec >= now_sec:
if filter_tail(full_file_path, G_scan_filter_tail):
ConColorShow().color_show(
'%24s %12s' %
(file_name, time.ctime(stat.st_mtime)),
ConColorShow.Green)
new_file_full_path_list.append(full_file_path)
changed_file_cnt += 1
print '### Dir %s Total[%d] changed [%d]###' % (
scan_folder, len(file_list), changed_file_cnt)
return new_file_full_path_list
pass
arg_parse.add_option('-d', 1, 'specific dir to scan')
arg_parse.add_option('-t', 1, '-t <sec_time>: min time specific')
arg_parse.add_option('-clear', [0, 1], '-clear [folder]: clear folder')
arg_parse.add_option('-p', 0, 'print default scan folder and des folder')
arg_parse.add_option('-r', 0, 'recursive scan folder')
arg_parse.add_option('-h', 0, 'print help')
return arg_parse
pass
if __name__ == '__main__':
G_default_filter_tail.append('.a')
arg_parser = arg_parser_init()
if not arg_parser.parse(sys.argv):
ConColorShow().error_show('please input valid args')
print arg_parser
exit(1)
pass
if arg_parser.check_option('-p'):
ConColorShow().highlight_show('scan folder base %s' % G_build_folder_path)
ConColorShow().highlight_show('store folder base %s'% G_store_folder_path)
exit(0)
if arg_parser.check_option('-clear'):
folder_args = arg_parser.get_option_args('-clear')
if len(folder_args):
clear_folder(folder_args[0])
else:
clear_folder(G_store_folder_path)