def remove_duplicates(self, path, report_only=False):
files = self.ls(path, True)
common.print_line('analyzing for duplications...')
keys = common.sort_dict_keys(files)
duplicates = []
for key in keys:
if key.endswith(ClSync.duplicate_suffix):
logging.debug('found duplicate file: ' + key)
date1 = common.get_datetime_from_iso8601(files[key].mod_time)
logging.debug(key + ' timestamp: ' + str(date1.timestamp()))
key2 = key.replace(ClSync.duplicate_suffix, '')
date2 = common.get_datetime_from_iso8601(files[key2].mod_time)
logging.debug(key2 + ' timestamp: ' + str(date2.timestamp()))
if date1.timestamp() > date2.timestamp():
logging.debug(key + ' is newer than ' + key2)
file_to_remove = files[key2].remote + key2
common.print_line('found duplicate file. Removing: ' +
file_to_remove + '...')
duplicates.append(key2)
if report_only is False:
self.delete_file(key2, files[key2].remote)
elif date1.timestamp() == date1.timestamp():
logging.debug(key + ' is equal to ' + key2)
file_to_remove = files[key2].remote + key2
common.print_line('found duplicate file. Removing: ' +
file_to_remove + '...')
duplicates.append(key2)
if report_only is False:
self.delete_file(key2, files[key2].remote)
else:
logging.debug(key + ' is older than ' + key2)
file_to_remove = files[key].remote + key
common.print_line('found duplicate file. Removing: ' +
file_to_remove + '...')
duplicates.append(key)
if report_only is False:
self.delete_file(key, files[key].remote)
logging.debug('file to remove: ' + file_to_remove)
return duplicates
def lsmd5():
global __cl_sync
if __cl_sync is None:
__cl_sync = clsync.ClSync(__config)
if len(__args) == 1:
logging.error('invalid lsmd5 command')
usage_lsmd5()
sys.exit(-1)
files = __cl_sync.lsmd5(common.remove_ending_slash(__args[1]))
largest_length = 25
keys = common.sort_dict_keys(files)
for tmp_file in keys:
filename_length = len(tmp_file)
if filename_length > largest_length:
largest_length = filename_length
common.print_line('NAME'.ljust(largest_length) + " " + 'MD5'.ljust(32))
common.print_line(''.join('-' for i in range(largest_length)) + " " +
''.join('-' for i in range(32)))
for tmp_file in keys:
file_name = tmp_file
common.print_line(
file_name.ljust(largest_length) + " " + files[tmp_file])
def find():
global __cl_sync
if __cl_sync is None:
__cl_sync = clsync.ClSync(__config)
if len(__args) == 1:
logging.error('invalid find command')
usage_find()
sys.exit(-1)
files = __cl_sync.find(common.remove_ending_slash(__args[1]))
largest_length = 25
keys = common.sort_dict_keys(files)
for tmp_file in keys:
filename_length = len(files[tmp_file].path)
if not files[tmp_file].is_dir and filename_length > largest_length:
largest_length = filename_length
common.print_line('---' + " " + 'NAME'.ljust(largest_length) + " " +
'SIZE'.rjust(9) + " " + 'MOD TIME'.ljust(19) + " " +
'REMOTE')
common.print_line('---' + " " + ''.join('-'
for i in range(largest_length)) +
" " + ''.join('-' for i in range(9)) + " " +
''.join('-' for i in range(19)) + " " +
''.join('-' for i in range(15)))
for tmp_file in keys:
if files[tmp_file].is_dir is True:
first_chars = '-d-'
else:
first_chars = '---'
file_name = files[tmp_file].path
if file_name.startswith('//'):
file_name = file_name[1:len(file_name)]
common.print_line(
first_chars + " " + file_name.ljust(largest_length) + " " +
str(files[tmp_file].size).rjust(9) + " " +
common.get_printable_datetime(files[tmp_file].mod_time).ljust(19) +
" " + files[tmp_file].remote)
def compare_clfiles(self,
local_dir,
local_clfiles,
remote_clfiles,
delete_file=True):
common.print_line('calculating differences...')
logging.debug('comparing clfiles')
logging.debug('local directory: ' + local_dir)
logging.debug('local clfiles size: ' + str(len(local_clfiles)))
logging.debug('remote clfiles size: ' + str(len(remote_clfiles)))
remote_dir = os.path.dirname(local_dir)
operations = []
for local_path in local_clfiles:
local_clfile = local_clfiles[local_path]
if local_clfile.is_dir:
continue
logging.debug('checking local clfile: ' + local_path + " name: " +
local_clfile.name)
rel_name = common.remove_localdir(
local_dir, local_clfile.path + '/' + local_clfile.name)
rel_path = common.remove_localdir(local_dir, local_clfile.path)
logging.debug('relative name: ' + rel_name)
if rel_name not in remote_clfiles:
logging.debug('not found in remote_clfiles')
local_clfile.remote_path = rel_path
op = operation.Operation(operation.Operation.ADD, local_clfile,
None)
operations.append(op)
else:
logging.debug('file found in remote_clfiles')
remote_clfile = remote_clfiles[rel_name]
if self._compare_method == 'size':
size_local = local_clfile.size
size_remote = remote_clfile.size
current_remote = remote_clfiles[rel_name].remote
logging.debug('local_file.size:' + str(local_clfile.size) +
', remote_clfile.size:' +
str(remote_clfile.size))
if size_local != size_remote:
logging.debug('file has changed')
local_clfile.remote_path = rel_path
local_clfile.remote = current_remote
op = operation.Operation(operation.Operation.UPDATE,
local_clfile, None)
operations.append(op)
elif self._compare_method == 'md5':
local_md5 = local_clfile.md5
remote_md5 = remote_clfile.md5
current_remote = remote_clfiles[rel_name].remote
logging.debug('local_file.md5:' + str(local_md5) +
', remote_clfile.md5:' + str(remote_md5))
if local_md5 != remote_md5:
logging.debug('file has changed')
local_clfile.remote_path = rel_path
local_clfile.remote = current_remote
op = operation.Operation(operation.Operation.UPDATE,
local_clfile, None)
operations.append(op)
else:
logging.error('compare_method: ' + self._compare_method +
' not valid!')
raise Exception('compare_method: ' + self._compare_method +
' not valid!')
if delete_file is True:
reverse_keys = common.sort_dict_keys(remote_clfiles, True)
for remote_path in reverse_keys:
remote_clfile = remote_clfiles[remote_path]
logging.debug('checking file ' + remote_dir + remote_path +
' for deletion')
rel_name = common.remove_localdir(
local_dir, remote_clfile.path + '/' + remote_clfile.name)
rel_path = common.remove_localdir(local_dir,
remote_clfile.path)
logging.debug('relative name: ' + rel_name)
if remote_dir + remote_path not in local_clfiles:
logging.debug('file ' + remote_path + ' has been deleted')
remote_clfile.remote_path = rel_path
op = operation.Operation(operation.Operation.REMOVE,
remote_clfile, None)
operations.append(op)
common.print_line('found ' + str(len(operations)) + ' differences')
return operations