def test_CDisk70_modeGet(self):
p = path_join(self.path, 'testfile')
chmod(p, 0o100666)
stat, res = self.cloud.task('getm', p)
self.assertTrue(stat)
self.assertTrue(res == ('getm', 'testfile'))
self.assertTrue(file_info(p).st_mode == 0o100640)
def _upload(self, cmd, path):
r_path = relpath(path, start=self.path)
status, res = super().task(cmd, r_path, path)
if status and pathExists(path):
fst = file_info(path)
self.h_data[path] = int(fst.st_mtime)
self._r_setMode(r_path, fst.st_mode)
return status, res
def ignore_path_down(path):
nonlocal ignore
ret = set()
while path not in ignore and path != self.path:
ret.add(path)
if pathExists(path):
self.h_data[path] = int(file_info(path).st_mtime)
path = path_split(path)[0]
ignore |= ret
return ret
def _move(self, cmd, pathto, pathfrom):
status, res = super().task(cmd, relpath(pathto, start=self.path),
relpath(pathfrom, start=self.path))
if status:
# update history date too
p = self.h_data.pop(pathfrom, None)
if p is not None:
self.h_data[pathto] = p
else:
if pathExists(pathto):
self.h_data[pathto] = int(file_info(pathto).st_mtime)
return status, res
def _download(
self, cmd, path
): # download via temporary file to make it in transaction manner
with tempFile(suffix='.temp',
delete=False) as f: # , dir=self.work_dir
temp = f.name
r_path = relpath(path, start=self.path)
status, res = super().task(cmd, r_path, temp)
if status:
try:
fileMove(temp, path)
self.h_data[path] = int(file_info(path).st_mtime)
self._getMode('', path)
except OSError as e:
status = False
res = {
'code': -1,
'error': 'OSError',
'path': path,
'errno': e.errno,
'description': e.strerror
}
return status, res
def _mkDir(self, cmd, path):
status, res = super().task(cmd, relpath(path, start=self.path))
if status:
self.h_data[path] = int(file_info(path).st_mtime)
return status, res
def _setMode(self, cmd, path):
st, res = self._r_setMode(relpath(path, start=self.path),
file_info(path).st_mode)
if st:
res = ('setm', res[1])
return st, res
def _fullSync(self):
def ignore_path_down(path):
nonlocal ignore
ret = set()
while path not in ignore and path != self.path:
ret.add(path)
if pathExists(path):
self.h_data[path] = int(file_info(path).st_mtime)
path = path_split(path)[0]
ignore |= ret
return ret
ignore = set(
) # set of files that shouldn't be synced or already in sync
exclude = set(self.watch.exclude)
# {colud} - {local} -> download from cloud or delete from cloud if it exist in the history
# ({cloud} & {local}) and hashes are equal = ignore
# ({cloud} & {local}) and hashes not equal -> decide conflict/upload/download depending on
# the update time of files and time stored in the history
for status, i in self.task('list', 40):
if status:
path = i[
'path'] # full file path !NOTE! getList doesn't return empty folders
p = path_split(path)[0] # containing folder
if in_paths(p, exclude):
continue
if pathExists(path):
if i['type'] == 'dir': # it is existing directory
# there is nothing to check for directories
# here we may check UGM and if they are different we have to decide:
# - store UGM to cloud or
# - restore UGM from cloud
# but for this decision we need last updated data for directories in history
####
# !!! Actually Yd don't return empty folders in file list !!!
# This section newer run
####
#ignore_path_down(path); continue
pass
else: # existig file
try:
with open(path, 'rb') as f:
hh = sha256(f.read()).hexdigest()
except:
hh = ''
c_t = i[
'modified'] # cloud file modified date-time
l_t = int(file_info(path).st_mtime
) # local file modified date-time
h_t = self.h_data.get(
path, l_t) # history file modified date-time
if hh == i['sha256']:
# Cloud and local hashes are equal
# here we may check UGM and if they are different we have to decide:
# - store UGM to cloud or
# - restore UGM from cloud
# depending on modified time (compare c_t and l_t)
ignore_path_down(
path
) # add in ignore and history all folders by way to file
continue
else:
# Cloud and local files are different. Need to decide what to do: upload,
# download, or it is conflict.
# Solutions:
# - conflict if both cloud and local files are newer than stored in the history
# - download if the cloud file newer than the local, or
# - upload if the local file newer than the cloud file.
if l_t > h_t and c_t > h_t: # conflict
info('conflict')
continue ### it is not fully designed and not tested yet !!!
# Concept: rename older file to file.older and copy both files --> cloud and local
path2 = path + '.older'
ignore.add(path2)
ignore.add(path)
if l_t > c_t: # older file is in cloud
self.downloads.add(path2)
self.task(
'move', path2,
path) # need to do before rest
self._submit('down', path2)
self._submit('up', path)
else: # local file is older than file in cloud
self.downloads.add(path)
fileMove(
path, path2
) # it will be captured as move from & move to !!!???
self._submit('down', path)
self._submit('up', path2)
continue
elif l_t > c_t: # local time greater than the cloud time
# upload (as file exists the dir exists too - no need to create dir in cloud)
self._submit('up', path)
ignore_path_down(
path
) # add in ignore and history all folders by way to file
continue
else: # download
# upload (as file exists the dir exists too - no need to create local dir)
self.downloads.add(
path
) # remember in downloads to avod events on this path
self._submit('down', path)
ignore_path_down(
path
) # add in ignore and history all folders by way to file
continue
# The file is not exists
# it means that it has to be downloaded or.... deleted from the cloud when local file
# was deleted and this deletion was not cached by active client (client was not
# connected to cloud or was not running at the moment of deletion).
if self.h_data.get(
path,
False): # do we have history data for this path?
# as we have history info for path but local path doesn't exists then we have to
# delete it from cloud
if not pathExists(
p): # containing directory is also removed?
while True: # go down to the shortest removed directory
p_ = path_split(p)[0]
if pathExists(p_):
break
p = p_
self.h_data.pop(p) # remove history
### !!! all files in this folder mast be removed too, but we
### can't walk as files/folders was deleted from local FS!
### NEED history database to do delete where path.startwith(p) - it can't be done in dict
self._submit('del', p)
# add d to exceptions to avoid unnecessary checks for other files which are within p
exclude.add(p)
else: # only file was deleted
self._submit('del', path)
del self.h_data[path] # remove history
else: # local file have to be downloaded from the cloud
if i['type'] == 'file':
if not pathExists(p):
self.downloads |= ignore_path_down(
p
) # store new dir in downloads to avoid upload
makedirs(p, exist_ok=True)
ignore.add(p)
self.downloads.add(
path
) # store downloaded file in downloads to avoid upload
self._submit('down', path)
ignore.add(path)
#else: # directory not exists !!! newer run !!!
# self.downloads.add(ignore_path_down(path)) # store new dir in downloads to avoid upload
# makedirs(path, exist_ok=True)
# ---- Done forward path (sync cloud to local) ------
# (local - ignored) -> upload to cloud
for root, dirs, files in walk(self.path):
if in_paths(root, exclude):
continue
for d in dirs:
d = path_join(root, d)
if d not in ignore | exclude:
# directory have to be created before start of uploading a file in it
# do it in-line as it rather fast operation
s, r = self.task('mkdir', d)
info('done in-line %s %s' % (str(s), str(r)))
### !need to check success of folder creation! !need to decide what to do in case of error!
for f in files:
f = path_join(root, f)
if f not in ignore:
self._submit('up', f)
return 'fullSync'