def _update_attr_when_hash_equal(self, item_local_path, item):
t = datetime_to_timestamp(item.modified_time)
try:
self.logger.info('Updating timestamp for file "%s" to "%s".', item_local_path,
str(item.modified_time))
os.utime(item_local_path, (t, t))
self.items_store.update_item(item, ItemRecordStatuses.OK)
except (IOError, OSError) as e:
self.logger.error('IO error when updating timestamp for file "%s": %s', item_local_path, e)
if not self.task_pool.has_pending_task(item_local_path):
self.logger.info('Update server timestamp for file "%s" instead.', item_local_path)
self.task_pool.add_task(UpdateMetadataTask(self, rel_parent_path=self.rel_path + '/',
item_name=item.name, new_mtime=t))
def handle(self):
local_item_tmp_path = self.local_parent_path + get_tmp_filename(self.item_name)
try:
with open(local_item_tmp_path, "wb") as f:
self.drive.download_file(file=f, size=self._item.size, item_id=self._item.id)
os.rename(local_item_tmp_path, self.local_path)
t = datetime_to_timestamp(self._item.modified_time)
os.utime(self.local_path, (t, t))
os.chown(self.local_path, OS_USER_ID, OS_USER_GID)
self.items_store.update_item(self._item, ItemRecordStatuses.DOWNLOADED)
except (IOError, OSError) as e:
self.logger.error('An IO error occurred when downloading "%s": %s.', self.local_path, e)
except errors.OneDriveError as e:
self.logger.error('An API error occurred when downloading "%s": %s.', self.local_path, e)
def test_handle(self, mock_request):
tmp_path = self.parent_task.drive.config.local_root + '/' + get_tmp_filename('test')
dest_path = self.parent_task.drive.config.local_root + '/test'
tmp_path2 = self.task.local_parent_path + get_tmp_filename('test')
ts = datetime_to_timestamp(self.item.modified_time)
mock_request.get(self.task.drive.drive_uri + self.task.drive.drive_path + '/items/' + self.item.id + '/content',
content=b'1', status_code=codes.ok)
m = mock.mock_open()
with mock.patch('builtins.open', m, create=True):
self.task.handle()
self.assertEqual([(tmp_path, dest_path)], self.calls_hist['os.rename'])
self.assertEqual([(dest_path, OS_USER_ID, OS_USER_GID)], self.calls_hist['os.chown'])
self.assertEqual([(dest_path, (ts, ts))], self.calls_hist['os.utime'])
self.assertEqual(tmp_path, tmp_path2)
m.assert_called_once_with(tmp_path2, 'wb')
handle = m()
handle.write.assert_called_once_with(b'1')
def _update_attr_when_hash_equal(self, item_local_path, item):
t = datetime_to_timestamp(item.modified_time)
try:
self.logger.info('Updating timestamp for file "%s" to "%s".',
item_local_path, str(item.modified_time))
os.utime(item_local_path, (t, t))
self.items_store.update_item(item, ItemRecordStatuses.OK)
except (IOError, OSError) as e:
self.logger.error(
'IO error when updating timestamp for file "%s": %s',
item_local_path, e)
if not self.task_pool.has_pending_task(item_local_path):
self.logger.info(
'Update server timestamp for file "%s" instead.',
item_local_path)
self.task_pool.add_task(
UpdateMetadataTask(self,
rel_parent_path=self.rel_path + '/',
item_name=item.name,
new_mtime=t))
def handle(self):
local_item_tmp_path = self.local_parent_path + get_tmp_filename(
self.item_name)
try:
with open(local_item_tmp_path, 'wb') as f:
self.drive.download_file(file=f,
size=self._item.size,
item_id=self._item.id)
os.rename(local_item_tmp_path, self.local_path)
t = datetime_to_timestamp(self._item.modified_time)
os.utime(self.local_path, (t, t))
os.chown(self.local_path, OS_USER_ID, OS_USER_GID)
self.items_store.update_item(self._item,
ItemRecordStatuses.DOWNLOADED)
except (IOError, OSError) as e:
self.logger.error(
'An IO error occurred when downloading "%s": %s.',
self.local_path, e)
except errors.OneDriveError as e:
self.logger.error(
'An API error occurred when downloading "%s": %s.',
self.local_path, e)
def _analyze_remote_item(self, remote_item, all_local_items):
"""
Analyze what to do with a remote item. Assume that this item passes ignore list.
This function is the core algorithm and probably the ugliest code I've written so far.
:param onedrived.api.items.OneDriveItem remote_item: The remote item object.
:param [str] all_local_items: All remaining untouched local items.
"""
item_local_path = self.local_path + '/' + remote_item.name
q = self.items_store.get_items_by_id(parent_path=self.remote_path, item_name=remote_item.name)
exists = os.path.exists(item_local_path)
has_record = len(q) > 0
if not has_record and not exists:
# There is no record in database. The item is not present. Probably a new file.
self._create_download_task(item_local_path, remote_item)
elif has_record and not exists:
# There is record but the file is gone.
item_id, item_record = _unpack_first_item(q)
if item_id == remote_item.id and remote_item.c_tag == item_record.c_tag \
and remote_item.e_tag == item_record.e_tag:
# Same record. The file is probably deleted when daemon is off.
self._create_delete_item_task(item_local_path, remote_item)
else:
# The record differs. For safety we download it.
self.logger.info('Cannot determine actions for "%s". Non-existent and records differ.', item_local_path)
self._create_download_task(item_local_path, remote_item)
else:
# The entry exists locally.
# First solve possible type conflict.
is_dir = os.path.isdir(item_local_path)
if is_dir != remote_item.is_folder:
self.logger.info('Type conflict on path "%s". One side is file and the other is dir.', item_local_path)
self._move_existing_and_download(item_local_path, remote_item, all_local_items, q)
elif is_dir:
# Both sides are directories. Just update the record and sync if needed.
if has_record:
item_id, item_record = _unpack_first_item(q)
has_record = item_id == remote_item.id
if not has_record:
self.logger.info('Fix database record for directory "%s".', item_local_path)
self.items_store.update_item(remote_item, ItemRecordStatuses.OK)
else:
self.logger.debug('Directory "%s" has intact record.', item_local_path)
else:
# Both sides are files. Examine file attributes.
file_size, file_mtime = stat_file(item_local_path)
if file_size == remote_item.size \
and compare_timestamps(file_mtime, datetime_to_timestamp(remote_item.modified_time)) == 0:
# Same file name. Same size. Same mtime. Guess they are the same for laziness.
# Just update the record.
need_update = not has_record
if has_record:
item_id, item_record = _unpack_first_item(q)
need_update = item_record.c_tag != remote_item.c_tag or item_record.e_tag != remote_item.e_tag
if need_update:
self.logger.info('Fix database record for file "%s" based on file size and mtime.',
item_local_path)
self.items_store.update_item(remote_item, ItemRecordStatuses.OK)
else:
self.logger.debug('File "%s" seems fine.', item_local_path)
else:
if has_record:
item_id, item_record = _unpack_first_item(q)
if item_id == remote_item.id and item_record.c_tag == remote_item.c_tag \
or item_record.e_tag == remote_item.e_tag:
# The remote item did not change since last update, but LOCAL file changed since then.
self.logger.info('File "%s" changed locally since last sync. Upload.', item_local_path)
self._create_upload_task(local_item_name=remote_item.name, is_dir=False)
elif file_size == item_record.size and compare_timestamps(
file_mtime, datetime_to_timestamp(item_record.modified_time)) == 0:
# The local item did not change since last update, but REMOTE file changed since then.
self.logger.info('File "%s" changed remotely since last sync. Download.', item_local_path)
self._create_download_task(item_local_path, remote_item)
else:
# Three ends mismatch. Keep both.
self.logger.info('Cannot determine file "%s". All information differ. Keep both.',
item_local_path)
self._move_existing_and_download(item_local_path, remote_item, all_local_items, q)
else:
# Examine file hash.
if _have_equal_hash(item_local_path, remote_item):
# Same hash means that they are the same file. Update local timestamp and database record.
self._update_attr_when_hash_equal(item_local_path, remote_item)
else:
self.logger.info('Cannot determine file "%s". Rename and keep both.', item_local_path)
self._move_existing_and_download(item_local_path, remote_item, all_local_items, q)
def test_convert(self):
self.assertEqual(self.d, dateparser.str_to_datetime(self.s))
self.assertEqual(self.s, dateparser.datetime_to_str(self.d))
self.assertEqual(self.t, dateparser.datetime_to_timestamp(self.d))
def _analyze_remote_item(self, remote_item, all_local_items):
"""
Analyze what to do with a remote item. Assume that this item passes ignore list.
This function is the core algorithm and probably the ugliest code I've written so far.
:param onedrived.api.items.OneDriveItem remote_item: The remote item object.
:param [str] all_local_items: All remaining untouched local items.
"""
item_local_path = self.local_path + '/' + remote_item.name
q = self.items_store.get_items_by_id(parent_path=self.remote_path,
item_name=remote_item.name)
exists = os.path.exists(item_local_path)
has_record = len(q) > 0
if not has_record and not exists:
# There is no record in database. The item is not present. Probably a new file.
self._create_download_task(item_local_path, remote_item)
elif has_record and not exists:
# There is record but the file is gone.
item_id, item_record = _unpack_first_item(q)
if item_id == remote_item.id and remote_item.c_tag == item_record.c_tag \
and remote_item.e_tag == item_record.e_tag:
# Same record. The file is probably deleted when daemon is off.
self._create_delete_item_task(item_local_path, remote_item)
else:
# The record differs. For safety we download it.
self.logger.info(
'Cannot determine actions for "%s". Non-existent and records differ.',
item_local_path)
self._create_download_task(item_local_path, remote_item)
else:
# The entry exists locally.
# First solve possible type conflict.
is_dir = os.path.isdir(item_local_path)
if is_dir != remote_item.is_folder:
self.logger.info(
'Type conflict on path "%s". One side is file and the other is dir.',
item_local_path)
self._move_existing_and_download(item_local_path, remote_item,
all_local_items, q)
elif is_dir:
# Both sides are directories. Just update the record and sync if needed.
if has_record:
item_id, item_record = _unpack_first_item(q)
has_record = item_id == remote_item.id
if not has_record:
self.logger.info('Fix database record for directory "%s".',
item_local_path)
self.items_store.update_item(remote_item,
ItemRecordStatuses.OK)
else:
self.logger.debug('Directory "%s" has intact record.',
item_local_path)
else:
# Both sides are files. Examine file attributes.
file_size, file_mtime = stat_file(item_local_path)
if file_size == remote_item.size \
and compare_timestamps(file_mtime, datetime_to_timestamp(remote_item.modified_time)) == 0:
# Same file name. Same size. Same mtime. Guess they are the same for laziness.
# Just update the record.
need_update = not has_record
if has_record:
item_id, item_record = _unpack_first_item(q)
need_update = item_record.c_tag != remote_item.c_tag or item_record.e_tag != remote_item.e_tag
if need_update:
self.logger.info(
'Fix database record for file "%s" based on file size and mtime.',
item_local_path)
self.items_store.update_item(remote_item,
ItemRecordStatuses.OK)
else:
self.logger.debug('File "%s" seems fine.',
item_local_path)
else:
if has_record:
item_id, item_record = _unpack_first_item(q)
if item_id == remote_item.id and item_record.c_tag == remote_item.c_tag \
or item_record.e_tag == remote_item.e_tag:
# The remote item did not change since last update, but LOCAL file changed since then.
self.logger.info(
'File "%s" changed locally since last sync. Upload.',
item_local_path)
self._create_upload_task(
local_item_name=remote_item.name, is_dir=False)
elif file_size == item_record.size and compare_timestamps(
file_mtime,
datetime_to_timestamp(
item_record.modified_time)) == 0:
# The local item did not change since last update, but REMOTE file changed since then.
self.logger.info(
'File "%s" changed remotely since last sync. Download.',
item_local_path)
self._create_download_task(item_local_path,
remote_item)
else:
# Three ends mismatch. Keep both.
self.logger.info(
'Cannot determine file "%s". All information differ. Keep both.',
item_local_path)
self._move_existing_and_download(
item_local_path, remote_item, all_local_items,
q)
else:
# Examine file hash.
if _have_equal_hash(item_local_path, remote_item):
# Same hash means that they are the same file. Update local timestamp and database record.
self._update_attr_when_hash_equal(
item_local_path, remote_item)
else:
self.logger.info(
'Cannot determine file "%s". Rename and keep both.',
item_local_path)
self._move_existing_and_download(
item_local_path, remote_item, all_local_items,
q)