def upload_files_in_folder(self, arg, dirname, fnames):
"""Handles the iteration across files within a folder."""
if utils.match_pattern(dirname, self.ignore):
return False
for fname in (nm for nm in fnames
if not utils.match_pattern(nm, self.ignore)):
if self.client._should_abort_folder_upload(self.upload_key):
return
full_path = os.path.join(dirname, fname)
if os.path.isdir(full_path):
# Skip folders; os.walk will include them in the next pass.
continue
obj_name = os.path.relpath(full_path, self.base_path)
obj_size = os.stat(full_path).st_size
self.client.upload_file(self.container, full_path, obj_name=obj_name,
return_none=True)
self.client._update_progress(self.upload_key, obj_size)
def upload_files_in_folder(self, arg, dirname, fnames):
"""Handles the iteration across files within a folder."""
if utils.match_pattern(dirname, self.ignore):
return False
for fname in (nm for nm in fnames
if not utils.match_pattern(nm, self.ignore)):
if self.client._should_abort_folder_upload(self.upload_key):
return
full_path = os.path.join(dirname, fname)
if os.path.isdir(full_path):
# Skip folders; os.walk will include them in the next pass.
continue
obj_name = os.path.relpath(full_path, self.base_path)
obj_size = os.stat(full_path).st_size
self.client.upload_file(self.container, full_path,
obj_name=obj_name, return_none=True)
self.client._update_progress(self.upload_key, obj_size)
def _sync_folder_to_container(self, folder_path, cont, prefix, delete,
include_hidden, ignore, ignore_timestamps):
"""
This is the internal method that is called recursively to handle
nested folder structures.
"""
fnames = os.listdir(folder_path)
ignore = utils.coerce_string_to_list(ignore)
if not include_hidden:
ignore.append(".*")
for fname in fnames:
if utils.match_pattern(fname, ignore):
continue
pth = os.path.join(folder_path, fname)
if os.path.isdir(pth):
subprefix = fname
if prefix:
subprefix = "%s/%s" % (prefix, subprefix)
self._sync_folder_to_container(pth, cont, prefix=subprefix,
delete=delete, include_hidden=include_hidden,
ignore=ignore, ignore_timestamps=ignore_timestamps)
continue
self._local_files.append(os.path.join(prefix, fname))
local_etag = utils.get_checksum(pth)
fullname = fname
if prefix:
fullname = "%s/%s" % (prefix, fname)
try:
obj = cont.get_object(fullname)
obj_etag = obj.etag
except exc.NoSuchObject:
obj = None
obj_etag = None
if local_etag != obj_etag:
if not ignore_timestamps:
if obj:
obj_time_str = obj.last_modified[:19]
else:
obj_time_str = EARLY_DATE_STR
local_mod = datetime.datetime.utcfromtimestamp(
os.stat(pth).st_mtime)
local_mod_str = local_mod.isoformat()
if obj_time_str >= local_mod_str:
# Remote object is newer
continue
cont.upload_file(pth, obj_name=fullname, etag=local_etag,
return_none=True)
if delete and not prefix:
self._delete_objects_not_in_list(cont)
def _sync_folder_to_container(self, folder_path, cont, prefix, delete,
include_hidden, ignore, ignore_timestamps):
"""
This is the internal method that is called recursively to handle
nested folder structures.
"""
fnames = os.listdir(folder_path)
ignore = utils.coerce_string_to_list(ignore)
if not include_hidden:
ignore.append(".*")
for fname in fnames:
if utils.match_pattern(fname, ignore):
continue
pth = os.path.join(folder_path, fname)
if os.path.isdir(pth):
subprefix = fname
if prefix:
subprefix = "%s/%s" % (prefix, subprefix)
self._sync_folder_to_container(pth, cont, prefix=subprefix,
delete=delete, include_hidden=include_hidden,
ignore=ignore, ignore_timestamps=ignore_timestamps)
continue
self._local_files.append(os.path.join(prefix, fname))
local_etag = utils.get_checksum(pth)
fullname = fname
if prefix:
fullname = "%s/%s" % (prefix, fname)
try:
obj = cont.get_object(fullname)
obj_etag = obj.etag
except exc.NoSuchObject:
obj = None
obj_etag = None
if local_etag != obj_etag:
if not ignore_timestamps:
if obj:
obj_time_str = obj.last_modified[:19]
else:
obj_time_str = EARLY_DATE_STR
local_mod = datetime.datetime.utcfromtimestamp(
os.stat(pth).st_mtime)
local_mod_str = local_mod.isoformat()
if obj_time_str >= local_mod_str:
# Remote object is newer
continue
cont.upload_file(pth, obj_name=fullname, etag=local_etag,
return_none=True)
if delete and not prefix:
self._delete_objects_not_in_list(cont)
def test_match_pattern(self):
ignore_pat = "*.bad"
self.assertTrue(utils.match_pattern("some.bad", ignore_pat))
self.assertFalse(utils.match_pattern("some.good", ignore_pat))
def test_match_pattern(self):
ignore_pat = "*.bad"
self.assertTrue(utils.match_pattern("some.bad", ignore_pat))
self.assertFalse(utils.match_pattern("some.good", ignore_pat))
