def test_local_directory(self): """ Generate an entire local directory. """ input_local_dir = {'src': {'path': self.local_dir, 'type': 'local'}, 'dest': {'path': 'bucket/', 'type': 's3'}, 'dir_op': True, 'use_src_name': True} params = {'region': 'us-east-1'} files = FileGenerator(self.service, self.endpoint,'').call(input_local_dir) result_list = [] for filename in files: result_list.append(filename) size, last_update = get_file_stat(self.local_file) file_info = FileInfo(src=self.local_file, dest='bucket/text1.txt', compare_key='text1.txt', size=size, last_update=last_update, src_type='local', dest_type='s3', operation_name='', service=None, endpoint=None) path = self.local_dir + 'another_directory' + os.sep \ + 'text2.txt' size, last_update = get_file_stat(path) file_info2 = FileInfo(src=path, dest='bucket/another_directory/text2.txt', compare_key='another_directory/text2.txt', size=size, last_update=last_update, src_type='local', dest_type='s3', operation_name='', service=None, endpoint=None) ref_list = [file_info2, file_info] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i])
def test_local_directory(self): """ Generate an entire local directory. """ input_local_dir = {'src': {'path': self.local_dir, 'type': 'local'}, 'dest': {'path': 'bucket/', 'type': 's3'}, 'dir_op': True, 'use_src_name': True} params = {'region': 'us-east-1'} files = FileGenerator(self.service, self.endpoint, '', params).call(input_local_dir) result_list = [] for filename in files: result_list.append(filename) size, last_update = get_file_stat(self.local_file) file_info = FileInfo(src=self.local_file, dest='bucket/text1.txt', compare_key='text1.txt', size=size, last_update=last_update, src_type='local', dest_type='s3', operation_name='', service=None, endpoint=None) path = self.local_dir + 'another_directory' + os.sep \ + 'text2.txt' size, last_update = get_file_stat(path) file_info2 = FileInfo(src=path, dest='bucket/another_directory/text2.txt', compare_key='another_directory/text2.txt', size=size, last_update=last_update, src_type='local', dest_type='s3', operation_name='', service=None, endpoint=None) ref_list = [file_info2, file_info] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i])
def test_get_file_stat_error_message(self): patch_attribute = 'awscli.customizations.s3.utils.datetime' with mock.patch(patch_attribute) as f: with mock.patch('os.stat'): f.fromtimestamp.side_effect = ValueError( "timestamp out of range for platform " "localtime()/gmtime() function") with self.assertRaisesRegexp(ValueError, 'myfilename\.txt'): get_file_stat('myfilename.txt')
def test_get_file_stat_error_message(self): patch_attribute = 'awscli.customizations.s3.utils.datetime' with mock.patch(patch_attribute) as f: with mock.patch('os.stat'): f.fromtimestamp.side_effect = ValueError( "timestamp out of range for platform " "localtime()/gmtime() function") with self.assertRaisesRegexp(ValueError, 'myfilename\.txt'): get_file_stat('myfilename.txt')
def list_files(self, path, dir_op): """ This function yields the appropriate local file or local files under a directory depending on if the operation is on a directory. For directories a depth first search is implemented in order to follow the same sorted pattern as a s3 list objects operation outputs. It yields the file's source path, size, and last update """ join, isdir, isfile = os.path.join, os.path.isdir, os.path.isfile error, listdir = os.error, os.listdir if not self.should_ignore_file(path): if not dir_op: size, last_update = get_file_stat(path) last_update = self._validate_update_time(last_update, path) yield path, {'Size': size, 'LastModified': last_update} else: # We need to list files in byte order based on the full # expanded path of the key: 'test/1/2/3.txt' However, # listdir() will only give us contents a single directory # at a time, so we'll get 'test'. At the same time we don't # want to load the entire list of files into memory. This # is handled by first going through the current directory # contents and adding the directory separator to any # directories. We can then sort the contents, # and ensure byte order. listdir_names = listdir(path) names = [] for name in listdir_names: if not self.should_ignore_file_with_decoding_warnings( path, name): file_path = join(path, name) if isdir(file_path): name = name + os.path.sep names.append(name) self.normalize_sort(names, os.sep, '/') for name in names: file_path = join(path, name) if isdir(file_path): # Anything in a directory will have a prefix of # this current directory and will come before the # remaining contents in this directory. This # means we need to recurse into this sub directory # before yielding the rest of this directory's # contents. for x in self.list_files(file_path, dir_op): yield x else: size, last_update = get_file_stat(file_path) last_update = self._validate_update_time( last_update, path) yield (file_path, { 'Size': size, 'LastModified': last_update })
def list_files(self, path, dir_op): """ This function yields the appropriate local file or local files under a directory depending on if the operation is on a directory. For directories a depth first search is implemented in order to follow the same sorted pattern as a s3 list objects operation outputs. It yields the file's source path, size, and last update """ join, isdir, isfile = os.path.join, os.path.isdir, os.path.isfile error, listdir = os.error, os.listdir if not self.should_ignore_file(path): if not dir_op: size, last_update = get_file_stat(path) last_update = self._validate_update_time(last_update, path) yield path, {'Size': size, 'LastModified': last_update} else: # We need to list files in byte order based on the full # expanded path of the key: 'test/1/2/3.txt' However, # listdir() will only give us contents a single directory # at a time, so we'll get 'test'. At the same time we don't # want to load the entire list of files into memory. This # is handled by first going through the current directory # contents and adding the directory separator to any # directories. We can then sort the contents, # and ensure byte order. listdir_names = listdir(path) names = [] for name in listdir_names: if not self.should_ignore_file_with_decoding_warnings( path, name): file_path = join(path, name) if isdir(file_path): name = name + os.path.sep names.append(name) self.normalize_sort(names, os.sep, '/') for name in names: file_path = join(path, name) if isdir(file_path): # Anything in a directory will have a prefix of # this current directory and will come before the # remaining contents in this directory. This # means we need to recurse into this sub directory # before yielding the rest of this directory's # contents. for x in self.list_files(file_path, dir_op): yield x else: size, last_update = get_file_stat(file_path) last_update = self._validate_update_time( last_update, path) yield ( file_path, {'Size': size, 'LastModified': last_update} )
def test_successfully_sets_utime(self): now = datetime.datetime.now(tzlocal()) epoch_now = time.mktime(now.timetuple()) with temporary_file('w') as f: set_file_utime(f.name, epoch_now) _, update_time = get_file_stat(f.name) self.assertEqual(time.mktime(update_time.timetuple()), epoch_now)
def test_local_file(self): """ Generate a single local file. """ input_local_file = { "src": {"path": self.local_file, "type": "local"}, "dest": {"path": "bucket/text1.txt", "type": "s3"}, "dir_op": False, "use_src_name": False, } params = {"region": "us-east-1"} files = FileGenerator(self.service, self.endpoint, "", params).call(input_local_file) result_list = [] for filename in files: result_list.append(filename) size, last_update = get_file_stat(self.local_file) file_info = FileInfo( src=self.local_file, dest="bucket/text1.txt", compare_key="text1.txt", size=size, last_update=last_update, src_type="local", dest_type="s3", operation_name="", service=None, endpoint=None, ) ref_list = [file_info] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i])
def test_local_file(self): """ Generate a single local file. """ input_local_file = { 'src': { 'path': self.local_file, 'type': 'local' }, 'dest': { 'path': 'bucket/text1.txt', 'type': 's3' }, 'dir_op': False, 'use_src_name': False } params = {'region': 'us-east-1'} files = FileGenerator(self.client, '').call(input_local_file) result_list = [] for filename in files: result_list.append(filename) size, last_update = get_file_stat(self.local_file) file_stat = FileStat(src=self.local_file, dest='bucket/text1.txt', compare_key='text1.txt', size=size, last_update=last_update, src_type='local', dest_type='s3', operation_name='') ref_list = [file_stat] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i])
def test_successfully_sets_utime(self): now = datetime.datetime.now(tzlocal()) epoch_now = time.mktime(now.timetuple()) with temporary_file("w") as f: set_file_utime(f.name, epoch_now) _, update_time = get_file_stat(f.name) self.assertEqual(time.mktime(update_time.timetuple()), epoch_now)
def test_local_file(self): """ Generate a single local file. """ input_local_file = {'src': {'path': self.local_file, 'type': 'local'}, 'dest': {'path': 'bucket/text1.txt', 'type': 's3'}, 'dir_op': False, 'use_src_name': False} params = {'region': 'us-east-1'} files = FileGenerator(self.service, self.endpoint, '', params).call(input_local_file) result_list = [] for filename in files: result_list.append(filename) size, last_update = get_file_stat(self.local_file) file_info = FileInfo(src=self.local_file, dest='bucket/text1.txt', compare_key='text1.txt', size=size, last_update=last_update, src_type='local', dest_type='s3', operation_name='', service=None, endpoint=None) ref_list = [file_info] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i])
def test_local_directory(self): """ Generate an entire local directory. """ input_local_dir = { "src": {"path": self.local_dir, "type": "local"}, "dest": {"path": "bucket/", "type": "s3"}, "dir_op": True, "use_src_name": True, } params = {"region": "us-east-1"} files = FileGenerator(self.service, self.endpoint, "", params).call(input_local_dir) result_list = [] for filename in files: result_list.append(filename) size, last_update = get_file_stat(self.local_file) file_info = FileInfo( src=self.local_file, dest="bucket/text1.txt", compare_key="text1.txt", size=size, last_update=last_update, src_type="local", dest_type="s3", operation_name="", service=None, endpoint=None, ) path = self.local_dir + "another_directory" + os.sep + "text2.txt" size, last_update = get_file_stat(path) file_info2 = FileInfo( src=path, dest="bucket/another_directory/text2.txt", compare_key="another_directory/text2.txt", size=size, last_update=last_update, src_type="local", dest_type="s3", operation_name="", service=None, endpoint=None, ) ref_list = [file_info2, file_info] self.assertEqual(len(result_list), len(ref_list)) for i in range(len(result_list)): compare_files(self, result_list[i], ref_list[i])
def _safely_get_file_stats(self, file_path): try: size, last_update = get_file_stat(file_path) except (OSError, ValueError): self.triggers_warning(file_path) else: last_update = self._validate_update_time(last_update, file_path) return file_path, {'Size': size, 'LastModified': last_update}
def assert_handles_fromtimestamp_error(self, error): patch_attribute = 'awscli.customizations.s3.utils.datetime' with mock.patch(patch_attribute) as datetime_mock: with temporary_file('w') as temp_file: temp_file.write('foo') temp_file.flush() datetime_mock.fromtimestamp.side_effect = error size, update_time = get_file_stat(temp_file.name) self.assertIsNone(update_time)
def test_get_file_stat_returns_epoch_on_invalid_timestamp_os_error(self): patch_attribute = "awscli.customizations.s3.utils.datetime" with mock.patch(patch_attribute) as datetime_mock: with temporary_file("w") as temp_file: temp_file.write("foo") temp_file.flush() datetime_mock.fromtimestamp.side_effect = OSError() size, update_time = get_file_stat(temp_file.name) self.assertIsNone(update_time)
def test_get_file_stat_returns_epoch_on_invalid_timestamp(self): patch_attribute = 'awscli.customizations.s3.utils.datetime' with mock.patch(patch_attribute) as datetime_mock: with temporary_file('w') as temp_file: temp_file.write('foo') temp_file.flush() datetime_mock.fromtimestamp.side_effect = ValueError() size, update_time = get_file_stat(temp_file.name) self.assertIsNone(update_time)
def assert_handles_fromtimestamp_error(self, error): patch_attribute = 'awscli.customizations.s3.utils.datetime' with mock.patch(patch_attribute) as datetime_mock: with temporary_file('w') as temp_file: temp_file.write('foo') temp_file.flush() datetime_mock.fromtimestamp.side_effect = error size, update_time = get_file_stat(temp_file.name) self.assertIsNone(update_time)
def test_get_file_stat(self): now = datetime.datetime.now(tzlocal()) epoch_now = time.mktime(now.timetuple()) with temporary_file('w') as f: f.write('foo') f.flush() os.utime(f.name, (epoch_now, epoch_now)) size, update_time = get_file_stat(f.name) self.assertEqual(size, 3) self.assertEqual(time.mktime(update_time.timetuple()), epoch_now)
def list_files(self, path, dir_op): """ This function yields the appropriate local file or local files under a directory depending on if the operation is on a directory. For directories a depth first search is implemented in order to follow the same sorted pattern as a s3 list objects operation outputs. It yields the file's source path, size, and last update """ join, isdir, isfile = os.path.join, os.path.isdir, os.path.isfile error, listdir = os.error, os.listdir if not dir_op: size, last_update = get_file_stat(path) yield path, size, last_update else: # We need to list files in byte order based on the full # expanded path of the key: 'test/1/2/3.txt' However, listdir() # will only give us contents a single directory at a time, so we'll # get 'test'. At the same time we don't want to load the entire # list of files into memory. This is handled by first going # through the current directory contents and adding the directory # separator to any directories. We can then sort the contents, # and ensure byte order. names = listdir(path) self._check_paths_decoded(path, names) for i, name in enumerate(names): file_path = join(path, name) if isdir(file_path): names[i] = name + os.path.sep names.sort() for name in names: file_path = join(path, name) if isdir(file_path): # Anything in a directory will have a prefix of this # current directory and will come before the # remaining contents in this directory. This means we need # to recurse into this sub directory before yielding the # rest of this directory's contents. for x in self.list_files(file_path, dir_op): yield x else: size, last_update = get_file_stat(file_path) yield file_path, size, last_update
def test_get_file_stat(self): now = datetime.datetime.now(tzlocal()) epoch_now = time.mktime(now.timetuple()) with temporary_file("w") as f: f.write("foo") f.flush() os.utime(f.name, (epoch_now, epoch_now)) size, update_time = get_file_stat(f.name) self.assertEqual(size, 3) self.assertEqual(time.mktime(update_time.timetuple()), epoch_now)
def list_files(self, path, dir_op): """ This function yields the appropriate local file or local files under a directory depending on if the operation is on a directory. For directories a depth first search is implemented in order to follow the same sorted pattern as a s3 list objects operation outputs. It yields the file's source path, size, and last update """ join, isdir, isfile = os.path.join, os.path.isdir, os.path.isfile error, listdir = os.error, os.listdir if not dir_op: size, last_update = get_file_stat(path) yield path, size, last_update else: # We need to list files in byte order based on the full # expanded path of the key: 'test/1/2/3.txt' However, listdir() # will only give us contents a single directory at a time, so we'll # get 'test'. At the same time we don't want to load the entire # list of files into memory. This is handled by first going # through the current directory contents and adding the directory # separator to any directories. We can then sort the contents, # and ensure byte order. names = listdir(path) self._check_paths_decoded(path, names) for i, name in enumerate(names): file_path = join(path, name) if isdir(file_path): names[i] = name + os.path.sep names.sort() for name in names: file_path = join(path, name) if isdir(file_path): # Anything in a directory will have a prefix of this # current directory and will come before the # remaining contents in this directory. This means we need # to recurse into this sub directory before yielding the # rest of this directory's contents. for x in self.list_files(file_path, dir_op): yield x else: size, last_update = get_file_stat(file_path) yield file_path, size, last_update
def list_files(self, path, dir_op): """ This function yields the appropriate local file or local files under a directory depending on if the operation is on a directory. For directories a depth first search is implemented in order to follow the same sorted pattern as a s3 list objects operation outputs. It yields the file's source path, size, and last update """ join, isdir, isfile = os.path.join, os.path.isdir, os.path.isfile error, listdir = os.error, os.listdir if not dir_op: size, last_update = get_file_stat(path) yield path, size, last_update else: names = sorted(listdir(path)) for name in names: file_path = join(path, name) if isdir(file_path): for x in self.list_files(file_path, dir_op): yield x else: size, last_update = get_file_stat(file_path) yield file_path, size, last_update
def list_files(self, path, dir_op): """ This function yields the appropriate local file or local files under a directory depending on if the operation is on a directory. For directories a depth first search is implemented in order to follow the same sorted pattern as a s3 list objects operation outputs. It yields the file's source path, size, and last update """ join, isdir, isfile = os.path.join, os.path.isdir, os.path.isfile error, listdir = os.error, os.listdir if not dir_op: size, last_update = get_file_stat(path) yield path, size, last_update else: names = sorted(listdir(path)) for name in names: file_path = join(path, name) if isdir(file_path): for x in self.list_files(file_path, dir_op): yield x else: size, last_update = get_file_stat(file_path) yield file_path, size, last_update
def test_on_success_modifies_utime(self): self.subscriber.on_done(self.future) _, utime = get_file_stat(self.filename) self.assertEqual(utime, self.desired_utime)
def test_error_message(self): with mock.patch('os.stat', mock.Mock(side_effect=IOError('msg'))): with self.assertRaisesRegexp(ValueError, 'myfilename\.txt'): get_file_stat('myfilename.txt')
def test_get_file_stat_error_message(self): with mock.patch("os.stat", mock.Mock(side_effect=IOError("msg"))): with self.assertRaisesRegexp(ValueError, "myfilename\.txt"): get_file_stat("myfilename.txt")
def test_error_message(self): with mock.patch('os.stat', mock.Mock(side_effect=IOError('msg'))): with self.assertRaisesRegexp(ValueError, 'myfilename\.txt'): get_file_stat('myfilename.txt')
def test_on_success_modifies_utime(self): self.subscriber.on_done(self.future) _, utime = get_file_stat(self.filename) self.assertEqual(utime, self.desired_utime)