def test_exception_when_bucket_not_found():
with raises(s3tree.exceptions.BucketNotFound):
s3tree.S3Tree(
bucket_name=DUMMY_BUCKET_NAME,
aws_access_key_id=DUMMY_ACCESS_KEY_ID,
aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY,
)
def test_exception_when_directory_not_found():
generate_dummy_bucket()
with raises(s3tree.exceptions.DirectoryNotFound):
s3tree.S3Tree(bucket_name=DUMMY_BUCKET_NAME,
aws_access_key_id=DUMMY_ACCESS_KEY_ID,
aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY,
path='/non-existent')
def test_s3tree_work_with_global_config():
s3tree.config.aws_access_key_id = "foo"
s3tree.config.aws_secret_access_key = "bar"
try:
s3tree.S3Tree(bucket_name="foo")
except s3tree.exceptions.ImproperlyConfiguredError:
raise fail("Initialization failed with global config.")
except Exception:
pass
def test_tree_as_json_property():
generate_dummy_bucket()
tree = s3tree.S3Tree(bucket_name=DUMMY_BUCKET_NAME,
aws_access_key_id=DUMMY_ACCESS_KEY_ID,
aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY)
json_data = tree.as_json
data = json.loads(json_data)
assert isinstance(json_data, string_types)
assert isinstance(data, list)
assert len(data) == 7
def test_file_reads():
generate_dummy_bucket()
tree = s3tree.S3Tree(bucket_name=DUMMY_BUCKET_NAME,
aws_access_key_id=DUMMY_ACCESS_KEY_ID,
aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY,
path='css')
# the first element in this tree is a file
dummy_file = tree[0]
assert isinstance(dummy_file, s3tree.models.File)
assert isinstance(dummy_file.read(), string_types)
assert dummy_file.read().startswith('abcd')
def test_directory_traversal():
generate_dummy_bucket()
tree = s3tree.S3Tree(bucket_name=DUMMY_BUCKET_NAME,
aws_access_key_id=DUMMY_ACCESS_KEY_ID,
aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY,
path='cache')
# get the first element, which is a directory
directory = tree[0]
child_tree = directory.get_tree()
assert isinstance(child_tree, s3tree.S3Tree)
assert len(child_tree) == 2
def preprocess(self):
"""Preprocess the CelebA attribute file."""
num_embeds = 10
embeddings = np.load('../embeddings.npy', allow_pickle=True)
ids = np.load('../ids.npy', allow_pickle=True)
speaker2idx = dict()
for idx in range(len(ids)):
if ids[idx][2:] in speaker2idx:
speaker2idx[ids[idx][2:]].append(idx)
else:
speaker2idx[ids[idx][2:]] = [idx]
tree = s3tree.S3Tree(bucket_name='face2speech',
path='voxceleb/dev/aac')
count = 0
for folder in tree.directories:
start = time.time()
print(folder)
voices = []
for video in folder.get_tree():
for file in video.get_tree().files:
if file.name.endswith('.npy'):
voices.append(file.path)
for voice in voices:
voice_id = folder.name[2:]
embed_id = voice_id
# print(voice_id)
if folder.name[2:] in speaker2idx:
ems = speaker2idx[folder.name[2:]]
# count1=0
for em in ems[0:num_embeds]:
embedding = embeddings[em]
# if count1 < 10:
self.train_dataset.append(
[voice_id, embed_id, voice, embedding])
# else:
# break
# print (voice_id, voice_id)
count += 1
print("Time taken is {}".format(time.time() - start))
def test_s3tree_tree_sanity():
generate_dummy_bucket()
tree = s3tree.S3Tree(bucket_name=DUMMY_BUCKET_NAME,
aws_access_key_id=DUMMY_ACCESS_KEY_ID,
aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY,
path='/')
# we're using some magic values for asserting tests here for the tree.
# refer to helpers.generate_dummy_bucket to see the origin of these values.
assert len(tree) == 7
assert tree.num_directories == 3
assert len(tree.directories) == 3
assert tree.num_files == 4
assert len(tree.files) == 4
# the first object in the tree should be a directory called `css`
assert isinstance(tree[0], s3tree.models.Directory)
assert tree[0].name == 'cache'
# the last object in the tree should be a file called `Makefile`
assert isinstance(tree[-1], s3tree.models.File)
assert tree[-1].name == 'index.js'
def test_s3tree_improperly_configured():
with raises(s3tree.exceptions.ImproperlyConfiguredError):
s3tree.S3Tree(bucket_name="foo")
file = csv.reader(open('/Users/username/Desktop/file3.csv', 'rb'))
list_of_account_numbers = set()
accounts_in_drive = set()
for i in file:
list_of_account_numbers.add((i[0]))
files = []
total_account = list()
path = '/Users/username/Desktop/File3-0128/'
for r, d, f in os.walk(path):
for file in f:
if '.pdf' in file:
files.append(os.path.join(r, file))
for f in files:
if '/Users/rpendela/Desktop/File3-0128/bbdfmstmt' in f:
pdf_name = str(f).split('/')[6]
account_id = str(pdf_name).split('_')[0]
# if account_id in list_of_account_numbers:
if not os.path.exists('/Users/username/Desktop/test3' + '/' +
account_id):
os.makedirs('/Users/username/Desktop/test3' + '/' + account_id)
shutil.copy(f, '/Users/username/Desktop/test3' + '/' + account_id)
import s3tree
s3tree.config.aws_access_key_id = 'access-key'
s3tree.config.aws_secret_access_key = 'secret-key'
tree = s3tree.S3Tree(bucket_name='bucket', path='/')
for obj in tree:
print(obj)
