def test_exception_when_bucket_not_found(): with raises(s3tree.exceptions.BucketNotFound): s3tree.S3Tree( bucket_name=DUMMY_BUCKET_NAME, aws_access_key_id=DUMMY_ACCESS_KEY_ID, aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY, )
def test_exception_when_directory_not_found(): generate_dummy_bucket() with raises(s3tree.exceptions.DirectoryNotFound): s3tree.S3Tree(bucket_name=DUMMY_BUCKET_NAME, aws_access_key_id=DUMMY_ACCESS_KEY_ID, aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY, path='/non-existent')
def test_s3tree_work_with_global_config(): s3tree.config.aws_access_key_id = "foo" s3tree.config.aws_secret_access_key = "bar" try: s3tree.S3Tree(bucket_name="foo") except s3tree.exceptions.ImproperlyConfiguredError: raise fail("Initialization failed with global config.") except Exception: pass
def test_tree_as_json_property(): generate_dummy_bucket() tree = s3tree.S3Tree(bucket_name=DUMMY_BUCKET_NAME, aws_access_key_id=DUMMY_ACCESS_KEY_ID, aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY) json_data = tree.as_json data = json.loads(json_data) assert isinstance(json_data, string_types) assert isinstance(data, list) assert len(data) == 7
def test_file_reads(): generate_dummy_bucket() tree = s3tree.S3Tree(bucket_name=DUMMY_BUCKET_NAME, aws_access_key_id=DUMMY_ACCESS_KEY_ID, aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY, path='css') # the first element in this tree is a file dummy_file = tree[0] assert isinstance(dummy_file, s3tree.models.File) assert isinstance(dummy_file.read(), string_types) assert dummy_file.read().startswith('abcd')
def test_directory_traversal(): generate_dummy_bucket() tree = s3tree.S3Tree(bucket_name=DUMMY_BUCKET_NAME, aws_access_key_id=DUMMY_ACCESS_KEY_ID, aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY, path='cache') # get the first element, which is a directory directory = tree[0] child_tree = directory.get_tree() assert isinstance(child_tree, s3tree.S3Tree) assert len(child_tree) == 2
def preprocess(self): """Preprocess the CelebA attribute file.""" num_embeds = 10 embeddings = np.load('../embeddings.npy', allow_pickle=True) ids = np.load('../ids.npy', allow_pickle=True) speaker2idx = dict() for idx in range(len(ids)): if ids[idx][2:] in speaker2idx: speaker2idx[ids[idx][2:]].append(idx) else: speaker2idx[ids[idx][2:]] = [idx] tree = s3tree.S3Tree(bucket_name='face2speech', path='voxceleb/dev/aac') count = 0 for folder in tree.directories: start = time.time() print(folder) voices = [] for video in folder.get_tree(): for file in video.get_tree().files: if file.name.endswith('.npy'): voices.append(file.path) for voice in voices: voice_id = folder.name[2:] embed_id = voice_id # print(voice_id) if folder.name[2:] in speaker2idx: ems = speaker2idx[folder.name[2:]] # count1=0 for em in ems[0:num_embeds]: embedding = embeddings[em] # if count1 < 10: self.train_dataset.append( [voice_id, embed_id, voice, embedding]) # else: # break # print (voice_id, voice_id) count += 1 print("Time taken is {}".format(time.time() - start))
def test_s3tree_tree_sanity(): generate_dummy_bucket() tree = s3tree.S3Tree(bucket_name=DUMMY_BUCKET_NAME, aws_access_key_id=DUMMY_ACCESS_KEY_ID, aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY, path='/') # we're using some magic values for asserting tests here for the tree. # refer to helpers.generate_dummy_bucket to see the origin of these values. assert len(tree) == 7 assert tree.num_directories == 3 assert len(tree.directories) == 3 assert tree.num_files == 4 assert len(tree.files) == 4 # the first object in the tree should be a directory called `css` assert isinstance(tree[0], s3tree.models.Directory) assert tree[0].name == 'cache' # the last object in the tree should be a file called `Makefile` assert isinstance(tree[-1], s3tree.models.File) assert tree[-1].name == 'index.js'
def test_s3tree_improperly_configured(): with raises(s3tree.exceptions.ImproperlyConfiguredError): s3tree.S3Tree(bucket_name="foo")
file = csv.reader(open('/Users/username/Desktop/file3.csv', 'rb')) list_of_account_numbers = set() accounts_in_drive = set() for i in file: list_of_account_numbers.add((i[0])) files = [] total_account = list() path = '/Users/username/Desktop/File3-0128/' for r, d, f in os.walk(path): for file in f: if '.pdf' in file: files.append(os.path.join(r, file)) for f in files: if '/Users/rpendela/Desktop/File3-0128/bbdfmstmt' in f: pdf_name = str(f).split('/')[6] account_id = str(pdf_name).split('_')[0] # if account_id in list_of_account_numbers: if not os.path.exists('/Users/username/Desktop/test3' + '/' + account_id): os.makedirs('/Users/username/Desktop/test3' + '/' + account_id) shutil.copy(f, '/Users/username/Desktop/test3' + '/' + account_id) import s3tree s3tree.config.aws_access_key_id = 'access-key' s3tree.config.aws_secret_access_key = 'secret-key' tree = s3tree.S3Tree(bucket_name='bucket', path='/') for obj in tree: print(obj)