def connection(): create_db('_test_db') connection = db_connection('_test_db') yield connection drop_tables(connection) connection.close()
def connection(): create_db('_test_db') connection = db_connection('_test_db') yield connection drop_tables(connection) connection.close()
def connection(): create_db("_test_db") connection = db_connection("_test_db") yield connection drop_tables(connection) connection.close() os.remove("_test_db")
def main(): config = configparser.ConfigParser() config.read('dwh.cfg') print("Connecting to Amazon Redshift...") conn = psycopg2.connect( "host={} dbname={} user={} password={} port={}".format( *config['CLUSTER'].values())) cur = conn.cursor() print("Successfully connected to Amazon Redshift") print("Dropping any existing tables...") drop_tables(cur, conn) print("Creating Tables...") create_tables(cur, conn) print("Done Creating Tables.") conn.close() print("Disconnected from Amazon Redshift.")
def dsm_mockup_db(postgres_service_url, s3_client, mock_files_factory): # db utils.create_tables(url=postgres_service_url) # s3 client bucket_name = BUCKET_NAME s3_client.create_bucket(bucket_name, delete_contents_if_exists=True) # TODO: use pip install Faker users = ["alice", "bob", "chuck", "dennis"] projects = [ "astronomy", "biology", "chemistry", "dermatology", "economics", "futurology", "geology", ] location = SIMCORE_S3_STR nodes = ["alpha", "beta", "gamma", "delta"] N = 100 files = mock_files_factory(count=N) counter = 0 data = {} for _file in files: idx = randrange(len(users)) user_name = users[idx] user_id = idx + 10 idx = randrange(len(projects)) project_name = projects[idx] project_id = idx + 100 idx = randrange(len(nodes)) node = nodes[idx] node_id = idx + 10000 file_name = str(counter) object_name = Path(str(project_id), str(node_id), str(counter)).as_posix() file_uuid = Path(object_name).as_posix() raw_file_path = file_uuid display_file_path = str(Path(project_name) / Path(node) / Path(file_name)) created_at = str(datetime.datetime.now()) file_size = 1234 assert s3_client.upload_file(bucket_name, object_name, _file) d = { "file_uuid": file_uuid, "location_id": "0", "location": location, "bucket_name": bucket_name, "object_name": object_name, "project_id": str(project_id), "project_name": project_name, "node_id": str(node_id), "node_name": node, "file_name": file_name, "user_id": str(user_id), "user_name": user_name, "file_id": str(uuid.uuid4()), "raw_file_path": file_uuid, "display_file_path": display_file_path, "created_at": created_at, "last_modified": created_at, "file_size": file_size, } counter = counter + 1 data[object_name] = FileMetaData(**d) # pylint: disable=no-member utils.insert_metadata(postgres_service_url, data[object_name]) total_count = 0 for _obj in s3_client.list_objects_v2(bucket_name, recursive=True): total_count = total_count + 1 assert total_count == N yield data # s3 client s3_client.remove_bucket(bucket_name, delete_contents=True) # db utils.drop_tables(url=postgres_service_url)
def drop_tables(): """Use PeeWee to drop tables.""" utils.drop_tables()
def dsm_mockup_db(postgres_service_url, s3_client, mock_files_factory): # db utils.create_tables(url=postgres_service_url) # s3 client bucket_name = BUCKET_NAME s3_client.create_bucket(bucket_name, delete_contents_if_exists=True) # TODO: use pip install Faker users = ['alice', 'bob', 'chuck', 'dennis'] projects = [ 'astronomy', 'biology', 'chemistry', 'dermatology', 'economics', 'futurology', 'geology' ] location = SIMCORE_S3_STR nodes = ['alpha', 'beta', 'gamma', 'delta'] N = 100 files = mock_files_factory(count=N) counter = 0 data = {} for _file in files: idx = randrange(len(users)) user_name = users[idx] user_id = idx + 10 idx = randrange(len(projects)) project_name = projects[idx] project_id = idx + 100 idx = randrange(len(nodes)) node = nodes[idx] node_id = idx + 10000 file_name = str(counter) object_name = Path(str(project_id), str(node_id), str(counter)).as_posix() file_uuid = Path(object_name).as_posix() assert s3_client.upload_file(bucket_name, object_name, _file) d = { 'file_uuid': file_uuid, 'location_id': "0", 'location': location, 'bucket_name': bucket_name, 'object_name': object_name, 'project_id': str(project_id), 'project_name': project_name, 'node_id': str(node_id), 'node_name': node, 'file_name': file_name, 'user_id': str(user_id), 'user_name': user_name } counter = counter + 1 data[object_name] = FileMetaData(**d) # pylint: disable=no-member utils.insert_metadata(postgres_service_url, data[object_name]) total_count = 0 for _obj in s3_client.list_objects_v2(bucket_name, recursive=True): total_count = total_count + 1 assert total_count == N yield data # s3 client s3_client.remove_bucket(bucket_name, delete_contents=True) # db utils.drop_tables(url=postgres_service_url)
def tearDown(self): utils.drop_tables()