def connection():
create_db('_test_db')
connection = db_connection('_test_db')
yield connection
drop_tables(connection)
connection.close()
def connection():
create_db('_test_db')
connection = db_connection('_test_db')
yield connection
drop_tables(connection)
connection.close()
def connection():
create_db("_test_db")
connection = db_connection("_test_db")
yield connection
drop_tables(connection)
connection.close()
os.remove("_test_db")
def main():
config = configparser.ConfigParser()
config.read('dwh.cfg')
print("Connecting to Amazon Redshift...")
conn = psycopg2.connect(
"host={} dbname={} user={} password={} port={}".format(
*config['CLUSTER'].values()))
cur = conn.cursor()
print("Successfully connected to Amazon Redshift")
print("Dropping any existing tables...")
drop_tables(cur, conn)
print("Creating Tables...")
create_tables(cur, conn)
print("Done Creating Tables.")
conn.close()
print("Disconnected from Amazon Redshift.")
def dsm_mockup_db(postgres_service_url, s3_client, mock_files_factory):
# db
utils.create_tables(url=postgres_service_url)
# s3 client
bucket_name = BUCKET_NAME
s3_client.create_bucket(bucket_name, delete_contents_if_exists=True)
# TODO: use pip install Faker
users = ["alice", "bob", "chuck", "dennis"]
projects = [
"astronomy",
"biology",
"chemistry",
"dermatology",
"economics",
"futurology",
"geology",
]
location = SIMCORE_S3_STR
nodes = ["alpha", "beta", "gamma", "delta"]
N = 100
files = mock_files_factory(count=N)
counter = 0
data = {}
for _file in files:
idx = randrange(len(users))
user_name = users[idx]
user_id = idx + 10
idx = randrange(len(projects))
project_name = projects[idx]
project_id = idx + 100
idx = randrange(len(nodes))
node = nodes[idx]
node_id = idx + 10000
file_name = str(counter)
object_name = Path(str(project_id), str(node_id), str(counter)).as_posix()
file_uuid = Path(object_name).as_posix()
raw_file_path = file_uuid
display_file_path = str(Path(project_name) / Path(node) / Path(file_name))
created_at = str(datetime.datetime.now())
file_size = 1234
assert s3_client.upload_file(bucket_name, object_name, _file)
d = {
"file_uuid": file_uuid,
"location_id": "0",
"location": location,
"bucket_name": bucket_name,
"object_name": object_name,
"project_id": str(project_id),
"project_name": project_name,
"node_id": str(node_id),
"node_name": node,
"file_name": file_name,
"user_id": str(user_id),
"user_name": user_name,
"file_id": str(uuid.uuid4()),
"raw_file_path": file_uuid,
"display_file_path": display_file_path,
"created_at": created_at,
"last_modified": created_at,
"file_size": file_size,
}
counter = counter + 1
data[object_name] = FileMetaData(**d)
# pylint: disable=no-member
utils.insert_metadata(postgres_service_url, data[object_name])
total_count = 0
for _obj in s3_client.list_objects_v2(bucket_name, recursive=True):
total_count = total_count + 1
assert total_count == N
yield data
# s3 client
s3_client.remove_bucket(bucket_name, delete_contents=True)
# db
utils.drop_tables(url=postgres_service_url)
def drop_tables():
"""Use PeeWee to drop tables."""
utils.drop_tables()
def dsm_mockup_db(postgres_service_url, s3_client, mock_files_factory):
# db
utils.create_tables(url=postgres_service_url)
# s3 client
bucket_name = BUCKET_NAME
s3_client.create_bucket(bucket_name, delete_contents_if_exists=True)
# TODO: use pip install Faker
users = ['alice', 'bob', 'chuck', 'dennis']
projects = [
'astronomy', 'biology', 'chemistry', 'dermatology', 'economics',
'futurology', 'geology'
]
location = SIMCORE_S3_STR
nodes = ['alpha', 'beta', 'gamma', 'delta']
N = 100
files = mock_files_factory(count=N)
counter = 0
data = {}
for _file in files:
idx = randrange(len(users))
user_name = users[idx]
user_id = idx + 10
idx = randrange(len(projects))
project_name = projects[idx]
project_id = idx + 100
idx = randrange(len(nodes))
node = nodes[idx]
node_id = idx + 10000
file_name = str(counter)
object_name = Path(str(project_id), str(node_id),
str(counter)).as_posix()
file_uuid = Path(object_name).as_posix()
assert s3_client.upload_file(bucket_name, object_name, _file)
d = {
'file_uuid': file_uuid,
'location_id': "0",
'location': location,
'bucket_name': bucket_name,
'object_name': object_name,
'project_id': str(project_id),
'project_name': project_name,
'node_id': str(node_id),
'node_name': node,
'file_name': file_name,
'user_id': str(user_id),
'user_name': user_name
}
counter = counter + 1
data[object_name] = FileMetaData(**d)
# pylint: disable=no-member
utils.insert_metadata(postgres_service_url, data[object_name])
total_count = 0
for _obj in s3_client.list_objects_v2(bucket_name, recursive=True):
total_count = total_count + 1
assert total_count == N
yield data
# s3 client
s3_client.remove_bucket(bucket_name, delete_contents=True)
# db
utils.drop_tables(url=postgres_service_url)
def tearDown(self):
utils.drop_tables()