def test_copy_to_s3_copy_from_s3_roundtrip(self):
with MinioServer() as minio:
t = threading.Thread(target=minio.run)
t.daemon = True
t.start()
wait_until(lambda: _is_up('127.0.0.1', 9000))
with connect(crate_node.http_url) as conn:
c = conn.cursor()
c.execute('CREATE TABLE t1 (x int)')
c.execute('INSERT INTO t1 (x) VALUES (1), (2), (3)')
c.execute('REFRESH TABLE t1')
c.execute("""
CREATE REPOSITORY r1 TYPE S3
WITH (access_key = 'minio', secret_key = 'miniostorage', bucket='backups', endpoint = '127.0.0.1:9000', protocol = 'http')
""")
c.execute(
'CREATE SNAPSHOT r1.s1 ALL WITH (wait_for_completion = true)'
)
c.execute('DROP TABLE t1')
c.execute(
'RESTORE SNAPSHOT r1.s1 ALL WITH (wait_for_completion = true)'
)
c.execute('SELECT COUNT(*) FROM t1')
rowcount = c.fetchone()[0]
self.assertEqual(rowcount, 3)
def test_snapshot_compatibility(self):
"""Test snapshot compatibility when upgrading 3.3.x -> 4.x.x
Using Minio as a S3 repository, the first cluster that runs
creates the repo, a table and inserts/selects some data, which
then is snapshotted and deleted. The next cluster recovers the
data from the last snapshot, performs further inserts/selects,
to then snapshot the data and delete it.
We are interested in the transition 3.3.x -> 4.x.x
"""
with MinioServer() as minio:
t = threading.Thread(target=minio.run)
t.daemon = True
t.start()
wait_until(lambda: _is_up('127.0.0.1', 9000))
num_nodes = 3
num_docs = 30
prev_version = None
num_snapshot = 1
path_data = 'data_test_snapshot_compatibility'
cluster_settings = {
'cluster.name': gen_id(),
'path.data': path_data
}
shutil.rmtree(path_data, ignore_errors=True)
for version in self.VERSION:
cluster = self._new_cluster(version,
num_nodes,
settings=cluster_settings)
cluster.start()
with connect(cluster.node().http_url,
error_trace=True) as conn:
c = conn.cursor()
if not prev_version:
c.execute(self.CREATE_REPOSITORY)
c.execute(CREATE_ANALYZER)
c.execute(CREATE_DOC_TABLE)
insert_data(conn, 'doc', 't1', num_docs)
else:
c.execute(
self.RESTORE_SNAPSHOT_TPT.format(num_snapshot - 1))
c.execute('SELECT COUNT(*) FROM t1')
rowcount = c.fetchone()[0]
self.assertEqual(rowcount, num_docs)
run_selects(c, version)
c.execute(self.CREATE_SNAPSHOT_TPT.format(num_snapshot))
c.execute(self.DROP_DOC_TABLE)
self._process_on_stop()
prev_version = version
num_snapshot += 1
shutil.rmtree(path_data, ignore_errors=True)
def test_copy_to_s3_copy_from_s3_roundtrip(self):
client = Minio('127.0.0.1:9000',
access_key=MinioServer.MINIO_ACCESS_KEY,
secret_key=MinioServer.MINIO_SECRET_KEY,
secure=False)
with MinioServer() as minio:
t = threading.Thread(target=minio.run)
t.daemon = True
t.start()
wait_until(lambda: _is_up('127.0.0.1', 9000))
with connect(crate_node.http_url) as conn:
c = conn.cursor()
c.execute('CREATE TABLE t1 (x int)')
c.execute('INSERT INTO t1 (x) VALUES (1), (2), (3)')
c.execute('REFRESH TABLE t1')
c.execute("""
CREATE REPOSITORY r1 TYPE S3
WITH (access_key = 'minio', secret_key = 'miniostorage/', bucket='backups', endpoint = '127.0.0.1:9000', protocol = 'http')
""")
# Make sure access_key and secret_key are masked in sys.repositories
c.execute(
'''SELECT settings['access_key'], settings['secret_key'] from sys.repositories where name = 'r1' '''
)
settings = c.fetchone()
self.assertEqual(settings[0], '[xxxxx]')
self.assertEqual(settings[1], '[xxxxx]')
c.execute(
'CREATE SNAPSHOT r1.s1 ALL WITH (wait_for_completion = true)'
)
c.execute('DROP TABLE t1')
c.execute(
'RESTORE SNAPSHOT r1.s1 ALL WITH (wait_for_completion = true)'
)
c.execute('SELECT COUNT(*) FROM t1')
rowcount = c.fetchone()[0]
self.assertEqual(rowcount, 3)
c.execute('DROP SNAPSHOT r1.s1')
c.execute(
'''SELECT COUNT(*) FROM sys.snapshots WHERE name = 's1' '''
)
rowcount = c.fetchone()[0]
self.assertEqual(rowcount, 0)
[
self.assertEqual(n.object_name.endswith('.dat'), False)
for n in client.list_objects('backups')
]
def test_basic_copy_to_and_copy_from(self):
client = Minio('127.0.0.1:9000',
access_key=MinioServer.MINIO_ACCESS_KEY,
secret_key=MinioServer.MINIO_SECRET_KEY,
region="myRegion",
secure=False)
with MinioServer() as minio:
t = threading.Thread(target=minio.run)
t.daemon = True
t.start()
wait_until(lambda: _is_up('127.0.0.1', 9000))
client.make_bucket("my-bucket")
with connect(crate_node.http_url) as conn:
c = conn.cursor()
c.execute('CREATE TABLE t1 (x int)')
c.execute('INSERT INTO t1 (x) VALUES (1), (2), (3)')
c.execute('REFRESH TABLE t1')
c.execute("""
COPY t1 TO DIRECTORY 's3://minio:miniostorage%[email protected]:9000/my-bucket/key' WITH (protocol = 'http')
""")
c.execute('CREATE TABLE r1 (x int)')
c.execute("""
COPY r1 FROM 's3://minio:miniostorage%[email protected]:9000/my-bucket/k*y/*' WITH (protocol = 'http')
""")
c.execute('REFRESH TABLE r1')
c.execute('SELECT x FROM r1 order by x')
result = c.fetchall()
self.assertEqual(result, [[1], [2], [3]])
c.execute("""
COPY r1 FROM ['s3://minio:miniostorage%[email protected]:9000/my-bucket/key/t1_0_.json',
's3://minio:miniostorage%[email protected]:9000/my-bucket/key/t1_1_.json',
's3://minio:miniostorage%[email protected]:9000/my-bucket/key/t1_2_.json',
's3://minio:miniostorage%[email protected]:9000/my-bucket/key/t1_3_.json'] WITH (protocol = 'http')
""")
c.execute('REFRESH TABLE r1')
c.execute('SELECT x FROM r1 order by x')
result = c.fetchall()
self.assertEqual(result, [[1], [1], [2], [2], [3], [3]])
c.execute('DROP TABLE t1')
c.execute('DROP TABLE r1')
def test_anonymous_read_write(self):
client = Minio('127.0.0.1:9000',
access_key=MinioServer.MINIO_ACCESS_KEY,
secret_key=MinioServer.MINIO_SECRET_KEY,
region="myRegion",
secure=False)
with MinioServer() as minio:
t = threading.Thread(target=minio.run)
t.daemon = True
t.start()
wait_until(lambda: _is_up('127.0.0.1', 9000))
client.make_bucket("my.bucket")
# https://docs.min.io/docs/python-client-api-reference.html#set_bucket_policy
# Example anonymous read-write bucket policy.
policy = {
"Version":
"2012-10-17",
"Statement": [
{
"Effect":
"Allow",
"Principal": {
"AWS": "*"
},
"Action": [
"s3:GetBucketLocation",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
],
"Resource":
"arn:aws:s3:::my.bucket",
},
{
"Effect":
"Allow",
"Principal": {
"AWS": "*"
},
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject",
"s3:ListMultipartUploadParts",
"s3:AbortMultipartUpload",
],
"Resource":
"arn:aws:s3:::my.bucket/*",
},
],
}
client.set_bucket_policy("my.bucket", json.dumps(policy))
with connect(crate_node.http_url) as conn:
c = conn.cursor()
c.execute('CREATE TABLE t1 (x int)')
c.execute('INSERT INTO t1 (x) VALUES (1), (2), (3)')
c.execute('REFRESH TABLE t1')
c.execute("""
COPY t1 TO DIRECTORY 's3://127.0.0.1:9000/my.bucket/' WITH (protocol = 'http')
""")
c.execute("""
COPY t1 FROM 's3://127.0.0.1:9000/my.bucket/*' WITH (protocol = 'http')
""")
c.execute('REFRESH TABLE t1')
c.execute('SELECT x FROM t1 order by x')
result = c.fetchall()
self.assertEqual(
result,
[[1], [1], [2], [2], [3], [3]
]) # two sets because copied to/from the same table
c.execute('DROP TABLE t1')
def test_snapshot_restore_and_drop_in_parallel(self):
"""Test to run the drop and restore operation on two different
snapshots in parallel.
The purpose of this test is to validate that the snapshot mechanism
of CrateDB can handle the two operations in parallel. Here, Minio is
used as s3 backend for the repository, but this should work on any
other backend as well.
"""
with MinioServer() as minio:
t = threading.Thread(target=minio.run)
t.daemon = True
t.start()
wait_until(lambda: _is_up('127.0.0.1', 9000))
num_nodes = random.randint(3, 5)
number_of_shards = random.randint(1, 3)
number_of_replicas = random.randint(0, 2)
num_docs = random.randint(1, 100)
cluster_settings = {
'cluster.name': gen_id(),
'path.data': self.PATH_DATA
}
shutil.rmtree(self.PATH_DATA, ignore_errors=True)
cluster = self._new_cluster('latest-nightly',
num_nodes,
settings=cluster_settings)
cluster.start()
with connect(cluster.node().http_url, error_trace=True) as conn:
c = conn.cursor()
wait_for_active_shards(c)
c.execute(
'''
create table doc.test(x int) clustered into ? shards with( number_of_replicas =?)
''', (
number_of_shards,
number_of_replicas,
))
insert_data(conn, 'doc', 'test', num_docs)
c.execute('''
CREATE REPOSITORY repo TYPE S3
WITH (access_key = 'minio',
secret_key = 'miniostorage',
bucket='backups',
endpoint = '127.0.0.1:9000',
protocol = 'http')
''')
c.execute(
'CREATE SNAPSHOT repo.snapshot1 TABLE doc.test WITH (wait_for_completion = true)'
)
c.execute(
'CREATE SNAPSHOT repo.snapshot2 TABLE doc.test WITH (wait_for_completion = true)'
)
c.execute('DROP TABLE doc.test')
# Drop snapshot2 while the restore of snapshot1 is still running
c.execute(
'RESTORE SNAPSHOT repo.snapshot1 ALL WITH (wait_for_completion = false)'
)
try:
c.execute('DROP SNAPSHOT repo.snapshot2')
except ProgrammingError:
self.fail(
"Restore and Drop Snapshot operation should work in parallel"
)
assert_busy(lambda: self._assert_num_docs(conn, num_docs))
cluster.stop()