# -*- coding: utf-8 -*-
# Minio Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname
# are dummy values, please replace them with original values.
from minio import Minio
client = Minio('s3.amazonaws.com',
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETACCESSKEY')
# Remove an object.
client.remove_object('my-bucketname', 'my-objectname')
def main():
"""
Functional testing of minio python library.
"""
fake = Factory.create()
client = Minio('s3.amazonaws.com',
os.getenv('ACCESS_KEY'),
os.getenv('SECRET_KEY'))
_http = urllib3.PoolManager(
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where()
)
# Get unique bucket_name, object_name.
bucket_name = uuid.uuid4().__str__()
object_name = uuid.uuid4().__str__()
# Enable trace
# client.trace_on(sys.stderr)
# Make a new bucket.
bucket_name = 'minio-pytest'
print(client.make_bucket(bucket_name))
print(client.make_bucket(bucket_name+'.unique',
location='us-west-1'))
## Check if return codes a valid from server.
try:
client.make_bucket(bucket_name+'.unique',
location='us-west-1')
except ResponseError as err:
if str(err.code) in ['BucketAlreadyOwnedByYou', 'BucketAlreadyExists']:
pass
else:
raise
# Check if bucket was created properly.
print(client.bucket_exists(bucket_name))
print(client.bucket_exists(bucket_name+'.unique'))
# List all buckets.
buckets = client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
with open('testfile', 'wb') as file_data:
file_data.write(fake.text().encode('utf-8'))
file_data.close()
# Put a file
file_stat = os.stat('testfile')
with open('testfile', 'rb') as file_data:
client.put_object(bucket_name, object_name, file_data, file_stat.st_size)
file_data.close()
# Fput a file
print(client.fput_object(bucket_name, object_name+'-f', 'testfile'))
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name))
# Get a full object
object_data = client.get_object(bucket_name, object_name)
with open('newfile', 'wb') as file_data:
for data in object_data:
file_data.write(data)
file_data.close()
# Get a full object locally.
print(client.fget_object(bucket_name, object_name, 'newfile-f'))
# List all object paths in bucket.
objects = client.list_objects(bucket_name, recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name, obj.last_modified, \
obj.etag, obj.size, obj.content_type)
presigned_get_object_url = client.presigned_get_object(bucket_name, object_name)
response = _http.urlopen('GET', presigned_get_object_url)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.get(bucket_name, object_name)
presigned_put_object_url = client.presigned_put_object(bucket_name, object_name)
value = fake.text().encode('utf-8')
data = io.BytesIO(value).getvalue()
response = _http.urlopen('PUT', presigned_put_object_url, body=data)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.put(bucket_name, object_name)
object_data = client.get_object(bucket_name, object_name)
if object_data.read() != value:
raise ValueError('Bytes not equal')
# Post policy.
policy = PostPolicy()
policy.set_bucket_name(bucket_name)
policy.set_key_startswith('objectPrefix/')
expires_date = datetime.utcnow()+timedelta(days=10)
policy.set_expires(expires_date)
print(client.presigned_post_policy(policy))
# Remove an object.
print(client.remove_object(bucket_name, object_name))
print(client.remove_object(bucket_name, object_name+'-f'))
# Remove a bucket. This operation will only work if your bucket is empty.
print(client.remove_bucket(bucket_name))
print(client.remove_bucket(bucket_name+'.unique'))
# Remove temporary files.
os.remove('testfile')
os.remove('newfile')
os.remove('newfile-f')
def test_remove_object_works(self, mock_connection):
mock_server = MockConnection()
mock_connection.return_value = mock_server
mock_server.mock_add_request(MockResponse('DELETE', 'http://localhost:9000/hello/world',
{'User-Agent': _DEFAULT_USER_AGENT}, 204))
client = Minio('http://localhost:9000')
client.remove_object('hello', 'world')
class UpdateTableFileCommand:
def __init__(self, table, file_url, **options):
self.table = table
self.file_url = file_url
self.file_url_info = urlparse(file_url)
self.hasher = hashlib.sha512()
self.file_size = 0
minio_endpoint = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc
self.should_upload = minio_endpoint != self.file_url_info.netloc
self.minio = Minio(minio_endpoint,
access_key=settings.AWS_ACCESS_KEY_ID,
secret_key=settings.AWS_SECRET_ACCESS_KEY)
self._output_file = None
self.delete_source = options["delete_source"]
@property
def output_file(self):
if not self._output_file:
self._output_file = NamedTemporaryFile(delete=False)
return self._output_file
def read_file_chunks(self, chunk_size):
response = requests.get(self.file_url, stream=True)
num_chunks = (math.ceil(
int(response.headers["Content-Length"]) /
chunk_size) if response.headers.get("Content-Length") else None)
chunks = response.iter_content(chunk_size=chunk_size)
for chunk in tqdm(chunks,
desc=f"Downloading {self.file_url} chunks...",
total=num_chunks):
self.file_size += len(chunk)
self.hasher.update(chunk)
yield chunk
def process_file_chunk(self, chunk, chunk_size):
if self.should_upload:
self.output_file.write(chunk)
def finish_process(self):
source = self.file_url_info.path # /BUCKET_NAME/OBJ_PATH
suffix = "".join(Path(source).suffixes)
dest_name = f"{self.table.dataset.slug}/{self.table.name}{suffix}"
bucket = settings.MINIO_STORAGE_DATASETS_BUCKET_NAME
is_same_file = source == f"/{bucket}/{dest_name}"
if self.should_upload:
self.output_file.close()
progress = MinioProgress()
self.log(f"Uploading file to bucket: {bucket}")
content_type, encoding = mimetypes.guess_type(dest_name)
if encoding == "gzip":
# quando é '.csv.gz' o retorno de guess_type é ('text/csv', 'gzip')
content_type = "application/gzip"
elif encoding is None:
content_type = "text/plain"
self.minio.fput_object(bucket,
dest_name,
self.output_file.name,
progress=progress,
content_type=content_type)
# TODO: use core.util.upload_file
elif not is_same_file:
self.log(f"Copying {source} to bucket {bucket}")
self.minio.copy_object(bucket, dest_name, source)
if self.delete_source:
self.log(f"Deleting {source}")
split_source = source.split("/")
source_bucket, source_obj = split_source[1], "/".join(
split_source[2:])
self.minio.remove_object(source_bucket, source_obj)
else:
self.log(f"Using {source} as the dataset file.", end="")
os.remove(self.output_file.name)
return f"{settings.AWS_S3_ENDPOINT_URL}{bucket}/{dest_name}"
@classmethod
def execute(cls, dataset_slug, tablename, file_url, **options):
table = Table.with_hidden.for_dataset(dataset_slug).named(tablename)
self = cls(table, file_url, **options)
chunk_size = settings.MINIO_DATASET_DOWNLOAD_CHUNK_SIZE
for chunk in self.read_file_chunks(chunk_size):
self.process_file_chunk(chunk, chunk_size)
new_file_url = self.finish_process()
table_file, created = self.create_table_file(new_file_url)
table_file_url = f"https://{settings.APP_HOST}{table_file.admin_url}"
if created:
self.log(f"\nNew TableFile entry: {table_file_url}")
else:
self.log(f"\nUsing existing TableFile entry: {table_file_url}")
self.log(f"File hash: {table_file.sha512sum}")
self.log(f"File size: {table_file.readable_size}")
def create_table_file(self, file_url):
filename = Path(urlparse(file_url).path).name
table_file, created = TableFile.objects.get_or_create(
table=self.table,
file_url=file_url,
sha512sum=self.hasher.hexdigest(),
size=str(self.file_size),
filename=filename,
)
return table_file, created
def log(self, msg, *args, **kwargs):
print(msg, *args, **kwargs)
def main():
"""
Functional testing of minio python library.
"""
fake = Factory.create()
client = Minio('s3.amazonaws.com', os.getenv('ACCESS_KEY'),
os.getenv('SECRET_KEY'))
_http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
# Get unique bucket_name, object_name.
bucket_name = uuid.uuid4().__str__()
object_name = uuid.uuid4().__str__()
# Enable trace
# client.trace_on(sys.stderr)
# Make a new bucket.
bucket_name = 'minio-pytest'
print(client.make_bucket(bucket_name))
print(client.make_bucket(bucket_name + '.unique', location='us-west-1'))
## Check if return codes a valid from server.
try:
client.make_bucket(bucket_name + '.unique', location='us-west-1')
except ResponseError as err:
if str(err.code) in ['BucketAlreadyOwnedByYou', 'BucketAlreadyExists']:
pass
else:
raise
# Check if bucket was created properly.
print(client.bucket_exists(bucket_name))
print(client.bucket_exists(bucket_name + '.unique'))
# List all buckets.
buckets = client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
with open('testfile', 'wb') as file_data:
file_data.write(fake.text().encode('utf-8'))
file_data.close()
# Put a file
file_stat = os.stat('testfile')
with open('testfile', 'rb') as file_data:
client.put_object(bucket_name, object_name, file_data,
file_stat.st_size)
file_data.close()
# Fput a file
print(client.fput_object(bucket_name, object_name + '-f', 'testfile'))
# Copy a file
print(
client.copy_object(bucket_name, object_name + '-copy',
'/' + bucket_name + '/' + object_name + '-f'))
try:
copy_conditions = CopyConditions()
copy_conditions.set_match_etag('test-etag')
print(
client.copy_object(bucket_name, object_name + '-copy',
'/' + bucket_name + '/' + object_name + '-f',
copy_conditions))
except ResponseError as err:
if err.code != 'PreconditionFailed':
raise
if err.message != 'At least one of the pre-conditions you specified did not hold':
raise
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name))
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name + '-f'))
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name + '-copy'))
# Get a full object
object_data = client.get_object(bucket_name, object_name)
with open('newfile', 'wb') as file_data:
for data in object_data:
file_data.write(data)
file_data.close()
# Get a full object locally.
print(client.fget_object(bucket_name, object_name, 'newfile-f'))
# List all object paths in bucket.
print("Listing using ListObjects API")
objects = client.list_objects(bucket_name, recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name, obj.last_modified, \
obj.etag, obj.size, obj.content_type)
# List all object paths in bucket using V2 API.
print("Listing using ListObjectsV2 API")
objects = client.list_objects_v2(bucket_name, recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name, obj.last_modified, \
obj.etag, obj.size, obj.content_type)
presigned_get_object_url = client.presigned_get_object(
bucket_name, object_name)
response = _http.urlopen('GET', presigned_get_object_url)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.get(bucket_name, object_name)
presigned_put_object_url = client.presigned_put_object(
bucket_name, object_name)
value = fake.text().encode('utf-8')
data = io.BytesIO(value).getvalue()
response = _http.urlopen('PUT', presigned_put_object_url, body=data)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.put(bucket_name, object_name)
object_data = client.get_object(bucket_name, object_name)
if object_data.read() != value:
raise ValueError('Bytes not equal')
# Post policy.
policy = PostPolicy()
policy.set_bucket_name(bucket_name)
policy.set_key_startswith('objectPrefix/')
expires_date = datetime.utcnow() + timedelta(days=10)
policy.set_expires(expires_date)
print(client.presigned_post_policy(policy))
# Remove an object.
print(client.remove_object(bucket_name, object_name))
print(client.remove_object(bucket_name, object_name + '-f'))
print(client.remove_object(bucket_name, object_name + '-copy'))
policy_name = client.get_bucket_policy(bucket_name)
if policy_name != Policy.NONE:
raise ValueError('Policy name is invalid ' + policy_name)
# Set read-write policy successfully.
client.set_bucket_policy(bucket_name, '', Policy.READ_WRITE)
# Reset policy to NONE.
client.set_bucket_policy(bucket_name, '', Policy.NONE)
# Validate if the policy is reverted back to NONE.
policy_name = client.get_bucket_policy(bucket_name)
if policy_name != Policy.NONE:
raise ValueError('Policy name is invalid ' + policy_name)
# Upload some new objects to prepare for multi-object delete test.
print("Prepare for remove_objects() test.")
object_names = []
for i in range(10):
curr_object_name = object_name + "-{}".format(i)
print("object-name: {}".format(curr_object_name))
print(client.fput_object(bucket_name, curr_object_name, "testfile"))
object_names.append(curr_object_name)
# delete the objects in a single library call.
print("Performing remove_objects() test.")
del_errs = client.remove_objects(bucket_name, object_names)
had_errs = False
for del_err in del_errs:
had_errs = True
print("Err is {}".format(del_err))
if had_errs:
print("remove_objects() FAILED - it had unexpected errors.")
else:
print("remove_objects() worked as expected.")
# Remove a bucket. This operation will only work if your bucket is empty.
print(client.remove_bucket(bucket_name))
print(client.remove_bucket(bucket_name + '.unique'))
# Remove temporary files.
os.remove('testfile')
os.remove('newfile')
os.remove('newfile-f')
def test_object_is_not_empty_string(self):
client = Minio('localhost:9000')
client.remove_object('hello', ' \t \n ')
# (C) 2015 MinIO, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from minio import Minio
client = Minio(
"play.min.io",
access_key="Q3AM3UQ867SPQQA43P2F",
secret_key="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
)
# Remove object.
client.remove_object("my-bucket", "my-object")
# Remove version of an object.
client.remove_object(
"my-bucket", "my-object",
version_id="dfbd25b3-abec-4184-a4e8-5a35a5c1174d",
)
from minio import Minio
from minio.error import ResponseError
import os
import sys
from os import listdir
##
# Follow bucket naming rules
#'10.233.54.122:9000'
#"cnb-ml-bucket"
if __name__ == "__main__":
minioserviceIP = sys.argv[1]
bucketname = sys.argv[2]
print("Deleting minio bucket \n")
minioClient = Minio(minioserviceIP,
access_key='minio',
secret_key='minio123',
secure=False)
try:
if minioClient.bucket_exists(bucketname):
objects = minioClient.list_objects(bucketname, recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name.encode('utf-8'),
obj.last_modified, obj.etag, obj.size, obj.content_type)
minioClient.remove_object(bucketname, obj.object_name)
minioClient.remove_bucket(bucketname)
except ResponseError as err:
print(err)
class KartonBackend:
def __init__(self, config):
self.config = config
self.redis = StrictRedis(
host=config["redis"]["host"],
port=int(config["redis"].get("port", 6379)),
decode_responses=True,
)
self.minio = Minio(
endpoint=config["minio"]["address"],
access_key=config["minio"]["access_key"],
secret_key=config["minio"]["secret_key"],
secure=bool(int(config["minio"].get("secure", True))),
)
@property
def default_bucket_name(self) -> str:
return self.config.minio_config["bucket"]
@staticmethod
def get_queue_name(identity: str, priority: TaskPriority) -> str:
"""
Return Redis routed task queue name for given identity and priority
:param identity: Karton service identity
:param priority: Queue priority (TaskPriority enum value)
:return: Queue name
"""
return f"karton.queue.{priority.value}:{identity}"
@staticmethod
def get_queue_names(identity: str) -> List[str]:
"""
Return all Redis routed task queue names for given identity,
ordered by priority (descending). Used internally by Consumer.
:param identity: Karton service identity
:return: List of queue names
"""
return [
identity, # Backwards compatibility (2.x.x)
KartonBackend.get_queue_name(identity, TaskPriority.HIGH),
KartonBackend.get_queue_name(identity, TaskPriority.NORMAL),
KartonBackend.get_queue_name(identity, TaskPriority.LOW),
]
@staticmethod
def serialize_bind(bind: KartonBind) -> str:
"""
Serialize KartonBind object (Karton service registration)
:param bind: KartonBind object with bind definition
:return: Serialized bind data
"""
return json.dumps(
{
"info": bind.info,
"version": bind.version,
"filters": bind.filters,
"persistent": bind.persistent,
"service_version": bind.service_version,
},
sort_keys=True,
)
@staticmethod
def unserialize_bind(identity: str, bind_data: str) -> KartonBind:
"""
Deserialize KartonBind object for given identity.
Compatible with Karton 2.x.x and 3.x.x
:param identity: Karton service identity
:param bind_data: Serialized bind data
:return: KartonBind object with bind definition
"""
bind = json.loads(bind_data)
if isinstance(bind, list):
# Backwards compatibility (v2.x.x)
return KartonBind(
identity=identity,
info=None,
version="2.x.x",
persistent=not identity.endswith(".test"),
filters=bind,
service_version=None,
)
return KartonBind(
identity=identity,
info=bind["info"],
version=bind["version"],
persistent=bind["persistent"],
filters=bind["filters"],
service_version=bind.get("service_version"),
)
def get_bind(self, identity: str) -> KartonBind:
"""
Get bind object for given identity
:param identity: Karton service identity
:return: KartonBind object
"""
return self.unserialize_bind(
identity, self.redis.hget(KARTON_BINDS_HSET, identity))
def get_binds(self) -> List[KartonBind]:
"""
Get all binds registered in Redis
:return: List of KartonBind objects for subsequent identities
"""
return [
self.unserialize_bind(identity, raw_bind) for identity, raw_bind in
self.redis.hgetall(KARTON_BINDS_HSET).items()
]
def register_bind(self, bind: KartonBind) -> Optional[KartonBind]:
"""
Register bind for Karton service and return the old one
:param bind: KartonBind object with bind definition
:return: Old KartonBind that was registered under this identity
"""
with self.redis.pipeline(transaction=True) as pipe:
pipe.hget(KARTON_BINDS_HSET, bind.identity)
pipe.hset(KARTON_BINDS_HSET, bind.identity,
self.serialize_bind(bind))
old_serialized_bind, _ = pipe.execute()
if old_serialized_bind:
return self.unserialize_bind(bind.identity, old_serialized_bind)
else:
return None
def unregister_bind(self, identity: str) -> None:
"""
Removes bind for identity
:param bind: Identity to be unregistered
"""
self.redis.hdel(KARTON_BINDS_HSET, identity)
def set_consumer_identity(self, identity: str) -> None:
"""
Sets identity for current Redis connection
"""
return self.redis.client_setname(identity)
def get_online_consumers(self) -> Dict[str, List[str]]:
"""
Gets all online consumer identities
:return: Dictionary {identity: [list of clients]}
"""
bound_identities = defaultdict(list)
for client in self.redis.client_list():
bound_identities[client["name"]].append(client)
return bound_identities
def get_task(self, task_uid: str) -> Optional[Task]:
"""
Get task object with given identifier
:param task_uid: Task identifier
:return: Task object
"""
task_data = self.redis.get(f"{KARTON_TASK_NAMESPACE}:{task_uid}")
if not task_data:
return None
return Task.unserialize(task_data, backend=self)
def get_tasks(self, task_uid_list: List[str]) -> List[Task]:
"""
Get multiple tasks for given identifier list
:param task_uid_list: List of task identifiers
:return: List of task objects
"""
task_list = self.redis.mget([
f"{KARTON_TASK_NAMESPACE}:{task_uid}" for task_uid in task_uid_list
])
return [
Task.unserialize(task_data, backend=self)
for task_data in task_list if task_data is not None
]
def get_all_tasks(self) -> List[Task]:
"""
Get all tasks registered in Redis
:return: List with Task objects
"""
tasks = self.redis.keys(f"{KARTON_TASK_NAMESPACE}:*")
return [
Task.unserialize(task_data) for task_data in self.redis.mget(tasks)
if task_data is not None
]
def register_task(self, task: Task) -> None:
"""
Register task in Redis.
Consumer should register only Declared tasks.
Status change should be done using set_task_status.
:param task: Task object
"""
self.redis.set(f"{KARTON_TASK_NAMESPACE}:{task.uid}", task.serialize())
def set_task_status(self,
task: Task,
status: TaskState,
consumer: Optional[str] = None) -> None:
"""
Request task status change to be applied by karton-system
:param task: Task object
:param status: New task status (TaskState)
:param consumer: Consumer identity
"""
self.redis.rpush(
KARTON_OPERATIONS_QUEUE,
json.dumps({
"status": status.value,
"identity": consumer,
"task": task.serialize(),
"type": "operation",
}),
)
def delete_task(self, task: Task) -> None:
"""
Remove task from Redis
:param task: Task object
"""
self.redis.delete(f"{KARTON_TASK_NAMESPACE}:{task.uid}")
def get_task_queue(self, queue: str) -> List[Task]:
"""
Return all tasks in provided queue
:param queue: Queue name
:return: List with Task objects contained in queue
"""
task_uids = self.redis.lrange(queue, 0, -1)
return self.get_tasks(task_uids)
def get_task_ids_from_queue(self, queue: str) -> List[str]:
"""
Return all task UIDs in a queue
:param queue: Queue name
:return: List with task identifiers contained in queue
"""
return self.redis.lrange(queue, 0, -1)
def remove_task_queue(self, queue: str) -> List[Task]:
"""
Remove task queue with all contained tasks
:param queue: Queue name
:return: List with Task objects contained in queue
"""
pipe = self.redis.pipeline()
pipe.lrange(queue, 0, -1)
pipe.delete(queue)
return self.get_tasks(pipe.execute()[0])
def produce_unrouted_task(self, task: Task) -> None:
"""
Add given task to unrouted task (``karton.tasks``) queue
Task must be registered before with :py:meth:`register_task`
:param task: Task object
"""
self.redis.rpush(KARTON_TASKS_QUEUE, task.uid)
def produce_routed_task(self, identity: str, task: Task) -> None:
"""
Add given task to routed task queue of given identity
Task must be registered using :py:meth:`register_task`
:param identity: Karton service identity
:param task: Task object
"""
self.redis.rpush(self.get_queue_name(identity, task.priority),
task.uid)
def consume_queues(self,
queues: Union[str, List[str]],
timeout: int = 0) -> Optional[Tuple[str, str]]:
"""
Get item from queues (ordered from the most to the least prioritized)
If there are no items, wait until one appear.
:param queues: Redis queue name or list of names
:param timeout: Waiting for item timeout (default: 0 = wait forever)
:return: Tuple of [queue_name, item] objects or None if timeout has been reached
"""
return self.redis.blpop(queues, timeout=timeout)
def consume_routed_task(self,
identity: str,
timeout: int = 5) -> Optional[Task]:
"""
Get routed task for given consumer identity.
If there are no tasks, blocks until new one appears or timeout is reached.
:param identity: Karton service identity
:param timeout: Waiting for task timeout (default: 5)
:return: Task object
"""
item = self.consume_queues(
self.get_queue_names(identity),
timeout=timeout,
)
if not item:
return None
queue, data = item
return self.get_task(data)
@staticmethod
def _log_channel(logger_name: Optional[str], level: Optional[str]) -> str:
return ".".join(
[KARTON_LOG_CHANNEL, (level or "*").lower(), logger_name or "*"])
def produce_log(
self,
log_record: Dict[str, Any],
logger_name: str,
level: str,
) -> bool:
"""
Push new log record to the logs channel
:param log_record: Dict with log record
:param logger_name: Logger name
:param level: Log level
:return: True if any active log consumer received log record
"""
return (self.redis.publish(self._log_channel(logger_name, level),
json.dumps(log_record)) > 0)
def consume_log(
self,
timeout: int = 5,
logger_filter: Optional[str] = None,
level: Optional[str] = None,
) -> Iterator[Optional[Dict[str, Any]]]:
"""
Subscribe to logs channel and yield subsequent log records
or None if timeout has been reached.
If you want to subscribe only to a specific logger name
and/or log level, pass them via logger_filter and level arguments.
:param timeout: Waiting for log record timeout (default: 5)
:param logger_filter: Filter for name of consumed logger
:param level: Log level
:return: Dict with log record
"""
with self.redis.pubsub() as pubsub:
pubsub.psubscribe(self._log_channel(logger_filter, level))
while pubsub.subscribed:
item = pubsub.get_message(ignore_subscribe_messages=True,
timeout=timeout)
if item and item["type"] == "pmessage":
body = json.loads(item["data"])
if "task" in body and isinstance(body["task"], str):
body["task"] = json.loads(body["task"])
yield body
yield None
def increment_metrics(self, metric: KartonMetrics, identity: str) -> None:
"""
Increments metrics for given operation type and identity
:param metric: Operation metric type
:param identity: Related Karton service identity
"""
self.redis.hincrby(metric.value, identity, 1)
def upload_object(
self,
bucket: str,
object_uid: str,
content: Union[bytes, BinaryIO],
length: int = None,
) -> None:
"""
Upload resource object to underlying object storage (Minio)
:param bucket: Bucket name
:param object_uid: Object identifier
:param content: Object content as bytes or file-like stream
:param length: Object content length (if file-like object provided)
"""
if isinstance(content, bytes):
length = len(content)
content = BytesIO(content)
self.minio.put_object(bucket, object_uid, content, length)
def upload_object_from_file(self, bucket: str, object_uid: str,
path: str) -> None:
"""
Upload resource object file to underlying object storage
:param bucket: Bucket name
:param object_uid: Object identifier
:param path: Path to the object content
"""
self.minio.fput_object(bucket, object_uid, path)
def get_object(self, bucket: str, object_uid: str) -> HTTPResponse:
"""
Get resource object stream with the content.
Returned response should be closed after use to release network resources.
To reuse the connection, it's required to call `response.release_conn()`
explicitly.
:param bucket: Bucket name
:param object_uid: Object identifier
:return: Response object with content
"""
return self.minio.get_object(bucket, object_uid)
def download_object(self, bucket: str, object_uid: str) -> bytes:
"""
Download resource object from object storage.
:param bucket: Bucket name
:param object_uid: Object identifier
:return: Content bytes
"""
reader = self.minio.get_object(bucket, object_uid)
try:
return reader.read()
finally:
reader.release_conn()
reader.close()
def download_object_to_file(self, bucket: str, object_uid: str,
path: str) -> None:
"""
Download resource object from object storage to file
:param bucket: Bucket name
:param object_uid: Object identifier
:param path: Target file path
"""
self.minio.fget_object(bucket, object_uid, path)
def list_objects(self, bucket: str) -> List[str]:
"""
List identifiers of stored resource objects
:param bucket: Bucket name
:return: List of object identifiers
"""
return [
object.object_name for object in self.minio.list_objects(bucket)
]
def remove_object(self, bucket: str, object_uid: str) -> None:
"""
Remove resource object from object storage
:param bucket: Bucket name
:param object_uid: Object identifier
"""
self.minio.remove_object(bucket, object_uid)
def check_bucket_exists(self, bucket: str, create: bool = False) -> bool:
"""
Check if bucket exists and optionally create it if it doesn't.
:param bucket: Bucket name
:param create: Create bucket if doesn't exist
:return: True if bucket exists yet
"""
if self.minio.bucket_exists(bucket):
return True
if create:
self.minio.make_bucket(bucket)
return False
def test_object_is_not_empty_string(self):
client = Minio('localhost:9000')
client.remove_object('hello', ' \t \n ')
def test_remove_bucket_invalid_name(self):
client = Minio('localhost:9000')
client.remove_object('AB*CD', 'world')
def test_object_is_string(self):
client = Minio('localhost:9000')
client.remove_object('hello', 1234)
port+=1
host2 = Minio(
ip+str(port) ,
access_key='hrithik',
secret_key='hrithik123',
secure = False
)
# buck = client.list_buckets()
# for x in buck:
# obj = client.list_objects(x.name)
# #obj = [x.object_name for x in obj]
# for y in obj:
# client.remove_object(x.name , y.object_name)
# client.remove_bucket(x.name)
buck = host1.list_buckets()
for x in buck:
obj = host1.list_objects(x.name)
#obj = [x.object_name for x in obj]
for y in obj:
host1.remove_object(x.name , y.object_name)
host1.remove_bucket(x.name)
buck = host2.list_buckets()
for x in buck:
obj = host2.list_objects(x.name)
for y in obj:
host2.remove_object(x.name , y.object_name)
host2.remove_bucket(x.name)
class Pull_Objects:
def __init__(self):
try:
self.minioClient = Minio(BACKEND,
access_key=ACCESS_KEY_ID,
secret_key=SECRET_KEY_ID,
secure=True)
except Exception as e:
print(e)
def single_object(self, body):
try:
bucket = self.minioClient.bucket_exists(BUCKET)
except Exception as e:
return {'code': 404, 'Error': "Bucket not found."}
else:
try:
version_of_package = body['version']
ver = list(version_of_package.split('.'))
first_version, second_version, third_version = ver[0], ver[
1], ver[2]
objects = self.minioClient.list_objects(BUCKET, recursive=True)
for obj in objects:
versions = list(obj.object_name.split('/'))
if first_version == versions[
0] and second_version == versions[
1] and third_version == versions[2]:
location = obj.object_name
break
result_url = self.minioClient.presigned_get_object(
BUCKET, location)
return {'code': 200, 'url': result_url}
except Exception as e:
return {'code': 404, 'Error': "File not found."}
def latest_object(self):
try:
bucket = self.minioClient.bucket_exists(BUCKET)
except Exception as e:
return {'code': 404, 'Error': "Bucket not found."}
else:
try:
objects = self.minioClient.list_objects(BUCKET, recursive=True)
last_object = list(objects)[-1].object_name
result_url = self.minioClient.presigned_get_object(
BUCKET, last_object)
return {'code': 200, 'url': result_url}
except Exception as e:
return {'code': 404, 'Error': "File not found."}
def get_All_objects(self):
try:
bucket = self.minioClient.bucket_exists(BUCKET)
except Exception as e:
return {'code': 404, 'Error': "Bucket not found."}
else:
try:
objects = self.minioClient.list_objects(BUCKET, recursive=True)
result_url = {}
for obj in objects:
result_url[
obj.
object_name] = self.minioClient.presigned_get_object(
BUCKET, obj.object_name)
return {'code': 200, 'url': result_url}
except Exception as e:
return {'code': 404, 'Error': "File not found."}
def delete_object(self, body):
try:
bucket = self.minioClient.bucket_exists(BUCKET)
except Exception as e:
return {'code': 404, 'Error': "Bucket not found."}
else:
try:
version_of_package = body['version']
ver = list(version_of_package.split('.'))
first_version, second_version, third_version = ver[0], ver[
1], ver[2]
objects = self.minioClient.list_objects(BUCKET, recursive=True)
for obj in objects:
versions = list(obj.object_name.split('/'))
if first_version == versions[
0] and second_version == versions[
1] and third_version == versions[2]:
break
result_url = self.minioClient.remove_object(
BUCKET, obj.object_name)
return {'code': 200, 'message': 'Object Deleted.'}
except Exception as e:
return {'code': 404, 'Error': "File not found."}
print("***************get object*****************")
object = minio_client2.get_object(
bucket_name="zcbucket",
object_name="bar",
version_id="26a34677-5c0f-45cc-b1d8-13fbc265f6c5")
print(object.data)
object = minio_client2.get_object(
bucket_name="zcbucket",
object_name="bar",
version_id="fd3bfab5-a2b2-4a78-8d37-9a261e9b908d")
print(object.data)
print("***************remove object*****************")
minio_client2.remove_object("zcbucket", "bar")
print("***************get object*****************")
try:
object = minio_client2.get_object(
bucket_name="zcbucket",
object_name="bar",
)
except NoSuchKey as err:
print("key bar is not exist!")
print(object.data)
object = minio_client2.get_object(
bucket_name="zcbucket",
object_name="bar",
version_id="fd3bfab5-a2b2-4a78-8d37-9a261e9b908d")
class MINIORepository(Repository):
"""
"""
client = None
def __init__(self, config):
super().__init__()
try:
access_key = config['storage_access_key']
except Exception:
access_key = 'minio'
try:
secret_key = config['storage_secret_key']
except Exception:
secret_key = 'minio123'
try:
self.bucket = config['storage_bucket']
except Exception:
self.bucket = 'fedn-models'
try:
self.context_bucket = config['context_bucket']
except Exception:
self.bucket = 'fedn-context'
try:
self.secure_mode = bool(config['storage_secure_mode'])
except Exception:
self.secure_mode = False
if not self.secure_mode:
print(
"\n\n\nWARNING : S3/MINIO RUNNING IN **INSECURE** MODE! THIS IS NOT FOR PRODUCTION!\n\n\n"
)
if self.secure_mode:
from urllib3.poolmanager import PoolManager
manager = PoolManager(num_pools=100,
cert_reqs='CERT_NONE',
assert_hostname=False)
self.client = Minio("{0}:{1}".format(config['storage_hostname'],
config['storage_port']),
access_key=access_key,
secret_key=secret_key,
secure=self.secure_mode,
http_client=manager)
else:
self.client = Minio("{0}:{1}".format(config['storage_hostname'],
config['storage_port']),
access_key=access_key,
secret_key=secret_key,
secure=self.secure_mode)
# TODO: generalize
self.context_bucket = 'fedn-context'
self.create_bucket(self.context_bucket)
self.create_bucket(self.bucket)
def create_bucket(self, bucket_name):
"""
:param bucket_name:
"""
found = self.client.bucket_exists(bucket_name)
if not found:
try:
response = self.client.make_bucket(bucket_name)
except InvalidResponseError as err:
raise
def set_artifact(self, instance_name, instance, is_file=False, bucket=''):
""" Instance must be a byte-like object. """
if bucket == '':
bucket = self.bucket
if is_file == True:
self.client.fput_object(bucket, instance_name, instance)
else:
try:
self.client.put_object(bucket, instance_name,
io.BytesIO(instance), len(instance))
except Exception as e:
raise Exception("Could not load data into bytes {}".format(e))
return True
def get_artifact(self, instance_name, bucket=''):
"""
:param instance_name:
:param bucket:
:return:
"""
if bucket == '':
bucket = self.bucket
try:
data = self.client.get_object(bucket, instance_name)
return data.read()
except Exception as e:
raise Exception("Could not fetch data from bucket, {}".format(e))
def get_artifact_stream(self, instance_name):
"""
:param instance_name:
:return:
"""
try:
data = self.client.get_object(self.bucket, instance_name)
return data
except Exception as e:
raise Exception("Could not fetch data from bucket, {}".format(e))
def list_artifacts(self):
"""
:return:
"""
objects_to_delete = []
try:
objs = self.client.list_objects(self.bucket)
for obj in objs:
print(obj.object_name)
objects_to_delete.append(obj.object_name)
except Exception as e:
raise Exception("Could not list models in bucket {}".format(
self.bucket))
return objects_to_delete
def delete_artifact(self, instance_name, bucket=[]):
"""
:param instance_name:
:param bucket:
"""
if not bucket:
bucket = self.bucket
try:
self.client.remove_object(bucket, instance_name)
except InvalidResponseError as err:
print(err)
print('Could not delete artifact: {}'.format(instance_name))
def delete_objects(self):
"""
"""
objects_to_delete = self.list_artifacts()
try:
# Remove list of objects.
errors = self.client.remove_objects(self.bucket, objects_to_delete)
for del_err in errors:
print("Deletion Error: {}".format(del_err))
except:
print('Could not delete objects: {}'.format(objects_to_delete))
def minio_object():
####################################################################################################################
#Extract an object from a specified bucket
####################################################################################################################
if request.method == 'GET':
# get keys from request args
if minio_keys(request):
auth = minio_keys(request)
if "err" in auth:
return jsonify(auth)
else:
return jsonify({"err": "No authentication keys provided"})
try:
minioClient = Minio(SERVER_ENDPOINT,
access_key=auth.get("access_key"),
secret_key=auth.get("secret_key"),
secure=False)
deposit_id = False
if not deposit_id:
config = json.loads(request.form.get('config'))
deposit_id = config.get('deposit_id')
bucket_name = config.get('bucket_name')
temp_obj_path = str(deposit_id)
obj = minioClient.get_object(bucket_name, deposit_id)
with open(temp_obj_path, 'wb') as file_data:
for d in obj.stream(32 * 1024):
file_data.write(d)
return send_file(temp_obj_path,
mimetype="application/octet-stream")
except NoSuchKey as err:
return jsonify(
{"err": "The requested deposit_id does not exist"})
except NoSuchBucket as err:
return jsonify({"err": str(err)})
except (InvalidBucketError, TypeError, NoSuchBucket):
return jsonify({"err": "Please provide valid bucket_name"})
except (AccessDenied, InvalidAccessKeyId, SignatureDoesNotMatch):
return jsonify({"err": "Invalid Authentication."})
except JSONDecodeError as err:
return jsonify({"err": "Incorrect formatting of data"})
except ResponseError as err:
return jsonify({"err": err})
####################################################################################################################
#Puts one object i.e. a file in the specified bucket only.
####################################################################################################################
elif request.method == 'POST':
# get data from request payload
try:
config = json.loads(request.form.get('config'))
data = json.loads(request.form.get('data'))
# extract authentication details
if minio_keys(request):
auth = minio_keys(request)
if "err" in auth:
return jsonify(auth)
else:
return jsonify({"err": "No authentication keys provided"})
# Initialize minioClient with an endpoint and keys.
minioClient = Minio(SERVER_ENDPOINT,
access_key=auth.get("access_key"),
secret_key=auth.get("secret_key"),
secure=False)
# extract deposit_id value
if data.get('deposit_id'):
deposit_id = data.get('deposit_id')
else:
return jsonify({"err": "Please provide a deposit_id"})
# extract file & temp save to disk
file = request.files['file']
file.save(deposit_id)
# extract bucket_name value
bucket_name = config.get('bucket_name')
minioClient.bucket_exists(bucket_name)
metadata = {}
metadata['BUCKET_NAME'] = bucket_name
r = minioClient.fput_object(bucket_name,
object_name=deposit_id,
file_path=deposit_id,
metadata=metadata)
# cleanup temp file
os.remove(deposit_id)
return jsonify({
"etag": r,
"deposit_id": deposit_id,
"metadata": metadata
})
except (NoSuchBucket, InvalidBucketError) as err:
return jsonify({
"err":
"This bucket does not exist. Please create this bucket at the bucket scoped endpoint."
})
except (InvalidAccessKeyId, AccessDenied,
SignatureDoesNotMatch) as err:
return jsonify({"err": "Invalid Authentication."})
except BadRequestKeyError as err:
return jsonify({"err": "Please attach a file to the request"})
except JSONDecodeError:
return jsonify({"err": "Incorrect formatting of data"})
except ResponseError as err:
return jsonify({"err": err})
except TypeError as err:
return jsonify({"err": "Please enter a bucket_name"})
####################################################################################################################
#Deletes one object i.e. a file from the specified bucket only.
####################################################################################################################
elif request.method == 'DELETE':
# extract authentication details
if minio_keys(request):
auth = minio_keys(request)
if "err" in auth:
return jsonify(auth)
else:
return jsonify({"err": "No authentication keys provided"})
# get data from request payload
config = json.loads(request.form.get('config'))
data = json.loads(request.form.get('data'))
# extract bucket_name value
bucket_name = config.get('bucket_name')
# extract object specific deposit_id
if data.get('deposit_id'):
deposit_id = data.get('deposit_id')
else:
return jsonify({"err": "Please enter valid deposit_id."})
try:
minioClient = Minio(SERVER_ENDPOINT,
access_key=auth.get("access_key"),
secret_key=auth.get("secret_key"),
secure=False)
except AccessDenied as err:
return jsonify({"err": str(err)})
try:
error = minioClient.get_object(bucket_name, deposit_id)
rem_object = minioClient.remove_object(bucket_name, deposit_id)
except (NoSuchBucket, InvalidBucketError, ResponseError,
TypeError):
return jsonify({"err": "Bucket does not exist."})
except NoSuchKey:
return jsonify({"err": "Please enter valid deposit_id."})
except (AccessDenied, InvalidAccessKeyId,
SignatureDoesNotMatch) as err:
return jsonify({"err": "Invalid Authentication."})
return jsonify({"deposit_id": deposit_id})
else:
return jsonify({"err": "Unsupported method"})
minioClient.fput_object(bucketName, path, dir + path)
except BaseException as err:
errorPutsFiles.append(file)
pass
else:
# добавляем в список текущих объектов
objectTime = datetime.datetime.fromtimestamp(
minioClient.stat_object(bucketName, path).last_modified)
objectTime = objectTime.replace(tzinfo=timeZone)
objectsNow.append({"name": path, "time": objectTime})
# удаление файлов с сервера
for file in deleteFiles:
path = file["name"]
# удаляем объект на сервере
minioClient.remove_object(bucketName, path)
# удаляем объект из списка текущих объектов
for object in objectsNow:
if object["name"] == path:
objectsNow.remove(object)
# обновление файла на сервере
for file in stableFiles:
if int(file["timeNow"]) / 10 > int(file["timeOld"]) / 10:
path = file["name"]
# отправляем новый объект на сервер
try:
minioClient.fput_object(bucketName, path, dir + path)
except BaseException as err:
# если неудачно то добавляем в список повторных
errorUpdatesFiles.append(file)
def main():
"""
Functional testing of minio python library.
"""
fake = Factory.create()
client = Minio('https://play.minio.io:9002',
'Q3AM3UQ867SPQQA43P2F',
'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG')
# Get unique bucket_name, object_name.
bucket_name = uuid.uuid4().__str__()
object_name = uuid.uuid4().__str__()
# Make a new bucket.
print(client.make_bucket(bucket_name))
# Check if bucket was created properly.
print(client.bucket_exists(bucket_name))
# Set bucket name to private.
print(client.set_bucket_acl(bucket_name, Acl.private()))
# Print current bucket acl.
print(client.get_bucket_acl(bucket_name))
# List all buckets.
buckets = client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
with open('testfile', 'wb') as file_data:
file_data.write(fake.text().encode('utf-8'))
file_data.close()
# Put a file
file_stat = os.stat('testfile')
with open('testfile', 'rb') as file_data:
client.put_object(bucket_name, object_name, file_data, file_stat.st_size)
file_data.close()
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name))
# Get a full object
data = client.get_object(bucket_name, object_name)
with open('newfile', 'wb') as file_data:
for d in data:
file_data.write(d)
file_data.close()
# List all object paths in bucket that begin with hello.
objects = client.list_objects(bucket_name)
for obj in objects:
print(obj.bucket_name, obj.object_name, obj.last_modified, \
obj.etag, obj.size, obj.content_type)
uploads = client.list_incomplete_uploads(bucket_name,
prefix='',
recursive=True)
for obj in uploads:
print(obj.bucket_name, obj.object_name, obj.upload_id)
print(client.presigned_get_object(bucket_name, object_name))
print(client.presigned_put_object(bucket_name, object_name))
# Remove an object.
print(client.remove_object(bucket_name, object_name))
# Remove a bucket.
# This operation will only work if your bucket is empty.
print(client.remove_bucket(bucket_name))
# Remove temporary files.
os.remove('testfile')
os.remove('newfile')
class MINIORepository(Repository):
client = None
def __init__(self, config):
super().__init__()
try:
access_key = config['storage_access_key']
except Exception:
access_key = 'minio'
try:
secret_key = config['storage_secret_key']
except Exception:
secret_key = 'minio123'
try:
self.bucket = config['storage_bucket']
except Exception:
self.bucket = 'models'
try:
self.secure_mode = bool(config['storage_secure_mode'])
except Exception:
self.secure_mode = False
if not self.secure_mode:
print(
"\n\n\nWARNING : RUNNING IN **INSECURE** MODE! THIS IS NOT FOR PRODUCTION!\n\n\n"
)
if self.secure_mode:
from urllib3.poolmanager import PoolManager
manager = PoolManager(num_pools=100,
cert_reqs='CERT_NONE',
assert_hostname=False)
self.client = Minio("{0}:{1}".format(config['storage_hostname'],
config['storage_port']),
access_key=access_key,
secret_key=secret_key,
secure=self.secure_mode,
http_client=manager)
else:
self.client = Minio("{0}:{1}".format(config['storage_hostname'],
config['storage_port']),
access_key=access_key,
secret_key=secret_key,
secure=self.secure_mode)
self.create_bucket(self.bucket)
def create_bucket(self, bucket_name):
try:
response = self.client.make_bucket(bucket_name)
except BucketAlreadyOwnedByYou as err:
pass
except BucketAlreadyExists as err:
pass
except ResponseError as err:
raise
def set_artifact(self, instance_name, instance, is_file=False, bucket=''):
""" Instance must be a byte-like object. """
if bucket == '':
bucket = self.bucket
if is_file == True:
self.client.fput_object(bucket, instance_name, instance)
else:
try:
self.client.put_object(bucket, instance_name,
io.BytesIO(instance), len(instance))
except Exception as e:
raise Exception("Could not load data into bytes {}".format(e))
return True
def get_artifact(self, instance_name):
try:
data = self.client.get_object(self.bucket, instance_name)
return data.read()
except Exception as e:
raise Exception("Could not fetch data from bucket, {}".format(e))
def get_artifact_stream(self, instance_name):
try:
data = self.client.get_object(self.bucket, instance_name)
return data
except Exception as e:
raise Exception("Could not fetch data from bucket, {}".format(e))
def list_artifacts(self):
try:
objs = self.client.list_objects(self.bucket)
for obj in objs:
print(obj)
except Exception as e:
raise Exception("Could not list models in bucket {}".format(
self.bucket))
def delete_artifact(self, instance_name, bucket=[]):
if not bucket:
bucket = self.bucket
try:
self.client.remove_object(bucket, instance_name)
except ResponseError as err:
print(err)
print('Could not delete artifact: {}'.format(instance_name))
# -*- coding: utf-8 -*-
# Minio Python Library for Amazon S3 compatible cloud storage, (C) 2015 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from minio import Minio
__author__ = 'minio'
client = Minio('https://s3.amazonaws.com',
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETACCESSKEY')
# Remove a bucket
client.remove_object('my-bucket', 'my_key')
class MinioObjectStore(ObjectStoreInterface):
def initialize(self, jsonArgs):
# The function should initialize any connections that need
# to be made or any variables that will be required
format = "%(asctime)s - %(levelname)s: %(threadName)s - %(message)s"
logging.basicConfig(format=format,
level=logging.DEBUG,
datefmt="%H:%M:%S")
args = json.loads(jsonArgs)
# Extract variables from JSON
self.host = args['host']
self.access_key = args['accessKey']
self.secret_key = args['secretKey']
self.https_enabled = args['httpsEnabled']
if 'bucketName' in args:
self.bucket_name = args['bucketName']
# Instantiate client
self.minio_client = Minio(self.host,
access_key=self.access_key,
secret_key=self.secret_key,
secure=self.https_enabled)
self.validate()
def upload(self, filepath, bucket_name=None):
"""
Uploads a file to Minio.
The function uploads a file from the path in the filesystem specified and uploads
it to a bucket in Minio.
Parameters:
filepath (string): The absolute path of the file to upload
bucket_name (string): Optional argument to indicate the bucket to use to upload the file
Returns:
string: The location of the image in Minio
"""
bucket = ""
if bucket_name is not None:
bucket = bucket_name
elif self.bucket_name is not None:
bucket = self.bucket_name
else:
raise Exception(
"The instance variable bucket_name was not initialized. You must either initialize it or pass it to the function"
)
# Upload file to Algorithmia
filename = os.path.basename(filepath)
logging.debug("Uploading file '%s' to bucket '%s'", filepath, bucket)
try:
self.minio_client.fput_object(bucket, filename, filepath)
except ResponseError as err:
logging.error("Upload of file '%s' failed", filepath)
raise err
logging.debug("Upload of file was successful")
return bucket + '/' + filename
def download(self, filename, download_path, bucket_name=None):
# Downloads an object from Minio
bucket = ""
if bucket_name is not None:
bucket = bucket_name
elif self.bucket_name is not None:
bucket = self.bucket_name
else:
raise Exception(
"The instance variable bucket_name was not initialized. You must either initialize it or pass it to the function"
)
logging.debug("Downloading file '%s' from bucket '%s'", filename,
bucket)
try:
self.minio_client.fget_object(bucket, filename, download_path)
except ResponseError as err:
logging.error("Download of file '%s' failed", filename)
raise err
logging.debug('File download was successful')
def delete(self, filename, bucket_name=None):
# Deletes a file from Minio
bucket = ""
if bucket_name is not None:
bucket = bucket_name
elif self.bucket_name is not None:
bucket = self.bucket_name
else:
raise Exception(
"The instance variable bucket_name was not initialized. You must either initialize it or pass it to the function"
)
logging.debug("Deleting file '%s' from bucket '%s'", filename, bucket)
try:
self.minio_client.remove_object(bucket, filename)
except ResponseError as err:
logging.error("Deletion of file '%s' failed", filename)
raise err
logging.debug('Deletion of file was successful')
def list_objects(self, bucket_name=None):
bucket = ""
if bucket_name is not None:
bucket = bucket_name
elif self.bucket_name is not None:
bucket = self.bucket_name
else:
raise Exception(
"The instance variable bucket_name was not initialized. You must either initialize it or pass it to the function"
)
objects = self.minio_client.list_objects(bucket)
object_list = []
for obj in objects:
object_list.append((obj.object_name, obj.last_modified))
return object_list
######################HELPER METHODS########################
def validate(self):
# The function should hold any validation that
# needs to be run before the class can be used
if self.bucket_name is not None:
try:
self.minio_client.bucket_exists(self.bucket_name)
except ResponseError as err:
logging.error(
"Validation failed for the input variable 'bucketName' with value '%s'",
self.bucket_name)
raise err
logging.debug("Testing file upload with test file: %s to Minio",
os.path.realpath(__file__))
self.upload(os.path.realpath(__file__))
logging.debug('Starting test file deletion')
self.delete(os.path.basename(__file__))
# -*- coding: utf-8 -*-
# Minio Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from minio import Minio
client = Minio('https://s3.amazonaws.com',
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETACCESSKEY')
# Remove an object.
client.remove_object('bucketName', 'objectName')
class MinioAdapter(BaseStorageAdapter):
def __init__(self, config, *args, **kwargs):
# Initialize minioClient with an endpoint and access/secret keys.
super().__init__(*args, **kwargs)
self._client = Minio(endpoint=config.get('HOSTNAME'),
access_key=config.get('AWS_ACCESS_KEY_ID'),
secret_key=config.get('AWS_SECRET_ACCESS_KEY'),
secure=False)
def bucket_exists(self, bucket_name):
if self._client.bucket_exists(bucket_name):
return True
return False
def create_bucket(self, bucket_name):
if self._client.bucket_exists(bucket_name):
raise Exception(f"{bucket_name} already exists")
else:
try:
self._client.make_bucket(bucket_name)
except ResponseError as err:
raise ResponseError
def remove_bucket(self, bucket_name):
try:
if self._client.bucket_exists(bucket_name):
self._client.remove_bucket(bucket_name)
except Exception as e:
raise e
def remove_file(self, bucket_name, file_name):
try:
if self._client.bucket_exists(self._client):
if (self._client.get_object(self._client, file_name)):
self._client.remove_object(bucket_name, file_name)
except Exception as e:
raise e
def get_bucket_list(self):
bucket_list = self._client.list_buckets()
return bucket_list
def get_all_files(self, bucket_name):
pass
def upload_file(self, bucket_name, file_name, file_path):
if (not self._client.bucket_exists(bucket_name)):
self._client.make_bucket(bucket_name=bucket_name)
try:
self._client.fput_object(bucket_name=bucket_name,
object_name=file_name,
file_path=file_path)
self.logger.info(f"Uploaded file {file_name}")
except ResponseError as e:
self.logger.error(e)
raise Exception(e)
def upload_data_stream(self, bucket_name, file_name, data_stream,
data_type):
length = len(data_stream)
if (not self._client.bucket_exists(bucket_name)):
self._client.make_bucket(bucket_name=bucket_name)
try:
self._client.put_object(bucket_name=bucket_name,
object_name=file_name,
data=data_stream,
length=length,
content_type=type)
except ResponseError as err:
self.logger.error(err)
raise err
def download_all_files(self, bucket_name, download_path):
try:
if (self._client.bucket_exists(bucket_name)):
obj_list = self._client.list_objects(bucket_name)
for obj in obj_list:
self._client.fget_object(bucket_name=bucket_name,
object_name=obj,
file_path=download_path)
except Exception as e:
self.logger.error(e)
raise e
def download_n_files(self, bucket_name, download_path, num_of_files):
try:
count = 0
for file in self._client.list_objects(bucket_name):
self._client.fget_object(bucket_name=bucket_name,
object_name=file,
file_path=download_path)
count = count + 1
if count == num_of_files:
break
except ResponseError as e:
self.logger.error(e)
raise e
def count_files(self, bucket_name):
list = self._client.list_objects(bucket_name)
return len(list)
def get_policy(self, bucket_name):
policy = self._client.get_bucket_policy(bucket_name)
return policy
def set_policy(self, bucket_name, policy):
self._client.set_bucket_policy(bucket_name, policy)
def test_object_is_string(self):
client = Minio('localhost:9000')
client.remove_object('hello', 1234)
# MinIO Python Library for Amazon S3 Compatible Cloud Storage,
# (C) 2015 MinIO, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname
# are dummy values, please replace them with original values.
from minio import Minio
from minio.error import ResponseError
client = Minio('s3.amazonaws.com',
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETACCESSKEY')
# Remove an object.
try:
client.remove_object('my-bucketname', 'my-objectname')
except ResponseError as err:
print(err)
def test_remove_bucket_invalid_name(self):
client = Minio('localhost:9000')
client.remove_object('ABCD', 'world')
class S3DataStore(DataStore):
"""
An implementation of the data store using S3 for storing policy checkpoints when using Coach in distributed mode.
The policy checkpoints are written by the trainer and read by the rollout worker.
"""
def __init__(self, params: S3DataStoreParameters):
"""
:param params: The parameters required to use the S3 data store.
"""
super(S3DataStore, self).__init__(params)
self.params = params
access_key = None
secret_key = None
if params.creds_file:
config = ConfigParser()
config.read(params.creds_file)
try:
access_key = config.get('default', 'aws_access_key_id')
secret_key = config.get('default', 'aws_secret_access_key')
except Error as e:
print("Error when reading S3 credentials file: %s", e)
else:
access_key = os.environ.get('ACCESS_KEY_ID')
secret_key = os.environ.get('SECRET_ACCESS_KEY')
self.mc = Minio(self.params.end_point,
access_key=access_key,
secret_key=secret_key)
def deploy(self) -> bool:
return True
def get_info(self):
return "s3://{}/{}".format(self.params.bucket_name)
def undeploy(self) -> bool:
return True
def save_to_store(self):
self._save_to_store(self.params.checkpoint_dir)
def _save_to_store(self, checkpoint_dir):
"""
save_to_store() uploads the policy checkpoint, gifs and videos to the S3 data store. It reads the checkpoint state files and
uploads only the latest checkpoint files to S3. It is used by the trainer in Coach when used in the distributed mode.
"""
try:
# remove lock file if it exists
self.mc.remove_object(self.params.bucket_name,
SyncFiles.LOCKFILE.value)
# Acquire lock
self.mc.put_object(self.params.bucket_name,
SyncFiles.LOCKFILE.value, io.BytesIO(b''), 0)
state_file = CheckpointStateFile(os.path.abspath(checkpoint_dir))
if state_file.exists():
ckpt_state = state_file.read()
checkpoint_file = None
for root, dirs, files in os.walk(checkpoint_dir):
for filename in files:
if filename == CheckpointStateFile.checkpoint_state_filename:
checkpoint_file = (root, filename)
continue
if filename.startswith(ckpt_state.name):
abs_name = os.path.abspath(
os.path.join(root, filename))
rel_name = os.path.relpath(abs_name,
checkpoint_dir)
self.mc.fput_object(self.params.bucket_name,
rel_name, abs_name)
abs_name = os.path.abspath(
os.path.join(checkpoint_file[0], checkpoint_file[1]))
rel_name = os.path.relpath(abs_name, checkpoint_dir)
self.mc.fput_object(self.params.bucket_name, rel_name,
abs_name)
# upload Finished if present
if os.path.exists(
os.path.join(checkpoint_dir, SyncFiles.FINISHED.value)):
self.mc.put_object(self.params.bucket_name,
SyncFiles.FINISHED.value, io.BytesIO(b''),
0)
# upload Ready if present
if os.path.exists(
os.path.join(checkpoint_dir,
SyncFiles.TRAINER_READY.value)):
self.mc.put_object(self.params.bucket_name,
SyncFiles.TRAINER_READY.value,
io.BytesIO(b''), 0)
# release lock
self.mc.remove_object(self.params.bucket_name,
SyncFiles.LOCKFILE.value)
if self.params.expt_dir and os.path.exists(self.params.expt_dir):
for filename in os.listdir(self.params.expt_dir):
if filename.endswith((".csv", ".json")):
self.mc.fput_object(
self.params.bucket_name, filename,
os.path.join(self.params.expt_dir, filename))
if self.params.expt_dir and os.path.exists(
os.path.join(self.params.expt_dir, 'videos')):
for filename in os.listdir(
os.path.join(self.params.expt_dir, 'videos')):
self.mc.fput_object(
self.params.bucket_name, filename,
os.path.join(self.params.expt_dir, 'videos', filename))
if self.params.expt_dir and os.path.exists(
os.path.join(self.params.expt_dir, 'gifs')):
for filename in os.listdir(
os.path.join(self.params.expt_dir, 'gifs')):
self.mc.fput_object(
self.params.bucket_name, filename,
os.path.join(self.params.expt_dir, 'gifs', filename))
except ResponseError as e:
print("Got exception: %s\n while saving to S3", e)
def load_from_store(self):
"""
load_from_store() downloads a new checkpoint from the S3 data store when it is not available locally. It is used
by the rollout workers when using Coach in distributed mode.
"""
try:
state_file = CheckpointStateFile(
os.path.abspath(self.params.checkpoint_dir))
# wait until lock is removed
while True:
objects = self.mc.list_objects_v2(self.params.bucket_name,
SyncFiles.LOCKFILE.value)
if next(objects, None) is None:
try:
# fetch checkpoint state file from S3
self.mc.fget_object(self.params.bucket_name,
state_file.filename,
state_file.path)
except Exception as e:
continue
break
time.sleep(10)
# Check if there's a finished file
objects = self.mc.list_objects_v2(self.params.bucket_name,
SyncFiles.FINISHED.value)
if next(objects, None) is not None:
try:
self.mc.fget_object(
self.params.bucket_name, SyncFiles.FINISHED.value,
os.path.abspath(
os.path.join(self.params.checkpoint_dir,
SyncFiles.FINISHED.value)))
except Exception as e:
pass
# Check if there's a ready file
objects = self.mc.list_objects_v2(self.params.bucket_name,
SyncFiles.TRAINER_READY.value)
if next(objects, None) is not None:
try:
self.mc.fget_object(
self.params.bucket_name, SyncFiles.TRAINER_READY.value,
os.path.abspath(
os.path.join(self.params.checkpoint_dir,
SyncFiles.TRAINER_READY.value)))
except Exception as e:
pass
checkpoint_state = state_file.read()
if checkpoint_state is not None:
objects = self.mc.list_objects_v2(self.params.bucket_name,
prefix=checkpoint_state.name,
recursive=True)
for obj in objects:
filename = os.path.abspath(
os.path.join(self.params.checkpoint_dir,
obj.object_name))
if not os.path.exists(filename):
self.mc.fget_object(obj.bucket_name, obj.object_name,
filename)
except ResponseError as e:
print("Got exception: %s\n while loading from S3", e)
def setup_checkpoint_dir(self, crd=None):
if crd:
self._save_to_store(crd)
