class MinIO(object):
''' Class for work with Minio '''
def __init__ (self, config, pathTmp, verbose):
""" Initialising object
Parameters
----------
config : dict
config of module
pathTmp : str
path to temporary files
verbose : bool
verbose output
"""
self.verbose = verbose
self.config = config
self.pathTmp = pathTmp
self.moduleName = self.config['NAME']
self.handle = None
self.host = 'localhost'
if 'S3_HOST' in self.config:
self.host = self.config['S3_HOST']
self.port = '9000'
if 'S3_PORT' in self.config:
self.port = self.config['S3_PORT']
self.url = "%s:%s" % (self.host, self.port)
self.access_key = ''
if 'S3_ACCESS_KEY' in self.config:
self.access_key = self.config['S3_ACCESS_KEY']
self.secret_key = ''
if 'S3_SECRET_KEY' in self.config:
self.secret_key = self.config['S3_SECRET_KEY']
def reconnect(self):
self.close()
timeout = 15
stop_time = 1
elapsed_time = 0
str_err = ''
while (self.handle is None) and elapsed_time < timeout:
time.sleep(stop_time)
elapsed_time += stop_time
try:
self.handle = Minio(self.url, access_key=self.access_key,
secret_key=self.secret_key, secure=False)
except Exception as e:
if self.verbose:
print("DBG: WAIT: %d: Connect to Minio '%s':%s" % (elapsed_time, self.url, str(e)))
str_err = str(e)
if self.handle is None:
print("FATAL: Connect to Minio '%s': %s" % (self.url, str_err))
return None
if self.verbose:
print("DBG: Connected to Minio '%s'" % (self.url))
return self.handle
def close(self):
if not self.handle is None:
self.handle.close()
self.handle = None
def init(self):
if not 'INIT_MINIO_CREATE_FOLDERS' in self.config:
return
if self.reconnect() is None:
return
folders = self.config['INIT_MINIO_CREATE_FOLDERS']
af = folders.split(';')
for folder in af:
f = folder.split(':')
if len(f) == 2:
self.mkDir(f[0], f[1])
def getBasketsList(self):
bs = self.handle.list_buckets()
res = []
for b in bs:
res.append(b.name)
res.sort()
return res
def uploadFile(self, bucketName, filename, fullPath):
try:
found = self.handle.bucket_exists(bucketName)
if not found:
self.handle.make_bucket(bucketName)
self.handle.fput_object(bucketName, filename, fullPath)
except Exception as e:
print("FATAL: uploadFile to Minio(%s): %s" % (self.url, str(e)))
return False
return True
def mkDir(self, bucketName, fullPath):
try:
found = self.handle.bucket_exists(bucketName)
if not found:
self.handle.make_bucket(bucketName)
self.handle.put_object(bucketName, fullPath, io.BytesIO(b''), 0, content_type='application/x-directory')
if self.verbose:
print("DBG: Make folder '%s:%s' into Minio(%s)" % (bucketName, fullPath, self.url))
except Exception as e:
print("FATAL: Make folder '%s' into Minio(%s): %s" % (fullPath, self.url, str(e)))
return False
return True
def getListObjects(self, bucketName, currentDir=''):
res = []
objects = []
try:
found = self.handle.bucket_exists(bucketName)
if not found:
self.handle.make_bucket(bucketName)
objects = self.handle.list_objects(bucketName, prefix=currentDir, recursive=True)
except Exception as e:
print("FATAL: List folder '%s' into Minio(%s): %s" % (currentDir, self.url, str(e)))
for obj in objects:
res.append(obj.object_name)
return res
def downloadFile(self, bucketName, filename, fullPath):
try:
self.handle.fget_object(bucketName, filename, fullPath)
except Exception as e:
print("FATAL: downloadFile to Minio(%s): %s" % (self.url, str(e)))
return False
return True
def compareFiles(self, bucketName, filename, filePath):
result = False
if not os.path.exists(filePath):
return result
try:
pathTmp = os.path.join(self.pathTmp, bucketName)
os.makedirs(pathTmp, exist_ok=True)
file1 = os.path.join(pathTmp, filename)
self.handle.fget_object(bucketName, filename, file1)
result = filecmp.cmp(file1, filePath, shallow=False)
os.remove(file1)
except Exception as e:
print("FATAL: compareFiles Minio(%s): %s" % (self.url, str(e)))
return False
return result
def main():
"""
Functional testing of minio python library.
"""
fake = Factory.create()
client = Minio('s3.amazonaws.com',
os.getenv('ACCESS_KEY'),
os.getenv('SECRET_KEY'))
_http = urllib3.PoolManager(
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where()
)
# Get unique bucket_name, object_name.
bucket_name = uuid.uuid4().__str__()
object_name = uuid.uuid4().__str__()
# Enable trace
# client.trace_on(sys.stderr)
# Make a new bucket.
bucket_name = 'minio-pytest'
print(client.make_bucket(bucket_name))
print(client.make_bucket(bucket_name+'.unique',
location='us-west-1'))
## Check if return codes a valid from server.
try:
client.make_bucket(bucket_name+'.unique',
location='us-west-1')
except ResponseError as err:
if str(err.code) in ['BucketAlreadyOwnedByYou', 'BucketAlreadyExists']:
pass
else:
raise
# Check if bucket was created properly.
print(client.bucket_exists(bucket_name))
print(client.bucket_exists(bucket_name+'.unique'))
# List all buckets.
buckets = client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
with open('testfile', 'wb') as file_data:
file_data.write(fake.text().encode('utf-8'))
file_data.close()
# Put a file
file_stat = os.stat('testfile')
with open('testfile', 'rb') as file_data:
client.put_object(bucket_name, object_name, file_data, file_stat.st_size)
file_data.close()
# Fput a file
print(client.fput_object(bucket_name, object_name+'-f', 'testfile'))
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name))
# Get a full object
object_data = client.get_object(bucket_name, object_name)
with open('newfile', 'wb') as file_data:
for data in object_data:
file_data.write(data)
file_data.close()
# Get a full object locally.
print(client.fget_object(bucket_name, object_name, 'newfile-f'))
# List all object paths in bucket.
objects = client.list_objects(bucket_name, recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name, obj.last_modified, \
obj.etag, obj.size, obj.content_type)
presigned_get_object_url = client.presigned_get_object(bucket_name, object_name)
response = _http.urlopen('GET', presigned_get_object_url)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.get(bucket_name, object_name)
presigned_put_object_url = client.presigned_put_object(bucket_name, object_name)
value = fake.text().encode('utf-8')
data = io.BytesIO(value).getvalue()
response = _http.urlopen('PUT', presigned_put_object_url, body=data)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.put(bucket_name, object_name)
object_data = client.get_object(bucket_name, object_name)
if object_data.read() != value:
raise ValueError('Bytes not equal')
# Post policy.
policy = PostPolicy()
policy.set_bucket_name(bucket_name)
policy.set_key_startswith('objectPrefix/')
expires_date = datetime.utcnow()+timedelta(days=10)
policy.set_expires(expires_date)
print(client.presigned_post_policy(policy))
# Remove an object.
print(client.remove_object(bucket_name, object_name))
print(client.remove_object(bucket_name, object_name+'-f'))
# Remove a bucket. This operation will only work if your bucket is empty.
print(client.remove_bucket(bucket_name))
print(client.remove_bucket(bucket_name+'.unique'))
# Remove temporary files.
os.remove('testfile')
os.remove('newfile')
os.remove('newfile-f')
def test_bucket_exists_works(self, mock_connection):
mock_server = MockConnection()
mock_connection.return_value = mock_server
mock_server.mock_add_request(MockResponse('HEAD',
'https://localhost:9000/hello/',
{'User-Agent': _DEFAULT_USER_AGENT},
200))
client = Minio('localhost:9000')
result = client.bucket_exists('hello')
eq_(True, result)
mock_server.mock_add_request(MockResponse('HEAD',
'https://localhost:9000/goodbye/',
{'User-Agent': _DEFAULT_USER_AGENT},
404))
false_result = client.bucket_exists('goodbye')
eq_(False, false_result)
# -*- coding: utf-8 -*-
# Minio Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from minio import Minio
client = Minio('https://s3.amazonaws.com',
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETACCESSKEY')
print(client.bucket_exists('bucketName'))
minioClient.make_bucket("mybucket")
except BucketAlreadyOwnedByYou as err:
pass
except BucketAlreadyExists as err:
pass
except ResponseError as err:
raise
# list_buckets()
buckets = minioClient.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
# bucket_existes()
try:
print(minioClient.bucket_exists("mybucket"))
except ResponseError as err:
print(err)
# remove_bucket()
# try:
# minioClient.remove_bucket("mybucket")
# except ResponseError as err:
# print(err)
# List all object paths in bucket that begin with my-prefixname.
objects = minioClient.list_objects('data1', prefix='data11', recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name.encode('utf-8'), obj.last_modified,
obj.etag, obj.size, obj.content_type)
# -*- coding: utf-8 -*-
# Minio Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 Minio, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
# dummy values, please replace them with original values.
from minio import Minio
client = Minio('s3.amazonaws.com',
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETACCESSKEY')
print(client.bucket_exists('my-bucketname'))
class MINIORepository(Repository):
client = None
def __init__(self, config):
super().__init__()
try:
access_key = config['storage_access_key']
except Exception:
access_key = 'minio'
try:
secret_key = config['storage_secret_key']
except Exception:
secret_key = 'minio123'
try:
self.bucket = config['storage_bucket']
except Exception:
self.bucket = 'fedn-models'
try:
self.context_bucket = config['context_bucket']
except Exception:
self.bucket = 'fedn-context'
try:
self.secure_mode = bool(config['storage_secure_mode'])
except Exception:
self.secure_mode = False
if not self.secure_mode:
print(
"\n\n\nWARNING : S3/MINIO RUNNING IN **INSECURE** MODE! THIS IS NOT FOR PRODUCTION!\n\n\n"
)
if self.secure_mode:
from urllib3.poolmanager import PoolManager
manager = PoolManager(num_pools=100,
cert_reqs='CERT_NONE',
assert_hostname=False)
self.client = Minio("{0}:{1}".format(config['storage_hostname'],
config['storage_port']),
access_key=access_key,
secret_key=secret_key,
secure=self.secure_mode,
http_client=manager)
else:
self.client = Minio("{0}:{1}".format(config['storage_hostname'],
config['storage_port']),
access_key=access_key,
secret_key=secret_key,
secure=self.secure_mode)
# TODO: generalize
self.context_bucket = 'fedn-context'
self.create_bucket(self.context_bucket)
self.create_bucket(self.bucket)
def create_bucket(self, bucket_name):
found = self.client.bucket_exists(bucket_name)
if not found:
try:
response = self.client.make_bucket(bucket_name)
except InvalidResponseError as err:
raise
def set_artifact(self, instance_name, instance, is_file=False, bucket=''):
""" Instance must be a byte-like object. """
if bucket == '':
bucket = self.bucket
if is_file == True:
self.client.fput_object(bucket, instance_name, instance)
else:
try:
self.client.put_object(bucket, instance_name,
io.BytesIO(instance), len(instance))
except Exception as e:
raise Exception("Could not load data into bytes {}".format(e))
return True
def get_artifact(self, instance_name, bucket=''):
if bucket == '':
bucket = self.bucket
try:
data = self.client.get_object(bucket, instance_name)
return data.read()
except Exception as e:
raise Exception("Could not fetch data from bucket, {}".format(e))
def get_artifact_stream(self, instance_name):
try:
data = self.client.get_object(self.bucket, instance_name)
return data
except Exception as e:
raise Exception("Could not fetch data from bucket, {}".format(e))
def list_artifacts(self):
objects_to_delete = []
try:
objs = self.client.list_objects(self.bucket)
for obj in objs:
print(obj.object_name)
objects_to_delete.append(obj.object_name)
except Exception as e:
raise Exception("Could not list models in bucket {}".format(
self.bucket))
return objects_to_delete
def delete_artifact(self, instance_name, bucket=[]):
if not bucket:
bucket = self.bucket
try:
self.client.remove_object(bucket, instance_name)
except ResponseError as err:
print(err)
print('Could not delete artifact: {}'.format(instance_name))
def delete_objects(self):
objects_to_delete = self.list_artifacts()
try:
# force evaluation of the remove_objects() call by iterating over
# the returned value.
for del_err in self.client.remove_objects(self.bucket,
objects_to_delete):
print("Deletion Error: {}".format(del_err))
except ResponseError as err:
print(err)
author: [email protected] ''' import os from minio import Minio from minio.error import ResponseError bucket_name = 'test101' client = Minio('your-s3-endpoint.com', access_key='xxxxxxxxxxxxxx', secret_key='xxxxxxxxxxxxxx', secure=True) print("\n=== check if bucket '%s' exists ===" % bucket_name) try: print(client.bucket_exists(bucket_name)) print('\n=== check bucket info ===') print(client.get_bucket_policy(bucket_name)) except ResponseError as err: raise print(err) sys.exit(1) print("\n=== list all buckets '%s' ===" % bucket_name) buckets = client.list_buckets() for bucket in buckets: print(bucket.name, bucket.creation_date) print("\n=== put object to buckets '%s' ===" % bucket_name) try: client.fput_object(bucket_name,
cursor.execute("DELETE FROM images where id = ?", (id, ))
bdd.commit()
minioClient.remove_object(BUCKET_NAME, data[0])
return {'success': 'Image has successfully deleted'}
api.add_resource(Image,
'/image',
'/image/<int:id>',
endpoint="image_get_delete")
if __name__ == '__main__':
if not os.path.exists(UPLOAD_FOLDER):
os.makedirs(UPLOAD_FOLDER)
bdd = sqlite3.connect('s3.db', check_same_thread=False)
bdd.execute("""
CREATE TABLE IF NOT EXISTS images(
`id` INTEGER PRIMARY KEY AUTOINCREMENT,
`imagename` TEXT NOT NULL,
`name` TEXT NOT NULL,
`description` TEXT DEFAULT NULL
);
""")
minioClient = Minio('minio:9000',
access_key='SCALEWAYS3LIKE',
secret_key='424242424242',
secure=False)
if minioClient.bucket_exists(BUCKET_NAME) == False:
minioClient.make_bucket(BUCKET_NAME)
app.run(host="0.0.0.0", debug=True)
def main():
"""
Functional testing of minio python library.
"""
fake = Factory.create()
client = Minio('https://play.minio.io:9002',
'Q3AM3UQ867SPQQA43P2F',
'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG')
# Get unique bucket_name, object_name.
bucket_name = uuid.uuid4().__str__()
object_name = uuid.uuid4().__str__()
# Make a new bucket.
print(client.make_bucket(bucket_name))
# Check if bucket was created properly.
print(client.bucket_exists(bucket_name))
# Set bucket name to private.
print(client.set_bucket_acl(bucket_name, Acl.private()))
# Print current bucket acl.
print(client.get_bucket_acl(bucket_name))
# List all buckets.
buckets = client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
with open('testfile', 'wb') as file_data:
file_data.write(fake.text().encode('utf-8'))
file_data.close()
# Put a file
file_stat = os.stat('testfile')
with open('testfile', 'rb') as file_data:
client.put_object(bucket_name, object_name, file_data, file_stat.st_size)
file_data.close()
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name))
# Get a full object
data = client.get_object(bucket_name, object_name)
with open('newfile', 'wb') as file_data:
for d in data:
file_data.write(d)
file_data.close()
# List all object paths in bucket that begin with hello.
objects = client.list_objects(bucket_name)
for obj in objects:
print(obj.bucket_name, obj.object_name, obj.last_modified, \
obj.etag, obj.size, obj.content_type)
uploads = client.list_incomplete_uploads(bucket_name,
prefix='',
recursive=True)
for obj in uploads:
print(obj.bucket_name, obj.object_name, obj.upload_id)
print(client.presigned_get_object(bucket_name, object_name))
print(client.presigned_put_object(bucket_name, object_name))
# Remove an object.
print(client.remove_object(bucket_name, object_name))
# Remove a bucket.
# This operation will only work if your bucket is empty.
print(client.remove_bucket(bucket_name))
# Remove temporary files.
os.remove('testfile')
os.remove('newfile')
def test_bucket_exists_invalid_name(self):
client = Minio('localhost:9000')
client.bucket_exists('AB*CD')
class KartonBackend:
def __init__(self, config):
self.config = config
self.redis = StrictRedis(
host=config["redis"]["host"],
port=int(config["redis"].get("port", 6379)),
decode_responses=True,
)
self.minio = Minio(
endpoint=config["minio"]["address"],
access_key=config["minio"]["access_key"],
secret_key=config["minio"]["secret_key"],
secure=bool(int(config["minio"].get("secure", True))),
)
@property
def default_bucket_name(self) -> str:
return self.config.minio_config["bucket"]
@staticmethod
def get_queue_name(identity: str, priority: TaskPriority) -> str:
"""
Return Redis routed task queue name for given identity and priority
:param identity: Karton service identity
:param priority: Queue priority (TaskPriority enum value)
:return: Queue name
"""
return f"karton.queue.{priority.value}:{identity}"
@staticmethod
def get_queue_names(identity: str) -> List[str]:
"""
Return all Redis routed task queue names for given identity,
ordered by priority (descending). Used internally by Consumer.
:param identity: Karton service identity
:return: List of queue names
"""
return [
identity, # Backwards compatibility (2.x.x)
KartonBackend.get_queue_name(identity, TaskPriority.HIGH),
KartonBackend.get_queue_name(identity, TaskPriority.NORMAL),
KartonBackend.get_queue_name(identity, TaskPriority.LOW),
]
@staticmethod
def serialize_bind(bind: KartonBind) -> str:
"""
Serialize KartonBind object (Karton service registration)
:param bind: KartonBind object with bind definition
:return: Serialized bind data
"""
return json.dumps(
{
"info": bind.info,
"version": bind.version,
"filters": bind.filters,
"persistent": bind.persistent,
"service_version": bind.service_version,
},
sort_keys=True,
)
@staticmethod
def unserialize_bind(identity: str, bind_data: str) -> KartonBind:
"""
Deserialize KartonBind object for given identity.
Compatible with Karton 2.x.x and 3.x.x
:param identity: Karton service identity
:param bind_data: Serialized bind data
:return: KartonBind object with bind definition
"""
bind = json.loads(bind_data)
if isinstance(bind, list):
# Backwards compatibility (v2.x.x)
return KartonBind(
identity=identity,
info=None,
version="2.x.x",
persistent=not identity.endswith(".test"),
filters=bind,
service_version=None,
)
return KartonBind(
identity=identity,
info=bind["info"],
version=bind["version"],
persistent=bind["persistent"],
filters=bind["filters"],
service_version=bind.get("service_version"),
)
def get_bind(self, identity: str) -> KartonBind:
"""
Get bind object for given identity
:param identity: Karton service identity
:return: KartonBind object
"""
return self.unserialize_bind(
identity, self.redis.hget(KARTON_BINDS_HSET, identity))
def get_binds(self) -> List[KartonBind]:
"""
Get all binds registered in Redis
:return: List of KartonBind objects for subsequent identities
"""
return [
self.unserialize_bind(identity, raw_bind) for identity, raw_bind in
self.redis.hgetall(KARTON_BINDS_HSET).items()
]
def register_bind(self, bind: KartonBind) -> Optional[KartonBind]:
"""
Register bind for Karton service and return the old one
:param bind: KartonBind object with bind definition
:return: Old KartonBind that was registered under this identity
"""
with self.redis.pipeline(transaction=True) as pipe:
pipe.hget(KARTON_BINDS_HSET, bind.identity)
pipe.hset(KARTON_BINDS_HSET, bind.identity,
self.serialize_bind(bind))
old_serialized_bind, _ = pipe.execute()
if old_serialized_bind:
return self.unserialize_bind(bind.identity, old_serialized_bind)
else:
return None
def unregister_bind(self, identity: str) -> None:
"""
Removes bind for identity
:param bind: Identity to be unregistered
"""
self.redis.hdel(KARTON_BINDS_HSET, identity)
def set_consumer_identity(self, identity: str) -> None:
"""
Sets identity for current Redis connection
"""
return self.redis.client_setname(identity)
def get_online_consumers(self) -> Dict[str, List[str]]:
"""
Gets all online consumer identities
:return: Dictionary {identity: [list of clients]}
"""
bound_identities = defaultdict(list)
for client in self.redis.client_list():
bound_identities[client["name"]].append(client)
return bound_identities
def get_task(self, task_uid: str) -> Optional[Task]:
"""
Get task object with given identifier
:param task_uid: Task identifier
:return: Task object
"""
task_data = self.redis.get(f"{KARTON_TASK_NAMESPACE}:{task_uid}")
if not task_data:
return None
return Task.unserialize(task_data, backend=self)
def get_tasks(self, task_uid_list: List[str]) -> List[Task]:
"""
Get multiple tasks for given identifier list
:param task_uid_list: List of task identifiers
:return: List of task objects
"""
task_list = self.redis.mget([
f"{KARTON_TASK_NAMESPACE}:{task_uid}" for task_uid in task_uid_list
])
return [
Task.unserialize(task_data, backend=self)
for task_data in task_list if task_data is not None
]
def get_all_tasks(self) -> List[Task]:
"""
Get all tasks registered in Redis
:return: List with Task objects
"""
tasks = self.redis.keys(f"{KARTON_TASK_NAMESPACE}:*")
return [
Task.unserialize(task_data) for task_data in self.redis.mget(tasks)
if task_data is not None
]
def register_task(self, task: Task) -> None:
"""
Register task in Redis.
Consumer should register only Declared tasks.
Status change should be done using set_task_status.
:param task: Task object
"""
self.redis.set(f"{KARTON_TASK_NAMESPACE}:{task.uid}", task.serialize())
def set_task_status(self,
task: Task,
status: TaskState,
consumer: Optional[str] = None) -> None:
"""
Request task status change to be applied by karton-system
:param task: Task object
:param status: New task status (TaskState)
:param consumer: Consumer identity
"""
self.redis.rpush(
KARTON_OPERATIONS_QUEUE,
json.dumps({
"status": status.value,
"identity": consumer,
"task": task.serialize(),
"type": "operation",
}),
)
def delete_task(self, task: Task) -> None:
"""
Remove task from Redis
:param task: Task object
"""
self.redis.delete(f"{KARTON_TASK_NAMESPACE}:{task.uid}")
def get_task_queue(self, queue: str) -> List[Task]:
"""
Return all tasks in provided queue
:param queue: Queue name
:return: List with Task objects contained in queue
"""
task_uids = self.redis.lrange(queue, 0, -1)
return self.get_tasks(task_uids)
def get_task_ids_from_queue(self, queue: str) -> List[str]:
"""
Return all task UIDs in a queue
:param queue: Queue name
:return: List with task identifiers contained in queue
"""
return self.redis.lrange(queue, 0, -1)
def remove_task_queue(self, queue: str) -> List[Task]:
"""
Remove task queue with all contained tasks
:param queue: Queue name
:return: List with Task objects contained in queue
"""
pipe = self.redis.pipeline()
pipe.lrange(queue, 0, -1)
pipe.delete(queue)
return self.get_tasks(pipe.execute()[0])
def produce_unrouted_task(self, task: Task) -> None:
"""
Add given task to unrouted task (``karton.tasks``) queue
Task must be registered before with :py:meth:`register_task`
:param task: Task object
"""
self.redis.rpush(KARTON_TASKS_QUEUE, task.uid)
def produce_routed_task(self, identity: str, task: Task) -> None:
"""
Add given task to routed task queue of given identity
Task must be registered using :py:meth:`register_task`
:param identity: Karton service identity
:param task: Task object
"""
self.redis.rpush(self.get_queue_name(identity, task.priority),
task.uid)
def consume_queues(self,
queues: Union[str, List[str]],
timeout: int = 0) -> Optional[Tuple[str, str]]:
"""
Get item from queues (ordered from the most to the least prioritized)
If there are no items, wait until one appear.
:param queues: Redis queue name or list of names
:param timeout: Waiting for item timeout (default: 0 = wait forever)
:return: Tuple of [queue_name, item] objects or None if timeout has been reached
"""
return self.redis.blpop(queues, timeout=timeout)
def consume_routed_task(self,
identity: str,
timeout: int = 5) -> Optional[Task]:
"""
Get routed task for given consumer identity.
If there are no tasks, blocks until new one appears or timeout is reached.
:param identity: Karton service identity
:param timeout: Waiting for task timeout (default: 5)
:return: Task object
"""
item = self.consume_queues(
self.get_queue_names(identity),
timeout=timeout,
)
if not item:
return None
queue, data = item
return self.get_task(data)
@staticmethod
def _log_channel(logger_name: Optional[str], level: Optional[str]) -> str:
return ".".join(
[KARTON_LOG_CHANNEL, (level or "*").lower(), logger_name or "*"])
def produce_log(
self,
log_record: Dict[str, Any],
logger_name: str,
level: str,
) -> bool:
"""
Push new log record to the logs channel
:param log_record: Dict with log record
:param logger_name: Logger name
:param level: Log level
:return: True if any active log consumer received log record
"""
return (self.redis.publish(self._log_channel(logger_name, level),
json.dumps(log_record)) > 0)
def consume_log(
self,
timeout: int = 5,
logger_filter: Optional[str] = None,
level: Optional[str] = None,
) -> Iterator[Optional[Dict[str, Any]]]:
"""
Subscribe to logs channel and yield subsequent log records
or None if timeout has been reached.
If you want to subscribe only to a specific logger name
and/or log level, pass them via logger_filter and level arguments.
:param timeout: Waiting for log record timeout (default: 5)
:param logger_filter: Filter for name of consumed logger
:param level: Log level
:return: Dict with log record
"""
with self.redis.pubsub() as pubsub:
pubsub.psubscribe(self._log_channel(logger_filter, level))
while pubsub.subscribed:
item = pubsub.get_message(ignore_subscribe_messages=True,
timeout=timeout)
if item and item["type"] == "pmessage":
body = json.loads(item["data"])
if "task" in body and isinstance(body["task"], str):
body["task"] = json.loads(body["task"])
yield body
yield None
def increment_metrics(self, metric: KartonMetrics, identity: str) -> None:
"""
Increments metrics for given operation type and identity
:param metric: Operation metric type
:param identity: Related Karton service identity
"""
self.redis.hincrby(metric.value, identity, 1)
def upload_object(
self,
bucket: str,
object_uid: str,
content: Union[bytes, BinaryIO],
length: int = None,
) -> None:
"""
Upload resource object to underlying object storage (Minio)
:param bucket: Bucket name
:param object_uid: Object identifier
:param content: Object content as bytes or file-like stream
:param length: Object content length (if file-like object provided)
"""
if isinstance(content, bytes):
length = len(content)
content = BytesIO(content)
self.minio.put_object(bucket, object_uid, content, length)
def upload_object_from_file(self, bucket: str, object_uid: str,
path: str) -> None:
"""
Upload resource object file to underlying object storage
:param bucket: Bucket name
:param object_uid: Object identifier
:param path: Path to the object content
"""
self.minio.fput_object(bucket, object_uid, path)
def get_object(self, bucket: str, object_uid: str) -> HTTPResponse:
"""
Get resource object stream with the content.
Returned response should be closed after use to release network resources.
To reuse the connection, it's required to call `response.release_conn()`
explicitly.
:param bucket: Bucket name
:param object_uid: Object identifier
:return: Response object with content
"""
return self.minio.get_object(bucket, object_uid)
def download_object(self, bucket: str, object_uid: str) -> bytes:
"""
Download resource object from object storage.
:param bucket: Bucket name
:param object_uid: Object identifier
:return: Content bytes
"""
reader = self.minio.get_object(bucket, object_uid)
try:
return reader.read()
finally:
reader.release_conn()
reader.close()
def download_object_to_file(self, bucket: str, object_uid: str,
path: str) -> None:
"""
Download resource object from object storage to file
:param bucket: Bucket name
:param object_uid: Object identifier
:param path: Target file path
"""
self.minio.fget_object(bucket, object_uid, path)
def list_objects(self, bucket: str) -> List[str]:
"""
List identifiers of stored resource objects
:param bucket: Bucket name
:return: List of object identifiers
"""
return [
object.object_name for object in self.minio.list_objects(bucket)
]
def remove_object(self, bucket: str, object_uid: str) -> None:
"""
Remove resource object from object storage
:param bucket: Bucket name
:param object_uid: Object identifier
"""
self.minio.remove_object(bucket, object_uid)
def check_bucket_exists(self, bucket: str, create: bool = False) -> bool:
"""
Check if bucket exists and optionally create it if it doesn't.
:param bucket: Bucket name
:param create: Create bucket if doesn't exist
:return: True if bucket exists yet
"""
if self.minio.bucket_exists(bucket):
return True
if create:
self.minio.make_bucket(bucket)
return False
def test_bucket_is_string(self):
client = Minio('localhost:9000')
client.bucket_exists(1234)
def test_bucket_is_not_empty_string(self):
client = Minio('localhost:9000')
client.bucket_exists(' \t \n ')
from minio import Minio
from minio.error import ResponseError
import os
import sys
from os import listdir
##
# Follow bucket naming rules
#'10.233.54.122:9000'
#"cnb-ml-bucket"
if __name__ == "__main__":
minioserviceIP = sys.argv[1]
bucketname = sys.argv[2]
print("Deleting minio bucket \n")
minioClient = Minio(minioserviceIP,
access_key='minio',
secret_key='minio123',
secure=False)
try:
if minioClient.bucket_exists(bucketname):
objects = minioClient.list_objects(bucketname, recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name.encode('utf-8'),
obj.last_modified, obj.etag, obj.size, obj.content_type)
minioClient.remove_object(bucketname, obj.object_name)
minioClient.remove_bucket(bucketname)
except ResponseError as err:
print(err)
class Pull_Objects:
def __init__(self):
try:
self.minioClient = Minio(BACKEND,
access_key=ACCESS_KEY_ID,
secret_key=SECRET_KEY_ID,
secure=True)
except Exception as e:
print(e)
def single_object(self, body):
try:
bucket = self.minioClient.bucket_exists(BUCKET)
except Exception as e:
return {'code': 404, 'Error': "Bucket not found."}
else:
try:
version_of_package = body['version']
ver = list(version_of_package.split('.'))
first_version, second_version, third_version = ver[0], ver[
1], ver[2]
objects = self.minioClient.list_objects(BUCKET, recursive=True)
for obj in objects:
versions = list(obj.object_name.split('/'))
if first_version == versions[
0] and second_version == versions[
1] and third_version == versions[2]:
location = obj.object_name
break
result_url = self.minioClient.presigned_get_object(
BUCKET, location)
return {'code': 200, 'url': result_url}
except Exception as e:
return {'code': 404, 'Error': "File not found."}
def latest_object(self):
try:
bucket = self.minioClient.bucket_exists(BUCKET)
except Exception as e:
return {'code': 404, 'Error': "Bucket not found."}
else:
try:
objects = self.minioClient.list_objects(BUCKET, recursive=True)
last_object = list(objects)[-1].object_name
result_url = self.minioClient.presigned_get_object(
BUCKET, last_object)
return {'code': 200, 'url': result_url}
except Exception as e:
return {'code': 404, 'Error': "File not found."}
def get_All_objects(self):
try:
bucket = self.minioClient.bucket_exists(BUCKET)
except Exception as e:
return {'code': 404, 'Error': "Bucket not found."}
else:
try:
objects = self.minioClient.list_objects(BUCKET, recursive=True)
result_url = {}
for obj in objects:
result_url[
obj.
object_name] = self.minioClient.presigned_get_object(
BUCKET, obj.object_name)
return {'code': 200, 'url': result_url}
except Exception as e:
return {'code': 404, 'Error': "File not found."}
def delete_object(self, body):
try:
bucket = self.minioClient.bucket_exists(BUCKET)
except Exception as e:
return {'code': 404, 'Error': "Bucket not found."}
else:
try:
version_of_package = body['version']
ver = list(version_of_package.split('.'))
first_version, second_version, third_version = ver[0], ver[
1], ver[2]
objects = self.minioClient.list_objects(BUCKET, recursive=True)
for obj in objects:
versions = list(obj.object_name.split('/'))
if first_version == versions[
0] and second_version == versions[
1] and third_version == versions[2]:
break
result_url = self.minioClient.remove_object(
BUCKET, obj.object_name)
return {'code': 200, 'message': 'Object Deleted.'}
except Exception as e:
return {'code': 404, 'Error': "File not found."}
open(filename, 'wb').write(response.content)
''' Read data with Spark SQL '''
spark = SparkSession.builder.getOrCreate()
df_data = spark.read.csv(path=filename,
sep=",",
header=True,
inferSchema=True)
df_data.head()
''' Upload data to Cloud object storage '''
cos = Minio(cos_endpoint,
access_key=cos_access_key,
secret_key=cos_secret_key,
secure=True)
if not cos.bucket_exists(cos_bucket_name):
try:
cos.make_bucket(cos_bucket_name)
except ResponseError as err:
print(err)
cos.fput_object(cos_bucket_name, filename, filename)
print('Data ' + filename + ' is uploaded to bucket at ' + cos_bucket_name)
with open("/tmp/filename", "w") as report:
report.write(filename)
df_data.printSchema()
print("Number of records: " + str(df_data.count()))
def main():
"""
Functional testing of minio python library.
"""
fake = Factory.create()
client = Minio('s3.amazonaws.com', os.getenv('ACCESS_KEY'),
os.getenv('SECRET_KEY'))
_http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
# Get unique bucket_name, object_name.
bucket_name = uuid.uuid4().__str__()
object_name = uuid.uuid4().__str__()
# Enable trace
# client.trace_on(sys.stderr)
# Make a new bucket.
bucket_name = 'minio-pytest'
print(client.make_bucket(bucket_name))
print(client.make_bucket(bucket_name + '.unique', location='us-west-1'))
## Check if return codes a valid from server.
try:
client.make_bucket(bucket_name + '.unique', location='us-west-1')
except ResponseError as err:
if str(err.code) in ['BucketAlreadyOwnedByYou', 'BucketAlreadyExists']:
pass
else:
raise
# Check if bucket was created properly.
print(client.bucket_exists(bucket_name))
print(client.bucket_exists(bucket_name + '.unique'))
# List all buckets.
buckets = client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
with open('testfile', 'wb') as file_data:
file_data.write(fake.text().encode('utf-8'))
file_data.close()
# Put a file
file_stat = os.stat('testfile')
with open('testfile', 'rb') as file_data:
client.put_object(bucket_name, object_name, file_data,
file_stat.st_size)
file_data.close()
# Fput a file
print(client.fput_object(bucket_name, object_name + '-f', 'testfile'))
# Copy a file
print(
client.copy_object(bucket_name, object_name + '-copy',
'/' + bucket_name + '/' + object_name + '-f'))
try:
copy_conditions = CopyConditions()
copy_conditions.set_match_etag('test-etag')
print(
client.copy_object(bucket_name, object_name + '-copy',
'/' + bucket_name + '/' + object_name + '-f',
copy_conditions))
except ResponseError as err:
if err.code != 'PreconditionFailed':
raise
if err.message != 'At least one of the pre-conditions you specified did not hold':
raise
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name))
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name + '-f'))
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name + '-copy'))
# Get a full object
object_data = client.get_object(bucket_name, object_name)
with open('newfile', 'wb') as file_data:
for data in object_data:
file_data.write(data)
file_data.close()
# Get a full object locally.
print(client.fget_object(bucket_name, object_name, 'newfile-f'))
# List all object paths in bucket.
print("Listing using ListObjects API")
objects = client.list_objects(bucket_name, recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name, obj.last_modified, \
obj.etag, obj.size, obj.content_type)
# List all object paths in bucket using V2 API.
print("Listing using ListObjectsV2 API")
objects = client.list_objects_v2(bucket_name, recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name, obj.last_modified, \
obj.etag, obj.size, obj.content_type)
presigned_get_object_url = client.presigned_get_object(
bucket_name, object_name)
response = _http.urlopen('GET', presigned_get_object_url)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.get(bucket_name, object_name)
presigned_put_object_url = client.presigned_put_object(
bucket_name, object_name)
value = fake.text().encode('utf-8')
data = io.BytesIO(value).getvalue()
response = _http.urlopen('PUT', presigned_put_object_url, body=data)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.put(bucket_name, object_name)
object_data = client.get_object(bucket_name, object_name)
if object_data.read() != value:
raise ValueError('Bytes not equal')
# Post policy.
policy = PostPolicy()
policy.set_bucket_name(bucket_name)
policy.set_key_startswith('objectPrefix/')
expires_date = datetime.utcnow() + timedelta(days=10)
policy.set_expires(expires_date)
print(client.presigned_post_policy(policy))
# Remove an object.
print(client.remove_object(bucket_name, object_name))
print(client.remove_object(bucket_name, object_name + '-f'))
print(client.remove_object(bucket_name, object_name + '-copy'))
policy_name = client.get_bucket_policy(bucket_name)
if policy_name != Policy.NONE:
raise ValueError('Policy name is invalid ' + policy_name)
# Set read-write policy successfully.
client.set_bucket_policy(bucket_name, '', Policy.READ_WRITE)
# Reset policy to NONE.
client.set_bucket_policy(bucket_name, '', Policy.NONE)
# Validate if the policy is reverted back to NONE.
policy_name = client.get_bucket_policy(bucket_name)
if policy_name != Policy.NONE:
raise ValueError('Policy name is invalid ' + policy_name)
# Upload some new objects to prepare for multi-object delete test.
print("Prepare for remove_objects() test.")
object_names = []
for i in range(10):
curr_object_name = object_name + "-{}".format(i)
print("object-name: {}".format(curr_object_name))
print(client.fput_object(bucket_name, curr_object_name, "testfile"))
object_names.append(curr_object_name)
# delete the objects in a single library call.
print("Performing remove_objects() test.")
del_errs = client.remove_objects(bucket_name, object_names)
had_errs = False
for del_err in del_errs:
had_errs = True
print("Err is {}".format(del_err))
if had_errs:
print("remove_objects() FAILED - it had unexpected errors.")
else:
print("remove_objects() worked as expected.")
# Remove a bucket. This operation will only work if your bucket is empty.
print(client.remove_bucket(bucket_name))
print(client.remove_bucket(bucket_name + '.unique'))
# Remove temporary files.
os.remove('testfile')
os.remove('newfile')
os.remove('newfile-f')
class MinioObjectStore(ObjectStoreInterface):
def initialize(self, jsonArgs):
# The function should initialize any connections that need
# to be made or any variables that will be required
format = "%(asctime)s - %(levelname)s: %(threadName)s - %(message)s"
logging.basicConfig(format=format,
level=logging.DEBUG,
datefmt="%H:%M:%S")
args = json.loads(jsonArgs)
# Extract variables from JSON
self.host = args['host']
self.access_key = args['accessKey']
self.secret_key = args['secretKey']
self.https_enabled = args['httpsEnabled']
if 'bucketName' in args:
self.bucket_name = args['bucketName']
# Instantiate client
self.minio_client = Minio(self.host,
access_key=self.access_key,
secret_key=self.secret_key,
secure=self.https_enabled)
self.validate()
def upload(self, filepath, bucket_name=None):
"""
Uploads a file to Minio.
The function uploads a file from the path in the filesystem specified and uploads
it to a bucket in Minio.
Parameters:
filepath (string): The absolute path of the file to upload
bucket_name (string): Optional argument to indicate the bucket to use to upload the file
Returns:
string: The location of the image in Minio
"""
bucket = ""
if bucket_name is not None:
bucket = bucket_name
elif self.bucket_name is not None:
bucket = self.bucket_name
else:
raise Exception(
"The instance variable bucket_name was not initialized. You must either initialize it or pass it to the function"
)
# Upload file to Algorithmia
filename = os.path.basename(filepath)
logging.debug("Uploading file '%s' to bucket '%s'", filepath, bucket)
try:
self.minio_client.fput_object(bucket, filename, filepath)
except ResponseError as err:
logging.error("Upload of file '%s' failed", filepath)
raise err
logging.debug("Upload of file was successful")
return bucket + '/' + filename
def download(self, filename, download_path, bucket_name=None):
# Downloads an object from Minio
bucket = ""
if bucket_name is not None:
bucket = bucket_name
elif self.bucket_name is not None:
bucket = self.bucket_name
else:
raise Exception(
"The instance variable bucket_name was not initialized. You must either initialize it or pass it to the function"
)
logging.debug("Downloading file '%s' from bucket '%s'", filename,
bucket)
try:
self.minio_client.fget_object(bucket, filename, download_path)
except ResponseError as err:
logging.error("Download of file '%s' failed", filename)
raise err
logging.debug('File download was successful')
def delete(self, filename, bucket_name=None):
# Deletes a file from Minio
bucket = ""
if bucket_name is not None:
bucket = bucket_name
elif self.bucket_name is not None:
bucket = self.bucket_name
else:
raise Exception(
"The instance variable bucket_name was not initialized. You must either initialize it or pass it to the function"
)
logging.debug("Deleting file '%s' from bucket '%s'", filename, bucket)
try:
self.minio_client.remove_object(bucket, filename)
except ResponseError as err:
logging.error("Deletion of file '%s' failed", filename)
raise err
logging.debug('Deletion of file was successful')
def list_objects(self, bucket_name=None):
bucket = ""
if bucket_name is not None:
bucket = bucket_name
elif self.bucket_name is not None:
bucket = self.bucket_name
else:
raise Exception(
"The instance variable bucket_name was not initialized. You must either initialize it or pass it to the function"
)
objects = self.minio_client.list_objects(bucket)
object_list = []
for obj in objects:
object_list.append((obj.object_name, obj.last_modified))
return object_list
######################HELPER METHODS########################
def validate(self):
# The function should hold any validation that
# needs to be run before the class can be used
if self.bucket_name is not None:
try:
self.minio_client.bucket_exists(self.bucket_name)
except ResponseError as err:
logging.error(
"Validation failed for the input variable 'bucketName' with value '%s'",
self.bucket_name)
raise err
logging.debug("Testing file upload with test file: %s to Minio",
os.path.realpath(__file__))
self.upload(os.path.realpath(__file__))
logging.debug('Starting test file deletion')
self.delete(os.path.basename(__file__))
def minio_object():
####################################################################################################################
#Extract an object from a specified bucket
####################################################################################################################
if request.method == 'GET':
# get keys from request args
if minio_keys(request):
auth = minio_keys(request)
if "err" in auth:
return jsonify(auth)
else:
return jsonify({"err": "No authentication keys provided"})
try:
minioClient = Minio(SERVER_ENDPOINT,
access_key=auth.get("access_key"),
secret_key=auth.get("secret_key"),
secure=False)
deposit_id = False
if not deposit_id:
config = json.loads(request.form.get('config'))
deposit_id = config.get('deposit_id')
bucket_name = config.get('bucket_name')
temp_obj_path = str(deposit_id)
obj = minioClient.get_object(bucket_name, deposit_id)
with open(temp_obj_path, 'wb') as file_data:
for d in obj.stream(32 * 1024):
file_data.write(d)
return send_file(temp_obj_path,
mimetype="application/octet-stream")
except NoSuchKey as err:
return jsonify(
{"err": "The requested deposit_id does not exist"})
except NoSuchBucket as err:
return jsonify({"err": str(err)})
except (InvalidBucketError, TypeError, NoSuchBucket):
return jsonify({"err": "Please provide valid bucket_name"})
except (AccessDenied, InvalidAccessKeyId, SignatureDoesNotMatch):
return jsonify({"err": "Invalid Authentication."})
except JSONDecodeError as err:
return jsonify({"err": "Incorrect formatting of data"})
except ResponseError as err:
return jsonify({"err": err})
####################################################################################################################
#Puts one object i.e. a file in the specified bucket only.
####################################################################################################################
elif request.method == 'POST':
# get data from request payload
try:
config = json.loads(request.form.get('config'))
data = json.loads(request.form.get('data'))
# extract authentication details
if minio_keys(request):
auth = minio_keys(request)
if "err" in auth:
return jsonify(auth)
else:
return jsonify({"err": "No authentication keys provided"})
# Initialize minioClient with an endpoint and keys.
minioClient = Minio(SERVER_ENDPOINT,
access_key=auth.get("access_key"),
secret_key=auth.get("secret_key"),
secure=False)
# extract deposit_id value
if data.get('deposit_id'):
deposit_id = data.get('deposit_id')
else:
return jsonify({"err": "Please provide a deposit_id"})
# extract file & temp save to disk
file = request.files['file']
file.save(deposit_id)
# extract bucket_name value
bucket_name = config.get('bucket_name')
minioClient.bucket_exists(bucket_name)
metadata = {}
metadata['BUCKET_NAME'] = bucket_name
r = minioClient.fput_object(bucket_name,
object_name=deposit_id,
file_path=deposit_id,
metadata=metadata)
# cleanup temp file
os.remove(deposit_id)
return jsonify({
"etag": r,
"deposit_id": deposit_id,
"metadata": metadata
})
except (NoSuchBucket, InvalidBucketError) as err:
return jsonify({
"err":
"This bucket does not exist. Please create this bucket at the bucket scoped endpoint."
})
except (InvalidAccessKeyId, AccessDenied,
SignatureDoesNotMatch) as err:
return jsonify({"err": "Invalid Authentication."})
except BadRequestKeyError as err:
return jsonify({"err": "Please attach a file to the request"})
except JSONDecodeError:
return jsonify({"err": "Incorrect formatting of data"})
except ResponseError as err:
return jsonify({"err": err})
except TypeError as err:
return jsonify({"err": "Please enter a bucket_name"})
####################################################################################################################
#Deletes one object i.e. a file from the specified bucket only.
####################################################################################################################
elif request.method == 'DELETE':
# extract authentication details
if minio_keys(request):
auth = minio_keys(request)
if "err" in auth:
return jsonify(auth)
else:
return jsonify({"err": "No authentication keys provided"})
# get data from request payload
config = json.loads(request.form.get('config'))
data = json.loads(request.form.get('data'))
# extract bucket_name value
bucket_name = config.get('bucket_name')
# extract object specific deposit_id
if data.get('deposit_id'):
deposit_id = data.get('deposit_id')
else:
return jsonify({"err": "Please enter valid deposit_id."})
try:
minioClient = Minio(SERVER_ENDPOINT,
access_key=auth.get("access_key"),
secret_key=auth.get("secret_key"),
secure=False)
except AccessDenied as err:
return jsonify({"err": str(err)})
try:
error = minioClient.get_object(bucket_name, deposit_id)
rem_object = minioClient.remove_object(bucket_name, deposit_id)
except (NoSuchBucket, InvalidBucketError, ResponseError,
TypeError):
return jsonify({"err": "Bucket does not exist."})
except NoSuchKey:
return jsonify({"err": "Please enter valid deposit_id."})
except (AccessDenied, InvalidAccessKeyId,
SignatureDoesNotMatch) as err:
return jsonify({"err": "Invalid Authentication."})
return jsonify({"deposit_id": deposit_id})
else:
return jsonify({"err": "Unsupported method"})
def aggregate_data(**kwargs):
"""
Aggregate data
"""
# logs information
execution_date = kwargs["execution_date"]
logging.info("Execution datetime={}".format(execution_date))
# temp data directory
temp_dir_path = os.path.join(os.path.dirname(__file__),
'..', '..',
'temp-data',
kwargs["dag"].dag_id,
kwargs["task"].task_id,
str(execution_date))
if not os.path.exists(temp_dir_path):
os.makedirs(temp_dir_path)
logging.info("Will write output to {}".format(temp_dir_path))
# connect to data lake object storage
dataLakeMinioClient = Minio(DATALAKE_MINIO_ENDPOINT,
access_key=DATALAKE_MINIO_ACCESS_KEY,
secret_key=DATALAKE_MINIO_SECRET_KEY,
secure=False)
try:
# check if source bucket exist
if dataLakeMinioClient.bucket_exists(DATALAKE_MINIO_CLEANED_BUCKET):
logging.info(
"[DATA-LAKE] Bucket `{}` is exist".format(DATALAKE_MINIO_CLEANED_BUCKET))
else:
raise ValueError('"[DATA-LAKE] Bucket is not exist"')
# download CSV file
source_object_name = "bank.xlsx_{}.csv".format(execution_date)
destination = '{}/{}'.format(temp_dir_path, source_object_name)
data = dataLakeMinioClient.get_object(
DATALAKE_MINIO_CLEANED_BUCKET, source_object_name)
with open(destination, 'wb') as file_data:
for d in data.stream(32*1024):
file_data.write(d)
logging.info("[DATA-LAKE] Successfully downloaded file from `{}/{}` to `{}`".format(
DATALAKE_MINIO_CLEANED_BUCKET, source_object_name, destination))
# add to pandas dataframe
df = pd.read_csv(destination)
logging.info("Successfully read data from {}".format(destination))
# transformation: group by 'Account No' and `VALUE DATE`, and aggregate with summarize
df = df.groupby(['Account No', 'VALUE DATE'])[["WITHDRAWAL AMT", "DEPOSIT AMT"]].sum().reset_index()
# logs transformation result
logging.info("Transformation result: \n{}\n{}".format(df, df.dtypes))
# store to data lake
df.to_csv(destination)
logging.info("Successfully save data to {}".format(destination))
if dataLakeMinioClient.bucket_exists(DATALAKE_MINIO_AGG_BUCKET):
logging.info("[DATA-LAKE] Bucket `{}` is exist".format(
DATALAKE_MINIO_AGG_BUCKET))
else:
logging.info("[DATA-LAKE] Bucket `{}` is not exist, will attempt to create".format(
DATALAKE_MINIO_AGG_BUCKET))
dataLakeMinioClient.make_bucket(DATALAKE_MINIO_AGG_BUCKET)
logging.info("[DATA-LAKE] Bucket `{}` is created".format(
DATALAKE_MINIO_AGG_BUCKET))
dataLakeMinioClient.fput_object(
DATALAKE_MINIO_AGG_BUCKET, source_object_name, destination, content_type='application/csv')
logging.info(
"[DATA-LAKE] Successfully stored `{}`".format(source_object_name))
# temp-data cleanup
shutil.rmtree(temp_dir_path)
logging.info("Delete directory `{}`".format(temp_dir_path))
except InvalidBucketError as err:
raise ValueError('"Error in bucket name"')
except NoSuchKey as err:
raise ValueError('"Object not found"')
except ResponseError as err:
raise ValueError('"Error In API call."')
class CosClient(LoggingConfigurable):
client = None
def __init__(self,
config=None,
endpoint=None,
access_key=None,
secret_key=None,
bucket=None):
super().__init__()
if config:
self.endpoint = urlparse(config.metadata['cos_endpoint'])
self.access_key = config.metadata['cos_username']
self.secret_key = config.metadata['cos_password']
self.bucket = config.metadata['cos_bucket']
else:
self.endpoint = urlparse(endpoint)
self.access_key = access_key
self.secret_key = secret_key
self.bucket = bucket
# Infer secure from the endpoint's scheme.
self.secure = self.endpoint.scheme == 'https'
self.client = self.__initialize_object_store()
def __initialize_object_store(self):
# Initialize minioClient with an endpoint and access/secret keys.
self.client = Minio(endpoint=self.endpoint.netloc,
access_key=self.access_key,
secret_key=self.secret_key,
secure=self.secure)
# Make a bucket with the make_bucket API call.
try:
if not self.client.bucket_exists(self.bucket):
self.client.make_bucket(self.bucket)
except BucketAlreadyOwnedByYou as ex:
self.log.warning("Object Storage bucket already owned by you",
exc_info=True)
raise ex from ex
except BucketAlreadyExists as ex:
self.log.warning("Object Storage bucket already exists",
exc_info=True)
raise ex from ex
except ResponseError as ex:
self.log.error("Object Storage error", exc_info=True)
raise ex from ex
return self.client
def upload_file(self, file_name, file_path):
"""
Uploads contents from a file, located on the local filesystem at `file_path`,
as `file_name` in object storage.
:param file_name: Name of the file object in object storage
:param file_path: Path on the local filesystem from which object data will be read.
:return:
"""
try:
self.client.fput_object(bucket_name=self.bucket,
object_name=file_name,
file_path=file_path)
except BaseException as ex:
self.log.error('Error uploading file {} to bucket {}'.format(
file_path, self.bucket),
exc_info=True)
raise ex from ex
def upload_file_to_dir(self, dir, file_name, file_path):
"""
Uploads contents from a file, located on the local filesystem at `file_path`,
as `file_name` in object storage.
:param: dir: the directory where the file should be uploaded to
:param file_name: Name of the file object in object storage
:param file_path: Path on the local filesystem from which object data will be read.
:return:
"""
self.upload_file(os.path.join(dir, file_name), file_path)
def download_file(self, file_name, file_path):
"""
Downloads and saves the object as a file in the local filesystem.
:param file_name: Name of the file object in object storage
:param file_path: Path on the local filesystem to which the object data will be written.
:return:
"""
try:
self.client.fget_object(bucket_name=self.bucket,
object_name=file_name,
file_path=file_path)
except BaseException as ex:
self.log.error('Error reading file {} from bucket {}'.format(
file_name, self.bucket),
exc_info=True)
raise ex from ex
def download_file_from_dir(self, dir, file_name, file_path):
"""
Downloads and saves the object as a file in the local filesystem.
:param dir: the directory where the file is located
:param file_name: Name of the file object in object storage
:param file_path: Path on the local filesystem to which the object data will be written.
:return:
"""
self.download_file(os.path.join(dir, file_name), file_path)
class MinioAdapter(BaseStorageAdapter):
def __init__(self, config, *args, **kwargs):
# Initialize minioClient with an endpoint and access/secret keys.
super().__init__(*args, **kwargs)
self._client = Minio(endpoint=config.get('HOSTNAME'),
access_key=config.get('AWS_ACCESS_KEY_ID'),
secret_key=config.get('AWS_SECRET_ACCESS_KEY'),
secure=False)
def bucket_exists(self, bucket_name):
if self._client.bucket_exists(bucket_name):
return True
return False
def create_bucket(self, bucket_name):
if self._client.bucket_exists(bucket_name):
raise Exception(f"{bucket_name} already exists")
else:
try:
self._client.make_bucket(bucket_name)
except ResponseError as err:
raise ResponseError
def remove_bucket(self, bucket_name):
try:
if self._client.bucket_exists(bucket_name):
self._client.remove_bucket(bucket_name)
except Exception as e:
raise e
def remove_file(self, bucket_name, file_name):
try:
if self._client.bucket_exists(self._client):
if (self._client.get_object(self._client, file_name)):
self._client.remove_object(bucket_name, file_name)
except Exception as e:
raise e
def get_bucket_list(self):
bucket_list = self._client.list_buckets()
return bucket_list
def get_all_files(self, bucket_name):
pass
def upload_file(self, bucket_name, file_name, file_path):
if (not self._client.bucket_exists(bucket_name)):
self._client.make_bucket(bucket_name=bucket_name)
try:
self._client.fput_object(bucket_name=bucket_name,
object_name=file_name,
file_path=file_path)
self.logger.info(f"Uploaded file {file_name}")
except ResponseError as e:
self.logger.error(e)
raise Exception(e)
def upload_data_stream(self, bucket_name, file_name, data_stream,
data_type):
length = len(data_stream)
if (not self._client.bucket_exists(bucket_name)):
self._client.make_bucket(bucket_name=bucket_name)
try:
self._client.put_object(bucket_name=bucket_name,
object_name=file_name,
data=data_stream,
length=length,
content_type=type)
except ResponseError as err:
self.logger.error(err)
raise err
def download_all_files(self, bucket_name, download_path):
try:
if (self._client.bucket_exists(bucket_name)):
obj_list = self._client.list_objects(bucket_name)
for obj in obj_list:
self._client.fget_object(bucket_name=bucket_name,
object_name=obj,
file_path=download_path)
except Exception as e:
self.logger.error(e)
raise e
def download_n_files(self, bucket_name, download_path, num_of_files):
try:
count = 0
for file in self._client.list_objects(bucket_name):
self._client.fget_object(bucket_name=bucket_name,
object_name=file,
file_path=download_path)
count = count + 1
if count == num_of_files:
break
except ResponseError as e:
self.logger.error(e)
raise e
def count_files(self, bucket_name):
list = self._client.list_objects(bucket_name)
return len(list)
def get_policy(self, bucket_name):
policy = self._client.get_bucket_policy(bucket_name)
return policy
def set_policy(self, bucket_name, policy):
self._client.set_bucket_policy(bucket_name, policy)
# -*- coding: utf-8 -*-
# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 MinIO, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
# dummy values, please replace them with original values.
from minio import Minio
from minio.error import ResponseError
client = Minio('s3.amazonaws.com',
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETACCESSKEY')
try:
print(client.bucket_exists('my-bucketname'))
except ResponseError as err:
print(err)
def configuration_wizard(config_filename: str) -> None:
config = ConfigParser()
log.info("Configuring MinIO")
while True:
minio_access_key = get_user_option(
"Enter the MinIO access key", default="minioadmin"
)
minio_secret_key = get_user_option(
"Enter the MinIO secret key", default="minioadmin"
)
minio_address = get_user_option(
"Enter the MinIO address", default="localhost:9000"
)
minio_bucket = get_user_option(
"Enter the MinIO bucket to use", default="karton"
)
minio_secure = get_user_option('Use SSL ("0", "1")?', default="0")
log.info("Testing MinIO connection...")
minio = Minio(
endpoint=minio_address,
access_key=minio_access_key,
secret_key=minio_secret_key,
secure=bool(int(minio_secure)),
)
bucket_exists = False
try:
bucket_exists = minio.bucket_exists(minio_bucket)
except Exception as e:
log.info("Error while connecting to MinIO: %s", e, exc_info=True)
retry = get_user_option(
'Do you want to try with different MinIO settings ("yes", "no")?',
default="yes",
)
if retry != "yes":
log.info("Quitting configuration")
return
else:
continue
log.info("Connected to MinIO successfully")
if not bucket_exists:
log.info(
(
"The required bucket %s does not exist. To create it automatically,"
" start karton-system with --setup-bucket flag"
),
minio_bucket,
)
break
config["minio"] = {
"access_key": minio_access_key,
"secret_key": minio_secret_key,
"address": minio_address,
"bucket": minio_bucket,
"secure": str(bool(int(minio_secure))),
}
log.info("Configuring Redis")
while True:
redis_host = get_user_option("Enter the Redis host", default="localhost")
redis_port = get_user_option("Enter the Redis port", default="6379")
log.info("Testing Redis connection...")
redis = StrictRedis(
host=redis_host,
port=int(redis_port),
decode_responses=True,
)
try:
redis.ping()
except Exception as e:
log.info("Error while connecting to Redis: %s", e, exc_info=True)
retry = get_user_option(
'Do you want to try with different Redis settings ("yes", "no")?',
default="yes",
)
if retry != "yes":
log.info("Quitting configuration")
return
else:
continue
log.info("Connected to Redis successfully")
break
config["redis"] = {
"host": redis_host,
"port": str(int(redis_port)),
}
with open(config_filename, "w") as configfile:
config.write(configfile)
log.info("Saved the new configuration file in %s", os.path.abspath(config_filename))
## Tough to diagnose what's happening with the unpacked archive
# for x in os.listdir( tempdir.name ):
# print(x)
#
# for x in os.listdir( unpackedInput ):
# print(x)
with runtime.Runtime() as covis:
result = covis.process(str(unpackedInput), str(matOutputPath))
## Reuse from above
if outputMinioClient:
outbucket = output.netloc
if not outputMinioClient.bucket_exists(outbucket):
outputMinioClient.make_bucket(outbucket)
## \TODO generate correct outputName
outpath = Path(output.path) / matOutput
outpath = str(outpath).strip(
"/") ## Minio client doesn't like leading slash
outputMinioClient.fput_object(file_path=matOutputPath,
bucket_name=outbucket,
object_name=outpath)
print("Uploaded to bucket %s, path %s" % (outbucket, outpath))
# if result:
# print("Saving results to %s" % result)
class Journal:
def __init__(self,
minio_server='10.30.24.123:9000',
access_key='sesuro5pka32vtt',
secret_key='c5977GQW2CHF6wsNG5bK',
debug=False):
if debug:
logging.basicConfig(
format='%(asctime)s %(levelname)s ==> %(message)s',
level=logging.DEBUG,
datefmt='%m/%d/%Y %I:%M:%S %p')
else:
logging.basicConfig(
format='%(asctime)s %(levelname)s ==> %(message)s',
level=logging.INFO,
datefmt='%m/%d/%Y %I:%M:%S %p')
self.__timezone__ = "America/New_York"
self.set_timezone()
logging.debug(f"Minio Info {minio_server}")
self.__mc__ = Minio(minio_server,
access_key=access_key,
secret_key=secret_key,
secure=False)
self.__netid__ = self.get_netid()
self.__notebook__ = self.get_notebook_path()
self.__notebook_full_path__ = f"{os.environ.get('HOME')}/{self.__notebook__}"
#self.__course__, self.__term__, self.__unit__, self.__assignment__, self.__assignment_type__ = self.parse_notebook_path()
self.__bucket__ = f"{os.environ['BUCKET']}"
self.initialize_bucket()
def set_timezone(self):
'''
save on a lot of date math.
'''
os.environ['TZ'] = self.__timezone__
time.tzset()
def initialize_bucket(self):
if not self.__mc__.bucket_exists(self.__bucket__):
self.__mc__.make_bucket(self.__bucket__)
def format_date(self, date):
return date.strftime("%Y-%m-%d %I:%M:%S %p")
def debug(self):
logging.debug(f"NETID = {self.__netid__}")
logging.debug(f"PATH = {self.__notebook__}")
logging.debug(f"FULL PATH = {self.__notebook_full_path__}")
logging.debug(f"COURSE = {self.__course__}")
logging.debug(f"TERM = {self.__term__}")
logging.debug(f"BUCKET = {self.__bucket__}")
logging.debug(f"JOURNAL = {self.get_journal_path(self.__netid__)}")
logging.debug(f"TIME ZONE = {self.__timezone__}")
return
# TODO: Remove netid from the arguments
def get_journal_path(self, netid=None):
if netid == None:
netid = self.__netid__
return f"journal/{netid}.csv"
def journal_exists(self, netid=None):
journal_path = self.get_journal_path(netid)
logging.debug(
f"CMD: journal_exists netid={netid}, journal_path={journal_path}")
def init_journal(self, netid=None):
tmp = "_tmp.csv"
journal_path = self.get_journal_path(netid)
logging.debug(
f"CMD: init_journal netid={netid}, journal_path={journal_path}")
dataframe = pd.DataFrame({'Date': [], 'Hours': [], 'Comments': []})
dataframe.to_csv(tmp, index=False)
etag = self.__mc__.fput_object(self.__bucket__, journal_path, tmp)
os.remove(tmp)
logging.debug("DONE: init_journal")
return dataframe
def load_journal(self, netid=None):
journal_path = self.get_journal_path(netid)
logging.debug(
f"CMD: load_journal netid={netid}, journal_path={journal_path}")
try:
dataframe = pd.read_csv(
self.__mc__.get_object(self.__bucket__, journal_path))
except: # ugh....
dataframe = self.init_journal(netid)
return dataframe
def save_journal(self, dataframe, netid=None):
tmp = "_tmp.csv"
journal_path = self.get_journal_path(netid)
logging.debug(
f"CMD: save_journal netid={netid}, journal_path={journal_path}")
dataframe.to_csv(tmp, index=False)
etag = self.__mc__.fput_object(self.__bucket__, journal_path, tmp)
os.remove(tmp)
return etag
def get_netid(self):
netid = os.environ.get("JUPYTERHUB_USER").lower()
hostname = socket.gethostname().lower()
callback_url = os.environ.get("JUPYTERHUB_OAUTH_CALLBACK_URL").lower()
activity_url = os.environ.get("JUPYTERHUB_ACTIVITY_URL").lower()
if callback_url.find(netid) >= 0 and activity_url.find(
netid) >= 0 and hostname.find(netid) >= 0:
return netid
else:
raise ValueError(
f"Unable to locate NetID={netid} for hostname {hostname}")
def get_notebook_path(self):
connection_file = os.path.basename(ipykernel.get_connection_file())
kernel_id = connection_file.split('-', 1)[1].split('.')[0]
token = os.environ.get("JUPYTERHUB_API_TOKEN")
netid = self.__netid__
response = requests.get(
f'http://127.0.0.1:8888/user/{netid}/api/sessions?token={token}')
response.raise_for_status()
sessions = response.json()
for sess in sessions:
if sess['kernel']['id'] == kernel_id:
return sess['notebook']['path']
break
def parse_notebook_path(self):
items = self.__notebook__.split("/")
if items[5].startswith("CCL"):
assign_type = "Lab"
elif items[5].startswith("HW") or items[5].startswith("NYC"):
assign_type = "Homework"
else:
assign_type = "Unknown"
return items[1], items[2], items[4], items[5], assign_type
@classmethod
def get_bucket_names(cls):
yield cls.files
yield cls.reports
client = Minio(
os.environ.get('MINIO_URL', 'localhost:9000'),
access_key=os.environ['MINIO_ACCESS_KEY'],
secret_key=os.environ['MINIO_SECRET_KEY'],
secure=False
)
for bucket_name in Bucket.get_bucket_names():
if not client.bucket_exists(bucket_name):
client.make_bucket(bucket_name)
def save(bucket: Bucket, name: str, file: io.BytesIO) -> None:
size = file.getbuffer().nbytes
client.put_object(
bucket_name=bucket,
object_name=name,
data=file,
length=size,
content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
def test_bucket_is_string(self):
client = Minio('localhost:9000')
client.bucket_exists(1234)
def _pull_from_minio(tag, bucket, project_name, force):
click.secho('Looking up remote..', fg='cyan')
home_dir = os.path.expanduser('~')
lab_dir = os.path.join(home_dir, '.lab')
project_dir = os.getcwd()
_clone = True
# Extract bucket name and project name from config if they are present
if (tag is None) & (bucket is None) & (project_name is None):
_clone = False
with open(os.path.join(project_dir, 'config', 'runtime.yaml'),
'r') as file:
project_config = yaml.load(file)
bucket = project_config['bucket']
project_name = project_config['name']
tag = project_config['tag']
check_minio_config(tag)
# Extract minio configuration
with open(os.path.join(lab_dir, 'config.yaml'), 'r') as file:
minio_config = yaml.load(file)[tag]
hostname = minio_config['minio_endpoint']
accesskey = minio_config['minio_accesskey']
secretkey = minio_config['minio_secretkey']
minioClient = Minio(hostname,
access_key=accesskey,
secret_key=secretkey,
secure=False)
if not minioClient.bucket_exists(bucket):
click.secho('Bucket ' + bucket + ' is not found on remote', fg='red')
raise click.Abort()
try:
objects = minioClient.list_objects(bucket,
prefix=project_name + '/',
recursive=True)
remote_objects = [o.object_name for o in objects]
if _clone is False:
if force:
local_objects = []
else:
local_objects = _list_dir('.')
local_objects = [
l.replace('./', project_name + '/') for l in local_objects
]
remote_objects = list(set(remote_objects) - set(local_objects))
if len(remote_objects) == 0:
click.secho(
'Project is in sync with remote. '
'Use <lab pull --force> to do a hard pull.',
fg='yellow')
raise click.Abort()
click.secho('Fetching ' + str(len(remote_objects)) +
' remote objects.',
fg='cyan')
for obj in remote_objects:
if _clone:
object_name = obj
else:
object_name = ''.join(obj.split(project_name + '/')[1:])
print('Downloading ' + object_name)
minioClient.fget_object(bucket, obj,
os.path.join(os.getcwd(), object_name))
except ResponseError as err:
print(err)
def test_bucket_exists_invalid_name(self):
client = Minio('localhost:9000')
client.bucket_exists('ABCD')
arg.add_argument("--s3", type = str, default = "")
arg.add_argument("--access_key", type = str, default = "")
arg.add_argument("--secret_key", type = str, default = "")
arg.add_argument("--dir", type = str, default = "")
options = vars(arg.parse_args())
for option in options.items():
if (not len(option[1])):
print("Пользователь не передал обязательный параметр командной строки")
exit(1)
history = {}
client = Minio(options['s3'],
options['access_key'],
options['secret_key'],
secure=False)
try:
client.bucket_exists("alexey")
except Exception:
print("Проверьте s3|access_key|secret_key")
exit(2)
if (not client.bucket_exists("alexey")):
client.make_bucket("alexey")
def get_list_of_files(path, list_of_files):
files = os.listdir(path)
for file in files:
if os.path.isdir(path+'/'+file):
get_list_of_files(path+'/'+file,list_of_files)
else:
list_of_files.append(path+'/'+file)
def sync_time(path, path_server):
time = client.stat_object('alexey',path_server).last_modified
os.utime(path,(time,time))
# Date: 2021/1/21 11:15 # Email [email protected] # ---------------------------------------------------------- import sys import uuid from pathlib import Path from minio import Minio # access_key: MinIo帐号 # secret_key: MinIo密码 minio_storage = Minio("192.168.30.180:9000", access_key="admin", secret_key="YouGuess", secure=False) images = sys.argv[1:] for image in images: print("File Uploading ...") suffix = Path(image).suffix file_name = str(uuid.uuid4()) + suffix # 存储桶名称 bucket_name = "hexo-blog" if not minio_storage.bucket_exists(bucket_name): # 如果存储桶不存在，则创建 minio_storage.make_bucket(bucket_name) minio_storage.fput_object(bucket_name, file_name, image, content_type="image/png", part_size=10485760) print("http://192.168.30.180/files/{}/{}".format(bucket_name, file_name))
class Storage():
bucket_name = os.getenv("S3_BUCKET", "notes")
app_root = os.getcwd()
storage_root = "%s/storage" % (app_root)
def __init__(self):
if not os.path.exists(self.storage_root):
os.makedirs(self.storage_root, exist_ok=True)
self.logger = logging.getLogger("uvicorn.error")
self.client = Minio(
os.getenv("MINIO_ADDRESS", ""),
access_key=os.getenv("MINIO_ACCESS_KEY", ""),
secret_key=os.getenv("MINIO_SECRET_KEY", ""),
region=os.getenv("MINIO_REGION_NAME", ""),
)
# Make bucket if not exist.
found = self.client.bucket_exists(self.bucket_name)
if not found:
self.client.make_bucket(self.bucket_name)
else:
self.logger.info("Bucket {} already exists".format(
self.bucket_name))
def get_files(self, doc_uuid, image_urls):
for item in image_urls:
create_path(self.storage_root, doc_uuid)
local_path, file_hash, error = get_file(self.logger,
self.storage_root,
doc_uuid, item["original"])
if not error:
filename = "%s/storage/images/%s.%s" % (
doc_uuid, file_hash, item["original"]["extension"])
try:
self.client.fput_object(self.bucket_name, filename,
local_path)
item["replacement"] = "/storage/%s" % (filename)
except Exception as e:
self.logger.exception(e)
def cleanup(self, doc_uuid):
# Remove a prefix recursively.
delete_object_list = map(
lambda x: DeleteObject(x.object_name),
self.client.list_objects(self.bucket_name,
"%s/" % (doc_uuid),
recursive=True))
errors = self.client.remove_objects("my-bucket", delete_object_list)
for error in errors:
self.logger.error("error occured when deleting object", error)
def get_object(self, filename, expiration=20):
# response = self.client.fget_object(self.bucket_name, filename, filename)
response = self.client.presigned_get_object(
self.bucket_name, filename, expires=timedelta(hours=expiration))
self.logger.info(filename)
self.logger.info(response)
return response
def handle(req):
client = Minio(
"10.20.1.54:30020",
access_key="admin",
secret_key="secretsecret",
secure = False
)
data = json.loads(req)
fname = data['fname']
file_uuid = data['file_uuid']
pipeline = data['pipeline']
function_name = data['function_name']
function_bucket_list = data['function_bucket']
train_data_build_func_bucket_name = function_bucket_list['lstm-pipeline-train-data-build']
train_data_build_func_file_name = train_data_build_func_bucket_name + '-' + fname.split('.')[0] + '-' + file_uuid + '.' + 'json'
uuid_renamed_file_h5 = function_name + '-' + fname.split('.')[0] + '-' + file_uuid + '.' + 'h5'
client.fget_object(train_data_build_func_bucket_name, 'xt-'+train_data_build_func_file_name, '/home/app/xt.json')
client.fget_object(train_data_build_func_bucket_name, 'yt-'+train_data_build_func_file_name, '/home/app/yt.json')
x_dict = ""
with open('/home/app/xt.json', 'r') as obj:
x_dict = json.load(obj)
y_dict = ""
with open('/home/app/yt.json', 'r') as obj:
y_dict = json.load(obj)
# print(x_dict)
# print(y_dict)
x_train = []
y_train = []
for i in x_dict:
x_train.append([])
for j in x_dict[i]:
x_train[int(i)].append([x_dict[i][j]])
for i in y_dict:
y_train.append([y_dict[i]])
x_train = np.array(x_train)
y_train = np.array(y_train)
# print(x_train)
# print(y_train)
buildModel(x_train, y_train)
# x_dict = dict()
# y_dict = dict()
# for i in range(len(x_train)):
# x_dict[str(i)] = dict()
# for j in range(len(x_train[i])):
# x_dict[str(i)][str(j)] = x_train[i][j][0]
# for i in range(len(y_train)):
# y_dict[str(i)] = y_train[i][0]
# with open('/home/app/x_train_data.json', 'w', encoding='utf-8') as f:
# json.dump(x_dict, f)
# with open('/home/app/y_train_data.json', 'w', encoding='utf-8') as f:
# json.dump(y_dict, f)
found = client.bucket_exists(os.environ['bucket_name'])
if not found:
client.make_bucket(os.environ['bucket_name'])
client.fput_object(os.environ['bucket_name'], uuid_renamed_file_h5, '/home/app/model_setup.h5')
return os.environ['bucket_name']
def test_bucket_is_not_empty_string(self):
client = Minio('localhost:9000')
client.bucket_exists(' \t \n ')
class MinioResourceContentRepository:
"""
"""
@classmethod
def from_config(cls, config):
return cls(minio_uri=config.MINIO_URI,
access_key=config.MINIO_ACCESS_KEY,
secret_key=config.MINIO_SECRET_KEY,
bucket_list=[config.MINIO_MESSAGE_RESOURCES_BUCKET])
def __init__(self,
minio_uri: str,
access_key: str,
secret_key: str,
bucket_list: Union[list, None] = None):
self.bucket_list = bucket_list or []
try:
self.client = Minio(minio_uri,
access_key=access_key,
secret_key=secret_key,
secure=False)
self.initialize()
except MaxRetryError:
self.client = None
def initialize(self):
for bucket in self.bucket_list:
if not self.client.bucket_exists(bucket):
self.client.make_bucket(bucket)
def _exists(self, bucket: str, object_path: str) -> bool:
stats = self.client.stat_object(
bucket_name=bucket,
object_name=object_path,
)
if stats:
return True
else:
return False
def list_objects(self, bucket: str, path: str):
if self.client:
objects = self.client.list_objects(bucket=bucket, prefix=path)
return objects
else:
raise ClientNotInitializedError
def get_presigned_get_url(self, bucket: str, object_path: str) -> str:
url = self.client.presigned_get_object(
bucket_name=bucket,
object_name=object_path,
)
return url
def get_presigned_put_url(self, bucket: str, object_path: str) -> str:
url = self.client.presigned_put_object(
bucket_name=bucket,
object_name=object_path,
)
return url
def get_presigned_delete_url(self, bucket: str, object_path: str) -> str:
url = self.client.get_presigned_url(
"DELETE",
bucket_name=bucket,
object_name=object_path,
)
return url
from minio import Minio
client = Minio("s3.embl.de", access_key="ysun-user", secret_key="PZx9Djtl7yeV7Kp5k8Gsm7y2SFJ2Gw6W", secure=True)
buckets = client.list_buckets()
for bucket in buckets:
print (bucket).name
print (client.bucket_exists('platybrowser'))
print (client.get_bucket_policy('platybrowser'))
def api_upload(pipeline, model_bucket, model_name=''):
file_dir = os.path.join(basedir, app.config['UPLOAD_FOLDER'])
if not os.path.exists(file_dir):
os.makedirs(file_dir)
client = Minio("10.20.1.54:30020",
access_key="admin",
secret_key="secretsecret",
secure=False)
kubernetes.config.load_kube_config()
api_instance = kubernetes.client.CustomObjectsApi()
api_response = api_instance.get_namespaced_custom_object(
group='kopf.dev',
version='v1',
plural='mlflows',
name=pipeline,
namespace='ml-faas')
f = request.files['file']
if f:
fname = f.filename
f.save(os.path.join(file_dir, fname))
found = client.bucket_exists(UPLOAD_FOLDER)
if not found:
client.make_bucket(UPLOAD_FOLDER)
try:
client.fput_object(UPLOAD_FOLDER, fname,
os.path.join(file_dir, fname))
except S3Error as err:
print(err)
USER_PIPELINE_LIST = []
function_bucket = {}
stage_list = api_response['status']['create_fn']['api_list']
for i in stage_list:
if 'user' in stage_list[i]['rule']:
for j in stage_list[i]['step']:
USER_PIPELINE_LIST.append(j)
function_bucket.update({j: j})
function_bucket.update({model_bucket: model_bucket})
file_uuid = str(uuid.uuid4())
bucket_name = UPLOAD_FOLDER
for i in USER_PIPELINE_LIST:
data = {
'fname': fname,
'file_uuid': file_uuid,
'pipeline': pipeline,
'model': model_name,
'function_name': i,
'function_bucket': function_bucket,
'user': '******'
}
print(data)
r = requests.post(f"""http://10.20.1.54:31112/function/{i}""",
json=data)
if r.status_code == 200:
res = requests.post(
f"""http://*****:*****@10.20.1.54:31112/system/scale-function/{i}""",
json={"replicas": 0})
continue
else:
return jsonify({
'Function': i,
'status_code': r.status_code,
'text': r.text
})
if r.status_code == 200:
uuid_renamed = i + '-' + fname.split(
'.')[0] + '-' + file_uuid + '.' + fname.split('.')[1]
url = client.presigned_get_object(
'lstm-pipeline-model-serving-fun', uuid_renamed)
return jsonify({'url': url})
else:
return 'no url'
