def main():
"""
Functional testing of minio python library.
"""
fake = Factory.create()
client = Minio('s3.amazonaws.com',
os.getenv('ACCESS_KEY'),
os.getenv('SECRET_KEY'))
_http = urllib3.PoolManager(
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where()
)
# Get unique bucket_name, object_name.
bucket_name = uuid.uuid4().__str__()
object_name = uuid.uuid4().__str__()
# Enable trace
# client.trace_on(sys.stderr)
# Make a new bucket.
bucket_name = 'minio-pytest'
print(client.make_bucket(bucket_name))
print(client.make_bucket(bucket_name+'.unique',
location='us-west-1'))
## Check if return codes a valid from server.
try:
client.make_bucket(bucket_name+'.unique',
location='us-west-1')
except ResponseError as err:
if str(err.code) in ['BucketAlreadyOwnedByYou', 'BucketAlreadyExists']:
pass
else:
raise
# Check if bucket was created properly.
print(client.bucket_exists(bucket_name))
print(client.bucket_exists(bucket_name+'.unique'))
# List all buckets.
buckets = client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
with open('testfile', 'wb') as file_data:
file_data.write(fake.text().encode('utf-8'))
file_data.close()
# Put a file
file_stat = os.stat('testfile')
with open('testfile', 'rb') as file_data:
client.put_object(bucket_name, object_name, file_data, file_stat.st_size)
file_data.close()
# Fput a file
print(client.fput_object(bucket_name, object_name+'-f', 'testfile'))
# Fetch stats on your object.
print(client.stat_object(bucket_name, object_name))
# Get a full object
object_data = client.get_object(bucket_name, object_name)
with open('newfile', 'wb') as file_data:
for data in object_data:
file_data.write(data)
file_data.close()
# Get a full object locally.
print(client.fget_object(bucket_name, object_name, 'newfile-f'))
# List all object paths in bucket.
objects = client.list_objects(bucket_name, recursive=True)
for obj in objects:
print(obj.bucket_name, obj.object_name, obj.last_modified, \
obj.etag, obj.size, obj.content_type)
presigned_get_object_url = client.presigned_get_object(bucket_name, object_name)
response = _http.urlopen('GET', presigned_get_object_url)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.get(bucket_name, object_name)
presigned_put_object_url = client.presigned_put_object(bucket_name, object_name)
value = fake.text().encode('utf-8')
data = io.BytesIO(value).getvalue()
response = _http.urlopen('PUT', presigned_put_object_url, body=data)
if response.status != 200:
response_error = ResponseError(response)
raise response_error.put(bucket_name, object_name)
object_data = client.get_object(bucket_name, object_name)
if object_data.read() != value:
raise ValueError('Bytes not equal')
# Post policy.
policy = PostPolicy()
policy.set_bucket_name(bucket_name)
policy.set_key_startswith('objectPrefix/')
expires_date = datetime.utcnow()+timedelta(days=10)
policy.set_expires(expires_date)
print(client.presigned_post_policy(policy))
# Remove an object.
print(client.remove_object(bucket_name, object_name))
print(client.remove_object(bucket_name, object_name+'-f'))
# Remove a bucket. This operation will only work if your bucket is empty.
print(client.remove_bucket(bucket_name))
print(client.remove_bucket(bucket_name+'.unique'))
# Remove temporary files.
os.remove('testfile')
os.remove('newfile')
os.remove('newfile-f')
def pull_last_backup_from_minio(self):
minio_client = Minio(
self.minio_url,
access_key=self.minio_acc,
secret_key=self.minio_sec,
secure=False,
)
download_fname = self.get_minio_list()[0]
print('File for pullin is: ' + download_fname)
# Get a full object
try:
minio_client.fget_object(self.bucket_name, download_fname,
"/tmp/restored/" + download_fname)
except ResponseError as err:
print(err)
#file wait
while not os.path.exists("/tmp/restored/" + download_fname):
time.sleep(1)
try:
if os.path.isfile("/tmp/restored/" + download_fname):
return download_fname
else:
raise ValueError("%s isn't a file!" % "/tmp/restored/" +
download_fname)
except ValueError as err:
print(err)
def minio_get(self):
if not os.path.exists(self.datadir) and not os.path.exists(self.cnfdir):
if not os.path.exists(self.perfix):
if not os.path.exists(f'/root/****-mongo-{self.mongoversion}.tar.gz'):
if self.uploadtype == 'local':
logger.error(f"文件不存在 /root/****-mongo-{self.mongoversion}.tar.gz")
exit()
else:
logger.info(f"正在下载****-mongo-{self.mongoversion}.tar.gz")
from minio import Minio
minioClient = Minio('********:9199',
access_key='****',
secret_key='****',
secure=False,
region='cn-****-01')
minioClient.fget_object('****-db-pkg',
f'****-mongo-{self.mongoversion}.tar.gz',
f'/root/****-mongo-{self.mongoversion}.tar.gz')
logger.info(f"解压缩文件到{self.perfix}")
self.extract(f'/root/****-mongo-{self.mongoversion}.tar.gz', self.perfix)
os.makedirs(self.datadir)
os.makedirs(self.cnfdir)
else:
logger.error(f"文件不为空{self.datadir} 或者 {self.cnfdir}，退出程序。。。。")
exit()
def handle(req):
client = Minio(os.environ['minio_hostname'],
access_key=os.environ['minio_access_key'],
secret_key=os.environ['minio_secret_key'],
secure=False)
client.fget_object("incoming", "q_2.png", "/tmp/q_2.png")
img = cv.imread("/tmp/q_2.png")
img = cv.cvtColor(img, cv.COLOR_BGR2RGB)
h, w = img.shape[:2]
img = cv.resize(img, (h // 2, w // 2), interpolation=cv.INTER_LINEAR)
img = imutils.rotate_bound(img, -90)
cv.imwrite("/tmp/q_2.png", img)
if not client.bucket_exists("processed"):
client.make_bucket("processed")
else:
print(f"Bucket 'processed' already exists.")
client.fput_object("processed", "q_2.png", "/tmp/q_2.png")
return req
def deploy(args):
from watson_machine_learning_client import WatsonMachineLearningAPIClient
from minio import Minio
import os
wml_model_name = args.model_name
wml_scoring_payload = args.scoring_payload
model_uid = args.model_uid
# retrieve credentials
wml_url = getSecret("/app/secrets/wml_url")
wml_username = getSecret("/app/secrets/wml_username")
wml_password = getSecret("/app/secrets/wml_password")
wml_instance_id = getSecret("/app/secrets/wml_instance_id")
cos_endpoint = getSecret("/app/secrets/cos_endpoint")
cos_access_key = getSecret("/app/secrets/cos_access_key")
cos_secret_key = getSecret("/app/secrets/cos_secret_key")
cos_input_bucket = getSecret("/app/secrets/cos_input_bucket")
# set up the WML client
wml_credentials = {
"url": wml_url,
"username": wml_username,
"password": wml_password,
"instance_id": wml_instance_id
}
client = WatsonMachineLearningAPIClient(wml_credentials)
# deploy the model
deployment_name = wml_model_name
deployment_desc = "deployment of %s" % wml_model_name
deployment = client.deployments.create(model_uid, deployment_name,
deployment_desc)
scoring_endpoint = client.deployments.get_scoring_url(deployment)
print("scoring_endpoint: ", scoring_endpoint)
# download scoring payload
payload_file = os.path.join('/app', wml_scoring_payload)
cos = Minio(cos_endpoint,
access_key=cos_access_key,
secret_key=cos_secret_key)
cos.fget_object(cos_input_bucket, wml_scoring_payload, payload_file)
# scoring the deployment
import json
with open(payload_file) as data_file:
test_data = json.load(data_file)
payload = test_data['payload']
data_file.close()
print("Scoring result: ")
result = client.deployments.score(scoring_endpoint, payload)
print(result)
with open("/tmp/output", "w") as f:
print(result, file=f)
f.close()
def main():
"""
This function will be used to get the data from out MinIO and copy it to
our flask folder.
"""
minioClient = Minio(
'127.0.0.1:9000',
access_key='YOUR_ACCESS_KEY',
secret_key='YOUR_SECRET_KEY',
secure=False, #Set secure = True if you are using a https connection
)
#try:
print("hi")
try:
minioClient.fget_object('test', 'asl_training.txt', 'fs.json')
os.rename('../flask/fs.json', '../flask/train.json')
except ResponseError as err:
print(err)
try:
minioClient.fget_object('test', 'asl_test.txt', 'fs.json')
os.rename('../flask/fs.json', '../flask/test.json')
except ResponseError as err:
print(err)
class DataStoreHandler:
def __init__(self, endpoint, access_key, secret_key, bucket_name):
self.bucket_name = bucket_name
self.minioClient = Minio(endpoint=endpoint,
access_key=access_key,
secret_key=secret_key,
secure=False)
print('Connected to DataStore')
try:
self.minioClient.make_bucket(bucket_name)
except BucketAlreadyOwnedByYou as err:
print('BucketAlreadyOwnedByYou')
pass
except BucketAlreadyExists as err:
print('BucketAlreadyExists')
pass
except ResponseError as err:
print('ResponseError')
pass
def upload(self, from_path, to_path):
try:
print("Uploading...")
self.minioClient.fput_object(self.bucket_name, to_path, from_path)
print("Upload Sucess")
except ResponseError as err:
return err
def download(self, from_path, to_path):
try:
print("Downloading...")
self.minioClient.fget_object(self.bucket_name, from_path, to_path)
print("Download Success")
except ResponseError as err:
print(err)
class MinioBlobMechanism:
def __init__(self):
self.minioClient = Minio(os.environ['MINIO_ENDPOINT'],
access_key=os.environ['MINIO_ACCESS_KEY'],
secret_key=os.environ['MINIO_SECRET_KEY'],
secure=False)
if not self.minioClient.bucket_exists(DataBlob):
self.minioClient.make_bucket(DataBlob)
if not self.minioClient.bucket_exists(ToolsBlob):
self.minioClient.make_bucket(ToolsBlob)
if not self.minioClient.bucket_exists(ResultsBlob):
self.minioClient.make_bucket(ResultsBlob)
def download_blob(self, container_name, blob_name):
return self.minioClient.get_object(container_name, blob_name)
def get_blob_to_path(self, container_name, blob_name, file_path):
self.minioClient.fget_object(container_name, blob_name, file_path)
def create_blob_from_path(self, container_name, blob_name, file_path):
self.minioClient.fput_object(container_name, blob_name, file_path)
def list_blobs(self, container_name):
blobObjects = self.minioClient.list_objects(container_name)
objects = [
BlobFile(blobObject.object_name) for blobObject in blobObjects
]
return objects
def delete_blob(self, container_name, blob_name):
self.minioClient.remove_object(container_name, blob_name)
class MinIO(ObjectStorage):
def create_client(self, *args, **kwargs):
ip = kwargs.get('ip', "localhost")
access_key = kwargs.get('access_key', None)
secret_key = kwargs.get('secret_key', None)
self.client = Minio(ip + ':9000',
access_key=access_key,
secret_key=secret_key,
secure=False)
def retrieve_from_bucket(self, source_bucket, file_name):
try:
self.client.fget_object(source_bucket, file_name,
"/tmp/" + file_name)
except Exception as e:
raise Exception(
"There was an error retrieving object from the bucket: " +
str(e))
def store_to_bucket(self, destination_bucket, file_name, img_path):
try:
self.client.fput_object(destination_bucket, file_name, img_path)
except Exception as e:
raise Exception(
"There was an error storing object to the bucket: " + str(e))
def get_artifact_in_minio(workflow_json, step_name, artifact_name, output_dir):
""" Minio is the S3 style object storage server for K8s. This method parses a pipeline run's workflow json to
fetch the output artifact location in Minio server for given step in the pipeline and downloads it
There are two types of nodes in the workflow_json: DAG and pod. DAG corresonds to the whole pipeline and
pod corresponds to a step in the DAG. Check `node["type"] != "DAG"` deals with case where name of component is
part of the pipeline name
"""
s3_data = {}
minio_access_key = "minio"
minio_secret_key = "minio123"
minio_port = utils.get_minio_service_port()
for node in workflow_json["status"]["nodes"].values():
if step_name in node["name"] and node["type"] != "DAG":
for artifact in node["outputs"]["artifacts"]:
if artifact["name"] == artifact_name:
s3_data = artifact["s3"]
minio_client = Minio(
"localhost:{}".format(minio_port),
access_key=minio_access_key,
secret_key=minio_secret_key,
secure=False,
)
output_file = os.path.join(output_dir, artifact_name + ".tgz")
minio_client.fget_object(s3_data["bucket"], s3_data["key"], output_file)
# https://docs.min.io/docs/python-client-api-reference.html#fget_object
return output_file
def pull_image(image_name,host,port=9000,key='e3net',passwd='e3credentials',force_pull=False):
'''
this version use minio as the backed object storage,
it's still convenient and even greater than what I imagined before
'''
bucket_name='images'
object_name=image_name
local_file='%s/%s'%(image_host_dir,image_name)
if force_pull:
try:
os.remove(local_file)
except:
pass
try:
if os.path.isfile(local_file):
return True
client=Minio('%s:%d'%(host,port),
access_key=key,
secret_key=passwd,
secure=False)
client.fget_object(bucket_name,object_name,local_file)
return True
except:
return False
def retrain_model():
logging.info("Retraining model DAG started")
# create minio connection
minio_client = Minio(endpoint="minio:9000",
access_key=os.environ["MINIO_ACCESS_KEY"],
secret_key=os.environ["MINIO_SECRET_KEY"],
secure=False)
# load existing model
with NamedTemporaryFile() as tmp:
minio_client.fget_object("models", "linearSVC.pkl", tmp.name)
model = joblib.load(tmp.name)
# get new data from DB
df = get_df_from_db(minio_client)
category_id_df = df[['category', 'category_id'
]].drop_duplicates().sort_values('category_id')
# get features and labels for model training
tfidf, features, labels = get_features_and_labels(df)
logging.info("Training model")
# train model
model.fit(features, labels)
# store new model
now_formated = datetime.today().strftime('%Y-%m-%d')
store_object_to_minio(minio_client, model, "models",
"linearSVC-{now_formated}.pkl".format(**locals()))
store_object_to_minio(minio_client, tfidf, "features",
"tfidf-{now_formated}.pkl".format(**locals()))
store_object_to_minio(
minio_client, category_id_df, "categories",
"categories-map-{now_formated}.pkl".format(**locals()))
logging.info("Retraining model DAG finished")
def proces_data(access_key='minio_access_key', secret_key='minio_secret_key'):
client = Minio('host.docker.internal:9000',
access_key=access_key,
secret_key=secret_key,
secure=False)
required_buckets = ["my-bucket", "good", "bad"]
for bucket in required_buckets:
if client.bucket_exists(bucket):
print(bucket + " exists")
else:
client.make_bucket(bucket)
objects = client.list_objects("my-bucket", recursive=True)
data_check = DataQualityCheck()
for obj in objects:
# print(obj.__dict__)
obj_name = obj.__dict__['_object_name']
response = client.get_object("my-bucket", obj.__dict__['_object_name'])
client.fget_object("my-bucket", obj_name, obj_name)
good_lines = []
bad_lines = []
with open(obj_name) as f:
for line in f:
line = str(line)
if data_check.is_valid_json(line):
line = data_check.quality_check(line)
else:
bad_lines.append(line)
continue
if len(line['error']) == 0:
good_lines.append(line)
else:
bad_lines.append(line)
badline_file = 'bad_line' + str(datetime.now()) + '.txt'
with open(badline_file, 'w') as filehandle:
for bad in bad_lines:
filehandle.write('%s\n' % bad)
goodline_file = 'good_line' + str(datetime.now()) + '.txt'
with open(goodline_file, 'w') as filehandle:
for good in goodline_file:
filehandle.write('%s\n' % good)
print(good_lines)
if len(good_lines) > 0:
make_pandas(good_lines)
client.fput_object(
"bad",
badline_file,
badline_file,
)
client.fput_object(
"good",
goodline_file,
goodline_file,
)
client.remove_object("my-bucket", obj_name)
def handle(req):
target_field = 'Temp'
direction = True
client = Minio("10.20.1.54:30020",
access_key="admin",
secret_key="secretsecret",
secure=False)
warnings.simplefilter(action="ignore", category=SettingWithCopyWarning)
data = json.loads(req)
fname = data['fname']
file_uuid = data['file_uuid']
pipeline = data['pipeline']
function_name = data['function_name']
function_bucket_list = data['function_bucket']
train_model_func_bucket_name = function_bucket_list[
'random-forest-pipeline-train-model']
data_clean_func_bucket_name = function_bucket_list[
'random-forest-pipeline-data-clean']
data_clean_pipeline_file_name = data_clean_func_bucket_name + '-' + fname.split(
'.')[0] + '-' + file_uuid + '.' + fname.split('.')[1]
uuid_renamed_h5 = train_model_func_bucket_name + '-' + fname.split(
'.')[0] + '-' + file_uuid + '.' + 'joblib'
uuid_renamed_file = os.environ['bucket_name'] + '-' + fname.split(
'.')[0] + '-' + file_uuid + '.' + fname.split('.')[1]
basic_basth = '/home/app'
file_path = os.path.join(basic_basth, fname)
client.fget_object(train_model_func_bucket_name, uuid_renamed_h5,
'/home/app/model.joblib')
client.fget_object(data_clean_func_bucket_name,
data_clean_pipeline_file_name, file_path)
data = pd.read_csv(file_path)
model = modelLoad(model_name='/home/app/model.joblib')
data = newField(data, target_field=target_field)
data_ok = correction(data=data,
past_day=24,
direction=direction,
model=model,
target_field=target_field)
data_ok.to_csv('/home/app/complete-data.csv')
found = client.bucket_exists(os.environ['bucket_name'])
if not found:
client.make_bucket(os.environ['bucket_name'])
client.fput_object(os.environ['bucket_name'], uuid_renamed_file,
'/home/app/complete-data.csv')
return os.environ['bucket_name']
def handle(req):
client = Minio("10.20.1.54:30020",
access_key="admin",
secret_key="secretsecret",
secure=False)
warnings.simplefilter(action="ignore", category=SettingWithCopyWarning)
data = json.loads(req)
fname = data['fname']
file_uuid = data['file_uuid']
pipeline = data['pipeline']
function_name = data['function_name']
function_bucket_list = data['function_bucket']
time_parser_func_bucket_name = function_bucket_list[
'lstm-pipeline-time-parser']
time_parser_func_file_name = time_parser_func_bucket_name + '-' + fname.split(
'.')[0] + '-' + file_uuid + '.' + fname.split('.')[1]
uuid_renamed_file_csv = function_name + '-' + fname.split(
'.')[0] + '-' + file_uuid + '.' + fname.split('.')[1]
# try:
client.fget_object(time_parser_func_bucket_name,
time_parser_func_file_name,
f"""/home/app/{time_parser_func_file_name}""")
# except S3Error as err:
# print(err)
data = pd.read_csv(f"""/home/app/{time_parser_func_file_name}""").copy()
data = data.round(2)
# 每個都要加
for i in range(data.shape[0]):
data["LocalTime"][i] = datetime.strptime(data["LocalTime"][i],
'%Y-%m-%d %H:%M:%S')
data = data.set_index('LocalTime')
#
data, condition = anomalyDetection(data)
data.to_csv(f"""/home/app/{uuid_renamed_file_csv}""")
found = client.bucket_exists(os.environ['bucket_name'])
if not found:
client.make_bucket(os.environ['bucket_name'])
client.fput_object(os.environ['bucket_name'], uuid_renamed_file_csv,
f"""/home/app/{uuid_renamed_file_csv}""")
return os.environ['bucket_name']
def handle(req):
dt = datetime.today()
file = dt.minute
# write_to_file("/tmp/file.mp3",event["body"])
mc = Minio("172.17.67.176:9000",
access_key="AKIAIOSFODNN7EXAMPLE",
secret_key='wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY',
secure=False)
mc.fget_object("aeneas", req, "/tmp/" + str(file) + ".xhtml")
with open("/tmp/" + str(file) + ".xhtml") as f:
contents = f.read()
return contents
class MinioManager:
def __init__(self, run_uuid):
self.client = Minio(
os.environ["MLFLOW_S3_ENDPOINT_URL"].split("//")[1],
os.environ["AWS_ACCESS_KEY_ID"],
os.environ["AWS_SECRET_ACCESS_KEY"],
secure=False)
self.run_uuid = run_uuid
def load_model_weights(self, weights_file_name):
weights_path = f"0/{self.run_uuid}/artifacts/model/data/{weights_file_name}"
self.client.fget_object("mlflow-storage", weights_path,
'./' + weights_file_name)
class Storage:
def __init__(self, args=None):
try:
self.client = Minio(args.minio_host,
access_key=args.minio_key,
secret_key=args.minio_secret,
secure=False)
except Exception as err:
logger.error(str(err))
traceback.print_stack()
self.client = None
self.bucket_name = args.minio_bucket
def list_buckets(self):
buckets = self.client.list_buckets()
for bucket in buckets:
print(bucket.name, bucket.creation_date)
def fget_object(self, bucket_name, obj_name, save_path):
try:
self.client.fget_object(bucket_name, obj_name, save_path)
except ResponseError as err:
logger.error(str(err))
traceback.print_stack()
def list_objects(self, bucket_name, prefix, recursive=True):
objects = self.client.list_objects(bucket_name,
prefix=prefix,
recursive=recursive)
return objects
def fput_obj(self, local_path, minio_path):
try:
self.client.fput_object(self.bucket_name, minio_path, local_path)
except Exception as err:
logger.error(str(err))
traceback.print_stack()
def make_targz(self, dst_path, src_dir):
with tarfile.open(dst_path, "w:gz") as tar:
tar.add(src_dir, arcname=os.path.basename(src_dir))
def tar_and_fput(self, src_dir, dest_obj_name):
try:
tar_path = src_dir + ".gz"
self.make_targz(tar_path, src_dir)
self.fput_obj(tar_path, dest_obj_name)
os.remove(tar_path)
except Exception as err:
logger.error(str(err))
traceback.print_stack()
def run(self):
thread_start_time = datetime.datetime.now()
context = f'Thread # {self.threadid}'
print(f'{context} | Started')
client = Minio(endpoint=self.endpoint,
access_key=self.access_key,
secret_key=self.secret_key)
bucket_exists = client.bucket_exists(self.bucket)
if not bucket_exists:
print(f'Bucket does not exist! [{self.bucket}]')
return
for iteration in range(self.download_iterations):
iteration_no = iteration + 1
iteration_start_time = datetime.datetime.now()
print(f'{context} | Iteration # {iteration_no} | Started')
for i in range(self.num_files):
download_start_time = datetime.datetime.now()
error = None
try:
index = i + 1
object_name = get_filename(self.file_size, index)
object_filepath = f'/tmp/{object_name}.{self.threadid}'
client.fget_object(self.bucket, object_name,
object_filepath)
object_size_bytes = os.path.getsize(object_filepath)
os.remove(object_filepath)
except Exception as e:
error = str(e)
download_end_time = datetime.datetime.now()
download_elapsed_time_seconds = (
download_end_time - download_start_time).total_seconds()
with self.csvfile_lock:
self.csvfile.write(
f'{self.threadid},{iteration_no},{object_name},{object_size_bytes},{download_elapsed_time_seconds},{error}\n'
)
iteration_end_time = datetime.datetime.now()
iteration_elapsed_seconds = (iteration_end_time -
iteration_start_time).total_seconds()
print(
f'{context} | Iteration # {iteration_no} | Finished [{iteration_elapsed_seconds} seconds]'
)
thread_end_time = datetime.datetime.now()
thread_elapsed_seconds = (thread_end_time -
thread_start_time).total_seconds()
print(f'{context} | Finished [{thread_elapsed_seconds} seconds]')
def handle(req):
span_ctx = tracer.extract(Format.HTTP_HEADERS, request.headers)
span_tags = {tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER}
with tracer.start_active_span(f"image-to-text-{req}", child_of=span_ctx, tags=span_tags) as scope:
scope.span.log_kv({"event": "mc_setup"})
client = Minio(os.environ['minio_hostname'],
access_key=os.environ['minio_access_key'],
secret_key=os.environ['minio_secret_key'],
secure=False)
scope.span.log_kv({"event": "get_object"})
filename = f"paragraph_{req}"
client.fget_object("incoming", f"{filename}.png", f"/tmp/{filename}.png")
scope.span.log_kv({"event": "image_to_string"})
text = pytesseract.image_to_string(Image.open(f"/tmp/{filename}.png"), lang="eng")
scope.span.log_kv({"event": "put_object"})
with open(f"/tmp/{filename}.txt", 'w') as file:
file.write(text)
if not client.bucket_exists("paragraphs"):
client.make_bucket("paragraphs")
client.fput_object("paragraphs", f"{filename}.txt", f"/tmp/{filename}.txt")
scope.span.log_kv({"event": "check_size_of_paragraph_bucket"})
if len(list(client.list_objects("paragraphs"))) == 4:
gateway_hostname = os.getenv("gateway_hostname", "gateway.openfaas")
invocation = os.getenv("INVOCATION", "sync")
if invocation == "async":
url = f"http://{gateway_hostname}:8080/async-function/merge"
elif invocation == "sync":
url = f"http://{gateway_hostname}:8080/function/merge"
else:
raise Exception("The only valid INVOCATION value is \"async\". If no value is given, synchronous "
"invocation is used.")
scope.span.log_kv({"event": "invoke_merge_function"})
requests.post(url)
return req
def get_string(minio: Minio, minio_file: str):
# get file from minio
filename_server = "./tmp/%s.txt" % str(uuid.uuid4())
try:
minio.fget_object(BUCKET_NAME, minio_file, filename_server)
except ResponseError as err:
print(err)
# file to string
with open(filename_server, 'r') as file:
data = file.read()
# delete file from server
os.remove(filename_server)
return data
def pull_backup_from_minio(self, download_fname):
minio_client = Minio(
self.minio_url,
access_key=self.minio_acc,
secret_key=self.minio_sec,
secure=False,
)
os.makedirs("/tmp/restored", exist_ok=True)
# Get a full object
try:
minio_client.fget_object(self.bucket_name, download_fname,
"/tmp/restored/" + download_fname)
except ResponseError as err:
print(err)
class MinioClient(BaseStorageClient):
def __init__(self, *args, **kwargs):
hostname = kwargs.get("hostname")
access_key = kwargs.get("access_key")
secret_key = kwargs.get("secret_key")
self.storage_client = Minio(hostname,
access_key=access_key,
secret_key=secret_key,
secure=False)
def stat_file(self, bucket_name, prefix=None, recursive=False):
return self.storage_client.list_objects(bucket_name,
prefix=prefix,
recursive=recursive)
def download_file(self, uri, file_path):
scheme = urlparse(uri).scheme
if scheme != "minio":
raise ValueError(f"Unknown URI scheme: {scheme}")
bucket_name = urlparse(uri).path.split("/")[1]
object_name = "/".join(urlparse(uri).path.split("/")[2:])
return self.storage_client.fget_object(bucket_name, object_name,
file_path)
def upload_file(self, bucket_name, object_name, file_path):
try:
self.storage_client.make_bucket(bucket_name)
except (BucketAlreadyExists, BucketAlreadyOwnedByYou):
pass
self.storage_client.fput_object(bucket_name, object_name, file_path)
return self.storage_client.stat_object(bucket_name, object_name)
def download_artifact(bucket_name, object_name, file_path, url, access_key, secret_key):
minioClient = Minio(url,
access_key=access_key,
secret_key=secret_key,
secure=False
)
return minioClient.fget_object(bucket_name, object_name, file_path)
def main():
# Create a client with the MinIO server playground, its access key
# and secret key.
# Note here secure is set False refer Readme.md for details:
# connection class to test directly
# client = Minio("localhost:9000",
# access_key="minio",
# secret_key="minio123",
# secure=False,
# )
# connetion class inside docker file
try:
client = Minio(
"minio:9000",
access_key="minio",
secret_key="minio123",
secure=False,
)
# Make 'asiatrip' bucket if not exist.
found = client.bucket_exists("buckettest")
print(found)
if found:
response = client.fget_object("buckettest", "images.zip",
"images.zip")
print("Last modified: ")
print(response.last_modified)
except Exception as e:
print(e)
finally:
print('Finally reached!')
def run(self):
logging.debug(
f' Executing PlotS3Bucket2FileMap Filter with params: {self.params}'
)
minioClient = Minio(self.params['endpoint'],
access_key=self.params['access_key'],
secret_key=self.params['secret_key'],
secure=True)
try:
minioClient.fget_object(self.params['bucket_name'],
self.params['object_name'],
self.get_outputs()[0])
except ResponseError as e:
logging.error(f' Error accessing S3 bucket: {e}')
def handle(req):
client = Minio(
"10.20.1.54:30020",
access_key="admin",
secret_key="secretsecret",
secure = False
)
data = json.loads(req)
fname = data['fname']
file_uuid = data['file_uuid']
pipeline = data['pipeline']
function_name = data['function_name']
function_bucket_list = data['function_bucket']
load_data_func_bucket_name = function_bucket_list['random-forest-pipeline-load-data']
load_data_func_file_name = load_data_func_bucket_name + '-' + fname.split('.')[0] + '-' + file_uuid + '.' + fname.split('.')[1]
uuid_renamed_file_csv = function_name + '-' + fname.split('.')[0] + '-' + file_uuid + '.' + fname.split('.')[1]
client.fget_object(load_data_func_bucket_name, load_data_func_file_name, f"""/home/app/{load_data_func_file_name}""")
data = pd.read_csv(f"""/home/app/{load_data_func_file_name}""").copy()
local_time = data['LocalTime'].copy()
time = []
for i in range(local_time.shape[0]):
time_split = local_time.iloc[i].split(' ')
date_split = time_split[0].split('/')
year = '20'+date_split[2]
month = date_split[0]
day = date_split[1]
today = year+'-'+month+'-'+day+' '+time_split[1]
time.append(datetime.strptime(today, '%Y-%m-%d %H:%M:%S'))
data['LocalTime'] = pd.Series(time)
data = data.set_index('LocalTime')
data.to_csv(f"""/home/app/{uuid_renamed_file_csv}""")
found = client.bucket_exists(os.environ['bucket_name'])
if not found:
client.make_bucket(os.environ['bucket_name'])
client.fput_object(os.environ['bucket_name'], uuid_renamed_file_csv, f"""/home/app/{uuid_renamed_file_csv}""")
return os.environ['bucket_name']
class ObjectStore(object):
def __init__(self, options):
from minio import Minio
self.client = Minio(
options['host'],
access_key = options['access_key'],
secret_key = options['secret_key'],
secure = True
)
self.bucket = options['bucket']
def put(self, identifier, local_path):
identifier = identifier.lstrip('/')
self.client.fput_object(self.bucket,identifier,local_path)
def get(self, identifier, local_path):
identifier = identifier.lstrip('/')
self.client.fget_object(self.bucket, identifier, local_path)
def get_file_from_minio(remote_file_path, end_point, access_key, secret_key):
minioClient = Minio(end_point,
access_key=access_key,
secret_key=secret_key,
secure=True)
try:
logger.error("Fetching file {}".format(remote_file_path))
minioClient.fget_object("data", remote_file_path, "file.pkl")
logger.error("Fetched file {}".format(remote_file_path))
logger.error("Saved file.pkl")
file = joblib.load("file.pkl")
os.remove("file.pkl")
logger.error("Returning file")
return file
except Exception as e:
logger.error(e)
logger.error("Couldn't fetch the file")
return None
def minio_download(self,
local_file_path,
endpoint,
access_key,
secret_key,
bucket_name,
object_name,
):
"""
Parameters
----------
local_file_path : str
relative or full path to the (future) local file.
endpoint : str
address of the MinIO server.
access_key : str
user.
secret_key : str
password.
bucket_name : str
preconfigureg bucket name.
object_name : str
a object name - can be None and will be auto-generated
Returns
-------
saved file name
"""
from minio import Minio
try:
start_up = time()
client = Minio(
endpoint=endpoint,
access_key=access_key,
secret_key=secret_key,
secure=False,
)
res = client.fget_object(
bucket_name=bucket_name,
object_name=object_name,
file_path=local_file_path,
)
self.P("Downloaded '{}' from {}/{}/{} in {:.2f}s".format(
local_file_path, endpoint, bucket_name, object_name,
time() - start_up), color='y')
except Exception as e:
self.P(str(e), color='error')
return None
return local_file_path
def handle(req):
span_ctx = tracer.extract(Format.HTTP_HEADERS, request.headers)
span_tags = {tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER}
with tracer.start_active_span('resize', child_of=span_ctx, tags=span_tags):
client = Minio(os.environ['minio_hostname'],
access_key=os.environ['minio_access_key'],
secret_key=os.environ['minio_secret_key'],
secure=False)
client.fget_object("incoming", "image.png", "/tmp/image.png")
img = Image.open("/tmp/image.png")
img = img.resize((img.width // 2, img.height // 2))
img.save("/tmp/image.png")
client.fput_object("processed", "image.png", "/tmp/image.png")
return req
# Initialize minioClient with an endpoint and access/secret keys.
minioClient = Minio(namespace.s3,
access_key=namespace.accesskey,
secret_key=namespace.secretkey,
secure=False)
# create bucket if is not exists
createBucket(minioClient, bucketName)
objectsOld = getObjects(minioClient, bucketName)
filesOld = traverseDir(dir, dir)
# load all objects from server to machine
for object in objectsOld:
objectName = object["name"]
minioClient.fget_object(bucketName, objectName, dir + objectName)
# delete files in machine wich is not found in the server
for file in filesOld:
for object in objectsOld:
if file["name"] == object["name"]:
break
else:
os.remove(dir + file["name"])
time.sleep(1)
objectsOld = getObjects(minioClient, bucketName)
filesOld = traverseDir(dir, dir)
errorPutsFiles = list([])
# -*- coding: utf-8 -*-
# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 MinIO, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname are
# dummy values, please replace them with original values.
from minio import Minio
from minio.error import ResponseError
client = Minio('s3.amazonaws.com',
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETACCESSKEY')
# Get a full object
try:
client.fget_object('my-bucketname', 'my-objectname', 'filepath')
except ResponseError as err:
print(err)
