def main(): """ Functional testing of minio python library. """ fake = Factory.create() client = Minio('s3.amazonaws.com', os.getenv('ACCESS_KEY'), os.getenv('SECRET_KEY')) _http = urllib3.PoolManager( cert_reqs='CERT_REQUIRED', ca_certs=certifi.where() ) # Get unique bucket_name, object_name. bucket_name = uuid.uuid4().__str__() object_name = uuid.uuid4().__str__() # Enable trace # client.trace_on(sys.stderr) # Make a new bucket. bucket_name = 'minio-pytest' print(client.make_bucket(bucket_name)) print(client.make_bucket(bucket_name+'.unique', location='us-west-1')) ## Check if return codes a valid from server. try: client.make_bucket(bucket_name+'.unique', location='us-west-1') except ResponseError as err: if str(err.code) in ['BucketAlreadyOwnedByYou', 'BucketAlreadyExists']: pass else: raise # Check if bucket was created properly. print(client.bucket_exists(bucket_name)) print(client.bucket_exists(bucket_name+'.unique')) # List all buckets. buckets = client.list_buckets() for bucket in buckets: print(bucket.name, bucket.creation_date) with open('testfile', 'wb') as file_data: file_data.write(fake.text().encode('utf-8')) file_data.close() # Put a file file_stat = os.stat('testfile') with open('testfile', 'rb') as file_data: client.put_object(bucket_name, object_name, file_data, file_stat.st_size) file_data.close() # Fput a file print(client.fput_object(bucket_name, object_name+'-f', 'testfile')) # Fetch stats on your object. print(client.stat_object(bucket_name, object_name)) # Get a full object object_data = client.get_object(bucket_name, object_name) with open('newfile', 'wb') as file_data: for data in object_data: file_data.write(data) file_data.close() # Get a full object locally. print(client.fget_object(bucket_name, object_name, 'newfile-f')) # List all object paths in bucket. objects = client.list_objects(bucket_name, recursive=True) for obj in objects: print(obj.bucket_name, obj.object_name, obj.last_modified, \ obj.etag, obj.size, obj.content_type) presigned_get_object_url = client.presigned_get_object(bucket_name, object_name) response = _http.urlopen('GET', presigned_get_object_url) if response.status != 200: response_error = ResponseError(response) raise response_error.get(bucket_name, object_name) presigned_put_object_url = client.presigned_put_object(bucket_name, object_name) value = fake.text().encode('utf-8') data = io.BytesIO(value).getvalue() response = _http.urlopen('PUT', presigned_put_object_url, body=data) if response.status != 200: response_error = ResponseError(response) raise response_error.put(bucket_name, object_name) object_data = client.get_object(bucket_name, object_name) if object_data.read() != value: raise ValueError('Bytes not equal') # Post policy. policy = PostPolicy() policy.set_bucket_name(bucket_name) policy.set_key_startswith('objectPrefix/') expires_date = datetime.utcnow()+timedelta(days=10) policy.set_expires(expires_date) print(client.presigned_post_policy(policy)) # Remove an object. print(client.remove_object(bucket_name, object_name)) print(client.remove_object(bucket_name, object_name+'-f')) # Remove a bucket. This operation will only work if your bucket is empty. print(client.remove_bucket(bucket_name)) print(client.remove_bucket(bucket_name+'.unique')) # Remove temporary files. os.remove('testfile') os.remove('newfile') os.remove('newfile-f')
def pull_last_backup_from_minio(self): minio_client = Minio( self.minio_url, access_key=self.minio_acc, secret_key=self.minio_sec, secure=False, ) download_fname = self.get_minio_list()[0] print('File for pullin is: ' + download_fname) # Get a full object try: minio_client.fget_object(self.bucket_name, download_fname, "/tmp/restored/" + download_fname) except ResponseError as err: print(err) #file wait while not os.path.exists("/tmp/restored/" + download_fname): time.sleep(1) try: if os.path.isfile("/tmp/restored/" + download_fname): return download_fname else: raise ValueError("%s isn't a file!" % "/tmp/restored/" + download_fname) except ValueError as err: print(err)
def minio_get(self): if not os.path.exists(self.datadir) and not os.path.exists(self.cnfdir): if not os.path.exists(self.perfix): if not os.path.exists(f'/root/****-mongo-{self.mongoversion}.tar.gz'): if self.uploadtype == 'local': logger.error(f"文件不存在 /root/****-mongo-{self.mongoversion}.tar.gz") exit() else: logger.info(f"正在下载****-mongo-{self.mongoversion}.tar.gz") from minio import Minio minioClient = Minio('********:9199', access_key='****', secret_key='****', secure=False, region='cn-****-01') minioClient.fget_object('****-db-pkg', f'****-mongo-{self.mongoversion}.tar.gz', f'/root/****-mongo-{self.mongoversion}.tar.gz') logger.info(f"解压缩文件到{self.perfix}") self.extract(f'/root/****-mongo-{self.mongoversion}.tar.gz', self.perfix) os.makedirs(self.datadir) os.makedirs(self.cnfdir) else: logger.error(f"文件不为空{self.datadir} 或者 {self.cnfdir},退出程序。。。。") exit()
def handle(req): client = Minio(os.environ['minio_hostname'], access_key=os.environ['minio_access_key'], secret_key=os.environ['minio_secret_key'], secure=False) client.fget_object("incoming", "q_2.png", "/tmp/q_2.png") img = cv.imread("/tmp/q_2.png") img = cv.cvtColor(img, cv.COLOR_BGR2RGB) h, w = img.shape[:2] img = cv.resize(img, (h // 2, w // 2), interpolation=cv.INTER_LINEAR) img = imutils.rotate_bound(img, -90) cv.imwrite("/tmp/q_2.png", img) if not client.bucket_exists("processed"): client.make_bucket("processed") else: print(f"Bucket 'processed' already exists.") client.fput_object("processed", "q_2.png", "/tmp/q_2.png") return req
def deploy(args): from watson_machine_learning_client import WatsonMachineLearningAPIClient from minio import Minio import os wml_model_name = args.model_name wml_scoring_payload = args.scoring_payload model_uid = args.model_uid # retrieve credentials wml_url = getSecret("/app/secrets/wml_url") wml_username = getSecret("/app/secrets/wml_username") wml_password = getSecret("/app/secrets/wml_password") wml_instance_id = getSecret("/app/secrets/wml_instance_id") cos_endpoint = getSecret("/app/secrets/cos_endpoint") cos_access_key = getSecret("/app/secrets/cos_access_key") cos_secret_key = getSecret("/app/secrets/cos_secret_key") cos_input_bucket = getSecret("/app/secrets/cos_input_bucket") # set up the WML client wml_credentials = { "url": wml_url, "username": wml_username, "password": wml_password, "instance_id": wml_instance_id } client = WatsonMachineLearningAPIClient(wml_credentials) # deploy the model deployment_name = wml_model_name deployment_desc = "deployment of %s" % wml_model_name deployment = client.deployments.create(model_uid, deployment_name, deployment_desc) scoring_endpoint = client.deployments.get_scoring_url(deployment) print("scoring_endpoint: ", scoring_endpoint) # download scoring payload payload_file = os.path.join('/app', wml_scoring_payload) cos = Minio(cos_endpoint, access_key=cos_access_key, secret_key=cos_secret_key) cos.fget_object(cos_input_bucket, wml_scoring_payload, payload_file) # scoring the deployment import json with open(payload_file) as data_file: test_data = json.load(data_file) payload = test_data['payload'] data_file.close() print("Scoring result: ") result = client.deployments.score(scoring_endpoint, payload) print(result) with open("/tmp/output", "w") as f: print(result, file=f) f.close()
def main(): """ This function will be used to get the data from out MinIO and copy it to our flask folder. """ minioClient = Minio( '127.0.0.1:9000', access_key='YOUR_ACCESS_KEY', secret_key='YOUR_SECRET_KEY', secure=False, #Set secure = True if you are using a https connection ) #try: print("hi") try: minioClient.fget_object('test', 'asl_training.txt', 'fs.json') os.rename('../flask/fs.json', '../flask/train.json') except ResponseError as err: print(err) try: minioClient.fget_object('test', 'asl_test.txt', 'fs.json') os.rename('../flask/fs.json', '../flask/test.json') except ResponseError as err: print(err)
class DataStoreHandler: def __init__(self, endpoint, access_key, secret_key, bucket_name): self.bucket_name = bucket_name self.minioClient = Minio(endpoint=endpoint, access_key=access_key, secret_key=secret_key, secure=False) print('Connected to DataStore') try: self.minioClient.make_bucket(bucket_name) except BucketAlreadyOwnedByYou as err: print('BucketAlreadyOwnedByYou') pass except BucketAlreadyExists as err: print('BucketAlreadyExists') pass except ResponseError as err: print('ResponseError') pass def upload(self, from_path, to_path): try: print("Uploading...") self.minioClient.fput_object(self.bucket_name, to_path, from_path) print("Upload Sucess") except ResponseError as err: return err def download(self, from_path, to_path): try: print("Downloading...") self.minioClient.fget_object(self.bucket_name, from_path, to_path) print("Download Success") except ResponseError as err: print(err)
class MinioBlobMechanism: def __init__(self): self.minioClient = Minio(os.environ['MINIO_ENDPOINT'], access_key=os.environ['MINIO_ACCESS_KEY'], secret_key=os.environ['MINIO_SECRET_KEY'], secure=False) if not self.minioClient.bucket_exists(DataBlob): self.minioClient.make_bucket(DataBlob) if not self.minioClient.bucket_exists(ToolsBlob): self.minioClient.make_bucket(ToolsBlob) if not self.minioClient.bucket_exists(ResultsBlob): self.minioClient.make_bucket(ResultsBlob) def download_blob(self, container_name, blob_name): return self.minioClient.get_object(container_name, blob_name) def get_blob_to_path(self, container_name, blob_name, file_path): self.minioClient.fget_object(container_name, blob_name, file_path) def create_blob_from_path(self, container_name, blob_name, file_path): self.minioClient.fput_object(container_name, blob_name, file_path) def list_blobs(self, container_name): blobObjects = self.minioClient.list_objects(container_name) objects = [ BlobFile(blobObject.object_name) for blobObject in blobObjects ] return objects def delete_blob(self, container_name, blob_name): self.minioClient.remove_object(container_name, blob_name)
class MinIO(ObjectStorage): def create_client(self, *args, **kwargs): ip = kwargs.get('ip', "localhost") access_key = kwargs.get('access_key', None) secret_key = kwargs.get('secret_key', None) self.client = Minio(ip + ':9000', access_key=access_key, secret_key=secret_key, secure=False) def retrieve_from_bucket(self, source_bucket, file_name): try: self.client.fget_object(source_bucket, file_name, "/tmp/" + file_name) except Exception as e: raise Exception( "There was an error retrieving object from the bucket: " + str(e)) def store_to_bucket(self, destination_bucket, file_name, img_path): try: self.client.fput_object(destination_bucket, file_name, img_path) except Exception as e: raise Exception( "There was an error storing object to the bucket: " + str(e))
def get_artifact_in_minio(workflow_json, step_name, artifact_name, output_dir): """ Minio is the S3 style object storage server for K8s. This method parses a pipeline run's workflow json to fetch the output artifact location in Minio server for given step in the pipeline and downloads it There are two types of nodes in the workflow_json: DAG and pod. DAG corresonds to the whole pipeline and pod corresponds to a step in the DAG. Check `node["type"] != "DAG"` deals with case where name of component is part of the pipeline name """ s3_data = {} minio_access_key = "minio" minio_secret_key = "minio123" minio_port = utils.get_minio_service_port() for node in workflow_json["status"]["nodes"].values(): if step_name in node["name"] and node["type"] != "DAG": for artifact in node["outputs"]["artifacts"]: if artifact["name"] == artifact_name: s3_data = artifact["s3"] minio_client = Minio( "localhost:{}".format(minio_port), access_key=minio_access_key, secret_key=minio_secret_key, secure=False, ) output_file = os.path.join(output_dir, artifact_name + ".tgz") minio_client.fget_object(s3_data["bucket"], s3_data["key"], output_file) # https://docs.min.io/docs/python-client-api-reference.html#fget_object return output_file
def pull_image(image_name,host,port=9000,key='e3net',passwd='e3credentials',force_pull=False): ''' this version use minio as the backed object storage, it's still convenient and even greater than what I imagined before ''' bucket_name='images' object_name=image_name local_file='%s/%s'%(image_host_dir,image_name) if force_pull: try: os.remove(local_file) except: pass try: if os.path.isfile(local_file): return True client=Minio('%s:%d'%(host,port), access_key=key, secret_key=passwd, secure=False) client.fget_object(bucket_name,object_name,local_file) return True except: return False
def retrain_model(): logging.info("Retraining model DAG started") # create minio connection minio_client = Minio(endpoint="minio:9000", access_key=os.environ["MINIO_ACCESS_KEY"], secret_key=os.environ["MINIO_SECRET_KEY"], secure=False) # load existing model with NamedTemporaryFile() as tmp: minio_client.fget_object("models", "linearSVC.pkl", tmp.name) model = joblib.load(tmp.name) # get new data from DB df = get_df_from_db(minio_client) category_id_df = df[['category', 'category_id' ]].drop_duplicates().sort_values('category_id') # get features and labels for model training tfidf, features, labels = get_features_and_labels(df) logging.info("Training model") # train model model.fit(features, labels) # store new model now_formated = datetime.today().strftime('%Y-%m-%d') store_object_to_minio(minio_client, model, "models", "linearSVC-{now_formated}.pkl".format(**locals())) store_object_to_minio(minio_client, tfidf, "features", "tfidf-{now_formated}.pkl".format(**locals())) store_object_to_minio( minio_client, category_id_df, "categories", "categories-map-{now_formated}.pkl".format(**locals())) logging.info("Retraining model DAG finished")
def proces_data(access_key='minio_access_key', secret_key='minio_secret_key'): client = Minio('host.docker.internal:9000', access_key=access_key, secret_key=secret_key, secure=False) required_buckets = ["my-bucket", "good", "bad"] for bucket in required_buckets: if client.bucket_exists(bucket): print(bucket + " exists") else: client.make_bucket(bucket) objects = client.list_objects("my-bucket", recursive=True) data_check = DataQualityCheck() for obj in objects: # print(obj.__dict__) obj_name = obj.__dict__['_object_name'] response = client.get_object("my-bucket", obj.__dict__['_object_name']) client.fget_object("my-bucket", obj_name, obj_name) good_lines = [] bad_lines = [] with open(obj_name) as f: for line in f: line = str(line) if data_check.is_valid_json(line): line = data_check.quality_check(line) else: bad_lines.append(line) continue if len(line['error']) == 0: good_lines.append(line) else: bad_lines.append(line) badline_file = 'bad_line' + str(datetime.now()) + '.txt' with open(badline_file, 'w') as filehandle: for bad in bad_lines: filehandle.write('%s\n' % bad) goodline_file = 'good_line' + str(datetime.now()) + '.txt' with open(goodline_file, 'w') as filehandle: for good in goodline_file: filehandle.write('%s\n' % good) print(good_lines) if len(good_lines) > 0: make_pandas(good_lines) client.fput_object( "bad", badline_file, badline_file, ) client.fput_object( "good", goodline_file, goodline_file, ) client.remove_object("my-bucket", obj_name)
def handle(req): target_field = 'Temp' direction = True client = Minio("10.20.1.54:30020", access_key="admin", secret_key="secretsecret", secure=False) warnings.simplefilter(action="ignore", category=SettingWithCopyWarning) data = json.loads(req) fname = data['fname'] file_uuid = data['file_uuid'] pipeline = data['pipeline'] function_name = data['function_name'] function_bucket_list = data['function_bucket'] train_model_func_bucket_name = function_bucket_list[ 'random-forest-pipeline-train-model'] data_clean_func_bucket_name = function_bucket_list[ 'random-forest-pipeline-data-clean'] data_clean_pipeline_file_name = data_clean_func_bucket_name + '-' + fname.split( '.')[0] + '-' + file_uuid + '.' + fname.split('.')[1] uuid_renamed_h5 = train_model_func_bucket_name + '-' + fname.split( '.')[0] + '-' + file_uuid + '.' + 'joblib' uuid_renamed_file = os.environ['bucket_name'] + '-' + fname.split( '.')[0] + '-' + file_uuid + '.' + fname.split('.')[1] basic_basth = '/home/app' file_path = os.path.join(basic_basth, fname) client.fget_object(train_model_func_bucket_name, uuid_renamed_h5, '/home/app/model.joblib') client.fget_object(data_clean_func_bucket_name, data_clean_pipeline_file_name, file_path) data = pd.read_csv(file_path) model = modelLoad(model_name='/home/app/model.joblib') data = newField(data, target_field=target_field) data_ok = correction(data=data, past_day=24, direction=direction, model=model, target_field=target_field) data_ok.to_csv('/home/app/complete-data.csv') found = client.bucket_exists(os.environ['bucket_name']) if not found: client.make_bucket(os.environ['bucket_name']) client.fput_object(os.environ['bucket_name'], uuid_renamed_file, '/home/app/complete-data.csv') return os.environ['bucket_name']
def handle(req): client = Minio("10.20.1.54:30020", access_key="admin", secret_key="secretsecret", secure=False) warnings.simplefilter(action="ignore", category=SettingWithCopyWarning) data = json.loads(req) fname = data['fname'] file_uuid = data['file_uuid'] pipeline = data['pipeline'] function_name = data['function_name'] function_bucket_list = data['function_bucket'] time_parser_func_bucket_name = function_bucket_list[ 'lstm-pipeline-time-parser'] time_parser_func_file_name = time_parser_func_bucket_name + '-' + fname.split( '.')[0] + '-' + file_uuid + '.' + fname.split('.')[1] uuid_renamed_file_csv = function_name + '-' + fname.split( '.')[0] + '-' + file_uuid + '.' + fname.split('.')[1] # try: client.fget_object(time_parser_func_bucket_name, time_parser_func_file_name, f"""/home/app/{time_parser_func_file_name}""") # except S3Error as err: # print(err) data = pd.read_csv(f"""/home/app/{time_parser_func_file_name}""").copy() data = data.round(2) # 每個都要加 for i in range(data.shape[0]): data["LocalTime"][i] = datetime.strptime(data["LocalTime"][i], '%Y-%m-%d %H:%M:%S') data = data.set_index('LocalTime') # data, condition = anomalyDetection(data) data.to_csv(f"""/home/app/{uuid_renamed_file_csv}""") found = client.bucket_exists(os.environ['bucket_name']) if not found: client.make_bucket(os.environ['bucket_name']) client.fput_object(os.environ['bucket_name'], uuid_renamed_file_csv, f"""/home/app/{uuid_renamed_file_csv}""") return os.environ['bucket_name']
def handle(req): dt = datetime.today() file = dt.minute # write_to_file("/tmp/file.mp3",event["body"]) mc = Minio("172.17.67.176:9000", access_key="AKIAIOSFODNN7EXAMPLE", secret_key='wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', secure=False) mc.fget_object("aeneas", req, "/tmp/" + str(file) + ".xhtml") with open("/tmp/" + str(file) + ".xhtml") as f: contents = f.read() return contents
class MinioManager: def __init__(self, run_uuid): self.client = Minio( os.environ["MLFLOW_S3_ENDPOINT_URL"].split("//")[1], os.environ["AWS_ACCESS_KEY_ID"], os.environ["AWS_SECRET_ACCESS_KEY"], secure=False) self.run_uuid = run_uuid def load_model_weights(self, weights_file_name): weights_path = f"0/{self.run_uuid}/artifacts/model/data/{weights_file_name}" self.client.fget_object("mlflow-storage", weights_path, './' + weights_file_name)
class Storage: def __init__(self, args=None): try: self.client = Minio(args.minio_host, access_key=args.minio_key, secret_key=args.minio_secret, secure=False) except Exception as err: logger.error(str(err)) traceback.print_stack() self.client = None self.bucket_name = args.minio_bucket def list_buckets(self): buckets = self.client.list_buckets() for bucket in buckets: print(bucket.name, bucket.creation_date) def fget_object(self, bucket_name, obj_name, save_path): try: self.client.fget_object(bucket_name, obj_name, save_path) except ResponseError as err: logger.error(str(err)) traceback.print_stack() def list_objects(self, bucket_name, prefix, recursive=True): objects = self.client.list_objects(bucket_name, prefix=prefix, recursive=recursive) return objects def fput_obj(self, local_path, minio_path): try: self.client.fput_object(self.bucket_name, minio_path, local_path) except Exception as err: logger.error(str(err)) traceback.print_stack() def make_targz(self, dst_path, src_dir): with tarfile.open(dst_path, "w:gz") as tar: tar.add(src_dir, arcname=os.path.basename(src_dir)) def tar_and_fput(self, src_dir, dest_obj_name): try: tar_path = src_dir + ".gz" self.make_targz(tar_path, src_dir) self.fput_obj(tar_path, dest_obj_name) os.remove(tar_path) except Exception as err: logger.error(str(err)) traceback.print_stack()
def run(self): thread_start_time = datetime.datetime.now() context = f'Thread # {self.threadid}' print(f'{context} | Started') client = Minio(endpoint=self.endpoint, access_key=self.access_key, secret_key=self.secret_key) bucket_exists = client.bucket_exists(self.bucket) if not bucket_exists: print(f'Bucket does not exist! [{self.bucket}]') return for iteration in range(self.download_iterations): iteration_no = iteration + 1 iteration_start_time = datetime.datetime.now() print(f'{context} | Iteration # {iteration_no} | Started') for i in range(self.num_files): download_start_time = datetime.datetime.now() error = None try: index = i + 1 object_name = get_filename(self.file_size, index) object_filepath = f'/tmp/{object_name}.{self.threadid}' client.fget_object(self.bucket, object_name, object_filepath) object_size_bytes = os.path.getsize(object_filepath) os.remove(object_filepath) except Exception as e: error = str(e) download_end_time = datetime.datetime.now() download_elapsed_time_seconds = ( download_end_time - download_start_time).total_seconds() with self.csvfile_lock: self.csvfile.write( f'{self.threadid},{iteration_no},{object_name},{object_size_bytes},{download_elapsed_time_seconds},{error}\n' ) iteration_end_time = datetime.datetime.now() iteration_elapsed_seconds = (iteration_end_time - iteration_start_time).total_seconds() print( f'{context} | Iteration # {iteration_no} | Finished [{iteration_elapsed_seconds} seconds]' ) thread_end_time = datetime.datetime.now() thread_elapsed_seconds = (thread_end_time - thread_start_time).total_seconds() print(f'{context} | Finished [{thread_elapsed_seconds} seconds]')
def handle(req): span_ctx = tracer.extract(Format.HTTP_HEADERS, request.headers) span_tags = {tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER} with tracer.start_active_span(f"image-to-text-{req}", child_of=span_ctx, tags=span_tags) as scope: scope.span.log_kv({"event": "mc_setup"}) client = Minio(os.environ['minio_hostname'], access_key=os.environ['minio_access_key'], secret_key=os.environ['minio_secret_key'], secure=False) scope.span.log_kv({"event": "get_object"}) filename = f"paragraph_{req}" client.fget_object("incoming", f"{filename}.png", f"/tmp/{filename}.png") scope.span.log_kv({"event": "image_to_string"}) text = pytesseract.image_to_string(Image.open(f"/tmp/{filename}.png"), lang="eng") scope.span.log_kv({"event": "put_object"}) with open(f"/tmp/{filename}.txt", 'w') as file: file.write(text) if not client.bucket_exists("paragraphs"): client.make_bucket("paragraphs") client.fput_object("paragraphs", f"{filename}.txt", f"/tmp/{filename}.txt") scope.span.log_kv({"event": "check_size_of_paragraph_bucket"}) if len(list(client.list_objects("paragraphs"))) == 4: gateway_hostname = os.getenv("gateway_hostname", "gateway.openfaas") invocation = os.getenv("INVOCATION", "sync") if invocation == "async": url = f"http://{gateway_hostname}:8080/async-function/merge" elif invocation == "sync": url = f"http://{gateway_hostname}:8080/function/merge" else: raise Exception("The only valid INVOCATION value is \"async\". If no value is given, synchronous " "invocation is used.") scope.span.log_kv({"event": "invoke_merge_function"}) requests.post(url) return req
def get_string(minio: Minio, minio_file: str): # get file from minio filename_server = "./tmp/%s.txt" % str(uuid.uuid4()) try: minio.fget_object(BUCKET_NAME, minio_file, filename_server) except ResponseError as err: print(err) # file to string with open(filename_server, 'r') as file: data = file.read() # delete file from server os.remove(filename_server) return data
def pull_backup_from_minio(self, download_fname): minio_client = Minio( self.minio_url, access_key=self.minio_acc, secret_key=self.minio_sec, secure=False, ) os.makedirs("/tmp/restored", exist_ok=True) # Get a full object try: minio_client.fget_object(self.bucket_name, download_fname, "/tmp/restored/" + download_fname) except ResponseError as err: print(err)
class MinioClient(BaseStorageClient): def __init__(self, *args, **kwargs): hostname = kwargs.get("hostname") access_key = kwargs.get("access_key") secret_key = kwargs.get("secret_key") self.storage_client = Minio(hostname, access_key=access_key, secret_key=secret_key, secure=False) def stat_file(self, bucket_name, prefix=None, recursive=False): return self.storage_client.list_objects(bucket_name, prefix=prefix, recursive=recursive) def download_file(self, uri, file_path): scheme = urlparse(uri).scheme if scheme != "minio": raise ValueError(f"Unknown URI scheme: {scheme}") bucket_name = urlparse(uri).path.split("/")[1] object_name = "/".join(urlparse(uri).path.split("/")[2:]) return self.storage_client.fget_object(bucket_name, object_name, file_path) def upload_file(self, bucket_name, object_name, file_path): try: self.storage_client.make_bucket(bucket_name) except (BucketAlreadyExists, BucketAlreadyOwnedByYou): pass self.storage_client.fput_object(bucket_name, object_name, file_path) return self.storage_client.stat_object(bucket_name, object_name)
def download_artifact(bucket_name, object_name, file_path, url, access_key, secret_key): minioClient = Minio(url, access_key=access_key, secret_key=secret_key, secure=False ) return minioClient.fget_object(bucket_name, object_name, file_path)
def main(): # Create a client with the MinIO server playground, its access key # and secret key. # Note here secure is set False refer Readme.md for details: # connection class to test directly # client = Minio("localhost:9000", # access_key="minio", # secret_key="minio123", # secure=False, # ) # connetion class inside docker file try: client = Minio( "minio:9000", access_key="minio", secret_key="minio123", secure=False, ) # Make 'asiatrip' bucket if not exist. found = client.bucket_exists("buckettest") print(found) if found: response = client.fget_object("buckettest", "images.zip", "images.zip") print("Last modified: ") print(response.last_modified) except Exception as e: print(e) finally: print('Finally reached!')
def run(self): logging.debug( f' Executing PlotS3Bucket2FileMap Filter with params: {self.params}' ) minioClient = Minio(self.params['endpoint'], access_key=self.params['access_key'], secret_key=self.params['secret_key'], secure=True) try: minioClient.fget_object(self.params['bucket_name'], self.params['object_name'], self.get_outputs()[0]) except ResponseError as e: logging.error(f' Error accessing S3 bucket: {e}')
def handle(req): client = Minio( "10.20.1.54:30020", access_key="admin", secret_key="secretsecret", secure = False ) data = json.loads(req) fname = data['fname'] file_uuid = data['file_uuid'] pipeline = data['pipeline'] function_name = data['function_name'] function_bucket_list = data['function_bucket'] load_data_func_bucket_name = function_bucket_list['random-forest-pipeline-load-data'] load_data_func_file_name = load_data_func_bucket_name + '-' + fname.split('.')[0] + '-' + file_uuid + '.' + fname.split('.')[1] uuid_renamed_file_csv = function_name + '-' + fname.split('.')[0] + '-' + file_uuid + '.' + fname.split('.')[1] client.fget_object(load_data_func_bucket_name, load_data_func_file_name, f"""/home/app/{load_data_func_file_name}""") data = pd.read_csv(f"""/home/app/{load_data_func_file_name}""").copy() local_time = data['LocalTime'].copy() time = [] for i in range(local_time.shape[0]): time_split = local_time.iloc[i].split(' ') date_split = time_split[0].split('/') year = '20'+date_split[2] month = date_split[0] day = date_split[1] today = year+'-'+month+'-'+day+' '+time_split[1] time.append(datetime.strptime(today, '%Y-%m-%d %H:%M:%S')) data['LocalTime'] = pd.Series(time) data = data.set_index('LocalTime') data.to_csv(f"""/home/app/{uuid_renamed_file_csv}""") found = client.bucket_exists(os.environ['bucket_name']) if not found: client.make_bucket(os.environ['bucket_name']) client.fput_object(os.environ['bucket_name'], uuid_renamed_file_csv, f"""/home/app/{uuid_renamed_file_csv}""") return os.environ['bucket_name']
class ObjectStore(object): def __init__(self, options): from minio import Minio self.client = Minio( options['host'], access_key = options['access_key'], secret_key = options['secret_key'], secure = True ) self.bucket = options['bucket'] def put(self, identifier, local_path): identifier = identifier.lstrip('/') self.client.fput_object(self.bucket,identifier,local_path) def get(self, identifier, local_path): identifier = identifier.lstrip('/') self.client.fget_object(self.bucket, identifier, local_path)
def get_file_from_minio(remote_file_path, end_point, access_key, secret_key): minioClient = Minio(end_point, access_key=access_key, secret_key=secret_key, secure=True) try: logger.error("Fetching file {}".format(remote_file_path)) minioClient.fget_object("data", remote_file_path, "file.pkl") logger.error("Fetched file {}".format(remote_file_path)) logger.error("Saved file.pkl") file = joblib.load("file.pkl") os.remove("file.pkl") logger.error("Returning file") return file except Exception as e: logger.error(e) logger.error("Couldn't fetch the file") return None
def minio_download(self, local_file_path, endpoint, access_key, secret_key, bucket_name, object_name, ): """ Parameters ---------- local_file_path : str relative or full path to the (future) local file. endpoint : str address of the MinIO server. access_key : str user. secret_key : str password. bucket_name : str preconfigureg bucket name. object_name : str a object name - can be None and will be auto-generated Returns ------- saved file name """ from minio import Minio try: start_up = time() client = Minio( endpoint=endpoint, access_key=access_key, secret_key=secret_key, secure=False, ) res = client.fget_object( bucket_name=bucket_name, object_name=object_name, file_path=local_file_path, ) self.P("Downloaded '{}' from {}/{}/{} in {:.2f}s".format( local_file_path, endpoint, bucket_name, object_name, time() - start_up), color='y') except Exception as e: self.P(str(e), color='error') return None return local_file_path
def handle(req): span_ctx = tracer.extract(Format.HTTP_HEADERS, request.headers) span_tags = {tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER} with tracer.start_active_span('resize', child_of=span_ctx, tags=span_tags): client = Minio(os.environ['minio_hostname'], access_key=os.environ['minio_access_key'], secret_key=os.environ['minio_secret_key'], secure=False) client.fget_object("incoming", "image.png", "/tmp/image.png") img = Image.open("/tmp/image.png") img = img.resize((img.width // 2, img.height // 2)) img.save("/tmp/image.png") client.fput_object("processed", "image.png", "/tmp/image.png") return req
# Initialize minioClient with an endpoint and access/secret keys. minioClient = Minio(namespace.s3, access_key=namespace.accesskey, secret_key=namespace.secretkey, secure=False) # create bucket if is not exists createBucket(minioClient, bucketName) objectsOld = getObjects(minioClient, bucketName) filesOld = traverseDir(dir, dir) # load all objects from server to machine for object in objectsOld: objectName = object["name"] minioClient.fget_object(bucketName, objectName, dir + objectName) # delete files in machine wich is not found in the server for file in filesOld: for object in objectsOld: if file["name"] == object["name"]: break else: os.remove(dir + file["name"]) time.sleep(1) objectsOld = getObjects(minioClient, bucketName) filesOld = traverseDir(dir, dir) errorPutsFiles = list([])
# -*- coding: utf-8 -*- # MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 MinIO, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname and my-objectname are # dummy values, please replace them with original values. from minio import Minio from minio.error import ResponseError client = Minio('s3.amazonaws.com', access_key='YOUR-ACCESSKEYID', secret_key='YOUR-SECRETACCESSKEY') # Get a full object try: client.fget_object('my-bucketname', 'my-objectname', 'filepath') except ResponseError as err: print(err)