def run(step): minio_addr = step['minio_addr'] minio_addr = os.environ.get('KRAKEN_MINIO_ADDR', minio_addr) minio_bucket = step['minio_bucket'] minio_access_key = step['minio_access_key'] minio_secret_key = step['minio_secret_key'] job_id = step['job_id'] action = step['action'] mc = minio.Minio(minio_addr, access_key=minio_access_key, secret_key=minio_secret_key, secure=False) # check connection try: mc.bucket_exists(minio_bucket) except Exception as e: log.exception('problem with connecting to minio %s', minio_addr) msg = 'problem with connecting to minio %s: %s' % (minio_addr, str(e)) return 1, msg if action == 'save': paths = step['paths'] cache_key, minio_folder = step['minio_folder'].popitem() dest_folder = '%s/%s' % (minio_folder, job_id) status, msg = _save_to_cache(mc, cache_key, minio_bucket, dest_folder, paths) else: minio_folders = step['minio_folders'] status, msg = _restore_from_cache(mc, minio_bucket, minio_folders) return status, msg
def getMinioClient(): minioClient = minio.Minio('s3.docker:9000', access_key=S3_ACCESS_KEY, secret_key=S3_SECRET_KEY, secure=False) return minioClient
def __init__(self, endpoint=None, accesskey=None, secretkey=None, bucket=None, secure=False): if bucket: self.default_bucket = bucket else: self.default_bucket = os.environ.get(envBucket, "hyperml") if not accesskey: accesskey = os.environ.get(envAccessKey) secretkey = os.environ.get(envSecretKey) if not endpoint: endpoint = os.environ.get(envURL) if not secure: if os.environ.get(envSecure, "True") == "True": secure = True else: secure = False print('minio:', endpoint, accesskey, secretkey, secure) self.client = minio.Minio(endpoint=endpoint, access_key=accesskey, secret_key=secretkey, secure=secure) self.access_key = accesskey self.secret_key = secretkey
def get_minio_client(url, access_key, secret_key, region='eu-central-1', bucket=None): """ Wrapper for getting minio.Minio class. Can optionally generate a bucket if parameter is provided. """ parsed = urlparse(url) secure = parsed.scheme == 'https' mc = minio.Minio(parsed.netloc, access_key, secret_key, region=region, secure=secure) if not bucket: return mc if not mc.bucket_exists(bucket): mc.make_bucket(bucket, location=region) return mc
def __init__( self, host_url: str, access_key: Optional[str] = None, secret_key: Optional[str] = None, default_bucket: Optional[str] = "sandcrawler", ): """ host is minio connection string (host:port) access and secret key are as expected default_bucket can be supplied so that it doesn't need to be repeated for each function call Example config: host="localhost:9000", access_key=os.environ['MINIO_ACCESS_KEY'], secret_key=os.environ['MINIO_SECRET_KEY'], """ self.mc = minio.Minio( host_url, access_key=access_key, secret_key=secret_key, secure=False, ) self.default_bucket = default_bucket
def _create_base_url_client(client: minio.Minio, bucket_name: str, base_url: str): """ Clone a Minio client, using a different endpoint from `base_url`. """ base_url_parts = urlsplit(base_url) # Clone from the normal client, but with base_url as the endpoint base_url_client = minio.Minio( base_url_parts.netloc, access_key=client._access_key, secret_key=client._secret_key, session_token=client._session_token, secure=base_url_parts.scheme == "https", # The bucket region may be auto-detected by client (via an HTTP # request), so don't just use client._region region=client._get_bucket_region(bucket_name), http_client=client._http, ) if hasattr(client, "_credentials"): # Client credentials do not exist prior to minio-py 5.0.7, but # they should be reused if possible base_url_client._credentials = client._credentials return base_url_client
def run( files_path, minio_host, minio_access_key, minio_secret_key, minio_bucket, google_credential_path, ): google_credentials = ServiceAccountCredentials.from_json_keyfile_name( google_credential_path, scopes=SCOPES) gauth = GoogleAuth() gauth.credentials = google_credentials gauth.ServiceAuth() drive = GoogleDrive(gauth) gspread_service = gspread.service_account(filename=google_credential_path) with open(files_path) as files_file: files = yaml.load(files_file, Loader=yaml.FullLoader) minio_client = minio.Minio(minio_host, minio_access_key, minio_secret_key, secure=False) download_dir = tempfile.mkdtemp() for file in files['files']: downloaded_path = get_file(drive, gspread_service, download_dir, file) minio_client.fput_object(minio_bucket, os.path.basename(downloaded_path), downloaded_path)
def connect_storage(config): import minio return minio.Minio(_conf_value(config, 'STORAGE_ENDPOINT'), access_key=_conf_value(config, 'STORAGE_ACCESS_KEY'), secret_key=_conf_value(config, 'STORAGE_SECRET_KEY'), secure=_conf_value(config, 'STORAGE_USE_TLS'), region=_conf_value(config, 'STORAGE_REGION'))
def connect(self): return minio.Minio(current_app.config['MINIO_ENDPOINT'], access_key=current_app.config['MINIO_ACCESS_KEY'], secret_key=current_app.config['MINIO_SECRET_KEY'], secure=current_app.config['MINIO_SECURE'], region=current_app.config['MINIO_REGION'], http_client=current_app.config['MINIO_HTTP_CLIENT'])
def init(): MINIO_CLIENT = minio_client.Minio(MINIO_URL, access_key=MINIO_ACCESSKEY, secret_key=MINIO_SECRET, secure=False, region=MINIO_REGION) return MINIO_CLIENT
def test_minio_lib(): pool_manager = urllib3.PoolManager( timeout=30.0, retries=urllib3.Retry(total=1), ) client = minio.Minio( NETLOC, access_key=ACCESS_KEY, secret_key=SECRET_KEY, secure=SCHEME == "https", http_client=pool_manager, ) client.make_bucket("test-minio-python-lib") with open("../testdata/small.txt", "rb") as f: client.put_object("test-minio-python-lib", "small", f, 1) with open("../testdata/large.txt", "rb") as f: client.put_object("test-minio-python-lib", "large", f, 65*1024*1024) res = client.list_objects_v2("test-minio-python-lib") assert set((o.object_name, o.size) for o in res) == EXPECTED_KEYS with open("../testdata/small.txt", "rb") as f: assert client.get_object("test-minio-python-lib", "small").read() == f.read() with open("../testdata/large.txt", "rb") as f: assert client.get_object("test-minio-python-lib", "large").read() == f.read() client.remove_object("test-minio-python-lib", "small") client.remove_object("test-minio-python-lib", "large") client.remove_bucket("test-minio-python-lib")
def samples_loader_from_minio(server, access_key, secret_key): minio_client = minio.Minio( server, access_key=access_key, secret_key=secret_key, secure=True, ) logger.info("downloading samples") objects = minio_client.list_objects("brat-data", prefix='wl_ground_truth/') documents = [] for o in objects: if o.object_name.endswith(".txt"): object_object = minio_client.get_object("brat-data", o.object_name) object_path = pathlib.Path(o.object_name) object_name = object_path.stem object_text = object_object.read().decode("utf-8") try: object_annotation = minio_client.get_object( "brat-data", o.object_name[:-3] + "ann").read().decode("utf-8") except: object_annotation = None document = Document(object_name, object_text, object_annotation) documents.append(document) logger.info(f"{len(documents)} samples were downloaded") return documents
def ensure_bucket_deleted(bucket=None): if not bucket: bucketname = settings.AWS_S3_BUCKET_NAME else: bucketname = bucket host = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc # GCS' S3 compatibility is broken, especially in bucket operations; # skip bucket creation there and just bug Aron to create buckets with # public-read access for you if "storage.googleapis.com" in host: logging.info( "Skipping storage deletion on googleapis; that sounds like a production bucket!" ) return minio_client = minio.Minio(host, access_key=settings.AWS_ACCESS_KEY_ID, secret_key=settings.AWS_SECRET_ACCESS_KEY, secure=False) if minio_client.bucket_exists(bucketname): try: # We need to delete all objects first, before we can actually delete the bucket. objs = ( o.object_name for o in minio_client.list_objects(bucketname, recursive=True)) list( minio_client.remove_objects(bucketname, objs) ) # evaluate the generator, or else remove_objects won't actually execute minio_client.remove_bucket(bucketname) except BucketAlreadyOwnedByYou: pass
def transfer_video(): client = minio.Minio( "localhost:9000", access_key="minio", secret_key="minio123", secure=False, ) video_path = request.args.get("path", "") prefix, filename = os.path.split(video_path) _, extension = os.path.splitext(filename) if extension == ".ts": new_path = os.path.join(prefix, filename.replace(".ts", ".mp4")) subprocess.run(["ffmpeg", "-i", video_path, "-c", "copy", "-y", new_path]) file_name = "{}.mp4".format(int(datetime.datetime.now().timestamp())) client.fput_object( "videos", file_name, new_path, content_type="video/mp4" ) return "/videos/{}".format(file_name) else: file_name = "{}{}".format(int(datetime.datetime.now().timestamp()), extension) client.fput_object( "pictures", file_name, video_path, ) return "/pictures/{}".format(file_name)
def _create_minio_client(self): http_client = None if not self.config.get("ssl_verify", False): http_client = urllib3.PoolManager( timeout=urllib3.Timeout.DEFAULT_TIMEOUT, cert_reqs="CERT_NONE", maxsize=10, retries=urllib3.Retry(total=5, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504])) if isinstance(self.config.get("ssl_verify", False), str): http_client = urllib3.PoolManager( timeout=urllib3.Timeout.DEFAULT_TIMEOUT, cert_reqs="CERT_REQUIRED", ca_certs=self.config.get("ssl_verify"), maxsize=10, retries=urllib3.Retry(total=5, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504])) client = minio.Minio(endpoint=self.config["endpoint"], access_key=self.config.get("access_key", None), secret_key=self.config.get("secret_key", None), secure=self.config.get("secure", True), region=self.config.get("region", None), http_client=http_client) # Test client auth client.bucket_exists(self.config.get("bucket", "carrier")) return client
def __init__(self, domain, secure_domain, hostname, port, access_key, secret_key): self.domain = domain self.secure_domain = secure_domain self.hostname = hostname self.port = port self.access_key = access_key self.secret_key = secret_key protocol = 'https' if self.secure_domain else 'http' self.base_url = f'{protocol}://{self.domain}' self.client = minio.Minio( f'{self.hostname}:{self.port}', self.access_key, self.secret_key, secure = False) bucket_names = ['scenes', 'images', 'sequences'] for bucket_name in bucket_names: if not self.client.bucket_exists(bucket_name): policy = { 'Version': '2012-10-17', 'Statement': [{ 'Sid': 'PublicReadOnly', 'Effect': 'Allow', 'Principal': '*', 'Action': 's3:GetObject', 'Resource': f'arn:aws:s3:::{bucket_name}/*' }] } self.client.make_bucket(bucket_name) self.client.set_bucket_policy(bucket_name, json.dumps(policy))
def dataUpload(self, id): try: # Look at attachments in data database r = requests.get(f"{settings.COUCHDB_URL}/data/{id}/file") data = io.BytesIO(r.content) # Upload to S3 client = minio.Minio(settings.S3_URL, settings.S3_ACCESS_KEY, settings.S3_SECRET_KEY, secure=False) res = client.put_object("blob", id, data, length=len(r.content)) assert type(res) == str, "Minio response to put not string" rev = requests.get( f"{settings.COUCHDB_URL}/data/{id}").json()["_rev"] # Make update to document: - detele attachment logging.info(f"Uploaded {id} to S3") r = requests.delete(f"{settings.COUCHDB_URL}/data/{id}", params={"rev": rev}) assert r.status_code == 200, "Delete status code not 200" return True except AssertionError as e: logging.error(f"id: {id} encountered error:{e}") return False except Exception as e: raise e
def download_file(self, filepath, access_key, secret_key): secure = True if self.schema == 'https' else False ca_certs = os.environ.get('SSL_CERT_FILE') or certifi.where() http_client = urllib3.PoolManager( timeout=3, maxsize=10, cert_reqs='CERT_REQUIRED', ca_certs=ca_certs, retries=urllib3.Retry(total=1, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504])) client = minio.Minio(self.host, access_key, secret_key, secure=secure, http_client=http_client) try: return client.fget_object(self.bucket, self.object_key, filepath) except Exception as e: raise exceptions.PluginError(message=_( 'failed to download file[%(filepath)s] from s3: %(reason)s') % { 'filepath': self.object_key, 'reason': str(e) })
def _create_minio_client(): url = urlparse(os.getenv("S3_ENDPOINT_URL", "http://s3.amazonaws.com")) use_ssl = url.scheme == 'https' if url.scheme else True return minio.Minio(url.netloc, access_key=os.getenv("ACCESS_KEY_ID", ""), secret_key=os.getenv("SECRET_ACCESS_KEY", ""), secure=use_ssl)
def client(self): return minio.Minio( endpoint=self.server_url, access_key=self.access_key, secret_key=self.secret_key, secure=False, )
def __init__(self): endpoint = settings.MINIO_STORAGE_ENDPOINT access_key = settings.MINIO_STORAGE_ACCESS_KEY secret_key = settings.MINIO_STORAGE_SECRET_KEY secure = settings.MINIO_STORAGE_USE_HTTPS minio_http_client = None if settings.DEBUG and settings.MINIO_DISABLE_CERT_CHECKS: # This is a copy of what the minio client does internally, # just with the cert checking disabled. minio_http_client = urllib3.PoolManager( timeout=urllib3.Timeout.DEFAULT_TIMEOUT, maxsize=10, cert_reqs="CERT_NONE", ca_certs=None, retries=urllib3.Retry( total=5, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504] ), ) self.client = minio.Minio( endpoint, access_key=access_key, secret_key=secret_key, secure=secure, http_client=minio_http_client, ) self.bucket = settings.MINIO_STORAGE_MEDIA_BUCKET_NAME
def __init__(self): self.minioClient = minio.Minio( os.environ["MINI_URL"], access_key=os.environ["MINI_KEY"], secret_key=os.environ["MINI_SECRET_KEY"], secure=False, )
def __init__( self, endpoint, bucket, access_key, secret_key, *, secure=False, proxy_server=None, **_ ): # from https://docs.min.io/docs/python-client-api-reference self.client = minio.Minio( endpoint, access_key=access_key, secret_key=secret_key, secure=secure, http_client=( urllib3.ProxyManager( proxy_server, timeout=urllib3.Timeout.DEFAULT_TIMEOUT, cert_reqs="CERT_REQUIRED", retries=urllib3.Retry( total=5, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504], ), ) if proxy_server else None ), ) self.bucket = bucket if not self.client.bucket_exists(bucket): raise errors.BucketInaccessible("Inaccessible s3 bucket %s" % bucket)
def get_client(config): """ Get configured MinIO client """ http_client = None if not config.get("verify", False): http_client = urllib3.PoolManager( timeout=urllib3.Timeout.DEFAULT_TIMEOUT, cert_reqs="CERT_NONE", maxsize=10, retries=urllib3.Retry(total=5, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504])) if isinstance(config.get("verify", False), str): http_client = urllib3.PoolManager( timeout=urllib3.Timeout.DEFAULT_TIMEOUT, cert_reqs="CERT_REQUIRED", ca_certs=config["verify"], maxsize=10, retries=urllib3.Retry(total=5, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504])) client = minio.Minio(endpoint=config["endpoint"], access_key=config.get("access_key", None), secret_key=config.get("secret_key", None), secure=config.get("secure", True), region=config.get("region", None), http_client=http_client) return client
def docker_run_minio(dc, no_volumes): image = 'minio/minio:RELEASE.2021-08-05T22-01-19Z' command = 'server --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 /data' name = '%s-%s' % (ioxperf_name, 'minio') ports = {'9000/tcp': 9000, '9001/tcp': 9001} if no_volumes: volumes = None else: volumes = { os.path.join(os.getcwd(), 'volumes/minio'): { 'bind': '/data', 'mode': 'rw', } } env = { 'MINIO_ROOT_USER': '******', 'MINIO_ROOT_PASSWORD': '******', 'MINIO_PROMETHEUS_AUTH_TYPE': 'public', 'MINIO_HTTP_TRACE': '/dev/stdout', } docker_pull_image_if_needed(dc, image) container = dc.containers.run(image=image, command=command, detach=True, environment=env, name=name, hostname='minio', labels=ioxperf_labels, network=ioxperf_name, ports=ports, volumes=volumes) docker_wait_container_running(container) while True: timeout = urllib3.util.Timeout(connect=0.1, read=0.1) http_client = urllib3.PoolManager(num_pools=1, timeout=timeout, retries=False) try: mc = minio.Minio(endpoint='127.0.0.1:9000', access_key='minio', secret_key='miniominio', secure=False, http_client=http_client) if not mc.bucket_exists('iox1'): mc.make_bucket('iox1') if not mc.bucket_exists('iox2'): mc.make_bucket('iox2') break except (urllib3.exceptions.ProtocolError, urllib3.exceptions.TimeoutError, minio.error.S3Error): pass print('waiting for Minio to become ready') time.sleep(0.5) pipe_container_logs_to_file(container, 'minio.log') print('Minio service ready') return container
def create_minio_client(): client = minio.Minio( endpoint=settings.MINIO_STORAGE_ENDPOINT, access_key=settings.MINIO_STORAGE_ACCESS_KEY, secret_key=settings.MINIO_STORAGE_SECRET_KEY, secure=settings.MINIO_STORAGE_USE_HTTPS, ) return client
def __init__(self, endpoint, bucket, access_key, secret_key, location, database, **_): self.client = minio.Minio(endpoint, access_key=access_key, secret_key=secret_key, secure=False) self.bucket = bucket self.remote_path = '/'.join((location.lstrip('/'), database))
def __init__(self): endpoint = settings.MINIO_STORAGE_ENDPOINT access_key = settings.MINIO_STORAGE_ACCESS_KEY secret_key = settings.MINIO_STORAGE_SECRET_KEY secure = settings.MINIO_STORAGE_USE_HTTPS self.client = minio.Minio( endpoint, access_key=access_key, secret_key=secret_key, secure=secure ) self.bucket = settings.MINIO_STORAGE_MEDIA_BUCKET_NAME