def run(step):
minio_addr = step['minio_addr']
minio_addr = os.environ.get('KRAKEN_MINIO_ADDR', minio_addr)
minio_bucket = step['minio_bucket']
minio_access_key = step['minio_access_key']
minio_secret_key = step['minio_secret_key']
job_id = step['job_id']
action = step['action']
mc = minio.Minio(minio_addr,
access_key=minio_access_key,
secret_key=minio_secret_key,
secure=False)
# check connection
try:
mc.bucket_exists(minio_bucket)
except Exception as e:
log.exception('problem with connecting to minio %s', minio_addr)
msg = 'problem with connecting to minio %s: %s' % (minio_addr, str(e))
return 1, msg
if action == 'save':
paths = step['paths']
cache_key, minio_folder = step['minio_folder'].popitem()
dest_folder = '%s/%s' % (minio_folder, job_id)
status, msg = _save_to_cache(mc, cache_key, minio_bucket, dest_folder,
paths)
else:
minio_folders = step['minio_folders']
status, msg = _restore_from_cache(mc, minio_bucket, minio_folders)
return status, msg
def getMinioClient():
minioClient = minio.Minio('s3.docker:9000',
access_key=S3_ACCESS_KEY,
secret_key=S3_SECRET_KEY,
secure=False)
return minioClient
def __init__(self, endpoint=None, accesskey=None, secretkey=None, bucket=None, secure=False):
if bucket:
self.default_bucket = bucket
else:
self.default_bucket = os.environ.get(envBucket, "hyperml")
if not accesskey:
accesskey = os.environ.get(envAccessKey)
secretkey = os.environ.get(envSecretKey)
if not endpoint:
endpoint = os.environ.get(envURL)
if not secure:
if os.environ.get(envSecure, "True") == "True":
secure = True
else:
secure = False
print('minio:', endpoint, accesskey, secretkey, secure)
self.client = minio.Minio(endpoint=endpoint,
access_key=accesskey,
secret_key=secretkey,
secure=secure)
self.access_key = accesskey
self.secret_key = secretkey
def get_minio_client(url,
access_key,
secret_key,
region='eu-central-1',
bucket=None):
"""
Wrapper for getting minio.Minio class.
Can optionally generate a bucket if parameter is provided.
"""
parsed = urlparse(url)
secure = parsed.scheme == 'https'
mc = minio.Minio(parsed.netloc,
access_key,
secret_key,
region=region,
secure=secure)
if not bucket:
return mc
if not mc.bucket_exists(bucket):
mc.make_bucket(bucket, location=region)
return mc
def __init__(
self,
host_url: str,
access_key: Optional[str] = None,
secret_key: Optional[str] = None,
default_bucket: Optional[str] = "sandcrawler",
):
"""
host is minio connection string (host:port)
access and secret key are as expected
default_bucket can be supplied so that it doesn't need to be repeated for each function call
Example config:
host="localhost:9000",
access_key=os.environ['MINIO_ACCESS_KEY'],
secret_key=os.environ['MINIO_SECRET_KEY'],
"""
self.mc = minio.Minio(
host_url,
access_key=access_key,
secret_key=secret_key,
secure=False,
)
self.default_bucket = default_bucket
def _create_base_url_client(client: minio.Minio, bucket_name: str,
base_url: str):
"""
Clone a Minio client, using a different endpoint from `base_url`.
"""
base_url_parts = urlsplit(base_url)
# Clone from the normal client, but with base_url as the endpoint
base_url_client = minio.Minio(
base_url_parts.netloc,
access_key=client._access_key,
secret_key=client._secret_key,
session_token=client._session_token,
secure=base_url_parts.scheme == "https",
# The bucket region may be auto-detected by client (via an HTTP
# request), so don't just use client._region
region=client._get_bucket_region(bucket_name),
http_client=client._http,
)
if hasattr(client, "_credentials"):
# Client credentials do not exist prior to minio-py 5.0.7, but
# they should be reused if possible
base_url_client._credentials = client._credentials
return base_url_client
def run(
files_path,
minio_host,
minio_access_key,
minio_secret_key,
minio_bucket,
google_credential_path,
):
google_credentials = ServiceAccountCredentials.from_json_keyfile_name(
google_credential_path, scopes=SCOPES)
gauth = GoogleAuth()
gauth.credentials = google_credentials
gauth.ServiceAuth()
drive = GoogleDrive(gauth)
gspread_service = gspread.service_account(filename=google_credential_path)
with open(files_path) as files_file:
files = yaml.load(files_file, Loader=yaml.FullLoader)
minio_client = minio.Minio(minio_host,
minio_access_key,
minio_secret_key,
secure=False)
download_dir = tempfile.mkdtemp()
for file in files['files']:
downloaded_path = get_file(drive, gspread_service, download_dir, file)
minio_client.fput_object(minio_bucket,
os.path.basename(downloaded_path),
downloaded_path)
def connect_storage(config):
import minio
return minio.Minio(_conf_value(config, 'STORAGE_ENDPOINT'),
access_key=_conf_value(config, 'STORAGE_ACCESS_KEY'),
secret_key=_conf_value(config, 'STORAGE_SECRET_KEY'),
secure=_conf_value(config, 'STORAGE_USE_TLS'),
region=_conf_value(config, 'STORAGE_REGION'))
def connect(self):
return minio.Minio(current_app.config['MINIO_ENDPOINT'],
access_key=current_app.config['MINIO_ACCESS_KEY'],
secret_key=current_app.config['MINIO_SECRET_KEY'],
secure=current_app.config['MINIO_SECURE'],
region=current_app.config['MINIO_REGION'],
http_client=current_app.config['MINIO_HTTP_CLIENT'])
def init():
MINIO_CLIENT = minio_client.Minio(MINIO_URL,
access_key=MINIO_ACCESSKEY,
secret_key=MINIO_SECRET,
secure=False,
region=MINIO_REGION)
return MINIO_CLIENT
def test_minio_lib():
pool_manager = urllib3.PoolManager(
timeout=30.0,
retries=urllib3.Retry(total=1),
)
client = minio.Minio(
NETLOC,
access_key=ACCESS_KEY,
secret_key=SECRET_KEY,
secure=SCHEME == "https",
http_client=pool_manager,
)
client.make_bucket("test-minio-python-lib")
with open("../testdata/small.txt", "rb") as f:
client.put_object("test-minio-python-lib", "small", f, 1)
with open("../testdata/large.txt", "rb") as f:
client.put_object("test-minio-python-lib", "large", f, 65*1024*1024)
res = client.list_objects_v2("test-minio-python-lib")
assert set((o.object_name, o.size) for o in res) == EXPECTED_KEYS
with open("../testdata/small.txt", "rb") as f:
assert client.get_object("test-minio-python-lib", "small").read() == f.read()
with open("../testdata/large.txt", "rb") as f:
assert client.get_object("test-minio-python-lib", "large").read() == f.read()
client.remove_object("test-minio-python-lib", "small")
client.remove_object("test-minio-python-lib", "large")
client.remove_bucket("test-minio-python-lib")
def samples_loader_from_minio(server, access_key, secret_key):
minio_client = minio.Minio(
server,
access_key=access_key,
secret_key=secret_key,
secure=True,
)
logger.info("downloading samples")
objects = minio_client.list_objects("brat-data", prefix='wl_ground_truth/')
documents = []
for o in objects:
if o.object_name.endswith(".txt"):
object_object = minio_client.get_object("brat-data", o.object_name)
object_path = pathlib.Path(o.object_name)
object_name = object_path.stem
object_text = object_object.read().decode("utf-8")
try:
object_annotation = minio_client.get_object(
"brat-data",
o.object_name[:-3] + "ann").read().decode("utf-8")
except:
object_annotation = None
document = Document(object_name, object_text, object_annotation)
documents.append(document)
logger.info(f"{len(documents)} samples were downloaded")
return documents
def ensure_bucket_deleted(bucket=None):
if not bucket:
bucketname = settings.AWS_S3_BUCKET_NAME
else:
bucketname = bucket
host = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc
# GCS' S3 compatibility is broken, especially in bucket operations;
# skip bucket creation there and just bug Aron to create buckets with
# public-read access for you
if "storage.googleapis.com" in host:
logging.info(
"Skipping storage deletion on googleapis; that sounds like a production bucket!"
)
return
minio_client = minio.Minio(host,
access_key=settings.AWS_ACCESS_KEY_ID,
secret_key=settings.AWS_SECRET_ACCESS_KEY,
secure=False)
if minio_client.bucket_exists(bucketname):
try:
# We need to delete all objects first, before we can actually delete the bucket.
objs = (
o.object_name
for o in minio_client.list_objects(bucketname, recursive=True))
list(
minio_client.remove_objects(bucketname, objs)
) # evaluate the generator, or else remove_objects won't actually execute
minio_client.remove_bucket(bucketname)
except BucketAlreadyOwnedByYou:
pass
def transfer_video():
client = minio.Minio(
"localhost:9000",
access_key="minio",
secret_key="minio123",
secure=False,
)
video_path = request.args.get("path", "")
prefix, filename = os.path.split(video_path)
_, extension = os.path.splitext(filename)
if extension == ".ts":
new_path = os.path.join(prefix, filename.replace(".ts", ".mp4"))
subprocess.run(["ffmpeg", "-i", video_path, "-c", "copy", "-y", new_path])
file_name = "{}.mp4".format(int(datetime.datetime.now().timestamp()))
client.fput_object(
"videos", file_name, new_path, content_type="video/mp4"
)
return "/videos/{}".format(file_name)
else:
file_name = "{}{}".format(int(datetime.datetime.now().timestamp()), extension)
client.fput_object(
"pictures", file_name, video_path,
)
return "/pictures/{}".format(file_name)
def _create_minio_client(self):
http_client = None
if not self.config.get("ssl_verify", False):
http_client = urllib3.PoolManager(
timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
cert_reqs="CERT_NONE",
maxsize=10,
retries=urllib3.Retry(total=5,
backoff_factor=0.2,
status_forcelist=[500, 502, 503, 504]))
if isinstance(self.config.get("ssl_verify", False), str):
http_client = urllib3.PoolManager(
timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
cert_reqs="CERT_REQUIRED",
ca_certs=self.config.get("ssl_verify"),
maxsize=10,
retries=urllib3.Retry(total=5,
backoff_factor=0.2,
status_forcelist=[500, 502, 503, 504]))
client = minio.Minio(endpoint=self.config["endpoint"],
access_key=self.config.get("access_key", None),
secret_key=self.config.get("secret_key", None),
secure=self.config.get("secure", True),
region=self.config.get("region", None),
http_client=http_client)
# Test client auth
client.bucket_exists(self.config.get("bucket", "carrier"))
return client
def __init__(self, domain, secure_domain, hostname, port, access_key, secret_key):
self.domain = domain
self.secure_domain = secure_domain
self.hostname = hostname
self.port = port
self.access_key = access_key
self.secret_key = secret_key
protocol = 'https' if self.secure_domain else 'http'
self.base_url = f'{protocol}://{self.domain}'
self.client = minio.Minio(
f'{self.hostname}:{self.port}', self.access_key, self.secret_key, secure = False)
bucket_names = ['scenes', 'images', 'sequences']
for bucket_name in bucket_names:
if not self.client.bucket_exists(bucket_name):
policy = {
'Version': '2012-10-17',
'Statement': [{
'Sid': 'PublicReadOnly',
'Effect': 'Allow',
'Principal': '*',
'Action': 's3:GetObject',
'Resource': f'arn:aws:s3:::{bucket_name}/*'
}]
}
self.client.make_bucket(bucket_name)
self.client.set_bucket_policy(bucket_name, json.dumps(policy))
def dataUpload(self, id):
try:
# Look at attachments in data database
r = requests.get(f"{settings.COUCHDB_URL}/data/{id}/file")
data = io.BytesIO(r.content)
# Upload to S3
client = minio.Minio(settings.S3_URL,
settings.S3_ACCESS_KEY,
settings.S3_SECRET_KEY,
secure=False)
res = client.put_object("blob", id, data, length=len(r.content))
assert type(res) == str, "Minio response to put not string"
rev = requests.get(
f"{settings.COUCHDB_URL}/data/{id}").json()["_rev"]
# Make update to document: - detele attachment
logging.info(f"Uploaded {id} to S3")
r = requests.delete(f"{settings.COUCHDB_URL}/data/{id}",
params={"rev": rev})
assert r.status_code == 200, "Delete status code not 200"
return True
except AssertionError as e:
logging.error(f"id: {id} encountered error:{e}")
return False
except Exception as e:
raise e
def download_file(self, filepath, access_key, secret_key):
secure = True if self.schema == 'https' else False
ca_certs = os.environ.get('SSL_CERT_FILE') or certifi.where()
http_client = urllib3.PoolManager(
timeout=3,
maxsize=10,
cert_reqs='CERT_REQUIRED',
ca_certs=ca_certs,
retries=urllib3.Retry(total=1,
backoff_factor=0.2,
status_forcelist=[500, 502, 503, 504]))
client = minio.Minio(self.host,
access_key,
secret_key,
secure=secure,
http_client=http_client)
try:
return client.fget_object(self.bucket, self.object_key, filepath)
except Exception as e:
raise exceptions.PluginError(message=_(
'failed to download file[%(filepath)s] from s3: %(reason)s') %
{
'filepath': self.object_key,
'reason': str(e)
})
def _create_minio_client():
url = urlparse(os.getenv("S3_ENDPOINT_URL", "http://s3.amazonaws.com"))
use_ssl = url.scheme == 'https' if url.scheme else True
return minio.Minio(url.netloc,
access_key=os.getenv("ACCESS_KEY_ID", ""),
secret_key=os.getenv("SECRET_ACCESS_KEY", ""),
secure=use_ssl)
def client(self):
return minio.Minio(
endpoint=self.server_url,
access_key=self.access_key,
secret_key=self.secret_key,
secure=False,
)
def __init__(self):
endpoint = settings.MINIO_STORAGE_ENDPOINT
access_key = settings.MINIO_STORAGE_ACCESS_KEY
secret_key = settings.MINIO_STORAGE_SECRET_KEY
secure = settings.MINIO_STORAGE_USE_HTTPS
minio_http_client = None
if settings.DEBUG and settings.MINIO_DISABLE_CERT_CHECKS:
# This is a copy of what the minio client does internally,
# just with the cert checking disabled.
minio_http_client = urllib3.PoolManager(
timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
maxsize=10,
cert_reqs="CERT_NONE",
ca_certs=None,
retries=urllib3.Retry(
total=5, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504]
),
)
self.client = minio.Minio(
endpoint,
access_key=access_key,
secret_key=secret_key,
secure=secure,
http_client=minio_http_client,
)
self.bucket = settings.MINIO_STORAGE_MEDIA_BUCKET_NAME
def __init__(self):
self.minioClient = minio.Minio(
os.environ["MINI_URL"],
access_key=os.environ["MINI_KEY"],
secret_key=os.environ["MINI_SECRET_KEY"],
secure=False,
)
def __init__(
self,
endpoint,
bucket,
access_key,
secret_key,
*,
secure=False,
proxy_server=None,
**_
):
# from https://docs.min.io/docs/python-client-api-reference
self.client = minio.Minio(
endpoint,
access_key=access_key,
secret_key=secret_key,
secure=secure,
http_client=(
urllib3.ProxyManager(
proxy_server,
timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
cert_reqs="CERT_REQUIRED",
retries=urllib3.Retry(
total=5,
backoff_factor=0.2,
status_forcelist=[500, 502, 503, 504],
),
)
if proxy_server
else None
),
)
self.bucket = bucket
if not self.client.bucket_exists(bucket):
raise errors.BucketInaccessible("Inaccessible s3 bucket %s" % bucket)
def get_client(config):
""" Get configured MinIO client """
http_client = None
if not config.get("verify", False):
http_client = urllib3.PoolManager(
timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
cert_reqs="CERT_NONE",
maxsize=10,
retries=urllib3.Retry(total=5,
backoff_factor=0.2,
status_forcelist=[500, 502, 503, 504]))
if isinstance(config.get("verify", False), str):
http_client = urllib3.PoolManager(
timeout=urllib3.Timeout.DEFAULT_TIMEOUT,
cert_reqs="CERT_REQUIRED",
ca_certs=config["verify"],
maxsize=10,
retries=urllib3.Retry(total=5,
backoff_factor=0.2,
status_forcelist=[500, 502, 503, 504]))
client = minio.Minio(endpoint=config["endpoint"],
access_key=config.get("access_key", None),
secret_key=config.get("secret_key", None),
secure=config.get("secure", True),
region=config.get("region", None),
http_client=http_client)
return client
def docker_run_minio(dc, no_volumes):
image = 'minio/minio:RELEASE.2021-08-05T22-01-19Z'
command = 'server --address 0.0.0.0:9000 --console-address 0.0.0.0:9001 /data'
name = '%s-%s' % (ioxperf_name, 'minio')
ports = {'9000/tcp': 9000, '9001/tcp': 9001}
if no_volumes:
volumes = None
else:
volumes = {
os.path.join(os.getcwd(), 'volumes/minio'): {
'bind': '/data',
'mode': 'rw',
}
}
env = {
'MINIO_ROOT_USER': '******',
'MINIO_ROOT_PASSWORD': '******',
'MINIO_PROMETHEUS_AUTH_TYPE': 'public',
'MINIO_HTTP_TRACE': '/dev/stdout',
}
docker_pull_image_if_needed(dc, image)
container = dc.containers.run(image=image,
command=command,
detach=True,
environment=env,
name=name,
hostname='minio',
labels=ioxperf_labels,
network=ioxperf_name,
ports=ports,
volumes=volumes)
docker_wait_container_running(container)
while True:
timeout = urllib3.util.Timeout(connect=0.1, read=0.1)
http_client = urllib3.PoolManager(num_pools=1,
timeout=timeout,
retries=False)
try:
mc = minio.Minio(endpoint='127.0.0.1:9000',
access_key='minio',
secret_key='miniominio',
secure=False,
http_client=http_client)
if not mc.bucket_exists('iox1'):
mc.make_bucket('iox1')
if not mc.bucket_exists('iox2'):
mc.make_bucket('iox2')
break
except (urllib3.exceptions.ProtocolError,
urllib3.exceptions.TimeoutError, minio.error.S3Error):
pass
print('waiting for Minio to become ready')
time.sleep(0.5)
pipe_container_logs_to_file(container, 'minio.log')
print('Minio service ready')
return container
def create_minio_client():
client = minio.Minio(
endpoint=settings.MINIO_STORAGE_ENDPOINT,
access_key=settings.MINIO_STORAGE_ACCESS_KEY,
secret_key=settings.MINIO_STORAGE_SECRET_KEY,
secure=settings.MINIO_STORAGE_USE_HTTPS,
)
return client
def __init__(self, endpoint, bucket, access_key, secret_key, location,
database, **_):
self.client = minio.Minio(endpoint,
access_key=access_key,
secret_key=secret_key,
secure=False)
self.bucket = bucket
self.remote_path = '/'.join((location.lstrip('/'), database))
def __init__(self):
endpoint = settings.MINIO_STORAGE_ENDPOINT
access_key = settings.MINIO_STORAGE_ACCESS_KEY
secret_key = settings.MINIO_STORAGE_SECRET_KEY
secure = settings.MINIO_STORAGE_USE_HTTPS
self.client = minio.Minio(
endpoint, access_key=access_key, secret_key=secret_key, secure=secure
)
self.bucket = settings.MINIO_STORAGE_MEDIA_BUCKET_NAME
