def init(self, miniobase, minioaccess, miniosecret, miniosecure):
print("Starting with minio at " + miniobase)
return Minio(miniobase,
access_key=minioaccess,
secret_key=miniosecret,
secure=miniosecure)
def __init__(self):
self.bucket_name = settings.MINIO.get('bucket_name')
self.connection = Minio(settings.MINIO.get('url'),
access_key=settings.MINIO.get('access_key'),
secret_key=settings.MINIO.get('secret_key'),
secure=False)
def test_object_is_non_string_iterable_3(self):
client = Minio('localhost:9000')
for err in client.remove_objects('hello', b'abc'):
print(err)
def test_object_is_string(self):
client = Minio('localhost:9000')
client.get_object('hello', 1234)
for msg in pubsub.listen():
print("Received the following messages:")
print(msg)
data_part = json.loads(msg['data'])
if data_part['id'] == 10:
pass
elif data_part['id'] == 200:
print("Test task finished successfully.")
print("Exiting Data-Preparation-Tester with exit code 0")
break
else:
print("Test task did not finish successfully.")
print("Exiting Data-Preparation-Tester with exit code 999")
exit_code = 999
break
exit(exit_code)
if __name__ == "__main__":
# Step 0: Setup connection with MinIO and Redis
redis_host = sys.argv[1]
redis_password = sys.argv[2]
minio_host = sys.argv[3]
minio_access_key = sys.argv[4]
minio_secret_key = sys.argv[5]
redis_client = redis.Redis(host=redis_host, password=redis_password)
minio_client = Minio(minio_host + ":9000", minio_access_key, minio_secret_key, secure=False)
test_data_prep_worker(redis_client, minio_client)
def stop():
global udriver
global recordlist
global device
global fileslist
global url
global client
global child_url
global upload_time
global get_gameID
global get_uuID
checking = True
obj_list = []
finish_time = []
different_time = []
try:
try:
print("停止采集1")
udriver.profile_stop()
print("停止采集2")
except:
print("stop_profile错误")
try:
screenShooter.stop()
except:
print("stop_screen错误")
while True:
time.sleep(1)
if checking:
record = udriver.profile_check()
checking = False
else:
break
try:
if device != "PC":
if len(record):
for file in record:
print("文件" + file['name'] + ".raw" + "已生成")
c = file['name'].split('-')
a = c[1] + '-' + c[2] + '-' + c[3] + " " + "" + c[
4] + ":" + c[5] + ":" + c[6]
timeArray = time.strptime(a, "%Y-%m-%d %H:%M:%S")
timestamp = time.mktime(timeArray)
files = str(timestamp).split('.')[0] + ".raw"
if files not in fileslist: # 将需要解析的文件名存起来
fileslist.append(files)
if not os.path.exists(ROOT_DIR + "/record"): # 将文件拉到本地
os.makedirs(ROOT_DIR + "/record")
cmd = os.popen(f"{adbPath} -s " + device + " pull " +
file['path'] + '/' + file['name'] +
".raw" + " " + ROOT_DIR +
"/record").read()
print("获取" + file['name'] + ".raw" + "到本地成功")
name = os.path.join(ROOT_DIR, "record",
file['name'] + ".raw") # 本地文件路径
cmd = os.popen(f"{adbPath} -s " + device +
" shell rm " + file['path'] + '/' +
file['name'] + ".raw").read()
print("删除设备中的" + file['name'] + ".raw" + "成功")
r = requests.get(upload_url + "requestupload?gameid=" +
str(get_gameID) + "&uuid=" +
str(get_uuID) + "&filename=" +
files) # 申请上传，得到空闲的服务器
r.encoding = 'utf-8'
result = r.text
if result.startswith(u'\ufeff'):
result = result.encode('utf8')[3:].decode('utf8')
result = json.loads(result)
try:
url = result['ip'] + ':' + result[
'port'] # 空闲的服务器路径
print("空闲的服务器：" + url)
child_url = str(
get_gameID) + "-" + get_uuID # 存储桶名
print("存储桶：" + child_url)
except:
print("当前无空闲的服务器")
return "当前无空闲的服务器"
client = Minio(url,
access_key=result['accessKey'],
secret_key=result['secretKey'],
secure=False) # 上传至存储桶
try:
if client.bucket_exists(bucket_name=child_url
): # bucket_exists：检查桶是否存在
pass
else:
client.make_bucket(child_url)
print("存储桶创建成功")
except Exception as err:
print(err)
start_upload_time = datetime.datetime.now()
upload_time.append({
'name': files,
'time': start_upload_time,
'size': "123456"
})
client.fput_object(child_url, files, name)
print("上传" + files + "到服务器成功")
if delete_files == "1":
os.remove("record/" + file['name'] + ".raw")
print("删除本地" + file['name'] + ".raw" + "成功")
else:
if len(record):
for file in record:
print("文件" + file['name'] + ".raw" + "已生成")
c = file['name'].split('-')
a = c[1] + '-' + c[2] + '-' + c[3] + " " + "" + c[
4] + ":" + c[5] + ":" + c[6]
timeArray = time.strptime(a, "%Y-%m-%d %H:%M:%S")
timestamp = time.mktime(timeArray)
files = str(timestamp).split('.')[0] + ".raw"
if files not in fileslist:
fileslist.append(files)
name = os.path.join(file['path'],
file['name'] + ".raw") # 本地文件路径
r = requests.get(upload_url + "requestupload?gameid=" +
str(get_gameID) + "&uuid=" +
str(get_uuID) + "&filename=" +
files) # 申请上传
r.encoding = 'utf-8'
result = r.text
if result.startswith(u'\ufeff'):
result = result.encode('utf8')[3:].decode('utf8')
result = json.loads(result)
print(result)
try:
url = result['ip'] + ':' + result[
'port'] # 空闲的服务器路径
print("空闲的服务器：" + url)
child_url = str(
get_gameID) + "-" + get_uuID # 存储桶名
print("存储桶：" + child_url)
except:
print("当前无空闲的服务器")
return "当前无空闲的服务器"
client = Minio(url,
access_key=result['accessKey'],
secret_key=result['secretKey'],
secure=False) # 上传至存储桶
try:
if client.bucket_exists(bucket_name=child_url
): # bucket_exists：检查桶是否存在
pass
else:
client.make_bucket(child_url)
print("存储桶创建成功")
except Exception as err:
print(err)
start_upload_time = datetime.datetime.now()
upload_time.append({
'name': files,
'time': start_upload_time,
'size': "123456"
})
client.fput_object(child_url, files, name)
print("上传" + files + "到服务器成功")
if delete_files == "1":
os.remove(file['path'] + "/" + file['name'] +
".raw")
print("删除本地" + file['name'] + ".raw" + "成功")
except:
print("上传错误")
# print(str(upload_time))
try:
while True:
time.sleep(2)
minio_files = client.list_objects(
child_url, prefix=None, recursive=False) # 获取上传成功了的文件名
for obj in minio_files:
obj_list.append(obj.object_name)
obj_list_set = set(obj_list)
fileslist_set = set(fileslist)
obj_alone = fileslist_set - obj_list_set # 找到没上传成功的文件
if len(obj_alone):
print("-----------------------------------------------")
for i in obj_alone:
print("文件" + str(i) + "上传中")
obj_list = []
else:
minio_files = client.list_objects(
child_url, prefix=None, recursive=False) # 获取上传成功了的文件名
for obj in minio_files:
obj_time = obj.last_modified.timestamp()
obj_time = datetime.datetime.fromtimestamp(obj_time)
finish_time.append({
"name": obj.object_name,
"time": obj_time,
"size": obj.size
})
for i in upload_time:
for j in finish_time:
if i['name'] == j['name']:
i['size'] = j['size']
timer = str(j['time'] - i['time'])
different_time.append({
'name': i['name'],
'time': timer,
'size': i['size']
})
# print(str(different_time))
print("-----------------------------------------------")
for file in different_time:
u_time = file['time'].split(':')
seconds = u_time[2].split('.')
if u_time[0] == "00" or u_time[0] == "0":
if u_time[1] == "00":
if seconds[0] == "00":
file['time'] = "0分0." + seconds[1] + "秒"
else:
file['time'] = "0分" + seconds[0] + "秒"
else:
file['time'] = u_time[1] + "分" + seconds[
0] + "秒"
else:
file['time'] = u_time[0] + "时" + u_time[
1] + "分" + seconds[0] + "秒"
file['size'] = str(round(file['size'] / 1024 / 1024,
2)) + "MB"
print("文件" + file['name'] + "上传时间：" + file['time'] +
"，文件大小：" + file['size'])
# print("differentFile" + str(different_time))
print("所有文件上传完毕")
break
data_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
except:
print("最后上传错误")
# print(type(data_time),data_time)
try:
# 数据库上传
data = {
"GameID": get_gameID,
"UUID": get_uuID,
"DataUrl": url,
"BucketName": child_url,
"UploadFiles": different_time,
"DateTime": data_time
}
res = requests.post('http://10.11.174.21:8890/SendUploadFile',
json=data)
print("数据已上传至数据库")
except:
print("数据库上传错误")
zip_screen()
shutil.rmtree("screen")
try:
r = requests.get(upload_url + "requestupload?gameid=" +
str(get_gameID) + "&uuid=" + str(get_uuID) +
"&filename=" + "screen.zip")
r.encoding = 'utf-8'
result = r.text
if result.startswith(u'\ufeff'):
result = result.encode('utf8')[3:].decode('utf8')
result = json.loads(result)
url = result['ip'] + ':' + result['port'] # 空闲的服务器路径
child_url = str(get_gameID) + "-" + get_uuID
client = Minio(url,
access_key=result['accessKey'],
secret_key=result['secretKey'],
secure=False) # 上传至存储桶
client.fput_object(child_url, "screen.zip", "screen.zip")
os.remove("screen.zip")
except:
print("截图上传错误")
recordlist = []
udriver.stop()
print("已停止采集")
return "已停止采集"
except:
print("停止采集失败")
return "停止采集失败"
import os, time, argparse
from minio import Minio
arg = argparse.ArgumentParser()
arg.add_argument("--s3", type = str, default = "")
arg.add_argument("--access_key", type = str, default = "")
arg.add_argument("--secret_key", type = str, default = "")
arg.add_argument("--dir", type = str, default = "")
options = vars(arg.parse_args())
for option in options.items():
if (not len(option[1])):
print("Пользователь не передал обязательный параметр командной строки")
exit(1)
history = {}
client = Minio(options['s3'],
options['access_key'],
options['secret_key'],
secure=False)
try:
client.bucket_exists("alexey")
except Exception:
print("Проверьте s3|access_key|secret_key")
exit(2)
if (not client.bucket_exists("alexey")):
client.make_bucket("alexey")
def get_list_of_files(path, list_of_files):
files = os.listdir(path)
for file in files:
if os.path.isdir(path+'/'+file):
get_list_of_files(path+'/'+file,list_of_files)
else:
list_of_files.append(path+'/'+file)
def load_dataf(bucket_name: str, file_name: str):
client = Minio(**MINIO_ARGS)
res = client.get_object(bucket_name, file_name)
return pd.read_json(BytesIO(res.data))
print('Took {} seconds to unpersist the graph'.format(
timeit.default_timer() - start_time))
initialSetup()
# ## Helper code for frame processing
def load_image_into_numpy_array(image):
(im_width, im_height) = image.size
return np.array(image.getdata()).reshape(
(im_height, im_width, 3)).astype(np.uint8)
minioClient = Minio(client, access_key=access, secret_key=secret, secure=True)
try:
minioClient.make_bucket("test", location="us-east-1")
except BucketAlreadyOwnedByYou as err:
pass
except BucketAlreadyExists as err:
pass
except ResponseError as err:
raise
count = 0
person_count = 0
location = '/pistol-detection/detect_pistol/Data' #!!!
os.chdir(location)
with tf.Session() as sess2:
def handle(req):
client = Minio("10.20.1.54:30020",
access_key="admin",
secret_key="secretsecret",
secure=False)
warnings.simplefilter(action="ignore", category=SettingWithCopyWarning)
data = json.loads(req)
fname = data['fname']
file_uuid = data['file_uuid']
pipeline = data['pipeline']
function_name = data['function_name']
function_bucket_list = data['function_bucket']
data_clean_func_bucket_name = function_bucket_list[
'random-forest-pipeline-data-clean']
data_clean_func_file_name = data_clean_func_bucket_name + '-' + fname.split(
'.')[0] + '-' + file_uuid + '.' + fname.split('.')[1]
uuid_renamed = function_name + '-' + fname.split(
'.')[0] + '-' + file_uuid + '.' + 'json'
client.fget_object(data_clean_func_bucket_name, data_clean_func_file_name,
'/home/app/data-clean.csv')
data = pd.read_csv("/home/app/data-clean.csv").copy()
# 每個都要加
for i in range(data.shape[0]):
data["LocalTime"][i] = datetime.strptime(data["LocalTime"][i],
'%Y-%m-%d %H:%M:%S')
data = data.set_index('LocalTime')
#
x_train, y_train = buildTrain(data=data)
x_train, y_train = shuffle(x_train=x_train, y_train=y_train)
x_dict = dict()
y_dict = dict()
for i in range(len(x_train)):
x_dict[str(i)] = dict()
for j in range(len(x_train[i])):
x_dict[str(i)][str(j)] = x_train[i][j][0]
for i in range(len(y_train)):
y_dict[str(i)] = y_train[i][0]
with open('/home/app/xt.json', 'w', encoding='utf-8') as f:
json.dump(x_dict, f)
with open('/home/app/yt.json', 'w', encoding='utf-8') as f:
json.dump(y_dict, f)
found = client.bucket_exists(os.environ['bucket_name'])
if not found:
client.make_bucket(os.environ['bucket_name'])
client.fput_object(os.environ['bucket_name'], 'xt-' + uuid_renamed,
'/home/app/xt.json')
client.fput_object(os.environ['bucket_name'], 'yt-' + uuid_renamed,
'/home/app/yt.json')
return os.environ['bucket_name']
from minio import Minio, ResponseError
from django.conf import settings
from datetime import timedelta
import io
_client = Minio(settings.MINIO_ADDRESS,
access_key=settings.MINIO_ACCESS_KEY,
secret_key=settings.MINIO_SECRET_KEY,
secure=False)
def upload(filename, filedata):
_client.put_object(settings.MINIO_BUCKET,
filename,
io.BytesIO(filedata),
len(filedata),
content_type='application/zip')
def copy(source_file, destination_file):
_client.copy_object(settings.MINIO_BUCKET, destination_file,
f'/{settings.MINIO_BUCKET}/{source_file}')
def download(filename, path):
with open(path, 'wb') as file:
data = _client.get_object(settings.MINIO_BUCKET, filename)
for chunck in data.stream(32 * 1024):
file.write(chunck)
import logging
import hashlib
import io
import sys
app = Flask(__name__)
# Log only in production mode.
# if not app.debug:
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
app.logger.addHandler(stream_handler)
# minioClient = Minio('minio.minio.svc:9000',
minioClient = Minio('minio.codeformuenster.org',
access_key='minio',
secret_key='minio123',
secure=True)
# secure=False)
try:
minioClient.make_bucket("parkleit")
except minio.error.BucketAlreadyOwnedByYou as err:
pass
except minio.error.BucketAlreadyExists as err:
pass
except minio.error.ResponseError as err:
app.logger.error(err)
raise
default = {
"type": "Feature",
import re
import sys
import os
import json
import random
from urllib.request import urlopen
from urllib.parse import urlencode
from bs4 import BeautifulSoup
from PIL import Image
from PIL import ImageOps
from unipath import Path
from minio import Minio
MINIO = Minio(os.getenv('MINIO_HOST'),
access_key=os.getenv('MINIO_ACCESS_KEY_ID'),
secret_key=os.getenv('MINIO_SECRET_ACCESS_KEY'),
secure=False)
BUCKET_NAME = 'natureasia-static'
WORKSPACE = Path(os.getenv('HOME')).child('Desktop')
WORKSPACE = WORKSPACE.child('THUMBNAIL_OUT')
LOCKFILE = Path('/tmp/thumbnail_maker.lock')
def exit_error(message):
"""
Exits shell with the status code of 1
:param message: error message to be shown
:type message: string
"""
print(message)
from minio import Minio
from minio import CopyConditions
import os
import glob
import uuid
ACCESS_KEY = "xx"
SECRET_KEY = "xx"
#Initialise MinioClient
minioClient = Minio('127.0.0.1:9000',
access_key=ACCESS_KEY,
secret_key=SECRET_KEY,
secure=False)
# Insert output from Deepspeech into this variable
metadata = {"Transcript": "xx", "Entities": "[[['xx','xx']],[['xx','xx']]]"}
# Append metadata onto original video and upload to new Minio Bucket
try:
copy_result = minioClient.copy_object("output_Bucket",
"updatedFileName",
"original_Bucket/fileToBeUpdated",
metadata=metadata)
except ResponseError as err:
print(err)
# Error-Checking to validate metadata has been appended to the corresponding video
try:
print(
minioClient.fget_object(bucket_name="output_Bucket",
def create_minio_client():
minioClient = Minio(os.environ.get('S3_ENDPOINT'),
access_key=os.environ.get('AWS_ACCESS_KEY'),
secret_key=os.environ.get('AWS_ACCESS_KEY'),
secure=False)
return minioClient
from minio import Minio
import joblib
from io import BytesIO
sys.path.append('/app/src')
#sys.path.insert(0, './src')
from modell import predicting
app = Flask(__name__)
api = Api(app)
api_key = 'secret_key'
minio_client = Minio(endpoint='minio:9000',
access_key='bence',
secret_key='tollasmadar',
secure=False)
def make_number_if_possible(s):
if s.replace('.', '', 1).isdigit():
if '.' in s:
return float(s)
else:
return int(s)
return s
def numberify_dict(d):
for k in d:
d[k] = make_number_if_possible(d[k])
def profile_check():
global udriver
global device
global startTime
global recordlist
global fileslist
global url
global client
global child_url
global get_gameID
global get_uuID
global delete_files
global is_checking
print("profile")
if is_checking:
print("checking")
return
else:
is_checking = True
record = udriver.profile_check()
print("check")
is_checking = False
if device != "PC":
if len(record):
for file in record:
print("文件" + file['name'] + ".raw" + "已生成")
c = file['name'].split('-')
a = c[1] + '-' + c[2] + '-' + c[3] + " " + "" + c[4] + ":" + c[
5] + ":" + c[6]
timeArray = time.strptime(a, "%Y-%m-%d %H:%M:%S")
timestamp = time.mktime(timeArray)
files = str(timestamp).split('.')[0] + ".raw"
if files not in fileslist: # 将需要解析的文件名存起来
fileslist.append(files)
if not os.path.exists(ROOT_DIR + "/record"): # 将文件拉到本地
os.makedirs(ROOT_DIR + "/record")
cmd = os.popen(f"{adbPath} -s " + device + " pull " +
file['path'] + '/' + file['name'] + ".raw" +
" " + ROOT_DIR + "/record").read()
print("获取" + file['name'] + ".raw" + "到本地成功")
name = os.path.join(ROOT_DIR, "record",
file['name'] + ".raw") # 本地文件路径
cmd = os.popen(f"{adbPath} -s " + device + " shell rm " +
file['path'] + '/' + file['name'] +
".raw").read()
print("删除设备中的" + file['name'] + ".raw" + "成功")
r = requests.get(upload_url + "requestupload?gameid=" +
str(get_gameID) + "&uuid=" + str(get_uuID) +
"&filename=" + files) # 申请上传，得到空闲的服务器
r.encoding = 'utf-8'
result = r.text
if result.startswith(u'\ufeff'):
result = result.encode('utf8')[3:].decode('utf8')
result = json.loads(result)
try:
url = result['ip'] + ':' + result['port'] # 空闲的服务器路径
print("空闲的服务器：" + url)
child_url = str(get_gameID) + "-" + get_uuID # 存储桶名
print("存储桶：" + child_url)
except:
print("当前无空闲的服务器")
return "当前无空闲的服务器"
client = Minio(url,
access_key=result['accessKey'],
secret_key=result['secretKey'],
secure=False) # 上传至存储桶
try:
if client.bucket_exists(
bucket_name=child_url): # bucket_exists：检查桶是否存在
pass
else:
client.make_bucket(child_url)
print("存储桶创建成功")
except Exception as err:
print(err)
start_upload_time = datetime.datetime.now()
upload_time.append({
'name': files,
'time': start_upload_time,
'size': "123456"
})
client.fput_object(child_url, files, name)
print("上传" + files + "到服务器成功")
if delete_files == "1":
os.remove("record/" + file['name'] + ".raw")
print("删除本地" + file['name'] + ".raw" + "成功")
else:
if len(record):
for file in record:
print("文件" + file['name'] + ".raw" + "已生成")
c = file['name'].split('-')
a = c[1] + '-' + c[2] + '-' + c[3] + " " + "" + c[4] + ":" + c[
5] + ":" + c[6]
timeArray = time.strptime(a, "%Y-%m-%d %H:%M:%S")
timestamp = time.mktime(timeArray)
files = str(timestamp).split('.')[0] + ".raw"
if files not in fileslist:
fileslist.append(files)
name = os.path.join(file['path'],
file['name'] + ".raw") # 本地文件路径
r = requests.get(upload_url + "requestupload?gameid=" +
str(get_gameID) + "&uuid=" + str(get_uuID) +
"&filename=" + files) # 申请上传
r.encoding = 'utf-8'
result = r.text
if result.startswith(u'\ufeff'):
result = result.encode('utf8')[3:].decode('utf8')
result = json.loads(result)
print(result)
try:
url = result['ip'] + ':' + result['port'] # 空闲的服务器路径
print("空闲的服务器：" + url)
child_url = str(get_gameID) + "-" + get_uuID # 存储桶名
print("存储桶：" + child_url)
except:
print("当前无空闲的服务器")
return "当前无空闲的服务器"
client = Minio(url,
access_key=result['accessKey'],
secret_key=result['secretKey'],
secure=False) # 上传至存储桶
try:
if client.bucket_exists(
bucket_name=child_url): # bucket_exists：检查桶是否存在
pass
else:
client.make_bucket(child_url)
print("存储桶创建成功")
except Exception as err:
print(err)
start_upload_time = datetime.datetime.now()
upload_time.append({
'name': files,
'time': start_upload_time,
'size': "123456"
})
client.fput_object(child_url, files, name)
print("上传" + files + "到服务器成功")
if delete_files == "1":
os.remove(file['path'] + "/" + file['name'] + ".raw")
print("删除本地" + file['name'] + ".raw" + "成功")
return None
import json
from minio import Minio
from minio.error import ResponseError
@contextlib.contextmanager
def nostdout():
save_stdout = sys.stdout
sys.stdout = io.BytesIO()
yield
sys.stdout = save_stdout
minioClient = Minio(os.environ['minio_url'],
access_key=os.environ['minio_access_key'],
secret_key=os.environ['minio_secret_key'],
secure=False)
caffe.set_mode_cpu()
# Select desired model
net = caffe.Net('./models/colorization_deploy_v2.prototxt',
'./models/colorization_release_v2.caffemodel', caffe.TEST)
(H_in, W_in) = net.blobs['data_l'].data.shape[2:] # get input shape
(H_out, W_out) = net.blobs['class8_ab'].data.shape[2:] # get output shape
pts_in_hull = np.load('./resources/pts_in_hull.npy') # load cluster centers
net.params['class8_ab'][0].data[:, :, 0, 0] = pts_in_hull.transpose(
(1, 0)) # populate cluster centers as 1x1 convolution kernel
"""
from sys import getsizeof
from PIL import Image
from io import BytesIO
from flask_jwt_extended import create_access_token, JWTManager, jwt_required, get_jwt_identity
from flask_cors import CORS
app = Flask(__name__)
CORS(app)
app.config["JWT_SECRET_KEY"] = "secreto"
jwt = JWTManager(app)
ALLOWED_EXTENSIONS = ['png', 'jpg']
UPLOAD_FOLDER = 'images'
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
client = Minio(
"play.min.io", #ini kudu diganti keknya karena ini cuma buat trial gitu
access_key="Q3AM3UQ867SPQQA43P2F",
secret_key="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG")
@app.route("/getToken", methods=['GET'])
def getToken():
# add login later if needed
return jsonify(
token=create_access_token(identity="admin"),
howto=
"tambahin header 'Authorization' trus valuenya 'Bearer *tokennya *'")
# request body = groupname
@app.route('/addGroup', methods=['POST'])
@jwt_required()
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from datetime import datetime, timedelta
from examples.progress import Progress
from minio import Minio
from minio.commonconfig import GOVERNANCE, Tags
from minio.retention import Retention
from minio.sse import SseCustomerKey, SseKMS, SseS3
client = Minio(
"play.min.io",
access_key="Q3AM3UQ867SPQQA43P2F",
secret_key="zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG",
)
# Upload data.
result = client.fput_object(
"my-bucket",
"my-object",
"my-filename",
)
print(
"created {0} object; etag: {1}, version-id: {2}".format(
result.object_name,
result.etag,
result.version_id,
), )
from minio import Minio
from minio.error import ResponseError
client = Minio('192.168.38.230:9000',
access_key='1234567890',
secret_key='1234567890',
secure=False)
try:
response = client.get_partial_object('data2_test', 'test (1).jpg', 1000)
file = open("test_fixed2.jpg", "w")
file.write(response.read())
file.close()
except ResponseError as err:
print(err)
return self.run(*args, **kwargs)
celery.Task = ContextTask
return celery
def get_mysql_connection():
mysqlClient = connect(user=MYSQL_USER,
password=MYSQL_PASSWORD,
host=MYSQL_HOST,
database=MYSQL_DATABASE)
return mysqlClient
minioClient = Minio(MINIO_URL,
access_key=MINIO_ACCESS_KEY,
secret_key=MINIO_SECRET_KEY,
secure=False)
def with_minio(fn):
def fn_inner(*args, **kwargs):
try:
minioClient.make_bucket(MINIO_BUCKET_NAME)
except BucketAlreadyOwnedByYou:
pass
except BucketAlreadyExists:
pass
except ResponseError:
raise
try:
def test_object_is_not_empty_string(self):
client = Minio('localhost:9000')
client.get_object('hello', ' \t \n ')
# -*- coding: utf-8 -*-
# MinIO Python Library for Amazon S3 Compatible Cloud Storage,
# (C) 2015 MinIO, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
# dummy values, please replace them with original values.
from minio import Minio
from minio.error import ResponseError
client = Minio('s3.amazonaws.com',
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETACCESSKEY')
try:
print(client.bucket_exists('my-bucketname'))
except ResponseError as err:
print(err)
from requests.exceptions import HTTPError
from tools.tools import *
from tools.http import HttpResponse
from authentication.authenticator import is_authorized
from tools.tools import render_markdown
from minio import Minio, error
hexaa = HEXAA()
app = hexaa._app
config = hexaa._config
http = HttpResponse()
minioconf = config["minio"]
minio_key = os.environ.get("MINIO_ACCESS_KEY") or minioconf["key"]
minio_secret = os.environ.get("MINIO_SECRET_KEY") or minioconf["secret"]
miniocl = Minio(minioconf["url"], minio_key, minio_secret, secure=False)
app.jinja_env.globals.update(timestamp_to_age=timestamp_to_age)
app.jinja_env.globals.update(max=max)
app.jinja_env.globals.update(min=min)
app.jinja_env.globals.update(render_markdown=render_markdown)
app.jinja_env.globals.update(datatimeFromTimestamp=datatimeFromTimestamp)
app.jinja_env.globals.update(dataFromTimestamp=datatimeFromTimestamp)
app.jinja_env.globals.update(get_object_attr=get_object_attr)
app.jinja_env.globals.update(search_for_object=search_for_object)
app.config['UPLOAD_FOLDER'] = config['dirs']['tmp_code_dir']
@app.errorhandler(HTTPError)
def errors(error):
status_code = error.response.status_code
# - The credentials will be valid for between 15 minutes and 12 hours.
# - An access policy can be applied to the temporary credentials. The
# resulting permissions are the intersection of the role's existing policy
# and the optionally provided policy. You cannot grant more permissions than
# those allowed by the policy of the role that is being assumed.
# - YOUR-ACCESSKEYID and YOUR-SECRETACCESSKEY are
# dummy values, please replace them with original values.
# - To use minio with AWS, the `Minio` client that is passed to the
# AssumeRoleProvider must have the endpoint 'sts.amazonaws.com', and the
# RoleARN argument must be provided.
from minio import Minio
from minio.credentials import AssumeRoleProvider, Credentials
client = Minio('localhost:9000',
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETACCESSKEY')
restricted_upload_policy = """{
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"s3:PutObject"
],
"Effect": "Allow",
"Resource": [
"arn:aws:s3:::uploads/2020/*"
],
"Sid": "Upload-access-to-specific-bucket-only"
}
def robustness_check(object_storage_url,
object_storage_username,
object_storage_password,
data_bucket_name,
result_bucket_name,
model_id,
feature_testset_path='processed_data/X_test.npy',
label_testset_path='processed_data/y_test.npy',
clip_values=(0, 1),
nb_classes=2,
input_shape=(1, 3, 64, 64),
model_class_file='model.py',
model_class_name='model',
LossFn='',
Optimizer='',
epsilon=0.2):
url = re.compile(r"https?://")
cos = Minio(url.sub('', object_storage_url),
access_key=object_storage_username,
secret_key=object_storage_password)
dataset_filenamex = "X_test.npy"
dataset_filenamey = "y_test.npy"
weights_filename = "model.pt"
model_files = model_id + '/_submitted_code/model.zip'
cos.fget_object(data_bucket_name, feature_testset_path, dataset_filenamex)
cos.fget_object(data_bucket_name, label_testset_path, dataset_filenamey)
cos.fget_object(result_bucket_name, model_id + '/' + weights_filename,
weights_filename)
cos.fget_object(result_bucket_name, model_files, 'model.zip')
# Load PyTorch model definition from the source code.
zip_ref = zipfile.ZipFile('model.zip', 'r')
zip_ref.extractall('model_files')
zip_ref.close()
modulename = 'model_files.' + model_class_file.split('.')[0].replace(
'-', '_')
'''
We required users to define where the model class is located or follow
some naming convention we have provided.
'''
model_class = getattr(importlib.import_module(modulename),
model_class_name)
# load & compile model
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
model = model_class().to(device)
model.load_state_dict(torch.load(weights_filename, map_location=device))
# Define Loss and optimizer function for the PyTorch model
if LossFn:
loss_fn = eval(LossFn)
else:
loss_fn = torch.nn.CrossEntropyLoss()
if Optimizer:
optimizer = eval(Optimizer)
else:
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
# create pytorch classifier
classifier = PyTorchClassifier(clip_values, model, loss_fn, optimizer,
input_shape, nb_classes)
# load test dataset
x = np.load(dataset_filenamex)
y = np.load(dataset_filenamey)
# craft adversarial samples using FGSM
crafter = FastGradientMethod(classifier, eps=epsilon)
x_samples = crafter.generate(x)
# obtain all metrics (robustness score, perturbation metric, reduction in confidence)
metrics, y_pred_orig, y_pred_adv = get_metrics(model, x, x_samples, y)
print("metrics:", metrics)
return metrics
from rtlsdr import RtlSdr
from minio import Minio
from minio.error import (ResponseError, BucketAlreadyOwnedByYou,
BucketAlreadyExists)
import argparse
import datetime
import numpy as np
import pyaudio
import scipy.signal as signal
import speech_recognition as sr
import threading
# Initialize minioClient with an endpoint and access/secret keys.
minioClient = Minio('localhost:9000',
access_key='PX7Y4M4GOAW6ZM3VIRKN',
secret_key='JBejQ+JU5ClGYkAGgROYJM9sN+LnLNxsA+cEE200',
secure=False)
SampleStream = List[float]
AudioStream = List[int]
stream_buf = bytes()
stream_counter = 0
audio_rate = 48000
recognizer = sr.Recognizer()
audio_output = pyaudio.PyAudio().open(format=pyaudio.paInt16,
channels=1,
rate=audio_rate,
output=True)
def test_bucket_invalid_name(self):
client = Minio('localhost:9000')
for err in client.remove_objects('AB&CD', 'world'):
print(err)
''' @Author: Xiuxu Jin(jyxk) @Date: 2020-05-26 19:11:04 @LastEditors: Xiuxu Jin @LastEditTime: 2020-05-26 19:17:40 @Description: file content @Email: [email protected] ''' from minio import Minio from minio.error import ResponseError from minio.error import (ResponseError, BucketAlreadyOwnedByYou,BucketAlreadyExists) minioClient = Minio('play.min.io:9000',access_key='Q3AM3UQ867SPQQA43P2F',secret_key='zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG',secure=True) # Make a bucket with the make_bucket API call. try: minioClient.make_bucket("maylogs", location="us-east-1") except BucketAlreadyOwnedByYou as err: pass except BucketAlreadyExists as err: pass except ResponseError as err: raise else: # Put an object 'pumaserver_debug.log' with contents from 'pumaserver_debug.log'. try: minioClient.fput_object('maylogs', 'pumaserver_debug.log', '/tmp/pumaserver_debug.log') except ResponseError as err: print(err)
