def test_sas_signed_identifier(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
file_name = self._create_file()
access_policy = AccessPolicy()
access_policy.start = '2011-10-11'
access_policy.expiry = '2018-10-12'
access_policy.permission = FilePermissions.READ
identifiers = {'testid': access_policy}
resp = self.fs.set_share_acl(self.share_name, identifiers)
token = self.fs.generate_file_shared_access_signature(
self.share_name,
None,
file_name,
id='testid'
)
# Act
service = FileService(
self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
request_session=requests.Session(),
)
self._set_test_proxy(service, self.settings)
result = service.get_file_to_bytes(self.share_name, None, file_name)
# Assert
self.assertEqual(self.short_byte_data, result.content)
def test_sas_access_file(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
file_name = self._create_file()
token = self.fs.generate_file_shared_access_signature(
self.share_name,
None,
file_name,
permission=FilePermissions.READ,
expiry=datetime.utcnow() + timedelta(hours=1),
)
# Act
service = FileService(
self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
request_session=requests.Session(),
)
self._set_test_proxy(service, self.settings)
result = service.get_file_to_bytes(self.share_name, None, file_name)
# Assert
self.assertEqual(self.short_byte_data, result.content)
def test_sas_access_file(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
file_name = self._create_file()
token = self.fs.generate_file_shared_access_signature(
self.share_name,
None,
file_name,
permission=FilePermissions.READ,
expiry=datetime.utcnow() + timedelta(hours=1),
)
# Act
service = FileService(
self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
request_session=requests.Session(),
)
self._set_test_proxy(service, self.settings)
result = service.get_file_to_bytes(self.share_name, None, file_name)
# Assert
self.assertEqual(self.short_byte_data, result.content)
def test_sas_signed_identifier(self):
# SAS URL is calculated from storage key, so this test runs live only
if TestMode.need_recording_file(self.test_mode):
return
# Arrange
file_name = self._create_file()
access_policy = AccessPolicy()
access_policy.start = '2011-10-11'
access_policy.expiry = '2018-10-12'
access_policy.permission = FilePermissions.READ
identifiers = {'testid': access_policy}
resp = self.fs.set_share_acl(self.share_name, identifiers)
token = self.fs.generate_file_shared_access_signature(self.share_name,
None,
file_name,
id='testid')
# Act
service = FileService(
self.settings.STORAGE_ACCOUNT_NAME,
sas_token=token,
request_session=requests.Session(),
)
self._set_service_options(service, self.settings)
result = service.get_file_to_bytes(self.share_name, None, file_name)
# Assert
self.assertEqual(self.short_byte_data, result.content)
def initialize(folder, filename, share="models-share") -> rt.InferenceSession:
file_service = FileService(account_name=FILE_ACCOUNT_NAME, account_key=FILE_ACCOUNT_KEY)
onnx_file = file_service.get_file_to_bytes(share, folder, filename)
print(f"Inference session initialized")
inference_session = rt.InferenceSession(onnx_file.content)
return inference_session
def load_data(source_dir='./data/final_project',
scale_configs=True,
data_from='local'):
data_dim = 1
configs = []
learning_curves = []
if data_from == 'local':
for fn in glob.glob(os.path.join(source_dir, "*.json")):
with open(fn, 'r') as fh:
tmp = json.load(fh)
configs.append(tmp['config']) # list of dicts
learning_curves.append(tmp['learning_curve'])
elif data_from == 'azure':
file_service = FileService(
account_name='jochenstorage',
account_key=
'7onD5l6X5dfmfUC+gXJCnKRn5AwPRXz8lEDaIsTVH5Di5y0wFTWgitc6Rq2TV85Zjx+EOMlRJYsxFHt1R41qPA=='
)
generator = file_service.list_directories_and_files(
'jochenfileshare/dlproject_data/final_project')
for file_or_dir in generator:
# print(file_or_dir.name)
myfile = file_service.get_file_to_bytes(
'jochenfileshare', 'dlproject_data/final_project',
file_or_dir.name)
tmp = json.loads(myfile.content.decode("utf-8"))
# tmp = json.loads(myfile.content)
configs.append(tmp['config']) # list of dicts
learning_curves.append(tmp['learning_curve'])
else:
print("invalid source ", data_from)
configs = configs_to_arr(configs) # from list of dicts to np.array
if scale_configs:
print("scaling configuration data")
configs = preprocessing.scale(configs)
lcs = np.array(learning_curves)
Y = lcs[:, -1]
# Keras LSTM expects data as [sample_no, timesteps, feature_no (X.shape[1]) ]
lcs = lcs.reshape(lcs.shape[0], lcs.shape[1], data_dim)
Y = Y.reshape(Y.shape[0], 1)
return configs, lcs, Y
# In[3]:
data = []
labels = []
# In[4]:
import os
kaggle_data = os.listdir('./kaggle_data/')
azure_data = file_service.list_directories_and_files('covid-share/data')
for i, file_or_dir in enumerate(azure_data):
print(f'COIVD - {file_or_dir.name} | NORMAL - {kaggle_data[i]}')
# Getting file from storage
byte_data = file_service.get_file_to_bytes('covid-share', 'data',
file_or_dir.name).content
np_bytes = np.fromstring(byte_data, np.uint8)
# Reshape
az_img = cv2.imdecode(np_bytes, cv2.COLOR_BGR2RGB)
az_img = cv2.resize(az_img, (224, 224))
data.append(az_img)
labels.append(1)
kag_img = cv2.imread('./kaggle_data/' + kaggle_data[i])
kag_img = cv2.cvtColor(kag_img, cv2.COLOR_BGR2RGB)
kag_img = cv2.resize(kag_img, (224, 224))
data.append(kag_img)
labels.append(0)
class DataSetAzure(DataSetBase):
def __init__(self):
super().__init__()
name = self.meta['account']['name']
key = self.meta['account']['key']
self.azure_files = FileService(account_name=name, account_key=key)
self.share_name = self.meta['account']['share-name']
# バイナリデータからラベルデータを取得
label_base = self.azure_files.get_file_to_bytes(
share_name=self.share_name,
directory_name=self.meta['dataset-dir'],
file_name=self.meta['dataset-file']['label'])
self.labels = np.frombuffer(label_base.content,
np.float64).reshape([-1, 3])
# バイナリデータからデータセットを取得
dataset_base = self.azure_files.get_file_to_bytes(
share_name=self.share_name,
directory_name=self.meta['dataset-dir'],
file_name=self.meta['dataset-file']['data'])
shape_base = self.azure_files.get_file_to_bytes(
share_name=self.share_name,
directory_name=self.meta['dataset-dir'],
file_name=self.meta['dataset-file']['shape'])
shape = np.frombuffer(shape_base.content, np.int64)
self.dataset = np.frombuffer(dataset_base.content,
np.float64).reshape(shape)
# バイナリデータからテストラベルデータを取得
label_test_base = self.azure_files.get_file_to_bytes(
share_name=self.share_name,
directory_name=self.meta['dataset-dir'],
file_name=self.meta['dataset-test-file']['label'])
self.test_labels = np.frombuffer(label_test_base.content,
np.float64).reshape([-1, 3])
# バイナリデータからテストデータセットを取得
dataset_base = self.azure_files.get_file_to_bytes(
share_name=self.share_name,
directory_name=self.meta['dataset-dir'],
file_name=self.meta['dataset-test-file']['data'])
shape_base = self.azure_files.get_file_to_bytes(
share_name=self.share_name,
directory_name=self.meta['dataset-dir'],
file_name=self.meta['dataset-test-file']['shape'])
shape = np.frombuffer(shape_base.content, np.int64)
self.test_dataset = np.frombuffer(dataset_base.content,
np.float64).reshape(shape)
# データ取得メソッド
def get_data(self, dir, file):
"""
:param dir: 目的のデータがあるディレクトリ
:param file: ファイル名
:return: 取得したデータの中身
"""
data = self.azure_files.get_file_to_bytes(share_name=self.share_name,
directory_name=dir,
file_name=file)
return data.content
# データ取得メソッド
def get_training_dataset(self, start=None, end=None):
datas = self.GET_DATAS(datas=self.dataset,
start=start,
end=end,
c_func=None)
return datas
# ラベル取得メソッド
def get_training_labels(self, start=None, end=None):
labels = self.GET_DATAS(datas=self.labels,
start=start,
end=end,
c_func=None)
return labels
def get_test_dataset(self):
return self.test_dataset
def get_test_labels(self):
return self.test_labels
# （テスト用）
def get_dir_metadatas(self, dir):
meta = self.azure_files.get_directory_properties(
share_name=self.share_name, directory_name=dir)
print(meta)
