def test_sas_signed_identifier(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange file_name = self._create_file() access_policy = AccessPolicy() access_policy.start = '2011-10-11' access_policy.expiry = '2018-10-12' access_policy.permission = FilePermissions.READ identifiers = {'testid': access_policy} resp = self.fs.set_share_acl(self.share_name, identifiers) token = self.fs.generate_file_shared_access_signature( self.share_name, None, file_name, id='testid' ) # Act service = FileService( self.settings.STORAGE_ACCOUNT_NAME, sas_token=token, request_session=requests.Session(), ) self._set_test_proxy(service, self.settings) result = service.get_file_to_bytes(self.share_name, None, file_name) # Assert self.assertEqual(self.short_byte_data, result.content)
def test_sas_access_file(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange file_name = self._create_file() token = self.fs.generate_file_shared_access_signature( self.share_name, None, file_name, permission=FilePermissions.READ, expiry=datetime.utcnow() + timedelta(hours=1), ) # Act service = FileService( self.settings.STORAGE_ACCOUNT_NAME, sas_token=token, request_session=requests.Session(), ) self._set_test_proxy(service, self.settings) result = service.get_file_to_bytes(self.share_name, None, file_name) # Assert self.assertEqual(self.short_byte_data, result.content)
def test_sas_access_file(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange file_name = self._create_file() token = self.fs.generate_file_shared_access_signature( self.share_name, None, file_name, permission=FilePermissions.READ, expiry=datetime.utcnow() + timedelta(hours=1), ) # Act service = FileService( self.settings.STORAGE_ACCOUNT_NAME, sas_token=token, request_session=requests.Session(), ) self._set_test_proxy(service, self.settings) result = service.get_file_to_bytes(self.share_name, None, file_name) # Assert self.assertEqual(self.short_byte_data, result.content)
def test_sas_signed_identifier(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange file_name = self._create_file() access_policy = AccessPolicy() access_policy.start = '2011-10-11' access_policy.expiry = '2018-10-12' access_policy.permission = FilePermissions.READ identifiers = {'testid': access_policy} resp = self.fs.set_share_acl(self.share_name, identifiers) token = self.fs.generate_file_shared_access_signature(self.share_name, None, file_name, id='testid') # Act service = FileService( self.settings.STORAGE_ACCOUNT_NAME, sas_token=token, request_session=requests.Session(), ) self._set_service_options(service, self.settings) result = service.get_file_to_bytes(self.share_name, None, file_name) # Assert self.assertEqual(self.short_byte_data, result.content)
def initialize(folder, filename, share="models-share") -> rt.InferenceSession: file_service = FileService(account_name=FILE_ACCOUNT_NAME, account_key=FILE_ACCOUNT_KEY) onnx_file = file_service.get_file_to_bytes(share, folder, filename) print(f"Inference session initialized") inference_session = rt.InferenceSession(onnx_file.content) return inference_session
def load_data(source_dir='./data/final_project', scale_configs=True, data_from='local'): data_dim = 1 configs = [] learning_curves = [] if data_from == 'local': for fn in glob.glob(os.path.join(source_dir, "*.json")): with open(fn, 'r') as fh: tmp = json.load(fh) configs.append(tmp['config']) # list of dicts learning_curves.append(tmp['learning_curve']) elif data_from == 'azure': file_service = FileService( account_name='jochenstorage', account_key= '7onD5l6X5dfmfUC+gXJCnKRn5AwPRXz8lEDaIsTVH5Di5y0wFTWgitc6Rq2TV85Zjx+EOMlRJYsxFHt1R41qPA==' ) generator = file_service.list_directories_and_files( 'jochenfileshare/dlproject_data/final_project') for file_or_dir in generator: # print(file_or_dir.name) myfile = file_service.get_file_to_bytes( 'jochenfileshare', 'dlproject_data/final_project', file_or_dir.name) tmp = json.loads(myfile.content.decode("utf-8")) # tmp = json.loads(myfile.content) configs.append(tmp['config']) # list of dicts learning_curves.append(tmp['learning_curve']) else: print("invalid source ", data_from) configs = configs_to_arr(configs) # from list of dicts to np.array if scale_configs: print("scaling configuration data") configs = preprocessing.scale(configs) lcs = np.array(learning_curves) Y = lcs[:, -1] # Keras LSTM expects data as [sample_no, timesteps, feature_no (X.shape[1]) ] lcs = lcs.reshape(lcs.shape[0], lcs.shape[1], data_dim) Y = Y.reshape(Y.shape[0], 1) return configs, lcs, Y
# In[3]: data = [] labels = [] # In[4]: import os kaggle_data = os.listdir('./kaggle_data/') azure_data = file_service.list_directories_and_files('covid-share/data') for i, file_or_dir in enumerate(azure_data): print(f'COIVD - {file_or_dir.name} | NORMAL - {kaggle_data[i]}') # Getting file from storage byte_data = file_service.get_file_to_bytes('covid-share', 'data', file_or_dir.name).content np_bytes = np.fromstring(byte_data, np.uint8) # Reshape az_img = cv2.imdecode(np_bytes, cv2.COLOR_BGR2RGB) az_img = cv2.resize(az_img, (224, 224)) data.append(az_img) labels.append(1) kag_img = cv2.imread('./kaggle_data/' + kaggle_data[i]) kag_img = cv2.cvtColor(kag_img, cv2.COLOR_BGR2RGB) kag_img = cv2.resize(kag_img, (224, 224)) data.append(kag_img) labels.append(0)
class DataSetAzure(DataSetBase): def __init__(self): super().__init__() name = self.meta['account']['name'] key = self.meta['account']['key'] self.azure_files = FileService(account_name=name, account_key=key) self.share_name = self.meta['account']['share-name'] # バイナリデータからラベルデータを取得 label_base = self.azure_files.get_file_to_bytes( share_name=self.share_name, directory_name=self.meta['dataset-dir'], file_name=self.meta['dataset-file']['label']) self.labels = np.frombuffer(label_base.content, np.float64).reshape([-1, 3]) # バイナリデータからデータセットを取得 dataset_base = self.azure_files.get_file_to_bytes( share_name=self.share_name, directory_name=self.meta['dataset-dir'], file_name=self.meta['dataset-file']['data']) shape_base = self.azure_files.get_file_to_bytes( share_name=self.share_name, directory_name=self.meta['dataset-dir'], file_name=self.meta['dataset-file']['shape']) shape = np.frombuffer(shape_base.content, np.int64) self.dataset = np.frombuffer(dataset_base.content, np.float64).reshape(shape) # バイナリデータからテストラベルデータを取得 label_test_base = self.azure_files.get_file_to_bytes( share_name=self.share_name, directory_name=self.meta['dataset-dir'], file_name=self.meta['dataset-test-file']['label']) self.test_labels = np.frombuffer(label_test_base.content, np.float64).reshape([-1, 3]) # バイナリデータからテストデータセットを取得 dataset_base = self.azure_files.get_file_to_bytes( share_name=self.share_name, directory_name=self.meta['dataset-dir'], file_name=self.meta['dataset-test-file']['data']) shape_base = self.azure_files.get_file_to_bytes( share_name=self.share_name, directory_name=self.meta['dataset-dir'], file_name=self.meta['dataset-test-file']['shape']) shape = np.frombuffer(shape_base.content, np.int64) self.test_dataset = np.frombuffer(dataset_base.content, np.float64).reshape(shape) # データ取得メソッド def get_data(self, dir, file): """ :param dir: 目的のデータがあるディレクトリ :param file: ファイル名 :return: 取得したデータの中身 """ data = self.azure_files.get_file_to_bytes(share_name=self.share_name, directory_name=dir, file_name=file) return data.content # データ取得メソッド def get_training_dataset(self, start=None, end=None): datas = self.GET_DATAS(datas=self.dataset, start=start, end=end, c_func=None) return datas # ラベル取得メソッド def get_training_labels(self, start=None, end=None): labels = self.GET_DATAS(datas=self.labels, start=start, end=end, c_func=None) return labels def get_test_dataset(self): return self.test_dataset def get_test_labels(self): return self.test_labels # (テスト用) def get_dir_metadatas(self, dir): meta = self.azure_files.get_directory_properties( share_name=self.share_name, directory_name=dir) print(meta)