def get_backup(gw_account_name, gw_account_key, gw_account_share, backup_local_path):
"""Upload directories and files from $account_name to local $backup_local_path using Azure FileService"""
print('\nRunning get_backup from the {} and file share {} to local path {}.\n'.format(gw_account_name, gw_account_share, backup_local_path))
file_service = FileService(account_name=gw_account_name, account_key=gw_account_key)
share_dirs_list = file_service.list_directories_and_files(gw_account_share)
for share_dir_name in share_dirs_list:
backup_local_dir = os.path.join(backup_local_path, share_dir_name.name)
if not os.path.isdir(backup_local_dir):
print('Local backup directory {} not found, creating...'.format(backup_local_dir))
os.makedirs(backup_local_dir)
share_files_list = file_service.list_directories_and_files(gw_account_share, share_dir_name.name)
for share_file in share_files_list:
try:
print('Getting file: {}'.format(os.path.join('/', share_dir_name.name, share_file.name)))
# example:
# file_service.get_file_to_path('gwdevproxydata', 'datanginx-conf.d', 'jm-gw-proxy-dev.domain.tld.conf', '/tmp/jm-gw-proxy-dev.domain.tld.conf-out')
file_service.get_file_to_path(gw_account_share, share_dir_name.name, share_file.name, os.path.join(backup_local_dir, share_file.name))
# to pass /data/datahtml/.well-known dir on master host
except azure.common.AzureMissingResourceHttpError as e:
print('\nWARNING: {}\n'.format(e))
class Azure:
def __init__(self, ac, key, fileshare):
self.account_name = ac
self.account_key = key
self.fileshare_name = fileshare
#Create a FileService that is used to call the File Service for the storage account
self.file_service = FileService(account_name=ac, account_key=key)
return
def List_directory(self, directory_path):
self.generator = self.file_service.list_directories_and_files(
directory_path)
print("Files in the directory: " + directory_path)
for file_or_dir in self.generator:
print("\t File/Directory name: " + file_or_dir.name)
return
def Download(self, loc_directory, fileshare_directory_name):
local_path = os.path.expanduser("~/" + loc_directory)
print("\nDownloading the following files to " + local_path)
self.generator = self.file_service.list_directories_and_files(
self.fileshare_name + "/" + fileshare_directory_name)
for file_or_dir in self.generator:
print("\t File/Directory name: " + file_or_dir.name)
for file_or_dir in self.generator:
#full_path_to_file2 = os.path.join(local_path, file_or_dir.name)
self.file_service.get_file_to_path(self.fileshare_name,
fileshare_directory_name,
file_or_dir.name,
local_path + file_or_dir.name)
print("\nFiles downloaded to " + local_path)
return
def Upload(self, loc_directory, fileshare_directory_name):
local_path = os.path.expanduser("~/" + loc_directory)
self.generator = self.file_service.list_directories_and_files(
self.fileshare_name + "/" + fileshare_directory_name)
print("\nUploading the following files to " + fileshare_directory_name)
entries = os.listdir(local_path)
# for entry in entries:
# print(entry)
for entry in entries:
self.file_service.create_file_from_path(
self.fileshare_name, #Fileshare name
fileshare_directory_name, # We want to create this blob in the root directory, so we specify None for the directory_name
entry, #name of the file that is created
local_path + entry, #file that needs to be uploaded
content_settings=ContentSettings(
content_type='application/vnd.ms-excel'))
print("The followig files have been uploaded")
#listing the files in the fileshare_name
obj.List_directory(fileshare_name + "/" +
fileshare_directory_name_upload)
def getLatestModel(customer, modelName, storage_account_name,
storage_account_key):
fileService = FileService(account_name=storage_account_name,
account_key=storage_account_key)
if fileService.exists('trainedmodels', customer):
modelTimestampArr = []
files = fileService.list_directories_and_files('trainedmodels',
customer + '/' +
modelName,
prefix=modelName)
for file in files:
date = file.name.split('.')[0].split('_')[1]
modelTimestampArr.append(date)
latestModelFileName = modelName + '_' + max(modelTimestampArr) + '.pkl'
print(latestModelFileName)
file = fileService.get_file_to_bytes('trainedmodels',
customer + '/' + modelName,
latestModelFileName)
model = pickle.loads(file.content)['model']
return model
else:
print('Customer or model not found.')
def _get_files_from_afs(cli_ctx, afs, path, expiry):
"""Returns a list of files and directories under given path on mounted Azure File share.
:param models.AzureFileShareReference afs: Azure file share reference.
:param str path: path to list files from.
:param int expiry: SAS expiration time in minutes.
"""
from azure.storage.file import FileService
from azure.storage.file.models import File, FilePermissions
result = []
service = FileService(afs.account_name, _get_storage_account_key(cli_ctx, afs.account_name, None))
share_name = afs.azure_file_url.split('/')[-1]
effective_path = _get_path_for_storage(path)
if not service.exists(share_name, effective_path):
return result
for f in service.list_directories_and_files(share_name, effective_path):
if isinstance(f, File):
sas = service.generate_file_shared_access_signature(
share_name, effective_path, f.name, permission=FilePermissions(read=True),
expiry=datetime.datetime.utcnow() + datetime.timedelta(minutes=expiry))
result.append(
LogFile(
f.name, service.make_file_url(share_name, effective_path, f.name, 'https', sas),
False, f.properties.content_length))
else:
result.append(LogFile(f.name, None, True, None))
return result
def upload(path: str):
from azure.storage.file import FileService
service = FileService(account_name=config['account_name'],
account_key=config['account_key'])
if shared_directory not in service.list_directories_and_files(
config['share_name']):
service.create_directory(config['share_name'], shared_directory)
service.create_file_from_path(config['share_name'], shared_directory,
path.split('/')[-1], path)
def file():
static_dir_path = "D:\home\site\wwwroot\static"
static_file_dir_path = static_dir_path + '\\' + 'files'
account_name = 'hanastragetest'
account_key = 'acount_key'
root_share_name = 'root'
share_name = 'images'
directory_url = 'https://hanastragetest.file.core.windows.net/' + root_share_name + '/' + share_name
# create local save directory
if os.path.exist(static_file_dir_path) is False:
os.mkdir(static_file_dir_path)
file_service = FileService(account_name=account_name,
account_key=account_key)
# create share
file_service.create_share(root_share_name)
# create directory
file_service.create_directory(root_share_name, share_name)
files = os.listdir(static_dir_path)
for file in files:
# delete
if file_service.exists(root_share_name, share_name, file):
file_service.delete_file(root_share_name, share_name, file)
# file upload
file_service.create_file_from_path(
root_share_name,
share_name, # We want to create this blob in the root directory, so we specify None for the directory_name
file,
static_dir_path + '\\' + file,
content_settings=ContentSettings(content_type='image/png'))
generator = file_service.list_directories_and_files(
root_share_name, share_name)
html = ""
for file in generator:
# file download
file_save_path = static_file_dir_path + '\\' + file
file_service.get_file_to_path(root_share_name, share_name, file,
file_save_path)
html = "{}<img src='{}'>".format(html, file_save_path)
result = {
"result": True,
"data": {
"file_or_dir_name":
[file_or_dir.name for file_or_dir in generator]
}
}
return make_response(json.dumps(result, ensure_ascii=False) + html)
def num1():
t = {}
i = 1
file_service = FileService(
account_name='mystorge',
account_key=
'0T4f/dzyV7AIw4a9bevK5ysML0qP55CEWEqJyJWXyr6fKRxowLq8tL7mep/MfSc//mcQggeH1+K79A4HUDug3w=='
)
generator = file_service.list_directories_and_files('image1')
for file_or_dir in generator:
t[i] = file_or_dir.name
i += 1
return render_template("table.html", t=t)
def clean(shared_directory, config_path, remove_directory):
config_path = os.path.expanduser(config_path)
with open(config_path) as f:
config = json.load(f)
from azure.storage.file import FileService
service = FileService(account_name=config['account_name'],
account_key=config['account_key'])
if service.exists(config['share_name'], shared_directory):
for file in service.list_directories_and_files(config['share_name'],
shared_directory):
service.delete_file(config['share_name'], shared_directory,
file.name)
if remove_directory:
service.delete_directory(config['share_name'], shared_directory)
def download_data():
file_service = FileService(
account_name='personimagedata',
account_key=
'RNlmOs2+a0l0Vb2Ogxi3fSMA/LKsHgutsgvQE6p/Nu5OrXhmma8XNfnT3qFobkpGqYTecz75MXrmkoTThScJ+w=='
)
generator = file_service.list_directories_and_files(SERVER_FOLDER)
downloaded_files = set(
filename for filename in os.listdir(os.getcwd() + '/' + DATAFILE))
for file_or_dir in generator:
print(file_or_dir.name)
if file_or_dir.name not in downloaded_files:
file_service.get_file_to_path(SERVER_FOLDER, None,
file_or_dir.name,
DATAFILE + file_or_dir.name)
def load_data(source_dir='./data/final_project',
scale_configs=True,
data_from='local'):
data_dim = 1
configs = []
learning_curves = []
if data_from == 'local':
for fn in glob.glob(os.path.join(source_dir, "*.json")):
with open(fn, 'r') as fh:
tmp = json.load(fh)
configs.append(tmp['config']) # list of dicts
learning_curves.append(tmp['learning_curve'])
elif data_from == 'azure':
file_service = FileService(
account_name='jochenstorage',
account_key=
'7onD5l6X5dfmfUC+gXJCnKRn5AwPRXz8lEDaIsTVH5Di5y0wFTWgitc6Rq2TV85Zjx+EOMlRJYsxFHt1R41qPA=='
)
generator = file_service.list_directories_and_files(
'jochenfileshare/dlproject_data/final_project')
for file_or_dir in generator:
# print(file_or_dir.name)
myfile = file_service.get_file_to_bytes(
'jochenfileshare', 'dlproject_data/final_project',
file_or_dir.name)
tmp = json.loads(myfile.content.decode("utf-8"))
# tmp = json.loads(myfile.content)
configs.append(tmp['config']) # list of dicts
learning_curves.append(tmp['learning_curve'])
else:
print("invalid source ", data_from)
configs = configs_to_arr(configs) # from list of dicts to np.array
if scale_configs:
print("scaling configuration data")
configs = preprocessing.scale(configs)
lcs = np.array(learning_curves)
Y = lcs[:, -1]
# Keras LSTM expects data as [sample_no, timesteps, feature_no (X.shape[1]) ]
lcs = lcs.reshape(lcs.shape[0], lcs.shape[1], data_dim)
Y = Y.reshape(Y.shape[0], 1)
return configs, lcs, Y
def discover_resources(**kwargs):
discovered_azure_sql = []
for handler in AzureARMHandler.objects.all():
set_progress('Connecting to Azure storage \
files for handler: {}'.format(handler))
credentials = ServicePrincipalCredentials(
client_id=handler.client_id,
secret=handler.secret,
tenant=handler.tenant_id
)
azure_client = storage.StorageManagementClient(
credentials, handler.serviceaccount)
azure_resources_client = resources.ResourceManagementClient(
credentials, handler.serviceaccount)
for resource_group in azure_resources_client.resource_groups.list():
try:
for st in azure_client.storage_accounts.list_by_resource_group(resource_group.name)._get_next().json()['value']:
res = azure_client.storage_accounts.list_keys(
resource_group.name, st['name'])
keys = res.keys
file_service = FileService(
account_name=st['name'], account_key=keys[1].value)
for share in file_service.list_shares():
for file in file_service.list_directories_and_files(share_name=share.name).items:
if type(file) is File:
discovered_azure_sql.append(
{
'name': share.name + '-' + file.name,
'azure_storage_file_name': file.name,
'azure_file_identifier': share.name + '-' + file.name,
'azure_storage_file_share_name': share.name,
'resource_group_name': resource_group.name,
'azure_rh_id': handler.id,
'azure_storage_account_name': st['name'],
'azure_account_key': keys[0].value,
'azure_account_key_fallback': keys[1].value
}
)
except:
continue
return discovered_azure_sql
def cleanupBackups(self):
result = messagebox.askquestion('You are about to save the changes', 'Are you sure?', icon='warning')
if result == 'yes':
# files = []
# for (dirpath, dirname, filenames) in walk(self.gfwlistFileDir):
# files.extend(filenames)
# for f in files:
# print (f)
# the following is for cleaning up files locally
# bkups = glob.glob(os.path.join(self.gfwlistFileDir, '*.bk'))
# for f in bkups[:len(bkups)-1]:
# os.remove(f)
azureFileService = FileService(account_name=self.azureAccountName, account_key=self.azureAccountKey)
generator = azureFileService.list_directories_and_files(self.azureFileShareName, self.azureFileShareFileDir)
for fileOrDir in generator:
if (fileOrDir.name.endswith('.bk')):
azureFileService.delete_file(self.azureFileShareName, self.azureFileShareFileDir, fileOrDir.name)
def downloadStoredFiles(config, accountKey, sourceDir, targetDir):
fs = FileService(account_name=config['STORAGE_ACCOUNT'],
account_key=accountKey)
storageLoc = config['STORAGE_LOCATION']
if not path.exists(targetDir):
makedirs(targetDir)
print(
f'\nFileService: reading files from Azure Storage location="{storageLoc}" directory="{sourceDir}"'
)
if not fs.exists(storageLoc, sourceDir):
return
dirsFiles = fs.list_directories_and_files(storageLoc, sourceDir)
fileNames = [
df.name for df in dirsFiles
if df.name.endswith('.txt') or df.name.endswith('.csv')
]
for fname in fileNames:
if path.exists(path.join(targetDir, fname)):
print(f'already got file={fname}')
else:
print(f'downloading file={fname}')
fs.get_file_to_path(storageLoc, sourceDir, fname,
path.join(targetDir, fname))
def list(self, container_or_share_name, container=None, account=None):
"""List the blobs/files inside a container/share_name.
Args:
container_or_share_name(str): Name of the container/share_name where we want to list the blobs/files.
container(bool): flag to know it you are listing files or blobs.
account(str): The name of the storage account.
"""
key = self.storage_client.storage_accounts.list_keys(
self.resource_group_name, account).keys[0].value
if container:
bs = BlockBlobService(account_name=account, account_key=key)
container_list = []
for i in bs.list_blobs(container_or_share_name).items:
container_list.append(i.name)
return container_list
elif not container:
fs = FileService(account_name=account, account_key=key)
container_list = []
for i in fs.list_directories_and_files(
container_or_share_name).items:
container_list.append(i.name)
return container_list
else:
raise ValueError("You have to pass a value for container param")
import configparser
from azure.storage.file import FileService
configs = configparser.ConfigParser()
configs.read("C:/ws/filescripts/config.ini")
storage_key = configs.get("Azure", "storage-key")
share_name = configs.get("Azure", "share-name")
storage_name = configs.get("Azure", "storage-name")
remote_data_path = 'prod/WeeklyReports'
local_data_path = 'C:/data/ford/WeeklyReports/'
file_service = FileService(account_name=share_name, account_key=storage_key)
file_service.set_proxy(host='localhost', port='3128')
shares = list(
file_service.list_directories_and_files(storage_name, remote_data_path))
file_list = list(map(lambda x: x.name, shares))
for fn in file_list:
file_service.get_file_to_path(storage_name, remote_data_path, fn,
local_data_path + fn)
# fn = 'MonthlyReport_June_2017.txt'
# file_service.get_file_to_path(storage_name, remote_data_path, fn, local_data_path+fn)
#connect to your storage account
from azure.storage.file import FileService
from azure.storage.file import ContentSettings
import os
# Set up
directory = os.fsencode('C:/Users/luopa/Desktop/Cognitive/result')
file_service = FileService(
account_name='eyc3blob',
account_key=
'8lWwXYYTbRtpg02HRd/DiVrSfyF4xNqtGmUzIy1GkLLOG1tXsXGLZG2KbDCz3XerLM1xPV4nl62YgmhAYAAUTQ=='
)
# Create result directory
# file_service.create_directory(share_name='eyc3file/speech', directory_name='result')
# Looping through the documents
generator = file_service.list_directories_and_files('eyc3file/speech')
for file in os.listdir(directory):
filename = os.fsdecode(file)
print(filename)
file_service.create_file_from_path(
share_name='eyc3file/speech',
directory_name='result',
file_name=filename,
local_file_path='C:/Users/luopa/Desktop/Cognitive/result' + '/' +
filename)
class StorageHelper(object):
"""Handle details related to a single storage account and share.
Instantiate this object with information sufficient to
uniquely identify a storage account and a file share within it.
Then .account can be used to retrieve the Azure SDK for Python
object corresponding to the account, and .key can be used to
get an access key for it.
For both those properties, if the value mentioned doesn't exist,
it will be created upon first property access.
"""
def __init__(self,
client_data,
resource_helper,
name,
account=None,
default_share='share'):
self.name = name
self.default_share = default_share
self._account = account
self._key = os.environ.get('AZURE_STORAGE_KEY')
self.resource_helper = resource_helper
self.client = StorageManagementClient(*client_data)
self.file_service = FileService(
account_name=self.account.name,
account_key=self.key,
)
@property
def account(self):
"""Return the managed StorageAccounts object.
If no such account exists, create it first.
"""
if self._account is None:
print('Creating storage account...')
# Error to create storage account if it already exists!
name_check = self.client.storage_accounts.check_name_availability(
self.name)
if name_check.name_available:
storage_creation = self.client.storage_accounts.create(
self.resource_helper.group.name, self.name,
StorageAccountCreateParameters(
sku=StorageAccountSku(StorageSkuName.standard_lrs),
kind=StorageKind.storage,
location=self.resource_helper.group.location,
))
storage = storage_creation.result()
else:
try:
storage = self.client.storage_accounts.get_properties(
self.resource_helper.group.name, self.name)
except CloudError:
print('Storage account {} already exists'
' in a resource group other than {}.'.format(
self.name, self.resource_helper.group.name))
print('Got storage account:', storage.name)
self._account = storage
return self._account
@property
def key(self):
"""Get the first available storage key.
This will crash if there are no available storage keys,
which is unlikely since two are created along with a storage account.
"""
if self._key is None:
storage_keys = self.client.storage_accounts.list_keys(
self.resource_helper.group.name, self.account.name)
self._key = next(iter(storage_keys.keys)).value
return self._key
def upload_file(self, path, sharename):
"""Upload a file into the default share on the storage account.
If the share doesn't exist, create it first.
"""
self.file_service.create_file_from_path(
self.default_share if sharename is None else sharename,
None,
os.path.basename(path),
path,
)
return '/'.join([self.default_share, os.path.basename(path)])
def download_file(self, sharename, filename):
file_service.get_file_to_path(sharename, None, filename, filename)
def delete_file(self, sharename, filename):
file_service.delete_file(sharename, None, filename)
def create_share(self, sharename):
self.file_service.create_share(sharename)
def create_directory(self, sharename, directoryname):
self.file_service.create_directory(sharename, directoryname)
def list_directories_and_files(self, sharename):
generator = self.file_service.list_directories_and_files(sharename)
return [file_or_dir.name for file_or_dir in generator]
def list_shares(self):
shares = list(self.file_service.list_shares(include_snapshots=True))
sharelist = [fileshare.name for fileshare in shares]
print(sharelist)
return sharelist
# In[2]:
file_service = FileService(account_name='covidmodels', account_key='')
# In[ ]:
# In[3]:
data = []
labels = []
# In[4]:
import os
kaggle_data = os.listdir('./kaggle_data/')
azure_data = file_service.list_directories_and_files('covid-share/data')
for i, file_or_dir in enumerate(azure_data):
print(f'COIVD - {file_or_dir.name} | NORMAL - {kaggle_data[i]}')
# Getting file from storage
byte_data = file_service.get_file_to_bytes('covid-share', 'data',
file_or_dir.name).content
np_bytes = np.fromstring(byte_data, np.uint8)
# Reshape
az_img = cv2.imdecode(np_bytes, cv2.COLOR_BGR2RGB)
az_img = cv2.resize(az_img, (224, 224))
data.append(az_img)
labels.append(1)
class AzureFileManager():
def __init__(self):
# fetch config data
conf = Configuration()
# create Azure File share service
self.file_service = FileService(
account_name=conf.account_name, account_key=conf.account_key)
# set azure share file name (container)
self.file_share = conf.file_share
def upload_file(self, upload_path, file_path):
if not os.path.isfile(file_path):
print("Your file is not exists, check your file path and try again.")
return
filename = os.path.basename(file_path)
# remove ' or " from path, if path was empty like "" or '' set upload_path=None, this make upload file to root directory
upload_path = upload_path.strip().replace("'", '').replace('"', '')
# remove start and end / or \
if upload_path.endswith('/') or upload_path.endswith('\\'):
upload_path = upload_path[:-1]
if upload_path.startswith('/') or upload_path.startswith('\\'):
upload_path = upload_path[1:]
# sanity check
upload_path = upload_path if len(upload_path) >= 1 else None
print("Start uploading...")
try:
# create sub directories
self.create_sub_directories(upload_path)
# upload
self.file_service.create_file_from_path(
share_name=self.file_share, # file_share name in azure
directory_name=upload_path, # server directories address. None => root directory
file_name=filename, # Name of file to create in azure
local_file_path=file_path)
print("'{0}' has been successfully uploaded".format(filename))
except:
print("Failed to upload '{0}', please try again".format(filename))
def download_file(self, file_path):
""" download file from azure, enter file path in azure """
# check file path was not empty
file_path = file_path.strip().replace("'", '').replace('"', '')
if len(file_path) == 0:
print("Please enter a file path")
return
filename = os.path.basename(file_path)
dir_path = os.path.dirname(file_path)
# if parent path was not available, use None => root directory
dir_path = dir_path if dir_path else None
print("Downloading...")
try:
self.file_service.get_file_to_path(
share_name=self.file_share,
directory_name=dir_path, # The path to the directory in azure
file_name=filename, # Name of existing file in azure
# Path of file to write to local machine
file_path="{0}".format(filename))
print(
"'{0}' has been successfully downloaded and saved in current directory.".format(filename))
except:
print("Failed to download '{0}', either file doesn't exist or you are offline.".format(
filename))
def get_list_of_files(self, dir_name=None):
""" show list of all files and all directories in azure"""
generator = self.file_service.list_directories_and_files(
share_name=self.file_share,
directory_name=dir_name)
parent = "" if dir_name == None else dir_name
for file_or_dir in generator:
if not re.match(r"(.[a-z]*[A-Z]*[0-9]*)$", file_or_dir.name):
# file
if len(parent) == 0:
print(file_or_dir.name)
else:
print("{0}/{1}".format(parent, file_or_dir.name))
else:
# dir
if len(parent) == 0:
self.get_list_of_files(file_or_dir.name)
else:
self.get_list_of_files(
"{0}/{1}".format(parent, file_or_dir.name))
def create_sub_directories(self, path):
""" create sub directories in Azure """
if path is None:
return
dirs = os.path.normpath(path).split(os.path.sep)
parent = ''
for dir in dirs:
parent += dir if len(parent) == 0 else '/'+dir
self.file_service.create_directory(self.file_share, parent)
from azure.storage.file import FileService
file_service = FileService(account_name="account_name",
account_key="account_key")
# ファイル一覧表示
generator = file_service.list_directories_and_files('my-file')
for file_or_dir in generator:
print(file_or_dir.name)
# ローカルのtest.txtをAzure Storageへアップロード
from azure.storage.file import ContentSettings
file_service.create_file_from_path(
'my-file',
None,
'test.txt',
'test.txt',
content_settings=ContentSettings(content_type='text/plain'))
class AFSLoader():
def __init__(self, local_root: Path, afs_creds: dict = None):
if afs_creds is None:
afs_creds = get_afs_creds()
self.afs_name = afs_creds["AFS_NAME"]
self.afs_key = afs_creds["AFS_KEY"]
self.afs_share = afs_creds["AFS_SHARE"]
self.file_service = FileService(account_name=self.afs_name,
account_key=self.afs_key)
self.local_root = Path(local_root)
def get_afs_creds(self):
return self.afs_name, self.afs_key, self.afs_share
def upload_data_afs(self, data_path: Path, push_data: bool = False):
"""
Copy data to the AFS directory.
:param data_path: <Path>. Specify your path to the local data folder.
:param push_data. If True upload data if it already exists.
:return: path of the directory in the AFS share.
"""
logging.info("Sending data to AFS")
checksum = md5_dir(data_path)[:10]
afs_path = time.strftime("%Y-%m-%d-%H.%M") + '-' + checksum
list_folder = self.file_service.list_directories_and_files(
self.afs_share)
for folder in list_folder:
if checksum == folder.name[-10:]:
logging.info("Folder for data already exist!")
afs_path = folder.name
logging.info("Data is in the AFS {}".format(folder.name))
if push_data:
logging.warning("Rewriting data")
afs_path = folder.name
else:
return afs_path
self.file_service.create_directory(share_name=self.afs_share,
directory_name=afs_path)
for file in Path(data_path).iterdir():
progress_callback = lambda current, total: logbar(
current, total, f"Uploading {file.name}")
self.file_service.create_file_from_path(
share_name=self.afs_share,
directory_name=afs_path,
file_name=file.name,
local_file_path=str(file),
max_connections=cpu_count(),
progress_callback=progress_callback)
logging.info("Sending is over")
return afs_path
def download_data_afs(self, afs_path: Path, dst_path: Path = None):
afs_path = Path(afs_path)
if not dst_path:
assert self.local_root is not None
dst_path = self.local_root
list_folder = self.file_service.list_directories_and_files(
self.afs_share, directory_name=afs_path)
try:
os.mkdir(self.local_root / afs_path)
except FileExistsError:
print(f"Directory {self.local_root / afs_path} was rewritten ")
for file in list_folder:
progress_callback = lambda current, total: logbar(
current, total, f"Downloading {file.name}")
self.file_service.get_file_to_path(
share_name=self.afs_share,
directory_name=afs_path,
file_name=file.name,
file_path=str(dst_path / afs_path / file.name),
progress_callback=progress_callback)
print("Copy des fichiers de sauvegarde sur le répertoire Microsoft AZURE OK !")
logging.debug("Copy des fichiers de sauvegarde sur le répertoire Microsoft AZURE OK !") # warning
# syslog.syslog(syslog.LOG_DEBUG,"Copy des fichiers de sauvegarde sur le répertoire Microsoft AZURE OK !") # warning
# suppression des fichiers de sauvegarde
os.remove(repertoire_de_sauvegarde+"/save_"+str(BACKUP_DATE)+"db.sql")
print("Suppression du fichier "+BACKUP_DATE+"db.sql")
os.remove(repertoire_de_sauvegarde+"/save_"+str(BACKUP_DATE)+".tar.bz2")
print("Suppression du fichier "+BACKUP_DATE+".tar.bz2")
# Liste des fichiers ou répertoires de Microsoft AZURE et suppression des anciennes sauvegardes en fonction du nombre de jour
print("Liste des sauvegardes: ")
list_file = file_service.list_directories_and_files(AZURE_REP_BKP)
for file_or_dir in list_file:
if ('save_'+str(BACKUP_DATE_OLD)) in file_or_dir.name:
file_service.delete_file(AZURE_REP_BKP,'save_'+str(BACKUP_DATE_OLD),'save_'+str(BACKUP_DATE_OLD)+'db.sql')
file_service.delete_file(AZURE_REP_BKP,'save_'+str(BACKUP_DATE_OLD),'save_'+str(BACKUP_DATE_OLD)+'.tar.bz2')
file_service.delete_directory(AZURE_REP_BKP,'save_'+str(BACKUP_DATE_OLD))
else:
print("")
print(file_or_dir.name)
logging.warning(file_or_dir.name) # warning
# syslog.syslog(syslog.LOG_WARNING, file_or_dir.name) # warning
print("")
print("La sauvegarde c'est terminé correctement !")
logging.warning("La sauvegarde c'est terminé correctement !") # warning
# syslog.syslog(syslog.LOG_WARNING,"La sauvegarde c'est terminé correctement !") # warning
class ModelGymClient:
config = {}
project_root = ""
project_name = ""
user = ""
def __init__(self,
config=None,
config_path=MODELGYM_CONFIG["default_config_path"]):
if config_path:
self.config = self.__config_by_path(config_path)
if type(config) is dict:
self.config.update(config)
else:
if config:
raise TypeError("config must be dictionary!")
project_root = Path(self.config["local_project_root"]).expanduser()
self.project_root = project_root
self.project_name = Path(self.project_root.parts[-1])
if not project_root.is_dir():
project_root.mkdir(parents=True, exist_ok=True)
user_folder = self.project_root / self.config["user"]
self.user = self.config["user"]
if not user_folder.is_dir():
user_folder.mkdir(parents=True, exist_ok=True)
# self.stub = new_client()
self.file_service = FileService(
account_name=self.config['azurefs_acc_name'],
account_key=self.config['azurefs_acc_key'])
self.afs_share = self.config['azurefs_share']
self.__get_client_transport_credentials(
str(Path(self.config["client_cert"]).expanduser()),
str(Path(self.config["client_key"]).expanduser()),
str(Path(self.config["ca_cert"]).expanduser()))
self.channel = grpc.secure_channel(
self.config["connect_to"],
self.creds,
options=(
('grpc.max_send_message_length',
self.config["max_msg_size_megabytes"]),
('grpc.max_receive_message_length',
self.config["max_msg_size_megabytes"]),
))
self.stub = wonderland_pb2_grpc.WonderlandStub(self.channel)
self.check_user()
def check_user(self):
list_folder = self.file_service.list_directories_and_files(
self.afs_share)
for folder in list_folder:
if self.user == folder.name:
return True
self.file_service.create_directory(share_name=self.afs_share,
directory_name=self.user)
return True
def __get_client_transport_credentials(self, client_cert_path,
client_key_path, ca_cert_path):
client_cert_path = Path(client_cert_path).expanduser()
client_key_path = Path(client_key_path).expanduser()
ca_cert_path = Path(ca_cert_path).expanduser()
path_ok = [
client_cert_path.exists(),
client_key_path.exists(),
ca_cert_path.exists()
]
if not all(path_ok):
raise ValueError("One of credentials files does not exist")
self.creds = grpc.ssl_channel_credentials(
ca_cert_path.read_bytes(), client_key_path.read_bytes(),
client_cert_path.read_bytes())
def __config_by_path(self, path):
path = Path(path).expanduser()
if path.exists():
with path.open() as file:
config = yaml.load(file)
return config
else:
raise FileNotFoundError(
"Config {} doesn't exist !!! Check ~/.wonder/config.yaml".
format(path))
def eval_model(self, model_info, data_path):
model_path = self.send_model(model_info)
job = Job(input=json.dumps({
"model_path": str(model_path),
"data_path": str(data_path)
}),
kind="hyperopt")
job = self.stub.CreateJob(job)
self.stub.GetJob(RequestWithId(id=job.id))
return job.id
def gather_results(self, job_id_list, timeout):
job_compeleted = {job_id: Job.PENDING for job_id in job_id_list}
deadline = time.time() + timeout
while True:
time.sleep(5)
for id in job_id_list:
job = self.stub.GetJob(RequestWithId(id=id))
job_compeleted[id] = job.status
if not any(s in job_compeleted.values()
for s in (Job.PENDING, Job.RUNNING, Job.PULLED)):
break
if time.time() > deadline:
print("Timeout was expired!")
break
results = []
for i, id in enumerate(job_id_list):
job = self.stub.GetJob(RequestWithId(id=id))
if job.status == Job.COMPLETED:
results += [{}]
else:
results.append(None)
files = {}
if job.output != "":
files = json.loads(job.output)
for file, path in files.items():
self.file_service.get_file_to_path(
share_name=self.afs_share,
directory_name=Path(path).parent,
file_name=Path(path).name,
file_path=str(self.project_root / path))
if file == 'output':
with open(self.project_root / path, "r") as f:
results[i]['output'] = json.load(f)
if file == 'result_model_path':
results[i]['result_model_path'] = self.project_root / path
if file == 'error':
with open(self.project_root / path, "r") as f:
logging.warning(f.read())
return results
def send_model(self, model_info):
folder = "model-" + ''.join([
random.choice(string.ascii_letters + string.digits)
for _ in range(12)
])
model_path = self.project_root / self.user / folder / MODELGYM_CONFIG[
"model_file"]
try:
model_folder = model_path.parent
model_folder.mkdir()
except FileExistsError:
logging.warning("Model folder {} is exist !".format(model_folder))
except FileNotFoundError:
logging.warning(
"Model folder {} is missing !".format(model_folder))
with (model_path).open(mode="w") as file:
json.dump(model_info, file, cls=NumpyEncoder)
afs_path = Path(self.user) / folder / MODELGYM_CONFIG["model_file"]
self.file_service.create_directory(share_name=self.afs_share,
directory_name=afs_path.parent)
self.file_service.create_file_from_path(share_name=self.afs_share,
directory_name=afs_path.parent,
file_name=afs_path.name,
local_file_path=model_path,
max_connections=cpu_count())
return afs_path
def send_data(self, data_path, push_data=False):
"""
Copy data to the AFS DATA directory.
:param data_path: <string>. Specify you data path by string.
:return: path in the AFS share.
"""
logging.info("Sending data to AFS")
checksum = get_data_hash(data_path)[:10]
data_folder = time.strftime("%Y-%m-%d-%H.%M") + '-' + checksum
afs_path = Path(MODELGYM_CONFIG["data_folder"]
) / data_folder / MODELGYM_CONFIG["data_file"]
list_folder = self.file_service.list_directories_and_files(
self.afs_share, directory_name="DATA")
for folder in list_folder:
if checksum == folder.name[-10:]:
logging.info("Folder for data already exist!")
afs_path = Path(
"DATA") / folder.name / MODELGYM_CONFIG["data_file"]
logging.info("Data is in the AFS {}".format(folder.name))
if push_data:
logging.warning("Rewriting data")
afs_path = Path(
MODELGYM_CONFIG["data_folder"]
) / folder.name / MODELGYM_CONFIG["data_file"]
else:
return afs_path
self.file_service.create_directory(share_name=self.afs_share,
directory_name=afs_path.parent)
self.file_service.create_file_from_path(share_name=self.afs_share,
directory_name=afs_path.parent,
file_name=afs_path.name,
local_file_path=data_path,
max_connections=cpu_count(),
progress_callback=logbar)
logging.info("Sending is over")
return afs_path
def from_project_root_path(self, path):
path = Path(path)
# if not path.exists():
# logging.warning("{} is missing !!".format(path))
try:
relative_path = path.relative_to(self.project_root.parent)
return str(relative_path)
except ValueError:
logging.warning("Path doesn't have project_root folder {}".format(
self.project_root))
# Check if AZURE_BACKUP_FOLDER exists, if not create it
if not file_service.exists(AZURE_SHARE_NAME, AZURE_BACKUP_FOLDER):
file_service.create_directory(AZURE_SHARE_NAME, AZURE_BACKUP_FOLDER)
# Upload
print("uploading to: '%s/%s/%s'" %
(AZURE_SHARE_NAME, AZURE_BACKUP_FOLDER, FILENAME))
file_service.create_file_from_path(AZURE_SHARE_NAME,
AZURE_BACKUP_FOLDER,
FILENAME,
FILENAME,
progress_callback=upload_callback)
# Cleaning Backup Files
backup_files = file_service.list_directories_and_files(
AZURE_SHARE_NAME, AZURE_BACKUP_FOLDER)
filenames = []
for file in backup_files:
filenames.append(file.name)
files_to_delete = []
if len(filenames) >= AZURE_KEEP_BACKUPS:
files_to_delete = filenames[:(len(filenames) - AZURE_KEEP_BACKUPS)]
for file in files_to_delete:
file_service.delete_file(AZURE_SHARE_NAME, AZURE_BACKUP_FOLDER,
file)
end = time.time()
duration = int(end - start)
slack_message(success=True, duration=duration, deleted=files_to_delete)
except Exception as e:
print(e)
