def upload_file(fname, fpath, coref):
daemonname = fname.split(".")[0]
i = 0
fail_msg = ""
while True:
try:
svc = FileService(account_name=acctname, account_key=acctkey)
l = [sonicversion, asicname, daemonname, hostname]
e = []
while len(e) != len(l):
e.append(l[len(e)])
svc.create_directory(sharename, "/".join(e))
log_debug("Remote dir created: " + "/".join(e))
svc.create_file_from_path(sharename, "/".join(l), fname, fpath)
log_debug("Remote file created: name{} path{}".format(
fname, fpath))
newcoref = os.path.dirname(
coref) + "/" + UPLOAD_PREFIX + os.path.basename(coref)
os.rename(coref, newcoref)
break
except Exception as ex:
log_err("core uploader failed: Failed during upload (" +
coref + ") err: (" + str(ex) + ") retry:" + str(i))
if not os.path.exists(fpath):
break
i += 1
time.sleep(PAUSE_ON_FAIL)
def main(files):
configfile = (os.path.dirname(os.path.abspath(__file__)) + '/upload.config')
print (configfile)
if not os.path.isfile(configfile):
print("Settings not found. Please create an upload.config file with the Azure file share account name, access key, file share name, and folder. Each value should be on its own line.")
exit()
if len(files) <1:
print("No files provided for upload. Please supply file paths as arguments.")
exit()
# get settings - account, key, share, and folder in subsequent lines
with open(configfile,"r") as config:
settings = config.readlines()
azure_account = settings[0].rstrip()
azure_key = settings[1].rstrip()
share = settings[2].rstrip()
folder = settings[3].rstrip()
file_service = FileService(account_name=azure_account, account_key=azure_key)
# Arguments should just be an array of filenames.
timestamp_suffix = datetime.now().strftime("%Y%m%d-%H%M_")
for file in files:
if not os.path.isfile(file):
print(file, "not found")
else:
print("Uploading:", file)
stampedfile = timestamp_suffix + os.path.basename(file)
file_service.create_file_from_path(share, folder, stampedfile, file, progress_callback=progress)
print(stampedfile," uploaded")
def upload_file(fname, fpath):
daemonname = fname.split(".")[0]
i = 0
fail_msg = ""
while i <= MAX_RETRIES:
try:
svc = FileService(account_name=acctname, account_key=acctkey)
l = [sonicversion, asicname, daemonname, hostname]
e = []
while len(e) != len(l):
e.append(l[len(e)])
svc.create_directory(sharename, "/".join(e))
log_debug("Remote dir created: " + "/".join(e))
svc.create_file_from_path(sharename, "/".join(l), fname, fpath)
log_debug("Remote file created: name{} path{}".format(
fname, fpath))
break
except Exception as e:
log_err("core uploader failed: Failed during upload (" +
str(e) + ")")
fail_msg = str(e)
i += 1
if i >= MAX_RETRIES:
raise Exception("Failed while uploading. msg(" + fail_msg +
") after " + str(i) + " retries")
time.sleep(PAUSE_ON_FAIL)
def move_to_storage_account(self, file, storage='blob'):
from hugme.__key__ import acc, key
from datetime import datetime
print(datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
'Movendo arquivo final para o {} storage...'.format(object))
if storage == 'blob':
from azure.storage.blob import BlockBlobService, PublicAccess
block_blob_service = BlockBlobService(account_name=acc,
account_key=key)
block_blob_service.set_container_acl(
'final', public_access=PublicAccess.Container)
block_blob_service.create_blob_from_path(
container_name='consumidorgov',
blob_name='comparativo',
file_path='consumidor_gov\\data\\' + file,
)
elif storage == 'files':
from azure.storage.file import FileService
file_service = FileService(account_name=acc, account_key=key)
file_service.create_file_from_path(
share_name='complains',
directory_name='hugme',
file_name='base.csv',
local_file_path='' + file,
)
else:
return False
def onRecordingComplete(self):
print("recording completed {}".format(self.filename))
file_service = FileService(account_name=cfg.storageAc,
account_key=cfg.accountkey)
file_service.create_file_from_path(cfg.fileShare, None,
'{}.flv'.format(self.filename),
'temp/{}.flv'.format(self.filename))
sharedAccessStorage = SharedAccessSignature(cfg.storageAc,
cfg.accountkey)
sasKey = sharedAccessStorage.generate_file(
cfg.fileShare,
file_name='{}.flv'.format(self.filename),
permission=FilePermissions.READ,
start=datetime.utcnow(),
expiry=datetime.utcnow() + timedelta(minutes=5))
downloadLink = cfg.downloadLinkFormat.format(cfg.storageAcUrl,
cfg.fileShare,
self.filename, sasKey)
self.recorderSock.emit('recordingcomplete', {
'sid': self.sid,
'download': downloadLink
})
os.remove('temp/{}.flv'.format(self.filename))
time.sleep(cfg.timeBeforeFileDelete)
file_service.delete_file(cfg.fileShare, None,
'{}.flv'.format(self.filename))
return
def run(job, **kwargs):
resource = kwargs.get('resource')
create_custom_fields_as_needed()
storage_account = '{{ storage_account }}'
file = "{{ file }}"
azure_storage_file_share_name = '{{ azure_storage_file_share_name }}'
file_name = Path(file).name
if file.startswith(settings.MEDIA_URL):
set_progress("Converting relative URL to filesystem path")
file = file.replace(settings.MEDIA_URL, settings.MEDIA_ROOT)
account_key = Resource.objects.filter(name__icontains='{{ storage_account }}')[0].azure_account_key
fallback_account_key = Resource.objects.filter(name__icontains="{{ storage_account }}")[0].azure_account_key_fallback
set_progress("Connecting To Azure...")
file_service = FileService(account_name=storage_account, account_key=account_key)
set_progress('Creating a file share...')
file_service.create_share(share_name=azure_storage_file_share_name, quota=1)
set_progress('Creating a file...')
if file_service.exists(share_name=azure_storage_file_share_name, file_name=file_name, directory_name=''):
file_service.create_file_from_path(share_name=azure_storage_file_share_name, file_name=file_name, directory_name='', local_file_path=file)
return "WARNING", "File with this name already exists", "The file will be updated."
else:
file_service.create_file_from_path(share_name=azure_storage_file_share_name, file_name=file_name, directory_name='', local_file_path=file)
resource.name = azure_storage_file_share_name + '-' + file_name
resource.azure_storage_account_name = storage_account
resource.azure_account_key = account_key
resource.azure_account_key_fallback = fallback_account_key
resource.azure_storage_file_share_name = azure_storage_file_share_name
resource.azure_storage_file_name = file_name
resource.save()
return "Success", "The File has succesfully been uploaded", ""
def shares():
# Create Container and Share
global storage_account_key, blob_service, blob_share, file_service, file_share
sak = storage_client.storage_accounts.list_keys(resourcegroupname,
storageaccountname)
storage_account_key = sak.keys[0].value
cloudstorage_client = CloudStorageAccount(storageaccountname,
storage_account_key)
blob_service = cloudstorage_client.create_block_blob_service()
blob_share = blob_service.create_container(
sharename, public_access=PublicAccess.Container)
file_service = FileService(account_name=storageaccountname,
account_key=storage_account_key)
file_share = file_service.create_share(sharename)
# Copy Setup Files to Container and Share
blob_service.create_blob_from_path(
sharename,
filename,
filename,
)
file_service.create_file_from_path(
sharename,
'',
filename,
filename,
)
def upload(path: str):
from azure.storage.file import FileService
service = FileService(account_name=config['account_name'],
account_key=config['account_key'])
if shared_directory not in service.list_directories_and_files(
config['share_name']):
service.create_directory(config['share_name'], shared_directory)
service.create_file_from_path(config['share_name'], shared_directory,
path.split('/')[-1], path)
def create_azure_fileshare(share_prefix, account_name, account_key):
"""
Generate a unique share name to avoid overlaps in shared infra
:param share_prefix:
:param account_name:
:param account_key:
:return:
"""
# FIXME - Need to remove hardcoded directoty link below
d_dir = './WebInDeploy/bootstrap'
share_name = "{0}-{1}".format(share_prefix.lower(), str(uuid.uuid4()))
print('using share_name of: {}'.format(share_name))
# archive_file_path = _create_archive_directory(files, share_prefix)
try:
# ignore SSL warnings - bad form, but SSL Decrypt causes issues with this
s = requests.Session()
s.verify = False
file_service = FileService(account_name=account_name,
account_key=account_key,
request_session=s)
# print(file_service)
if not file_service.exists(share_name):
file_service.create_share(share_name)
for d in ['config', 'content', 'software', 'license']:
print('creating directory of type: {}'.format(d))
if not file_service.exists(share_name, directory_name=d):
file_service.create_directory(share_name, d)
# FIXME - We only handle bootstrap files. May need to handle other dirs
if d == 'config':
for filename in os.listdir(d_dir):
print('creating file: {0}'.format(filename))
file_service.create_file_from_path(
share_name, d, filename, os.path.join(d_dir, filename))
except AttributeError as ae:
# this can be returned on bad auth information
print(ae)
return "Authentication or other error creating bootstrap file_share in Azure"
except AzureException as ahe:
print(ahe)
return str(ahe)
except ValueError as ve:
print(ve)
return str(ve)
print('all done')
return share_name
class Azure:
def __init__(self, ac, key, fileshare):
self.account_name = ac
self.account_key = key
self.fileshare_name = fileshare
#Create a FileService that is used to call the File Service for the storage account
self.file_service = FileService(account_name=ac, account_key=key)
return
def List_directory(self, directory_path):
self.generator = self.file_service.list_directories_and_files(
directory_path)
print("Files in the directory: " + directory_path)
for file_or_dir in self.generator:
print("\t File/Directory name: " + file_or_dir.name)
return
def Download(self, loc_directory, fileshare_directory_name):
local_path = os.path.expanduser("~/" + loc_directory)
print("\nDownloading the following files to " + local_path)
self.generator = self.file_service.list_directories_and_files(
self.fileshare_name + "/" + fileshare_directory_name)
for file_or_dir in self.generator:
print("\t File/Directory name: " + file_or_dir.name)
for file_or_dir in self.generator:
#full_path_to_file2 = os.path.join(local_path, file_or_dir.name)
self.file_service.get_file_to_path(self.fileshare_name,
fileshare_directory_name,
file_or_dir.name,
local_path + file_or_dir.name)
print("\nFiles downloaded to " + local_path)
return
def Upload(self, loc_directory, fileshare_directory_name):
local_path = os.path.expanduser("~/" + loc_directory)
self.generator = self.file_service.list_directories_and_files(
self.fileshare_name + "/" + fileshare_directory_name)
print("\nUploading the following files to " + fileshare_directory_name)
entries = os.listdir(local_path)
# for entry in entries:
# print(entry)
for entry in entries:
self.file_service.create_file_from_path(
self.fileshare_name, #Fileshare name
fileshare_directory_name, # We want to create this blob in the root directory, so we specify None for the directory_name
entry, #name of the file that is created
local_path + entry, #file that needs to be uploaded
content_settings=ContentSettings(
content_type='application/vnd.ms-excel'))
print("The followig files have been uploaded")
#listing the files in the fileshare_name
obj.List_directory(fileshare_name + "/" +
fileshare_directory_name_upload)
def file():
static_dir_path = "D:\home\site\wwwroot\static"
static_file_dir_path = static_dir_path + '\\' + 'files'
account_name = 'hanastragetest'
account_key = 'acount_key'
root_share_name = 'root'
share_name = 'images'
directory_url = 'https://hanastragetest.file.core.windows.net/' + root_share_name + '/' + share_name
# create local save directory
if os.path.exist(static_file_dir_path) is False:
os.mkdir(static_file_dir_path)
file_service = FileService(account_name=account_name,
account_key=account_key)
# create share
file_service.create_share(root_share_name)
# create directory
file_service.create_directory(root_share_name, share_name)
files = os.listdir(static_dir_path)
for file in files:
# delete
if file_service.exists(root_share_name, share_name, file):
file_service.delete_file(root_share_name, share_name, file)
# file upload
file_service.create_file_from_path(
root_share_name,
share_name, # We want to create this blob in the root directory, so we specify None for the directory_name
file,
static_dir_path + '\\' + file,
content_settings=ContentSettings(content_type='image/png'))
generator = file_service.list_directories_and_files(
root_share_name, share_name)
html = ""
for file in generator:
# file download
file_save_path = static_file_dir_path + '\\' + file
file_service.get_file_to_path(root_share_name, share_name, file,
file_save_path)
html = "{}<img src='{}'>".format(html, file_save_path)
result = {
"result": True,
"data": {
"file_or_dir_name":
[file_or_dir.name for file_or_dir in generator]
}
}
return make_response(json.dumps(result, ensure_ascii=False) + html)
def update_datanginxconfd(gw_account_name, gw_account_key, gw_account_share):
"""Upload data from cloned repo to the GW Storage account into the datanginx-conf.d directory with overwriting"""
print('\nRunning update_confd to the {} and file share {} to the path datanginx-conf.d.\n'.format(gw_account_name, gw_account_share))
file_service = FileService(account_name=gw_account_name, account_key=gw_account_key)
configs = glob.glob('*.conf')
for config in configs:
print('Uploading config: {}'.format(config))
file_service.create_file_from_path(gw_account_share, 'datanginx-conf.d', config, config)
def store_trained_model_in_azure(model):
file_service = FileService(
account_name='soilhumiditydata293s',
account_key=
'4PSsEO1xBAIdq3/MppWm+t6eYHi+CWhVn6xNZ6i4mLVgm50K8+NK6lA94v8MxG0bvVEfYCvsv1suxCyCnUYd0A=='
)
file_service.delete_file('model', None, 'model')
file_service.create_file_from_path(
'model', None, 'model',
'/fs/student/aditya_wadaskar/iot/ML_training/model')
def num3():
file = request.form['filename']
file += ".jpg"
file_service = FileService(
account_name='mystorge',
account_key=
'0T4f/dzyV7AIw4a9bevK5ysML0qP55CEWEqJyJWXyr6fKRxowLq8tL7mep/MfSc//mcQggeH1+K79A4HUDug3w=='
)
file_service.create_file_from_path(
'image1',
None,
file,
file,
content_settings=ContentSettings(content_type='image/png'))
return "<h1> File Uploaded sucessfully</h1>"
def upload_file(self, path):
"""Upload a file into the default share on the storage account.
If the share doesn't exist, create it first.
"""
file_service = FileService(
account_name=self.account.name,
account_key=self.key,
)
file_service.create_share(self.default_share)
file_service.create_file_from_path(
self.default_share,
None,
os.path.basename(path),
path,
)
return '/'.join([self.default_share, os.path.basename(path)])
def create_azure_fileshare(files, share_prefix, account_name, account_key):
# generate a unique share name to avoid overlaps in shared infra
share_name = "{0}-{1}".format(share_prefix.lower(), str(uuid.uuid4()))
print('using share_name of: {}'.format(share_name))
archive_file_path = _create_archive_directory(files, share_prefix)
try:
# ignore SSL warnings - bad form, but SSL Decrypt causes issues with this
s = requests.Session()
s.verify = False
file_service = FileService(account_name=account_name,
account_key=account_key,
request_session=s)
# print(file_service)
if not file_service.exists(share_name):
file_service.create_share(share_name)
for d in ['config', 'content', 'software', 'license']:
print('creating directory of type: {}'.format(d))
if not file_service.exists(share_name, directory_name=d):
file_service.create_directory(share_name, d)
d_dir = os.path.join(archive_file_path, d)
for filename in os.listdir(d_dir):
print('creating file: {0}'.format(filename))
file_service.create_file_from_path(
share_name, d, filename, os.path.join(d_dir, filename))
except AttributeError as ae:
# this can be returned on bad auth information
print(ae)
return "Authentication or other error creating bootstrap file_share in Azure"
except AzureException as ahe:
print(ahe)
return str(ahe)
except ValueError as ve:
print(ve)
return str(ve)
print('all done')
return 'Azure file-share {} created successfully'.format(share_name)
def upload_scripts(config, job_name, filenames):
service = FileService(config.storage_account['name'],
config.storage_account['key'])
service.create_directory(config.fileshare_name,
job_name,
fail_on_exist=False)
trasfer_file = lambda fname: service.create_file_from_path(
config.fileshare_name, job_name, os.path.basename(fname), fname)
for filename in filenames:
trasfer_file(filename)
def copy(self, source_path, dest_path, account=None, group_name=None):
"""Copy file from a path to another path. The azure url format is https://myaccount.blob.core.windows.net/mycontainer/myblob.
Args:
source_path(str): The path of the file to be copied.
dest_path(str): The destination path where the file is going to be allocated.
Raises:
:exc:`~..DriverError`: if the file is not uploaded correctly.
"""
if 'core.windows.net' not in source_path and 'core.windows.net' not in dest_path:
self.logger.error(
"Source or destination must be a azure storage url (format "
"https://myaccount.blob.core.windows.net/mycontainer/myblob")
raise DriverError
# Check if source exists and can read
if 'core.windows.net' in source_path:
parse_url = _parse_url(source_path)
key = self.storage_client.storage_accounts.list_keys(
self.resource_group_name, parse_url.account).keys[0].value
if parse_url.file_type == 'blob':
bs = BlockBlobService(account_name=parse_url.account,
account_key=key)
return bs.get_blob_to_path(parse_url.container_or_share_name,
parse_url.file, dest_path)
elif parse_url.file_type == 'file':
fs = FileService(account_name=parse_url.account,
account_key=key)
return fs.get_file_to_path(parse_url.container_or_share_name,
parse_url.path, parse_url.file,
dest_path)
else:
raise ValueError(
"This azure storage type is not valid. It should be blob or file."
)
else:
parse_url = _parse_url(dest_path)
key = self.storage_client.storage_accounts.list_keys(
self.resource_group_name, parse_url.account).keys[0].value
if parse_url.file_type == 'blob':
bs = BlockBlobService(account_name=parse_url.account,
account_key=key)
return bs.create_blob_from_path(
parse_url.container_or_share_name, parse_url.file,
source_path)
elif parse_url.file_type == 'file':
fs = FileService(account_name=parse_url.account,
account_key=key)
return fs.create_file_from_path(
parse_url.container_or_share_name, parse_url.path,
parse_url.file, source_path)
else:
raise ValueError(
"This azure storage type is not valid. It should be blob or file."
)
def saveChanges(self):
if (self.currentState != Application.DONE_SCANNING):
return
gfwlistBackupFile = self.azureFileShareFileName + '.' + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S" + '.bk')
# messagebox.showinfo('You are about to save the changes', 'Are you sure?')
result = messagebox.askquestion('You are about to save the changes', 'Are you sure?', icon='warning')
if result == 'yes':
# this is for copying file locally: copyfile(self.gfwlistFile, gfwlistBackupFile)
azureFileService = FileService(account_name=self.azureAccountName, account_key=self.azureAccountKey)
sourceUrl = 'https://%s.%s/%s/%s/%s' % (self.azureAccountName, self.azureFileServiceDomain, self.azureFileShareName, self.azureFileShareFileDir, self.azureFileShareFileName)
azureFileService.copy_file(self.azureFileShareName, self.azureFileShareFileDir, gfwlistBackupFile, sourceUrl)
# the folliwng is for writing file locally
with open(self.gfwlistFile, 'w') as f:
f.write(self.sectionBeforeRules.getvalue())
f.write(',\n'.join(' "' + str(e) + '"' for e in self.listBox.get(0, END)))
f.write('\n')
f.write(self.sectionAfterRules.getvalue())
# then write it to the file share
azureFileService.create_file_from_path(self.azureFileShareName, self.azureFileShareFileDir, self.azureFileShareFileName, self.gfwlistFile)
def upload():
AzureStorageAccount = 'effiles'
key = 'axLykwdLsUwKTDY5flU6ivGrt9obV38k2UMVDCSpLYE3K6jAkwsjWOThQydhuMSWHfx6lTq102gdkas/GyKhEA=='
up_path = 'uploads'
path1 = 'efficientfrontier'
file_service = FileService(account_name = AzureStorageAccount, account_key = key)
target = os.path.join(APP_ROOT, 'uploads/')
if request.method == 'POST':
if not os.path.isdir(target):
os.mkdir(target)
for file in request.files.getlist("file"):
filename = file.filename
destination = '/'.join([target, filename])
#print (destination)
file.save(destination) # Save the file to local server
now = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
filename_new = os.path.splitext(filename)[0] + '_' + now + os.path.splitext(filename)[1]
file_service.create_file_from_path (path1, up_path, filename_new, destination) # Upload the file to Azure Storage Account
# Create Cursor
cur = mysql.connection.cursor()
# Execute
cur.execute ('INSERT INTO files_uploaded (filename, USER, AzureAccount, AzureShare, Directory) VALUES(%s, %s, %s, %s, %s)', (filename_new, session['email'], AzureStorageAccount, path1, up_path))
# Commit to DB
mysql.connection.commit()
# Close connection
cur.close()
flash ('File Uploaded', 'success')
os.remove(destination) # Remove locally saved file
return redirect(url_for('files'))
def main(req: func.HttpRequest) -> func.HttpResponse:
acc_name = req.params.get('account_name')
acc_key = req.params.get('account_key')
container_name = req.params.get('container_name')
blob_name = req.params.get('blob_name')
file_service = FileService(account_name=acc_name, account_key=acc_key)
download_path = "/tmp/" + blob_name
file_service.get_file_to_path(container_name, None, blob_name,
download_path)
logging.info("Downloading blob to " + download_path)
latency, upload_path = video_processing(blob_name, download_path)
file_service.create_file_from_path(container_name, None,
upload_path.split("/")[FILE_PATH_INDEX],
upload_path)
logging.info(latency)
return func.HttpResponse(str(latency))
def _main(args):
with open(args.config_file_path) as data_file:
file_service = FileService(account_name=args.afs_storage_account,
account_key=args.afs_storage_account_key,
endpoint_suffix=args.afs_endpoint_suffix)
config_json = json.load(data_file)
if _check_setups_local(config_json['Installers'], args.ignore):
for installer in config_json['Installers']:
print("Uploading {} to AFS ".format(installer['SourcePath']))
file_service.create_file_from_path(args.afs_name, None,
installer['RemotePath'],
installer['SourcePath'])
if installer['IsMsi'] is False:
for patch in installer['Patches']:
patch_file_name = os.path.basename(patch)
print("Uploading {} to AFS ".format(patch_file_name))
file_service.create_file_from_path(
args.afs_name, None, patch_file_name, patch)
#archive_command = r'"{}" cp {} {}'.format(args.azcopy_path, installer['SourcePath'], args.afs_sas_uri)
#print(archive_command)
#subprocess.call(archive_command, shell=True)
else:
sys.exit(1)
class ModelGymClient:
config = {}
project_root = ""
project_name = ""
user = ""
def __init__(self,
config=None,
config_path=MODELGYM_CONFIG["default_config_path"]):
if config_path:
self.config = self.__config_by_path(config_path)
if type(config) is dict:
self.config.update(config)
else:
if config:
raise TypeError("config must be dictionary!")
project_root = Path(self.config["local_project_root"]).expanduser()
self.project_root = project_root
self.project_name = Path(self.project_root.parts[-1])
if not project_root.is_dir():
project_root.mkdir(parents=True, exist_ok=True)
user_folder = self.project_root / self.config["user"]
self.user = self.config["user"]
if not user_folder.is_dir():
user_folder.mkdir(parents=True, exist_ok=True)
# self.stub = new_client()
self.file_service = FileService(
account_name=self.config['azurefs_acc_name'],
account_key=self.config['azurefs_acc_key'])
self.afs_share = self.config['azurefs_share']
self.__get_client_transport_credentials(
str(Path(self.config["client_cert"]).expanduser()),
str(Path(self.config["client_key"]).expanduser()),
str(Path(self.config["ca_cert"]).expanduser()))
self.channel = grpc.secure_channel(
self.config["connect_to"],
self.creds,
options=(
('grpc.max_send_message_length',
self.config["max_msg_size_megabytes"]),
('grpc.max_receive_message_length',
self.config["max_msg_size_megabytes"]),
))
self.stub = wonderland_pb2_grpc.WonderlandStub(self.channel)
self.check_user()
def check_user(self):
list_folder = self.file_service.list_directories_and_files(
self.afs_share)
for folder in list_folder:
if self.user == folder.name:
return True
self.file_service.create_directory(share_name=self.afs_share,
directory_name=self.user)
return True
def __get_client_transport_credentials(self, client_cert_path,
client_key_path, ca_cert_path):
client_cert_path = Path(client_cert_path).expanduser()
client_key_path = Path(client_key_path).expanduser()
ca_cert_path = Path(ca_cert_path).expanduser()
path_ok = [
client_cert_path.exists(),
client_key_path.exists(),
ca_cert_path.exists()
]
if not all(path_ok):
raise ValueError("One of credentials files does not exist")
self.creds = grpc.ssl_channel_credentials(
ca_cert_path.read_bytes(), client_key_path.read_bytes(),
client_cert_path.read_bytes())
def __config_by_path(self, path):
path = Path(path).expanduser()
if path.exists():
with path.open() as file:
config = yaml.load(file)
return config
else:
raise FileNotFoundError(
"Config {} doesn't exist !!! Check ~/.wonder/config.yaml".
format(path))
def eval_model(self, model_info, data_path):
model_path = self.send_model(model_info)
job = Job(input=json.dumps({
"model_path": str(model_path),
"data_path": str(data_path)
}),
kind="hyperopt")
job = self.stub.CreateJob(job)
self.stub.GetJob(RequestWithId(id=job.id))
return job.id
def gather_results(self, job_id_list, timeout):
job_compeleted = {job_id: Job.PENDING for job_id in job_id_list}
deadline = time.time() + timeout
while True:
time.sleep(5)
for id in job_id_list:
job = self.stub.GetJob(RequestWithId(id=id))
job_compeleted[id] = job.status
if not any(s in job_compeleted.values()
for s in (Job.PENDING, Job.RUNNING, Job.PULLED)):
break
if time.time() > deadline:
print("Timeout was expired!")
break
results = []
for i, id in enumerate(job_id_list):
job = self.stub.GetJob(RequestWithId(id=id))
if job.status == Job.COMPLETED:
results += [{}]
else:
results.append(None)
files = {}
if job.output != "":
files = json.loads(job.output)
for file, path in files.items():
self.file_service.get_file_to_path(
share_name=self.afs_share,
directory_name=Path(path).parent,
file_name=Path(path).name,
file_path=str(self.project_root / path))
if file == 'output':
with open(self.project_root / path, "r") as f:
results[i]['output'] = json.load(f)
if file == 'result_model_path':
results[i]['result_model_path'] = self.project_root / path
if file == 'error':
with open(self.project_root / path, "r") as f:
logging.warning(f.read())
return results
def send_model(self, model_info):
folder = "model-" + ''.join([
random.choice(string.ascii_letters + string.digits)
for _ in range(12)
])
model_path = self.project_root / self.user / folder / MODELGYM_CONFIG[
"model_file"]
try:
model_folder = model_path.parent
model_folder.mkdir()
except FileExistsError:
logging.warning("Model folder {} is exist !".format(model_folder))
except FileNotFoundError:
logging.warning(
"Model folder {} is missing !".format(model_folder))
with (model_path).open(mode="w") as file:
json.dump(model_info, file, cls=NumpyEncoder)
afs_path = Path(self.user) / folder / MODELGYM_CONFIG["model_file"]
self.file_service.create_directory(share_name=self.afs_share,
directory_name=afs_path.parent)
self.file_service.create_file_from_path(share_name=self.afs_share,
directory_name=afs_path.parent,
file_name=afs_path.name,
local_file_path=model_path,
max_connections=cpu_count())
return afs_path
def send_data(self, data_path, push_data=False):
"""
Copy data to the AFS DATA directory.
:param data_path: <string>. Specify you data path by string.
:return: path in the AFS share.
"""
logging.info("Sending data to AFS")
checksum = get_data_hash(data_path)[:10]
data_folder = time.strftime("%Y-%m-%d-%H.%M") + '-' + checksum
afs_path = Path(MODELGYM_CONFIG["data_folder"]
) / data_folder / MODELGYM_CONFIG["data_file"]
list_folder = self.file_service.list_directories_and_files(
self.afs_share, directory_name="DATA")
for folder in list_folder:
if checksum == folder.name[-10:]:
logging.info("Folder for data already exist!")
afs_path = Path(
"DATA") / folder.name / MODELGYM_CONFIG["data_file"]
logging.info("Data is in the AFS {}".format(folder.name))
if push_data:
logging.warning("Rewriting data")
afs_path = Path(
MODELGYM_CONFIG["data_folder"]
) / folder.name / MODELGYM_CONFIG["data_file"]
else:
return afs_path
self.file_service.create_directory(share_name=self.afs_share,
directory_name=afs_path.parent)
self.file_service.create_file_from_path(share_name=self.afs_share,
directory_name=afs_path.parent,
file_name=afs_path.name,
local_file_path=data_path,
max_connections=cpu_count(),
progress_callback=logbar)
logging.info("Sending is over")
return afs_path
def from_project_root_path(self, path):
path = Path(path)
# if not path.exists():
# logging.warning("{} is missing !!".format(path))
try:
relative_path = path.relative_to(self.project_root.parent)
return str(relative_path)
except ValueError:
logging.warning("Path doesn't have project_root folder {}".format(
self.project_root))
class StorageHelper(object):
"""Handle details related to a single storage account and share.
Instantiate this object with information sufficient to
uniquely identify a storage account and a file share within it.
Then .account can be used to retrieve the Azure SDK for Python
object corresponding to the account, and .key can be used to
get an access key for it.
For both those properties, if the value mentioned doesn't exist,
it will be created upon first property access.
"""
def __init__(self,
client_data,
resource_helper,
name,
account=None,
default_share='share'):
self.name = name
self.default_share = default_share
self._account = account
self._key = os.environ.get('AZURE_STORAGE_KEY')
self.resource_helper = resource_helper
self.client = StorageManagementClient(*client_data)
self.file_service = FileService(
account_name=self.account.name,
account_key=self.key,
)
@property
def account(self):
"""Return the managed StorageAccounts object.
If no such account exists, create it first.
"""
if self._account is None:
print('Creating storage account...')
# Error to create storage account if it already exists!
name_check = self.client.storage_accounts.check_name_availability(
self.name)
if name_check.name_available:
storage_creation = self.client.storage_accounts.create(
self.resource_helper.group.name, self.name,
StorageAccountCreateParameters(
sku=StorageAccountSku(StorageSkuName.standard_lrs),
kind=StorageKind.storage,
location=self.resource_helper.group.location,
))
storage = storage_creation.result()
else:
try:
storage = self.client.storage_accounts.get_properties(
self.resource_helper.group.name, self.name)
except CloudError:
print('Storage account {} already exists'
' in a resource group other than {}.'.format(
self.name, self.resource_helper.group.name))
print('Got storage account:', storage.name)
self._account = storage
return self._account
@property
def key(self):
"""Get the first available storage key.
This will crash if there are no available storage keys,
which is unlikely since two are created along with a storage account.
"""
if self._key is None:
storage_keys = self.client.storage_accounts.list_keys(
self.resource_helper.group.name, self.account.name)
self._key = next(iter(storage_keys.keys)).value
return self._key
def upload_file(self, path, sharename):
"""Upload a file into the default share on the storage account.
If the share doesn't exist, create it first.
"""
self.file_service.create_file_from_path(
self.default_share if sharename is None else sharename,
None,
os.path.basename(path),
path,
)
return '/'.join([self.default_share, os.path.basename(path)])
def download_file(self, sharename, filename):
file_service.get_file_to_path(sharename, None, filename, filename)
def delete_file(self, sharename, filename):
file_service.delete_file(sharename, None, filename)
def create_share(self, sharename):
self.file_service.create_share(sharename)
def create_directory(self, sharename, directoryname):
self.file_service.create_directory(sharename, directoryname)
def list_directories_and_files(self, sharename):
generator = self.file_service.list_directories_and_files(sharename)
return [file_or_dir.name for file_or_dir in generator]
def list_shares(self):
shares = list(self.file_service.list_shares(include_snapshots=True))
sharelist = [fileshare.name for fileshare in shares]
print(sharelist)
return sharelist
class AFSLoader():
def __init__(self, local_root: Path, afs_creds: dict = None):
if afs_creds is None:
afs_creds = get_afs_creds()
self.afs_name = afs_creds["AFS_NAME"]
self.afs_key = afs_creds["AFS_KEY"]
self.afs_share = afs_creds["AFS_SHARE"]
self.file_service = FileService(account_name=self.afs_name,
account_key=self.afs_key)
self.local_root = Path(local_root)
def get_afs_creds(self):
return self.afs_name, self.afs_key, self.afs_share
def upload_data_afs(self, data_path: Path, push_data: bool = False):
"""
Copy data to the AFS directory.
:param data_path: <Path>. Specify your path to the local data folder.
:param push_data. If True upload data if it already exists.
:return: path of the directory in the AFS share.
"""
logging.info("Sending data to AFS")
checksum = md5_dir(data_path)[:10]
afs_path = time.strftime("%Y-%m-%d-%H.%M") + '-' + checksum
list_folder = self.file_service.list_directories_and_files(
self.afs_share)
for folder in list_folder:
if checksum == folder.name[-10:]:
logging.info("Folder for data already exist!")
afs_path = folder.name
logging.info("Data is in the AFS {}".format(folder.name))
if push_data:
logging.warning("Rewriting data")
afs_path = folder.name
else:
return afs_path
self.file_service.create_directory(share_name=self.afs_share,
directory_name=afs_path)
for file in Path(data_path).iterdir():
progress_callback = lambda current, total: logbar(
current, total, f"Uploading {file.name}")
self.file_service.create_file_from_path(
share_name=self.afs_share,
directory_name=afs_path,
file_name=file.name,
local_file_path=str(file),
max_connections=cpu_count(),
progress_callback=progress_callback)
logging.info("Sending is over")
return afs_path
def download_data_afs(self, afs_path: Path, dst_path: Path = None):
afs_path = Path(afs_path)
if not dst_path:
assert self.local_root is not None
dst_path = self.local_root
list_folder = self.file_service.list_directories_and_files(
self.afs_share, directory_name=afs_path)
try:
os.mkdir(self.local_root / afs_path)
except FileExistsError:
print(f"Directory {self.local_root / afs_path} was rewritten ")
for file in list_folder:
progress_callback = lambda current, total: logbar(
current, total, f"Downloading {file.name}")
self.file_service.get_file_to_path(
share_name=self.afs_share,
directory_name=afs_path,
file_name=file.name,
file_path=str(dst_path / afs_path / file.name),
progress_callback=progress_callback)
file_service = FileService(account_name=STORAGE_ACCOUNT_NAME, account_key=STORAGE_ACCOUNT_KEY)
file_service.create_share(share_name='azureml-project', quota=1)
file_service.create_share(share_name='azureml-share', quota=1)
file_service.create_directory('azureml-share', 'Solution1')
file_service.create_directory('azureml-share', 'Solution2')
block_blob_service = BlockBlobService(account_name=STORAGE_ACCOUNT_NAME, account_key=STORAGE_ACCOUNT_KEY)
container_name ='telemetry'
block_blob_service.create_container(container_name)
source=os.environ['AML_ASSETS_URL']
dest='azureml_project.zip'
urllib.request.urlretrieve(source, dest)
with zipfile.ZipFile(dest,"r") as zip_ref:
zip_ref.extractall("azureml-project")
for root, dirs, files in os.walk('azureml-project', topdown=True):
directory = os.path.relpath(root, 'azureml-project')
if directory != '.':
file_service.create_directory('azureml-project', directory)
for f in files:
file_service.create_file_from_path(
'azureml-project',
directory,
f,
os.path.join(root, f))
for i in range(1, len(split_dir)+1, 1):
combined_dir = '\\'.join(split_dir[:i])
file_service.create_directory(NOTEBOOK_CONFIG['file_share_name'], combined_dir, fail_on_exist=False)
for root, directories, files in os.walk('Share'):
for file in files:
regex_pattern = '{0}[\\\\]?'.format('Share').replace('\\', '\\\\')
upload_directory = re.sub(regex_pattern, '', root)
print('Uploading {0} to {1}...'.format(os.path.join(root, file), upload_directory))
if (len(upload_directory) == 0):
upload_directory = None
if (upload_directory != None):
create_directories(upload_directory, file_service)
file_service.create_file_from_path(
NOTEBOOK_CONFIG['file_share_name'],
upload_directory,
file,
os.path.join(root, file)
)
block_blob_service = BlockBlobService(account_name = NOTEBOOK_CONFIG['storage_account_name'], account_key = NOTEBOOK_CONFIG['storage_account_key'])
block_blob_service.create_container('prereq', public_access = PublicAccess.Container)
for root, directories, files in os.walk('Blob'):
for file in files:
block_blob_service.create_blob_from_path(
'prereq',
file,
os.path.join(root, file)
)
# create image
containername = os.environ['AZURE_CONTAINER_NAME']
subscription_id = os.environ['AZURE_SUBSCRIPTION_ID']
resource_group_params = {'location' : location}
sku = 'standard_ragrs)'
kind = 'BlobStorage'
storage_account_params = {sku:sku,kind:kind,location:location}
# Configure Credentials
credentials = ServicePrincipalCredentials(client_id=os.environ['AZURE_CLIENT_ID'],secret=os.environ['AZURE_CLIENT_SECRET'],tenant=os.environ['AZURE_TENANT_ID'])
resource_client = ResourceManagementClient(credentials, subscription_id)
storage_client = StorageManagementClient(credentials, subscription_id)
# Create Resource Group & Storage Account
resource_client.resource_groups.create_or_update(resourcegroupname, resource_group_params)
create_sa = storage_client.storage_accounts.create(resourcegroupname, storageaccountname, {'location':'eastus','kind':'storage','sku':{'name':'standard_ragrs'}})
create_sa.wait()
# Create Container
sak = storage_client.storage_accounts.list_keys(resourcegroupname, storageaccountname)
storageaccountkey = sak.keys[0].value
storage_client = CloudStorageAccount(storageaccountname, storageaccountkey)
blob_service = storage_client.create_block_blob_service()
blob_service.create_container(containername,public_access=PublicAccess.Blob)
# Copy Files
file_service = FileService(account_name=storageaccountname, account_key=storageaccountkey)
file_service.create_share(containername)
file_service.create_directory(containername, 'directory1')
file_service.create_file_from_path(containername,'directory1','55224azuresetup.ps1','55224azuresetup.ps1',)
class AzureFileWriter(FilebaseBaseWriter):
"""
Writes items to azure file shares. It is a File Based writer, so it has filebase
option available
- account_name (str)
Public acces name of the azure account.
- account_key (str)
Public acces key to the azure account.
- share (str)
File share name.
- filebase (str)
Base path to store the items in the share.
"""
supported_options = {
"account_name": {"type": six.string_types, "env_fallback": "EXPORTERS_AZUREWRITER_NAME"},
"account_key": {"type": six.string_types, "env_fallback": "EXPORTERS_AZUREWRITER_KEY"},
"share": {"type": six.string_types},
}
def __init__(self, options, meta, *args, **kw):
from azure.storage.file import FileService
super(AzureFileWriter, self).__init__(options, meta, *args, **kw)
account_name = self.read_option("account_name")
account_key = self.read_option("account_key")
self.azure_service = FileService(account_name, account_key)
self.share = self.read_option("share")
self.azure_service.create_share(self.share)
self.logger.info("AzureWriter has been initiated." "Writing to share {}".format(self.share))
self.set_metadata("files_counter", Counter())
self.set_metadata("files_written", [])
def write(self, dump_path, group_key=None, file_name=None):
if group_key is None:
group_key = []
self._write_file(dump_path, group_key, file_name)
def _update_metadata(self, dump_path, filebase_path, file_name):
buffer_info = self.write_buffer.metadata[dump_path]
file_info = {
"file_name": file_name,
"filebase_path": filebase_path,
"size": buffer_info["size"],
"number_of_records": buffer_info["number_of_records"],
}
files_written = self.get_metadata("files_written")
files_written.append(file_info)
self.set_metadata("files_written", files_written)
self.get_metadata("files_counter")[filebase_path] += 1
def _ensure_path(self, filebase):
path = filebase.split("/")
folders_added = []
for sub_path in path:
folders_added.append(sub_path)
parent = "/".join(folders_added)
self.azure_service.create_directory(self.share, parent)
@retry_long
def _write_file(self, dump_path, group_key, file_name=None):
filebase_path, file_name = self.create_filebase_name(group_key, file_name=file_name)
self._ensure_path(filebase_path)
self.azure_service.create_file_from_path(self.share, filebase_path, file_name, dump_path, max_connections=5)
self._update_metadata(dump_path, filebase_path, file_name)
def get_file_suffix(self, path, prefix):
number_of_keys = self.get_metadata("files_counter").get(path, 0)
suffix = "{}".format(str(number_of_keys))
return suffix
def _check_write_consistency(self):
from azure.common import AzureMissingResourceHttpError
for file_info in self.get_metadata("files_written"):
try:
afile = self.azure_service.get_file_properties(
self.share, file_info["filebase_path"], file_info["file_name"]
)
file_size = afile.properties.content_length
if str(file_size) != str(file_info["size"]):
raise InconsistentWriteState(
"File {} has unexpected size. (expected {} - got {})".format(
file_info["file_name"], file_info["size"], file_size
)
)
except AzureMissingResourceHttpError:
raise InconsistentWriteState("Missing file {}".format(file_info["file_name"]))
self.logger.info("Consistency check passed")
)
# Copy Setup Files to Container and Share
blob_service.create_blob_from_path(
containername,
configfilename,
configfilename,
)
blob_service.create_blob_from_path(
containername,
zipfilename,
zipfilename,
)
file_service.create_file_from_path(
containername,
'',
configfilename,
configfilename,
)
file_service.create_file_from_path(
containername,
'',
zipfilename,
zipfilename,
)
# Create Public IP Address
def create_public_ip_address(network_client):
public_ip_addess_params = {
'location': location,
'public_ip_allocation_method': 'Dynamic'
if step % 100 == 0:
print(step,
sess.run([cost, accuracy], feed_dict={
X: x_data,
Y: y_data
}))
saver.save(
sess, "./saver/save.{}.ckpt".format(
datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S")))
saver.save(sess, "./saver/save.last.ckpt")
pred = sess.run(prediction, feed_dict={X: x_data})
pw, pb = sess.run([W, b])
result = {'W': pw.tolist(), 'b': pb.tolist()}
print(result)
with open(RESULT_FILE, 'w') as outfile:
json.dump(result, outfile)
for p, y in zip(pred, y_data.flatten()):
print("[{}] Prediction: {} True Y: {}".format(p == int(y), p, int(y)))
file_service.create_file_from_path(FILE_SHARE, None, RESULT_FILE, RESULT_FILE)
for file in os.listdir(SAVER_FOLDER):
print(file)
file_service.create_file_from_path(FILE_SHARE, "saver", file,
SAVER_FOLDER + "/" + file)
class AzureFileWriter(FilebaseBaseWriter):
"""
Writes items to azure file shares. It is a File Based writer, so it has filebase
option available
- account_name (str)
Public acces name of the azure account.
- account_key (str)
Public acces key to the azure account.
- share (str)
File share name.
- filebase (str)
Base path to store the items in the share.
"""
supported_options = {
'account_name': {
'type': six.string_types,
'env_fallback': 'EXPORTERS_AZUREWRITER_NAME'
},
'account_key': {
'type': six.string_types,
'env_fallback': 'EXPORTERS_AZUREWRITER_KEY'
},
'share': {
'type': six.string_types
}
}
def __init__(self, options, meta, *args, **kw):
from azure.storage.file import FileService
super(AzureFileWriter, self).__init__(options, meta, *args, **kw)
account_name = self.read_option('account_name')
account_key = self.read_option('account_key')
self.azure_service = FileService(account_name, account_key)
self.share = self.read_option('share')
self.azure_service.create_share(self.share)
self.logger.info('AzureWriter has been initiated.'
'Writing to share {}'.format(self.share))
self.set_metadata('files_counter', Counter())
self.set_metadata('files_written', [])
def write(self, dump_path, group_key=None, file_name=None):
if group_key is None:
group_key = []
self._write_file(dump_path, group_key, file_name)
def _update_metadata(self, dump_path, filebase_path, file_name):
buffer_info = self.write_buffer.metadata[dump_path]
file_info = {
'file_name': file_name,
'filebase_path': filebase_path,
'size': buffer_info['size'],
'number_of_records': buffer_info['number_of_records']
}
files_written = self.get_metadata('files_written')
files_written.append(file_info)
self.set_metadata('files_written', files_written)
self.get_metadata('files_counter')[filebase_path] += 1
def _ensure_path(self, filebase):
path = filebase.split('/')
folders_added = []
for sub_path in path:
folders_added.append(sub_path)
parent = '/'.join(folders_added)
self.azure_service.create_directory(self.share, parent)
@retry_long
def _write_file(self, dump_path, group_key, file_name=None):
filebase_path, file_name = self.create_filebase_name(
group_key, file_name=file_name)
self._ensure_path(filebase_path)
self.azure_service.create_file_from_path(
self.share,
filebase_path,
file_name,
dump_path,
max_connections=5,
)
self._update_metadata(dump_path, filebase_path, file_name)
def get_file_suffix(self, path, prefix):
number_of_keys = self.get_metadata('files_counter').get(path, 0)
suffix = '{}'.format(str(number_of_keys))
return suffix
def _check_write_consistency(self):
from azure.common import AzureMissingResourceHttpError
for file_info in self.get_metadata('files_written'):
try:
afile = self.azure_service.get_file_properties(
self.share, file_info['filebase_path'],
file_info['file_name'])
file_size = afile.properties.content_length
if str(file_size) != str(file_info['size']):
raise InconsistentWriteState(
'File {} has unexpected size. (expected {} - got {})'.
format(file_info['file_name'], file_info['size'],
file_size))
except AzureMissingResourceHttpError:
raise InconsistentWriteState('Missing file {}'.format(
file_info['file_name']))
self.logger.info('Consistency check passed')
