def run(job, **kwargs): resource = kwargs.get('resource') create_custom_fields_as_needed() storage_account = '{{ storage_account }}' file = "{{ file }}" azure_storage_file_share_name = '{{ azure_storage_file_share_name }}' file_name = Path(file).name if file.startswith(settings.MEDIA_URL): set_progress("Converting relative URL to filesystem path") file = file.replace(settings.MEDIA_URL, settings.MEDIA_ROOT) account_key = Resource.objects.filter(name__icontains='{{ storage_account }}')[0].azure_account_key fallback_account_key = Resource.objects.filter(name__icontains="{{ storage_account }}")[0].azure_account_key_fallback set_progress("Connecting To Azure...") file_service = FileService(account_name=storage_account, account_key=account_key) set_progress('Creating a file share...') file_service.create_share(share_name=azure_storage_file_share_name, quota=1) set_progress('Creating a file...') if file_service.exists(share_name=azure_storage_file_share_name, file_name=file_name, directory_name=''): file_service.create_file_from_path(share_name=azure_storage_file_share_name, file_name=file_name, directory_name='', local_file_path=file) return "WARNING", "File with this name already exists", "The file will be updated." else: file_service.create_file_from_path(share_name=azure_storage_file_share_name, file_name=file_name, directory_name='', local_file_path=file) resource.name = azure_storage_file_share_name + '-' + file_name resource.azure_storage_account_name = storage_account resource.azure_account_key = account_key resource.azure_account_key_fallback = fallback_account_key resource.azure_storage_file_share_name = azure_storage_file_share_name resource.azure_storage_file_name = file_name resource.save() return "Success", "The File has succesfully been uploaded", ""
def _configure_auto_storage(cli_ctx, location): """Configures auto storage account for the cluster :param str location: location for the auto-storage account. :return (str, str): a tuple with auto storage account name and key. """ from azure.mgmt.resource.resources.models import ResourceGroup from azure.storage.file import FileService from azure.storage.blob import BlockBlobService resource_group = _get_auto_storage_resource_group() resource_client = get_mgmt_service_client(cli_ctx, ResourceType.MGMT_RESOURCE_RESOURCES) if resource_client.resource_groups.check_existence(resource_group): logger.warning('BatchAI will use existing %s resource group for auto-storage account', resource_group) else: logger.warning('Creating %s resource for auto-storage account', resource_group) resource_client.resource_groups.create_or_update( resource_group, ResourceGroup(location=location)) storage_client = _get_storage_management_client(cli_ctx) account = None for a in storage_client.storage_accounts.list_by_resource_group(resource_group): if a.primary_location == location.lower().replace(' ', ''): account = a.name logger.warning('Using existing %s storage account as an auto-storage account', account) break if account is None: account = _create_auto_storage_account(storage_client, resource_group, location) logger.warning('Created auto storage account %s', account) key = _get_storage_account_key(cli_ctx, account, None) file_service = FileService(account, key) file_service.create_share(AUTO_STORAGE_SHARE_NAME, fail_on_exist=False) blob_service = BlockBlobService(account, key) blob_service.create_container(AUTO_STORAGE_CONTAINER_NAME, fail_on_exist=False) return account, key
def create_file_share(self, storage_account_name: str, share_name: str, size: int, key: str) -> FileService: self.logger.info("Creating file share") file_service = FileService(account_name=storage_account_name, account_key=key) file_service.create_share(share_name, quota=size) return file_service
def prepare_azure_file_share_service(config, dataset_directory='dataset_directory'): # Create a file share service = FileService(config.storage_account_name, config.storage_account_key) service.create_share(config.workspace_file_share, fail_on_exist=False) # Create a directory in the file share service.create_directory(config.workspace_file_share, dataset_directory, fail_on_exist=False) return service
def create_azure_fileshare(share_prefix, account_name, account_key): """ Generate a unique share name to avoid overlaps in shared infra :param share_prefix: :param account_name: :param account_key: :return: """ # FIXME - Need to remove hardcoded directoty link below d_dir = './WebInDeploy/bootstrap' share_name = "{0}-{1}".format(share_prefix.lower(), str(uuid.uuid4())) print('using share_name of: {}'.format(share_name)) # archive_file_path = _create_archive_directory(files, share_prefix) try: # ignore SSL warnings - bad form, but SSL Decrypt causes issues with this s = requests.Session() s.verify = False file_service = FileService(account_name=account_name, account_key=account_key, request_session=s) # print(file_service) if not file_service.exists(share_name): file_service.create_share(share_name) for d in ['config', 'content', 'software', 'license']: print('creating directory of type: {}'.format(d)) if not file_service.exists(share_name, directory_name=d): file_service.create_directory(share_name, d) # FIXME - We only handle bootstrap files. May need to handle other dirs if d == 'config': for filename in os.listdir(d_dir): print('creating file: {0}'.format(filename)) file_service.create_file_from_path( share_name, d, filename, os.path.join(d_dir, filename)) except AttributeError as ae: # this can be returned on bad auth information print(ae) return "Authentication or other error creating bootstrap file_share in Azure" except AzureException as ahe: print(ahe) return str(ahe) except ValueError as ve: print(ve) return str(ve) print('all done') return share_name
def _create_file_share(storage_account, storage_account_key): """Creates Azure Files in the storage account to be mounted into a cluster :param str storage_account: name of the storage account. :param str storage_account_key: storage account key. """ if storage_account == FAKE_STORAGE.name: return service = FileService(storage_account, storage_account_key) service.create_share(AZURE_FILES_NAME)
def _create_file_share(storage_account, storage_account_key): """Creates Azure Files in the storage account to be mounted into a cluster :param str storage_account: name of the storage account. :param str storage_account_key: storage account key. """ if storage_account == Helpers.FAKE_STORAGE.name: return service = FileService(storage_account, storage_account_key) service.create_share(Helpers.AZURE_FILES_NAME)
def file(): static_dir_path = "D:\home\site\wwwroot\static" static_file_dir_path = static_dir_path + '\\' + 'files' account_name = 'hanastragetest' account_key = 'acount_key' root_share_name = 'root' share_name = 'images' directory_url = 'https://hanastragetest.file.core.windows.net/' + root_share_name + '/' + share_name # create local save directory if os.path.exist(static_file_dir_path) is False: os.mkdir(static_file_dir_path) file_service = FileService(account_name=account_name, account_key=account_key) # create share file_service.create_share(root_share_name) # create directory file_service.create_directory(root_share_name, share_name) files = os.listdir(static_dir_path) for file in files: # delete if file_service.exists(root_share_name, share_name, file): file_service.delete_file(root_share_name, share_name, file) # file upload file_service.create_file_from_path( root_share_name, share_name, # We want to create this blob in the root directory, so we specify None for the directory_name file, static_dir_path + '\\' + file, content_settings=ContentSettings(content_type='image/png')) generator = file_service.list_directories_and_files( root_share_name, share_name) html = "" for file in generator: # file download file_save_path = static_file_dir_path + '\\' + file file_service.get_file_to_path(root_share_name, share_name, file, file_save_path) html = "{}<img src='{}'>".format(html, file_save_path) result = { "result": True, "data": { "file_or_dir_name": [file_or_dir.name for file_or_dir in generator] } } return make_response(json.dumps(result, ensure_ascii=False) + html)
def create_and_attach_file_storage(cfg, ws): if len(cfg.DataReference.localDirectoryFilesList) > 0: for ref in cfg.DataReference.localDirectoryFilesList: log.info("Attempting to create file share '%s' on storage account '%s'.", ref.remoteFileShare, ref.storageAccountName) file_service = FileService(ref.storageAccountName, ref.storageAccountKey) exist = file_service.create_share(ref.remoteFileShare, fail_on_exist=False) if exist: log.info("File Share '%s' on storage account '%s' created.", ref.remoteFileShare, ref.storageAccountName) else: log.info("File Share '%s' on storage account '%s' already existed.", ref.remoteFileShare, ref.storageAccountName) # Get most recent list of datastores linked to current workspace datastores = ws.datastores() # Validate if share_ds is created ds = None if ref.dataref_id not in datastores else Datastore(workspace = ws, name = ref.dataref_id) # Register the DS to the workspace if ds: if ds.account_name == ref.storageAccountName and ds.container_name == ref.remoteFileShare: recreate = False else: recreate = True # also remove the existing reference ds.unregister() else: recreate = True if recreate: log.info('Registering file share "{}" to AML datastore for AML workspace "{}" under datastore id "{}".'.format(ref.remoteFileShare, ws.name, ref.dataref_id)) ds = Datastore.register_azure_file_share(workspace = ws, datastore_name = ref.dataref_id, file_share_name = ref.remoteFileShare, account_name = ref.storageAccountName, account_key= ref.storageAccountKey, overwrite=True, ) else: log.info('File share "{}" under AML workspace "{}" already registered under datastore id "{}".'.format(ref.remoteFileShare, ws.name, ref.dataref_id))
def shares(): # Create Container and Share global storage_account_key, blob_service, blob_share, file_service, file_share sak = storage_client.storage_accounts.list_keys(resourcegroupname, storageaccountname) storage_account_key = sak.keys[0].value cloudstorage_client = CloudStorageAccount(storageaccountname, storage_account_key) blob_service = cloudstorage_client.create_block_blob_service() blob_share = blob_service.create_container( sharename, public_access=PublicAccess.Container) file_service = FileService(account_name=storageaccountname, account_key=storage_account_key) file_share = file_service.create_share(sharename) # Copy Setup Files to Container and Share blob_service.create_blob_from_path( sharename, filename, filename, ) file_service.create_file_from_path( sharename, '', filename, filename, )
class FileShare(object): """ Information from Azure files service """ def __init__(self, account): self.account_name = account.storage_name() self.account_key = account.storage_key() self.files = FileService( self.account_name, self.account_key ) def list(self): """ list file shares """ result = [] try: for share in self.files.list_shares(): result.append(format(share.name)) except Exception as e: raise AzureFileShareListError( '%s: %s' % (type(e).__name__, format(e)) ) return result def create(self, share_name): """ create a file share """ try: self.files.create_share(share_name) except Exception as e: raise AzureFileShareCreateError( '%s: %s' % (type(e).__name__, format(e)) ) def delete(self, share_name): """ delete a file share """ try: self.files.delete_share(share_name) except Exception as e: raise AzureFileShareDeleteError( '%s: %s' % (type(e).__name__, format(e)) )
def upload_file(self, path): """Upload a file into the default share on the storage account. If the share doesn't exist, create it first. """ file_service = FileService( account_name=self.account.name, account_key=self.key, ) file_service.create_share(self.default_share) file_service.create_file_from_path( self.default_share, None, os.path.basename(path), path, ) return '/'.join([self.default_share, os.path.basename(path)])
def create_azure_fileshare(files, share_prefix, account_name, account_key): # generate a unique share name to avoid overlaps in shared infra share_name = "{0}-{1}".format(share_prefix.lower(), str(uuid.uuid4())) print('using share_name of: {}'.format(share_name)) archive_file_path = _create_archive_directory(files, share_prefix) try: # ignore SSL warnings - bad form, but SSL Decrypt causes issues with this s = requests.Session() s.verify = False file_service = FileService(account_name=account_name, account_key=account_key, request_session=s) # print(file_service) if not file_service.exists(share_name): file_service.create_share(share_name) for d in ['config', 'content', 'software', 'license']: print('creating directory of type: {}'.format(d)) if not file_service.exists(share_name, directory_name=d): file_service.create_directory(share_name, d) d_dir = os.path.join(archive_file_path, d) for filename in os.listdir(d_dir): print('creating file: {0}'.format(filename)) file_service.create_file_from_path( share_name, d, filename, os.path.join(d_dir, filename)) except AttributeError as ae: # this can be returned on bad auth information print(ae) return "Authentication or other error creating bootstrap file_share in Azure" except AzureException as ahe: print(ahe) return str(ahe) except ValueError as ve: print(ve) return str(ve) print('all done') return 'Azure file-share {} created successfully'.format(share_name)
def saveModel(customer, modelName, model, storage_account_name, storage_account_key): fileService = FileService(account_name=storage_account_name, account_key=storage_account_key) if not fileService.exists('trainedmodels', customer): fileService.create_share('trainedmodels') fileService.create_directory('trainedmodels', customer) if not fileService.exists('trainedmodels', customer + '/' + modelName): fileService.create_directory('trainedmodels', customer + '/' + modelName) modelPickle = pickle.dumps(model) timestr = time.strftime('%Y%m%d-%H%M%S') fileName = modelName + '_' + timestr + '.pkl' fileService.create_file_from_bytes('trainedmodels', customer + '/' + modelName, fileName, modelPickle) print(fileName + ' saved.')
import zipfile from azure.storage.blob import BlockBlobService from azure.storage.blob import PublicAccess from azure.storage.file import FileService ACCOUNT_NAME = os.environ['StorageAccountName'] ACCOUNT_KEY = os.environ['StorageAccountKey'] CONTAINER_NAME = os.environ['TelemetryContainerName'] az_blob_service = BlockBlobService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY) az_blob_service.create_container(CONTAINER_NAME, fail_on_exist=False) file_service = FileService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY) file_service.create_share(share_name='model', quota=1) source = os.environ['ModelZipUrl'] dest = 'model.zip' urllib.request.urlretrieve(source, dest) with zipfile.ZipFile(dest, "r") as zip_ref: zip_ref.extractall("model") for root, dirs, files in os.walk('model', topdown=True): directory = os.path.relpath(root, 'model') if directory != '.': file_service.create_directory('model', directory) for f in files: file_service.create_file_from_path('model', directory, f,
smtp = smtplib.SMTP(host="smtp.gmail.com", port=587) smtp.starttls() smtp.login(username, password) smtp.sendmail(send_from, send_to, msg.as_string()) smtp.close() print('Email sent') # Gmail login details username = '******' password = '******' default_address = [] # Login Details for Azure Storage file_service = FileService(account_name='killianoneachtain', account_key='KEY') file_service.create_share('security') file_service.create_directory('security', 'securityPhotos') cwd = os.getcwd() # Get the current working directory (cwd) path = cwd + "/securityPhotos" # change file permissions access_rights = 0o755 # create a photo directory if none exists try: os.mkdir(path, access_rights) except OSError: print("Creation of the directory %s failed. \nFolder already exists!" % path) else: print("Successfully created the directory %s " % path)
def main(path, debug, remote_directory, typhoonname): initialize.setup_cartopy() start_time = datetime.now() print( '---------------------AUTOMATION SCRIPT STARTED---------------------------------' ) print(str(start_time)) #%% check for active typhoons print( '---------------------check for active typhoons---------------------------------' ) print(str(start_time)) remote_dir = remote_directory if debug: typhoonname = 'SURIGAE' remote_dir = '20210421120000' logger.info(f"DEBUGGING piepline for typhoon{typhoonname}") Activetyphoon = [typhoonname] else: # If passed typhoon name is None or empty string if not typhoonname: Activetyphoon = Check_for_active_typhoon.check_active_typhoon() if not Activetyphoon: logger.info("No active typhoon in PAR stop pipeline") sys.exit() logger.info(f"Running on active Typhoon(s) {Activetyphoon}") else: Activetyphoon = [typhoonname] remote_dir = remote_directory logger.info(f"Running on custom Typhoon {Activetyphoon}") Alternative_data_point = (start_time - timedelta(hours=24)).strftime("%Y%m%d") date_dir = start_time.strftime("%Y%m%d%H") Input_folder = os.path.join(path, f'forecast/Input/{date_dir}/Input/') Output_folder = os.path.join(path, f'forecast/Output/{date_dir}/Output/') if not os.path.exists(Input_folder): os.makedirs(Input_folder) if not os.path.exists(Output_folder): os.makedirs(Output_folder) #download NOAA rainfall try: #Rainfall_data_window.download_rainfall_nomads(Input_folder,path,Alternative_data_point) Rainfall_data.download_rainfall_nomads(Input_folder, path, Alternative_data_point) rainfall_error = False except: traceback.print_exc() #logger.warning(f'Rainfall download failed, performing download in R script') logger.info( f'Rainfall download failed, performing download in R script') rainfall_error = True ###### download UCL data try: ucl_data.create_ucl_metadata(path, os.environ['UCL_USERNAME'], os.environ['UCL_PASSWORD']) ucl_data.process_ucl_data(path, Input_folder, os.environ['UCL_USERNAME'], os.environ['UCL_PASSWORD']) except: logger.info(f'UCL download failed') #%% ##Create grid points to calculate Winfield cent = Centroids() cent.set_raster_from_pnt_bounds((118, 6, 127, 19), res=0.05) #this option is added to make the script scaleable globally To Do #cent.set_raster_from_pnt_bounds((LonMin,LatMin,LonMax,LatMax), res=0.05) cent.check() cent.plot() #### admin = gpd.read_file( os.path.join(path, "./data-raw/phl_admin3_simpl2.geojson")) df = pd.DataFrame(data=cent.coord) df["centroid_id"] = "id" + (df.index).astype(str) centroid_idx = df["centroid_id"].values ncents = cent.size df = df.rename(columns={0: "lat", 1: "lon"}) df = gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df.lon, df.lat)) #df.to_crs({'init': 'epsg:4326'}) df.crs = {'init': 'epsg:4326'} df_admin = sjoin(df, admin, how="left").dropna() # Sometimes the ECMWF ftp server complains about too many requests # This code allows several retries with some sleep time in between n_tries = 0 while True: try: logger.info("Downloading ECMWF typhoon tracks") bufr_files = TCForecast.fetch_bufr_ftp(remote_dir=remote_dir) fcast = TCForecast() fcast.fetch_ecmwf(files=bufr_files) except ftplib.all_errors as e: n_tries += 1 if n_tries >= ECMWF_MAX_TRIES: logger.error( f' Data downloading from ECMWF failed: {e}, ' f'reached limit of {ECMWF_MAX_TRIES} tries, exiting') sys.exit() logger.error( f' Data downloading from ECMWF failed: {e}, retrying after {ECMWF_SLEEP} s' ) time.sleep(ECMWF_SLEEP) continue break #%% filter data downloaded in the above step for active typhoons in PAR # filter tracks with name of current typhoons and drop tracks with only one timestep fcast.data = [ track_data_clean.track_data_clean(tr) for tr in fcast.data if (tr.time.size > 1 and tr.name in Activetyphoon) ] # fcast.data = [tr for tr in fcast.data if tr.name in Activetyphoon] # fcast.data = [tr for tr in fcast.data if tr.time.size>1] for typhoons in Activetyphoon: #typhoons=Activetyphoon[0] logger.info(f'Processing data {typhoons}') fname = open( os.path.join(path, 'forecast/Input/', "typhoon_info_for_model.csv"), 'w') fname.write('source,filename,event,time' + '\n') if not rainfall_error: line_ = 'Rainfall,' + '%srainfall' % Input_folder + ',' + typhoons + ',' + date_dir #StormName # fname.write(line_ + '\n') line_ = 'Output_folder,' + '%s' % Output_folder + ',' + typhoons + ',' + date_dir #StormName # #line_='Rainfall,'+'%sRainfall/' % Input_folder +','+ typhoons + ',' + date_dir #StormName # fname.write(line_ + '\n') #typhoons='SURIGAE' # to run it manually for any typhoon # select windspeed for HRS model fcast.data = [tr for tr in fcast.data if tr.name == typhoons] tr_HRS = [tr for tr in fcast.data if (tr.is_ensemble == 'False')] if tr_HRS != []: HRS_SPEED = (tr_HRS[0].max_sustained_wind.values / 0.84).tolist( ) ############# 0.84 is conversion factor for ECMWF 10MIN TO 1MIN AVERAGE dfff = tr_HRS[0].to_dataframe() dfff[['VMAX', 'LAT', 'LON']] = dfff[['max_sustained_wind', 'lat', 'lon']] dfff['YYYYMMDDHH'] = dfff.index.values dfff['YYYYMMDDHH'] = dfff['YYYYMMDDHH'].apply( lambda x: x.strftime("%Y%m%d%H%M")) dfff['STORMNAME'] = typhoons dfff[['YYYYMMDDHH', 'VMAX', 'LAT', 'LON', 'STORMNAME']].to_csv(os.path.join(Input_folder, 'ecmwf_hrs_track.csv'), index=False) line_ = 'ecmwf,' + '%secmwf_hrs_track.csv' % Input_folder + ',' + typhoons + ',' + date_dir #StormName # #line_='Rainfall,'+'%sRainfall/' % Input_folder +','+ typhoons + ',' + date_dir #StormName # fname.write(line_ + '\n') # Adjust track time step data_forced = [ tr.where(tr.time <= max(tr_HRS[0].time.values), drop=True) for tr in fcast.data ] # data_forced = [track_data_clean.track_data_force_HRS(tr,HRS_SPEED) for tr in data_forced] # forced with HRS windspeed #data_forced= [track_data_clean.track_data_clean(tr) for tr in fcast.data] # taking speed of ENS # interpolate to 3h steps from the original 6h #fcast.equal_timestep(3) else: len_ar = np.min([len(var.lat.values) for var in fcast.data]) lat_ = np.ma.mean([var.lat.values[:len_ar] for var in fcast.data], axis=0) lon_ = np.ma.mean([var.lon.values[:len_ar] for var in fcast.data], axis=0) YYYYMMDDHH = pd.date_range(fcast.data[0].time.values[0], periods=len_ar, freq="H") vmax_ = np.ma.mean( [var.max_sustained_wind.values[:len_ar] for var in fcast.data], axis=0) d = { 'YYYYMMDDHH': YYYYMMDDHH, "VMAX": vmax_, "LAT": lat_, "LON": lon_ } dfff = pd.DataFrame(d) dfff['STORMNAME'] = typhoons dfff['YYYYMMDDHH'] = dfff['YYYYMMDDHH'].apply( lambda x: x.strftime("%Y%m%d%H%M")) dfff[['YYYYMMDDHH', 'VMAX', 'LAT', 'LON', 'STORMNAME']].to_csv(os.path.join(Input_folder, 'ecmwf_hrs_track.csv'), index=False) line_ = 'ecmwf,' + '%secmwf_hrs_track.csv' % Input_folder + ',' + typhoons + ',' + date_dir #StormName # #line_='Rainfall,'+'%sRainfall/' % Input_folder +','+ typhoons + ',' + date_dir #StormName # fname.write(line_ + '\n') data_forced = fcast.data # calculate windfields for each ensamble threshold = 0 #(threshold to filter dataframe /reduce data ) df = pd.DataFrame(data=cent.coord) df["centroid_id"] = "id" + (df.index).astype(str) centroid_idx = df["centroid_id"].values ncents = cent.size df = df.rename(columns={0: "lat", 1: "lon"}) #calculate wind field for each ensamble members list_intensity = [] distan_track = [] for tr in data_forced: logger.info( f"Running on ensemble # {tr.ensemble_number} for typhoon {tr.name}" ) track = TCTracks() typhoon = TropCyclone() track.data = [tr] #track.equal_timestep(3) tr = track.data[0] typhoon.set_from_tracks(track, cent, store_windfields=True) # Make intensity plot using the high resolution member if tr.is_ensemble == 'False': logger.info("High res member: creating intensity plot") plot_intensity.plot_inensity(typhoon=typhoon, event=tr.sid, output_dir=Output_folder, date_dir=date_dir, typhoon_name=tr.name) windfield = typhoon.windfields nsteps = windfield[0].shape[0] centroid_id = np.tile(centroid_idx, nsteps) intensity_3d = windfield[0].toarray().reshape(nsteps, ncents, 2) intensity = np.linalg.norm(intensity_3d, axis=-1).ravel() timesteps = np.repeat(track.data[0].time.values, ncents) #timesteps = np.repeat(tr.time.values, ncents) timesteps = timesteps.reshape((nsteps, ncents)).ravel() inten_tr = pd.DataFrame({ 'centroid_id': centroid_id, 'value': intensity, 'timestamp': timesteps, }) inten_tr = inten_tr[inten_tr.value > threshold] inten_tr['storm_id'] = tr.sid inten_tr['ens_id'] = tr.sid + '_' + str(tr.ensemble_number) inten_tr['name'] = tr.name inten_tr = (pd.merge(inten_tr, df_admin, how='outer', on='centroid_id').dropna().groupby( ['adm3_pcode', 'ens_id'], as_index=False).agg( {"value": ['count', 'max']})) inten_tr.columns = [ x for x in ['adm3_pcode', 'storm_id', 'value_count', 'v_max'] ] list_intensity.append(inten_tr) distan_track1 = [] for index, row in df.iterrows(): dist = np.min( np.sqrt( np.square(tr.lat.values - row['lat']) + np.square(tr.lon.values - row['lon']))) distan_track1.append(dist * 111) dist_tr = pd.DataFrame({ 'centroid_id': centroid_idx, 'value': distan_track1 }) dist_tr['storm_id'] = tr.sid dist_tr['name'] = tr.name dist_tr['ens_id'] = tr.sid + '_' + str(tr.ensemble_number) dist_tr = (pd.merge(dist_tr, df_admin, how='outer', on='centroid_id').dropna().groupby( ['adm3_pcode', 'name', 'ens_id'], as_index=False).agg({'value': 'min'})) dist_tr.columns = [ x for x in ['adm3_pcode', 'name', 'storm_id', 'dis_track_min'] ] # join_left_df_.columns.ravel()] distan_track.append(dist_tr) df_intensity_ = pd.concat(list_intensity) distan_track1 = pd.concat(distan_track) typhhon_df = pd.merge(df_intensity_, distan_track1, how='left', on=['adm3_pcode', 'storm_id']) typhhon_df.to_csv(os.path.join(Input_folder, 'windfield.csv'), index=False) line_ = 'windfield,' + '%swindfield.csv' % Input_folder + ',' + typhoons + ',' + date_dir #StormName # #line_='Rainfall,'+'%sRainfall/' % Input_folder +','+ typhoons + ',' + date_dir #StormName # fname.write(line_ + '\n') fname.close() ############################################################# #### Run IBF model ############################################################# os.chdir(path) if platform == "linux" or platform == "linux2": #check if running on linux or windows os # linux try: p = subprocess.check_call( ["Rscript", "run_model_V2.R", str(rainfall_error)]) except subprocess.CalledProcessError as e: logger.error(f'failed to excute R sript') raise ValueError(str(e)) elif platform == "win32": #if OS is windows edit the path for Rscript try: p = subprocess.check_call([ "C:/Program Files/R/R-4.1.0/bin/Rscript", "run_model_V2.R", str(rainfall_error) ]) except subprocess.CalledProcessError as e: logger.error(f'failed to excute R sript') raise ValueError(str(e)) ############################################################# # send email in case of landfall-typhoon ############################################################# image_filenames = list(Path(Output_folder).glob('*.png')) data_filenames = list(Path(Output_folder).glob('*.csv')) if image_filenames or data_filenames: message_html = """\ <html> <body> <h1>IBF model run result </h1> <p>Please find attached a map and data with updated model run</p> <img src="cid:Impact_Data"> </body> </html> """ Sendemail.sendemail( smtp_server=os.environ["SMTP_SERVER"], smtp_port=int(os.environ["SMTP_PORT"]), email_username=os.environ["EMAIL_LOGIN"], email_password=os.environ["EMAIL_PASSWORD"], email_subject='Updated impact map for a new Typhoon in PAR', from_address=os.environ["EMAIL_FROM"], to_address_list=os.environ["EMAIL_TO_LIST"].split(','), cc_address_list=os.environ["EMAIL_CC_LIST"].split(','), message_html=message_html, filename_list=image_filenames + data_filenames) else: raise FileNotFoundError( f'No .png or .csv found in {Output_folder}') ##################### upload model output to 510 datalack ############## file_service = FileService( account_name=os.environ["AZURE_STORAGE_ACCOUNT"], protocol='https', connection_string=os.environ["AZURE_CONNECTING_STRING"]) file_service.create_share('forecast') OutPutFolder = date_dir file_service.create_directory('forecast', OutPutFolder) for img_file in image_filenames: file_service.create_file_from_path( 'forecast', OutPutFolder, os.fspath(img_file.parts[-1]), img_file, content_settings=ContentSettings(content_type='image/png')) for data_file in data_filenames: file_service.create_file_from_path( 'forecast', OutPutFolder, os.fspath(data_file.parts[-1]), data_file, content_settings=ContentSettings(content_type='text/csv')) ##################### upload model input(Rainfall+wind intensity) to 510 datalack ############## # To DO print( '---------------------AUTOMATION SCRIPT FINISHED---------------------------------' ) print(str(datetime.now()))
class AzureFileWriter(FilebaseBaseWriter): """ Writes items to azure file shares. It is a File Based writer, so it has filebase option available - account_name (str) Public acces name of the azure account. - account_key (str) Public acces key to the azure account. - share (str) File share name. - filebase (str) Base path to store the items in the share. """ supported_options = { 'account_name': { 'type': six.string_types, 'env_fallback': 'EXPORTERS_AZUREWRITER_NAME' }, 'account_key': { 'type': six.string_types, 'env_fallback': 'EXPORTERS_AZUREWRITER_KEY' }, 'share': { 'type': six.string_types } } def __init__(self, options, meta, *args, **kw): from azure.storage.file import FileService super(AzureFileWriter, self).__init__(options, meta, *args, **kw) account_name = self.read_option('account_name') account_key = self.read_option('account_key') self.azure_service = FileService(account_name, account_key) self.share = self.read_option('share') self.azure_service.create_share(self.share) self.logger.info('AzureWriter has been initiated.' 'Writing to share {}'.format(self.share)) self.set_metadata('files_counter', Counter()) self.set_metadata('files_written', []) def write(self, dump_path, group_key=None, file_name=None): if group_key is None: group_key = [] self._write_file(dump_path, group_key, file_name) def _update_metadata(self, dump_path, filebase_path, file_name): buffer_info = self.write_buffer.metadata[dump_path] file_info = { 'file_name': file_name, 'filebase_path': filebase_path, 'size': buffer_info['size'], 'number_of_records': buffer_info['number_of_records'] } files_written = self.get_metadata('files_written') files_written.append(file_info) self.set_metadata('files_written', files_written) self.get_metadata('files_counter')[filebase_path] += 1 def _ensure_path(self, filebase): path = filebase.split('/') folders_added = [] for sub_path in path: folders_added.append(sub_path) parent = '/'.join(folders_added) self.azure_service.create_directory(self.share, parent) @retry_long def _write_file(self, dump_path, group_key, file_name=None): filebase_path, file_name = self.create_filebase_name( group_key, file_name=file_name) self._ensure_path(filebase_path) self.azure_service.create_file_from_path( self.share, filebase_path, file_name, dump_path, max_connections=5, ) self._update_metadata(dump_path, filebase_path, file_name) def get_file_suffix(self, path, prefix): number_of_keys = self.get_metadata('files_counter').get(path, 0) suffix = '{}'.format(str(number_of_keys)) return suffix def _check_write_consistency(self): from azure.common import AzureMissingResourceHttpError for file_info in self.get_metadata('files_written'): try: afile = self.azure_service.get_file_properties( self.share, file_info['filebase_path'], file_info['file_name']) file_size = afile.properties.content_length if str(file_size) != str(file_info['size']): raise InconsistentWriteState( 'File {} has unexpected size. (expected {} - got {})'. format(file_info['file_name'], file_info['size'], file_size)) except AzureMissingResourceHttpError: raise InconsistentWriteState('Missing file {}'.format( file_info['file_name'])) self.logger.info('Consistency check passed')
import twitter import hashlib import requests import time import os """ Auxiliary """ from azure.storage.file import FileService, ContentSettings fileService = FileService( account_name='cs7b04dc31e3552x4267x9c3', account_key= 'ihwu6KLKRkUv3/dF3ELyhqbsB34jUJGyyVexD3gr2PUhcL3X5XFg/aFumVEHZCUHqqfP+m2UBM1Lni3uw26WcA==' ) filesDir = 'fileshare' fileService.create_share(filesDir) """ VT """ apiKey = '811a31748544dd8d3a2d8a13785c2e78ffb2c351b5d56b37168ab6ff6315dc1f' def getFile(dirName, fileName): file_ = fileService.get_file_to_text(filesDir, dirName, fileName) return file_ """ Main logic """ def getAzureFileReport(githubUser, githubProject, fileName): if githubUser == "nada" or githubProject == "nada" or fileName == "nada": return "Please provide all the fields!"
RECEIPTS_DIRECTORY = 'delivery-receipts' RECEIPTS_URL = ('http://www.cmegroup.com/delivery_reports/' 'deliverable-commodities-under-registration.xls') RECEIPTS_FILENAME_SUFFIX = '-deliverable-commodities-under-registration.xls' STOCKS_DIRECTORY = 'stocks-of-grain' STOCKS_URL = 'http://www.cmegroup.com/delivery_reports/stocks-of-grain-updated-tuesday.xls' STOCKS_FILENAME_SUFFIX = '-stocks-of-grain-updated-tuesday.xls' def filename(suffix): return datetime.now().strftime('%Y%m%d') + suffix def get_bytes(url): return requests.get(url).content f = FileService(account_name=ACCOUNT_NAME, account_key=KEY) f.create_share(SHARE_NAME) f.create_directory(SHARE_NAME, RECEIPTS_DIRECTORY) f.create_file_from_bytes(SHARE_NAME, RECEIPTS_DIRECTORY, filename(RECEIPTS_FILENAME_SUFFIX), get_bytes(RECEIPTS_URL)) f.create_directory(SHARE_NAME, STOCKS_DIRECTORY) f.create_file_from_bytes(SHARE_NAME, STOCKS_DIRECTORY, filename(STOCKS_FILENAME_SUFFIX), get_bytes(STOCKS_URL))
def test_job_level_mounting(self, resource_group, location, cluster, storage_account, storage_account_key): """Tests if it's possible to mount external file systems for a job.""" job_name = 'job' # Create file share and container to mount on the job level if storage_account.name != helpers.FAKE_STORAGE.name: files = FileService(storage_account.name, storage_account_key) files.create_share('jobshare', fail_on_exist=False) blobs = BlockBlobService(storage_account.name, storage_account_key) blobs.create_container('jobcontainer', fail_on_exist=False) job = self.client.jobs.create( resource_group.name, job_name, parameters=models.JobCreateParameters( location=location, cluster=models.ResourceId(id=cluster.id), node_count=1, mount_volumes=models.MountVolumes( azure_file_shares=[ models.AzureFileShareReference( account_name=storage_account.name, azure_file_url='https://{0}.file.core.windows.net/{1}'.format( storage_account.name, 'jobshare'), relative_mount_path='job_afs', credentials=models.AzureStorageCredentialsInfo( account_key=storage_account_key ), ) ], azure_blob_file_systems=[ models.AzureBlobFileSystemReference( account_name=storage_account.name, container_name='jobcontainer', relative_mount_path='job_bfs', credentials=models.AzureStorageCredentialsInfo( account_key=storage_account_key ), ) ] ), # Put standard output on cluster level AFS to check that the job has access to it. std_out_err_path_prefix='$AZ_BATCHAI_MOUNT_ROOT/{0}'.format(helpers.AZURE_FILES_MOUNTING_PATH), # Create two output directories on job level AFS and blobfuse. output_directories=[ models.OutputDirectory(id='OUTPUT1', path_prefix='$AZ_BATCHAI_JOB_MOUNT_ROOT/job_afs'), models.OutputDirectory(id='OUTPUT2', path_prefix='$AZ_BATCHAI_JOB_MOUNT_ROOT/job_bfs') ], # Check that the job preparation has access to job level file systems. job_preparation=models.JobPreparation( command_line='echo afs > $AZ_BATCHAI_OUTPUT_OUTPUT1/prep_afs.txt; ' 'echo bfs > $AZ_BATCHAI_OUTPUT_OUTPUT2/prep_bfs.txt; ' 'echo done' ), # Check that the job has access to job custom_toolkit_settings=models.CustomToolkitSettings( command_line='echo afs > $AZ_BATCHAI_OUTPUT_OUTPUT1/job_afs.txt; ' 'echo bfs > $AZ_BATCHAI_OUTPUT_OUTPUT2/job_bfs.txt; ' 'mkdir $AZ_BATCHAI_OUTPUT_OUTPUT1/afs; ' 'echo afs > $AZ_BATCHAI_OUTPUT_OUTPUT1/afs/job_afs.txt; ' 'mkdir $AZ_BATCHAI_OUTPUT_OUTPUT2/bfs; ' 'echo bfs > $AZ_BATCHAI_OUTPUT_OUTPUT2/bfs/job_bfs.txt; ' 'echo done' ) ) ).result() self.assertEqual( helpers.wait_for_job_completion(self.is_live, self.client, resource_group.name, job.name, helpers.MINUTE), models.ExecutionState.succeeded) job = self.client.jobs.get(resource_group.name, job.name) # Assert job and job prep standard output is populated on cluster level filesystem helpers.assert_job_files_are(self, self.client, resource_group.name, job.name, helpers.STANDARD_OUTPUT_DIRECTORY_ID, {u'stdout.txt': u'done\n', u'stderr.txt': u'', u'stdout-job_prep.txt': u'done\n', u'stderr-job_prep.txt': u''}) # Assert files are generated on job level AFS helpers.assert_job_files_are(self, self.client, resource_group.name, job.name, 'OUTPUT1', {u'job_afs.txt': u'afs\n', u'prep_afs.txt': u'afs\n', u'afs': None}) # Assert files are generated on job level blobfuse helpers.assert_job_files_are(self, self.client, resource_group.name, job.name, 'OUTPUT2', {u'job_bfs.txt': u'bfs\n', u'prep_bfs.txt': u'bfs\n', u'bfs': None}) # Assert subfolders are available via API helpers.assert_job_files_in_path_are(self, self.client, resource_group.name, job.name, 'OUTPUT1', 'afs', {u'job_afs.txt': u'afs\n'}) helpers.assert_job_files_in_path_are(self, self.client, resource_group.name, job.name, 'OUTPUT2', 'bfs', {u'job_bfs.txt': u'bfs\n'}) # Assert that we can access the output files created on job level mount volumes directly in storage using path # segment returned by the server. if storage_account.name != helpers.FAKE_STORAGE.name: files = FileService(storage_account.name, storage_account_key) self.assertTrue( files.exists('jobshare', job.job_output_directory_path_segment + '/' + helpers.OUTPUT_DIRECTORIES_FOLDER_NAME, 'job_afs.txt')) blobs = BlockBlobService(storage_account.name, storage_account_key) self.assertTrue( blobs.exists('jobcontainer', job.job_output_directory_path_segment + '/' + helpers.OUTPUT_DIRECTORIES_FOLDER_NAME + '/job_bfs.txt')) # After the job is done the filesystems should be unmounted automatically, check this by submitting a new job. checker = self.client.jobs.create( resource_group.name, 'checker', parameters=models.JobCreateParameters( location=location, cluster=models.ResourceId(id=cluster.id), node_count=1, std_out_err_path_prefix='$AZ_BATCHAI_MOUNT_ROOT/{0}'.format(helpers.AZURE_FILES_MOUNTING_PATH), custom_toolkit_settings=models.CustomToolkitSettings( command_line='echo job; df | grep -E "job_bfs|job_afs"' ) ) ).result() # Check the job failed because there are not job level mount volumes anymore self.assertEqual( helpers.wait_for_job_completion(self.is_live, self.client, resource_group.name, checker.name, helpers.MINUTE), models.ExecutionState.failed) # Check that the cluster level AFS was still mounted helpers.assert_job_files_are(self, self.client, resource_group.name, checker.name, helpers.STANDARD_OUTPUT_DIRECTORY_ID, {u'stdout.txt': u'job\n', u'stderr.txt': u''})
class AzureFileWriter(FilebaseBaseWriter): """ Writes items to azure file shares. It is a File Based writer, so it has filebase option available - account_name (str) Public acces name of the azure account. - account_key (str) Public acces key to the azure account. - share (str) File share name. - filebase (str) Base path to store the items in the share. """ supported_options = { "account_name": {"type": six.string_types, "env_fallback": "EXPORTERS_AZUREWRITER_NAME"}, "account_key": {"type": six.string_types, "env_fallback": "EXPORTERS_AZUREWRITER_KEY"}, "share": {"type": six.string_types}, } def __init__(self, options, meta, *args, **kw): from azure.storage.file import FileService super(AzureFileWriter, self).__init__(options, meta, *args, **kw) account_name = self.read_option("account_name") account_key = self.read_option("account_key") self.azure_service = FileService(account_name, account_key) self.share = self.read_option("share") self.azure_service.create_share(self.share) self.logger.info("AzureWriter has been initiated." "Writing to share {}".format(self.share)) self.set_metadata("files_counter", Counter()) self.set_metadata("files_written", []) def write(self, dump_path, group_key=None, file_name=None): if group_key is None: group_key = [] self._write_file(dump_path, group_key, file_name) def _update_metadata(self, dump_path, filebase_path, file_name): buffer_info = self.write_buffer.metadata[dump_path] file_info = { "file_name": file_name, "filebase_path": filebase_path, "size": buffer_info["size"], "number_of_records": buffer_info["number_of_records"], } files_written = self.get_metadata("files_written") files_written.append(file_info) self.set_metadata("files_written", files_written) self.get_metadata("files_counter")[filebase_path] += 1 def _ensure_path(self, filebase): path = filebase.split("/") folders_added = [] for sub_path in path: folders_added.append(sub_path) parent = "/".join(folders_added) self.azure_service.create_directory(self.share, parent) @retry_long def _write_file(self, dump_path, group_key, file_name=None): filebase_path, file_name = self.create_filebase_name(group_key, file_name=file_name) self._ensure_path(filebase_path) self.azure_service.create_file_from_path(self.share, filebase_path, file_name, dump_path, max_connections=5) self._update_metadata(dump_path, filebase_path, file_name) def get_file_suffix(self, path, prefix): number_of_keys = self.get_metadata("files_counter").get(path, 0) suffix = "{}".format(str(number_of_keys)) return suffix def _check_write_consistency(self): from azure.common import AzureMissingResourceHttpError for file_info in self.get_metadata("files_written"): try: afile = self.azure_service.get_file_properties( self.share, file_info["filebase_path"], file_info["file_name"] ) file_size = afile.properties.content_length if str(file_size) != str(file_info["size"]): raise InconsistentWriteState( "File {} has unexpected size. (expected {} - got {})".format( file_info["file_name"], file_info["size"], file_size ) ) except AzureMissingResourceHttpError: raise InconsistentWriteState("Missing file {}".format(file_info["file_name"])) self.logger.info("Consistency check passed")
containername = os.environ['AZURE_CONTAINER_NAME'] subscription_id = os.environ['AZURE_SUBSCRIPTION_ID'] resource_group_params = {'location' : location} sku = 'standard_ragrs)' kind = 'BlobStorage' storage_account_params = {sku:sku,kind:kind,location:location} # Configure Credentials credentials = ServicePrincipalCredentials(client_id=os.environ['AZURE_CLIENT_ID'],secret=os.environ['AZURE_CLIENT_SECRET'],tenant=os.environ['AZURE_TENANT_ID']) resource_client = ResourceManagementClient(credentials, subscription_id) storage_client = StorageManagementClient(credentials, subscription_id) # Create Resource Group & Storage Account resource_client.resource_groups.create_or_update(resourcegroupname, resource_group_params) create_sa = storage_client.storage_accounts.create(resourcegroupname, storageaccountname, {'location':'eastus','kind':'storage','sku':{'name':'standard_ragrs'}}) create_sa.wait() # Create Container sak = storage_client.storage_accounts.list_keys(resourcegroupname, storageaccountname) storageaccountkey = sak.keys[0].value storage_client = CloudStorageAccount(storageaccountname, storageaccountkey) blob_service = storage_client.create_block_blob_service() blob_service.create_container(containername,public_access=PublicAccess.Blob) # Copy Files file_service = FileService(account_name=storageaccountname, account_key=storageaccountkey) file_service.create_share(containername) file_service.create_directory(containername, 'directory1') file_service.create_file_from_path(containername,'directory1','55224azuresetup.ps1','55224azuresetup.ps1',)
def test_job_level_mounting(self, resource_group, location, cluster, storage_account, storage_account_key): """Tests if it's possible to mount external file systems for a job.""" job_name = 'job' # Create file share and container to mount on the job level if storage_account.name != FAKE_STORAGE.name: files = FileService(storage_account.name, storage_account_key) files.create_share('jobshare', fail_on_exist=False) blobs = BlockBlobService(storage_account.name, storage_account_key) blobs.create_container('jobcontainer', fail_on_exist=False) job = self.client.jobs.create( resource_group.name, job_name, parameters=models.JobCreateParameters( location=location, cluster=models.ResourceId(id=cluster.id), node_count=1, mount_volumes=models. MountVolumes(azure_file_shares=[ models.AzureFileShareReference( account_name=storage_account.name, azure_file_url='https://{0}.file.core.windows.net/{1}'. format(storage_account.name, 'jobshare'), relative_mount_path='job_afs', credentials=models.AzureStorageCredentialsInfo( account_key=storage_account_key), ) ], azure_blob_file_systems=[ models.AzureBlobFileSystemReference( account_name=storage_account.name, container_name='jobcontainer', relative_mount_path='job_bfs', credentials=models. AzureStorageCredentialsInfo( account_key=storage_account_key), ) ]), # Put standard output on cluster level AFS to check that the job has access to it. std_out_err_path_prefix='$AZ_BATCHAI_MOUNT_ROOT/{0}'.format( AZURE_FILES_MOUNTING_PATH), # Create two output directories on job level AFS and blobfuse. output_directories=[ models.OutputDirectory( id='OUTPUT1', path_prefix='$AZ_BATCHAI_JOB_MOUNT_ROOT/job_afs'), models.OutputDirectory( id='OUTPUT2', path_prefix='$AZ_BATCHAI_JOB_MOUNT_ROOT/job_bfs') ], # Check that the job preparation has access to job level file systems. job_preparation=models.JobPreparation( command_line= 'echo afs > $AZ_BATCHAI_OUTPUT_OUTPUT1/prep_afs.txt; ' 'echo bfs > $AZ_BATCHAI_OUTPUT_OUTPUT2/prep_bfs.txt; ' 'echo done'), # Check that the job has access to job custom_toolkit_settings=models.CustomToolkitSettings( command_line= 'echo afs > $AZ_BATCHAI_OUTPUT_OUTPUT1/job_afs.txt; ' 'echo bfs > $AZ_BATCHAI_OUTPUT_OUTPUT2/job_bfs.txt; ' 'mkdir $AZ_BATCHAI_OUTPUT_OUTPUT1/afs; ' 'echo afs > $AZ_BATCHAI_OUTPUT_OUTPUT1/afs/job_afs.txt; ' 'mkdir $AZ_BATCHAI_OUTPUT_OUTPUT2/bfs; ' 'echo bfs > $AZ_BATCHAI_OUTPUT_OUTPUT2/bfs/job_bfs.txt; ' 'echo done'))).result() self.assertEqual( wait_for_job_completion(self.is_live, self.client, resource_group.name, job.name, MINUTE), models.ExecutionState.succeeded) job = self.client.jobs.get(resource_group.name, job.name) # Assert job and job prep standard output is populated on cluster level filesystem assert_job_files_are( self, self.client, resource_group.name, job.name, STANDARD_OUTPUT_DIRECTORY_ID, { u'stdout.txt': u'done\n', u'stderr.txt': u'', u'stdout-job_prep.txt': u'done\n', u'stderr-job_prep.txt': u'' }) # Assert files are generated on job level AFS assert_job_files_are(self, self.client, resource_group.name, job.name, 'OUTPUT1', { u'job_afs.txt': u'afs\n', u'prep_afs.txt': u'afs\n', u'afs': None }) # Assert files are generated on job level blobfuse assert_job_files_are(self, self.client, resource_group.name, job.name, 'OUTPUT2', { u'job_bfs.txt': u'bfs\n', u'prep_bfs.txt': u'bfs\n', u'bfs': None }) # Assert subfolders are available via API assert_job_files_in_path_are(self, self.client, resource_group.name, job.name, 'OUTPUT1', 'afs', {u'job_afs.txt': u'afs\n'}) assert_job_files_in_path_are(self, self.client, resource_group.name, job.name, 'OUTPUT2', 'bfs', {u'job_bfs.txt': u'bfs\n'}) # Assert that we can access the output files created on job level mount volumes directly in storage using path # segment returned by the server. if storage_account.name != FAKE_STORAGE.name: files = FileService(storage_account.name, storage_account_key) self.assertTrue( files.exists( 'jobshare', job.job_output_directory_path_segment + '/' + OUTPUT_DIRECTORIES_FOLDER_NAME, 'job_afs.txt')) blobs = BlockBlobService(storage_account.name, storage_account_key) self.assertTrue( blobs.exists( 'jobcontainer', job.job_output_directory_path_segment + '/' + OUTPUT_DIRECTORIES_FOLDER_NAME + '/job_bfs.txt')) # After the job is done the filesystems should be unmounted automatically, check this by submitting a new job. checker = self.client.jobs.create( resource_group.name, 'checker', parameters=models.JobCreateParameters( location=location, cluster=models.ResourceId(id=cluster.id), node_count=1, std_out_err_path_prefix='$AZ_BATCHAI_MOUNT_ROOT/{0}'.format( AZURE_FILES_MOUNTING_PATH), custom_toolkit_settings=models.CustomToolkitSettings( command_line='echo job; df | grep -E "job_bfs|job_afs"')) ).result() # Check the job failed because there are not job level mount volumes anymore self.assertEqual( wait_for_job_completion(self.is_live, self.client, resource_group.name, checker.name, MINUTE), models.ExecutionState.failed) # Check that the cluster level AFS was still mounted assert_job_files_are(self, self.client, resource_group.name, checker.name, STANDARD_OUTPUT_DIRECTORY_ID, { u'stdout.txt': u'job\n', u'stderr.txt': u'' })
with open('Blob\\setup_machine.py', 'w') as f: f.write(setup_machine_py) #Generate run_airsim_on_user_login.xml with open('Template\\run_airsim_on_user_login_xml.template', 'r', encoding='utf-16') as f: startup_task_xml = f.read() startup_task_xml = startup_task_xml\ .replace('{batch_job_user_name}', NOTEBOOK_CONFIG['batch_job_user_name']) with open('Share\\scripts_downpour\\run_airsim_on_user_login.xml', 'w', encoding='utf-16') as f: f.write(startup_task_xml) # create file share file_service = FileService(account_name = NOTEBOOK_CONFIG['storage_account_name'], account_key=NOTEBOOK_CONFIG['storage_account_key']) file_service.create_share(NOTEBOOK_CONFIG['file_share_name'], fail_on_exist=False) # upload all files to share def create_directories(path, file_service): split_dir = path.split('\\') for i in range(1, len(split_dir)+1, 1): combined_dir = '\\'.join(split_dir[:i]) file_service.create_directory(NOTEBOOK_CONFIG['file_share_name'], combined_dir, fail_on_exist=False) for root, directories, files in os.walk('Share'): for file in files: regex_pattern = '{0}[\\\\]?'.format('Share').replace('\\', '\\\\') upload_directory = re.sub(regex_pattern, '', root) print('Uploading {0} to {1}...'.format(os.path.join(root, file), upload_directory)) if (len(upload_directory) == 0): upload_directory = None
import logging from azure.storage.file import FileService from config import Config logger = logging.getLogger('azure.storage') logger.setLevel(logging.ERROR) environment = Config.ENVIRONMENT file_service = FileService(account_name=Config.AZURE_FILES_ACCOUNT, account_key=Config.AZURE_FILES_KEY) if environment != 'CI': file_service.create_share(environment) class AzureFilesService: @staticmethod def create_file_from_bytes(bytes_file, filename): file_service.create_file_from_bytes(share_name=environment, directory_name=None, file_name=filename, file=bytes_file) @staticmethod def download_azure_file(filename): return file_service.get_file_to_text(environment, None, filename)
'name': 'standard_ragrs' } }) storageaccount.wait() # Create Container and Share sak = storage_client.storage_accounts.list_keys(resourcegroupname, storageaccountname) storageaccountkey = sak.keys[0].value cloudstorage_client = CloudStorageAccount(storageaccountname, storageaccountkey) blob_service = cloudstorage_client.create_block_blob_service() blob_service.create_container(sharename, public_access=PublicAccess.Container) file_service = FileService(account_name=storageaccountname, account_key=storageaccountkey) file_service.create_share(sharename) # Copy Setup Files to Container and Share blob_service.create_blob_from_path( sharename, filename, filename, ) file_service.create_file_from_path( sharename, '', filename, filename, ) # Create an App Service Plan
import zipfile from datetime import datetime from azure.storage.table import TableService, Entity, TablePermissions from azure.storage.blob import BlockBlobService from azure.storage.blob import PublicAccess from azure.storage.file import FileService STORAGE_ACCOUNT_NAME = os.environ['STORAGE_ACCOUNT_NAME'] STORAGE_ACCOUNT_KEY = os.environ['STORAGE_ACCOUNT_KEY'] table_service = TableService(account_name=STORAGE_ACCOUNT_NAME, account_key=STORAGE_ACCOUNT_KEY) table_service.create_table('cluster') file_service = FileService(account_name=STORAGE_ACCOUNT_NAME, account_key=STORAGE_ACCOUNT_KEY) file_service.create_share(share_name='azureml-project', quota=1) file_service.create_share(share_name='azureml-share', quota=1) file_service.create_directory('azureml-share', 'Solution1') file_service.create_directory('azureml-share', 'Solution2') block_blob_service = BlockBlobService(account_name=STORAGE_ACCOUNT_NAME, account_key=STORAGE_ACCOUNT_KEY) container_name ='telemetry' block_blob_service.create_container(container_name) source=os.environ['AML_ASSETS_URL'] dest='azureml_project.zip' urllib.request.urlretrieve(source, dest)
def run(job, **kwargs): resource = kwargs.get('resource') create_custom_fields_as_needed() storage_account = '{{ storage_account }}' file_path = "{{ file }}" azure_storage_file_share_name = '{{ azure_storage_file_share_name }}' overwrite_files = {{overwrite_files}} file_name = Path(file_path).name if file_path.startswith(settings.MEDIA_URL): set_progress("Converting relative URL to filesystem path") file_path = file_path.replace(settings.MEDIA_URL, settings.MEDIA_ROOT) if not file_path.startswith(settings.MEDIA_ROOT): file_path = os.path.join(settings.MEDIA_ROOT, file_path) try: set_progress("Connecting To Azure...") account_key = Resource.objects.filter( name__icontains=storage_account)[0].azure_account_key fallback_account_key = Resource.objects.filter( name__icontains=storage_account)[0].azure_account_key_fallback file_service = FileService(account_name=storage_account, account_key=account_key) set_progress( 'Creating file share {file_share_name} if it doesn\'t already exist...' .format(file_share_name=azure_storage_file_share_name)) file_service.create_share(share_name=azure_storage_file_share_name, quota=1) set_progress('Connecting to file share') file_name_on_azure = file_name count = 0 while (not overwrite_files) and file_service.exists( share_name=azure_storage_file_share_name, file_name=file_name_on_azure, directory_name=''): count += 1 file_name_on_azure = '{file_name}({duplicate_number})'.format( file_name=file_name, duplicate_number=count) set_progress( 'File with name already exists on given file share, testing new name: {new_name}' .format(new_name=file_name_on_azure)) local_resource_name = azure_storage_file_share_name + '-' + file_name_on_azure if overwrite_files and file_service.exists( share_name=azure_storage_file_share_name, file_name=file_name_on_azure, directory_name=''): set_progress( 'File with name already exists on given file share, overwriting' ) old_resource_to_overwite = Resource.objects.filter( name=local_resource_name, lifecycle='ACTIVE').first() if old_resource_to_overwite: old_resource_to_overwite.delete() set_progress( 'Creating the file with name {file_name} on the Storage Account {storage_account} using the share named {share_name}' .format(file_name=file_name_on_azure, storage_account=storage_account, share_name=azure_storage_file_share_name)) file_service.create_file_from_path( share_name=azure_storage_file_share_name, file_name=file_name_on_azure, directory_name='', local_file_path=file_path) os.remove(file_path) set_progress( 'Creating local storage resource named {resource_name}'.format( resource_name=local_resource_name)) resource.name = local_resource_name resource.azure_storage_account_name = storage_account resource.azure_account_key = account_key resource.azure_account_key_fallback = fallback_account_key resource.azure_storage_file_share_name = azure_storage_file_share_name resource.azure_storage_file_name = file_name_on_azure resource.save() return "Success", "The File has succesfully been uploaded", "" except Exception as e: if os.path.exists(file_path): os.remove(file_path) if resource: resource.delete() raise CloudBoltException( "File could not be uploaded because of the following error: {error}" .format(error=e))
from azure.storage.file import FileService, ContentSettings import yaml config_file = 'config.yml' with open(config_file, 'r') as stream: try: f = yaml.safe_load(stream) except yaml.YAMLError as exc: print(exc) file_service = FileService(account_name=f['account_name'], account_key=f['account_key']) file_service.create_share('myshare') file_service.create_file_from_path( 'housing_prices', None, # We want to create this blob in the root directory, so we specify None for the directory_name 'myfile', 'test.txt', content_settings=ContentSettings(content_type='txt'))
except: print("Problème d'autorisation d'accès au compte Microsoft AZURE") logging.error("Problème d'autorisation d'accès au compte Microsoft AZURE") # syslog.syslog(syslog.LOG_ERR,"Problème d'autorisation d'accès au compte Microsoft AZURE") exit(2) # sortie avec erreur ! # Création du répertoire: backup6 sur Microsoft AZURE de notre exemple # # Vérifier si le répertoire de sauvegarde backup6 sur Microsoft AZURE existe ou non # try: file_service.exists(AZURE_REP_BKP) print("Le répertoire de sauvegarde AZURE existe !") logging.debug("Le répertoire de sauvegarde AZURE existe !") # syslog.syslog(syslog.LOG_DEBUG,"Le répertoire de sauvegarde AZURE existe !") except FileNotFoundError: file_service.create_share(AZURE_REP_BKP) print("Création du répertoire de sauvegarde AZURE ") logging.warning("Création du répertoire de sauvegarde AZURE ") # syslog.syslog(syslog.LOG_WARNING,"Création du répertoire de sauvegarde AZURE ") ############################## Temps ################################ BACKUP_DATE = date.today().strftime("%d-%m-%Y") # date d'aujourd'hui au format Jour/Mois/Année BACKUP_DATE_OLD = (date.today()-datetime.timedelta(days=int(NBjourDEretention))).strftime("%d-%m-%Y") # date d'aujourd'hui - le nb de jour de rétention au format Jour/Mois/Année ############################# Fonction ############################## # Fonction de récupération du Nom de l'image de la Base De Donnée du fichier docker-compose.yml # def get_database_name(): with open(repertoire_de_sauvegarde+"/docker-compose.yml",'r') as file: # Ouverture du fichier YML en lecture
class StorageHelper(object): """Handle details related to a single storage account and share. Instantiate this object with information sufficient to uniquely identify a storage account and a file share within it. Then .account can be used to retrieve the Azure SDK for Python object corresponding to the account, and .key can be used to get an access key for it. For both those properties, if the value mentioned doesn't exist, it will be created upon first property access. """ def __init__(self, client_data, resource_helper, name, account=None, default_share='share'): self.name = name self.default_share = default_share self._account = account self._key = os.environ.get('AZURE_STORAGE_KEY') self.resource_helper = resource_helper self.client = StorageManagementClient(*client_data) self.file_service = FileService( account_name=self.account.name, account_key=self.key, ) @property def account(self): """Return the managed StorageAccounts object. If no such account exists, create it first. """ if self._account is None: print('Creating storage account...') # Error to create storage account if it already exists! name_check = self.client.storage_accounts.check_name_availability( self.name) if name_check.name_available: storage_creation = self.client.storage_accounts.create( self.resource_helper.group.name, self.name, StorageAccountCreateParameters( sku=StorageAccountSku(StorageSkuName.standard_lrs), kind=StorageKind.storage, location=self.resource_helper.group.location, )) storage = storage_creation.result() else: try: storage = self.client.storage_accounts.get_properties( self.resource_helper.group.name, self.name) except CloudError: print('Storage account {} already exists' ' in a resource group other than {}.'.format( self.name, self.resource_helper.group.name)) print('Got storage account:', storage.name) self._account = storage return self._account @property def key(self): """Get the first available storage key. This will crash if there are no available storage keys, which is unlikely since two are created along with a storage account. """ if self._key is None: storage_keys = self.client.storage_accounts.list_keys( self.resource_helper.group.name, self.account.name) self._key = next(iter(storage_keys.keys)).value return self._key def upload_file(self, path, sharename): """Upload a file into the default share on the storage account. If the share doesn't exist, create it first. """ self.file_service.create_file_from_path( self.default_share if sharename is None else sharename, None, os.path.basename(path), path, ) return '/'.join([self.default_share, os.path.basename(path)]) def download_file(self, sharename, filename): file_service.get_file_to_path(sharename, None, filename, filename) def delete_file(self, sharename, filename): file_service.delete_file(sharename, None, filename) def create_share(self, sharename): self.file_service.create_share(sharename) def create_directory(self, sharename, directoryname): self.file_service.create_directory(sharename, directoryname) def list_directories_and_files(self, sharename): generator = self.file_service.list_directories_and_files(sharename) return [file_or_dir.name for file_or_dir in generator] def list_shares(self): shares = list(self.file_service.list_shares(include_snapshots=True)) sharelist = [fileshare.name for fileshare in shares] print(sharelist) return sharelist
from azure.storage.file import ContentSettings import os omnipresence_storage_account_name = 'cloudinfraprovision' omnipresence_storage_account_key = 'WVIc4TiKPDLxjtIWLpnk5fITbI6AFoZahvfTz4SgSjyP+fE3/qwgSgIo/UNavXPPjQDWrCfT4da6vnL209pThQ==' omnipresence_storage_file_share = 'azure-provision' #Azure Storage Account File Share Name allows only lowercase letters, numbers and hypen. remote_dir_path = '' #Initialize an Azure Storage Account File Service Instance omnipresence_storage_account = FileService(account_name=omnipresence_storage_account_name, account_key=omnipresence_storage_account_key) #test if your storage file share exists on Azure or not, if not, create it if (not omnipresence_storage_account.exists(omnipresence_storage_file_share)): omnipresence_storage_account.create_share(omnipresence_storage_file_share, quota='10') #walk through current directory, make directorys under Azure File Share and upload local files onto your Azure storage account File Share except for hiden files and directory for base_dir, dirs, file_names in os.walk(".", topdown=True): file_names = [ f for f in file_names if not f[0] == '.'] #parse out files whose name begins with a dot dirs[:] = [d for d in dirs if not d[0] == '.'] #parse out directorys whose name begins with a dot for local_file_name in file_names: remote_file_name = os.path.join(base_dir, local_file_name)[2:] local_file_name = remote_file_name if (omnipresence_storage_account.exists(omnipresence_storage_file_share)): omnipresence_storage_account.create_file_from_path( omnipresence_storage_file_share, None, # We want to create files under current remote directory, so we specify None for the directory_name remote_file_name, local_file_name, content_settings=ContentSettings(content_type='file'))