def save_text_file(data, containerName, blobName, accountName, accountKey): ''' save a textfile to azure block blob storage. Parameters ---------- data: str (text)data to upload containerName: str container in storage account blobName: str name of blob in container accountName: str name of storage account accountKey access key for storage account Returns ------- ''' # Create BlockBlockService block_blob_service = BlockBlobService( account_name=accountName, account_key=accountKey ) block_blob_service.create_blob_from_text(containerName, blobName, data)
def save_to(result, season, table, season_type, save_destination): if save_destination == "local": result.to_csv('/wc/data/' + season + '_' + table + '.csv', index=False, sep=';') #result.to_csv('/home/martin/temp/nbapc_crawled_data/' + season + '_' + table + '_' + season_type + '.csv', index=False, sep=';') print('CSV was successfully saved as:' + season + '_' + table + '_' + season_type + '.csv') elif save_destination == "s3": bucket = os.environ['s3bucket'] csv_buffer = StringIO() result.to_csv(csv_buffer, index=False, sep=';') s3 = boto3.resource('s3') s3.Object(bucket, 'crawled_data/' + season + '_' + table + '.csv').put(Body=csv_buffer.getvalue()) print('CSV was successfully uploaded to s3: crawled_data/' + season + '_' + table + '.csv') elif save_destination == "wasb": wasbaccountname = os.environ['wasbaccountname'] containername = os.environ['containername'] wasbaccountkey = os.environ['wasbaccountkey'] csv_buffer = StringIO() result.to_csv(csv_buffer, index=False, sep=';') block_blob_service = BlockBlobService(account_name=wasbaccountname, account_key=wasbaccountkey) block_blob_service.create_blob_from_text(containername, season + '_' + table + '.csv', csv_buffer.getvalue()) print( 'CSV was successfully uploaded to Azure Storage Account container: ' + containername + '/' + season + '_' + table + '.csv')
def publishPowerBI(blockBlobService: BlockBlobService, x: int, imgByteArr: Image, folder: str, eventVideo: list, text: str, caption: str): blobOutputTrainNumber = 'trainnumber/' + eventVideo[ "filename"] + "_" + text + "_" + str(x) + ".png" blockBlobService.create_blob_from_text( os.environ['remoteStorageOutputContainer'], blobOutputTrainNumber, imgByteArr) # create SAS url such that video can be read from blob storage sasTokenRead = blockBlobService.generate_blob_shared_access_signature( os.environ['remoteStorageOutputContainer'], blobOutputTrainNumber, BlobPermissions.READ, datetime.utcnow() + timedelta(hours=1)) sasPictureTrainNumber = os.environ['storUrl'] + os.environ[ 'remoteStorageOutputContainer'] + "/" + blobOutputTrainNumber + "?" + sasTokenRead #write data to Power BI as streaming data set streamingDataStr = '[{{"location": "{0}", "track": "{1}", "time": "{2}", "trainNumber": "{3}", "probGraffiti": "{4}", "caption": "{5}", "sasPictureTrainNumber":"{6}", "sasPictureGraffiti":"{7}"}}]'.format( eventVideo["location"], eventVideo["track"], eventVideo["timestamp"], text, 0.0, caption, sasPictureTrainNumber, "").encode("utf-8") try: req = urllib2.Request(os.environ['powerBIConnectionString'], streamingDataStr) gcontext = ssl.SSLContext() response = urllib2.urlopen(req, context=gcontext) logging.info( "POST request to Power BI with data:{0}".format(streamingDataStr)) logging.info("Response: HTTP {0} {1}\n".format(response.getcode(), response.read())) except: logging.info( "Writing to Power BI dashboard failed, no breaking error, continue" )
def import_historic_csv_files(url, account_name, account_key, container_name): block_blob_service = BlockBlobService(account_name, account_key) blobs_already_imported = get_blob_names_already_imported( block_blob_service, container_name) counter = 0 historic_csv_links = get_historic_csv_file_links_to_download(url) for csv_link in historic_csv_links: filename = os.path.basename(csv_link) if filename not in blobs_already_imported: counter += 1 # import csv file - read file's content and upload it to Azure Blob Storage r = requests.get(csv_link) # change the encoding for the request content r.encoding = 'utf-8' block_blob_service.create_blob_from_text(container_name, filename, r.text) logging.info( f"Created blob file '{filename}' in the container '{container_name}'" ) if counter == 0: logging.info( f"Nothing new to import. All CSV files are already uploaded to Blob Storage." )
def save_json(data, blob_path, filename, container_name, bbs=None): if not bbs: bbs = BlockBlobService(account_name=config["account_name"], account_key=config["account_key"]) blob_name = os.path.join(blob_path, filename) blob_name = remove_container_name_from_blob_path(blob_name, container_name) bbs.create_blob_from_text(container_name, blob_name, json.dumps(data))
def main(config_filename, output_model_name, sample_frac): ''' Coordinate application of trained models to large static image set ''' config = ConfigFile(config_filename, output_model_name) if config.model_source == 'batchaitraining': config = load_batchaitraining_model_components(config) elif config.model_source == 'mmlspark': config = load_mmlspark_model_components(config) else: raise Exception('Model source not recognized') df = load_data(config, sample_frac) if config.model_source == 'batchaitraining': predictions = df.withColumn('pred_label', config.label_udf('features')) \ .select('filepath', 'pred_label') elif config.model_source == 'mmlspark': predictions = config.mmlspark_model.transform(df) predictions = config.tf.transform(predictions).select( 'filepath', 'pred_label') output_str = predictions.toPandas().to_csv(index=False) blob_service = BlockBlobService(config.storage_account_name, config.storage_account_key) blob_service.create_blob_from_text( config.container_prediction_results, config.predictions_filename, output_str) return
def test_create_blob_with_container_sas(self): # SAS URL is calculated from storage key, so this test runs live only if TestMode.need_recording_file(self.test_mode): return # Arrange token = self.bs.generate_container_shared_access_signature( self.container_name, permission=ContainerPermissions.READ + ContainerPermissions.CREATE, expiry=datetime.utcnow() + timedelta(hours=1), ) bs_with_sas = BlockBlobService( account_name=self.settings.STORAGE_ACCOUNT_NAME, sas_token=token, protocol=self.settings.PROTOCOL) # Act test_blob_name = "testblobname" test_blob_content = "test-blob-content" bs_with_sas.create_blob_from_text(self.container_name, blob_name=test_blob_name, text=test_blob_content) blob_created_successfully = bs_with_sas.exists( container_name=self.container_name, blob_name=test_blob_name) # Assert self.assertTrue(blob_created_successfully, "blob:" + test_blob_name + "was created successfully")
def write_full_transcript_to_blob(id, transcript): print("saving transcript as blob...") blob_name = id block_blob_service = BlockBlobService(account_name, account_key) block_blob_service.create_blob_from_text(corpus_container_name, blob_name, json.dumps(transcript)) print("blob: %s saved." % blob_name)
def WriteBlob(blob_name, txt): """ 単一 BLOB ファイルを作成しテキストを保存する。 """ try: #blob_name = r'sample.txt'; hellopython = "BlockBlobService" blob_service = BlockBlobService(account_name, account_key) hellopython = "create_container" blob_service.create_container(log_container_name, public_access=PublicAccess.Blob) hellopython = "create_blob_from_bytes" #blob_service.create_blob_from_bytes( # log_container_name, # log_blob_name, # b'<center><h1>Hello World!</h1></center>', # content_settings=ContentSettings('text/html') #) hellopython = "create_blob_from_text" blob_service.create_blob_from_text(log_container_name, blob_name, txt) hellopython = "make_blob_url" print(blob_service.make_blob_url(log_container_name, log_blob_name)) hellopython = "Hello Python!" except: print(r"Exception.")
def main(msg: func.QueueMessage) -> None: logging.info('Python queue trigger function processed a queue item: %s', msg.get_body().decode('utf-8')) # get bearer token and authenticate to ADLSgen2 using Managed Identity of Azure Function credentials = MSIAuthentication(resource='https://storage.azure.com/') blob_service = BlockBlobService("testedlstorgen", token_credential=credentials) # get timestamp now = datetime.now() nowstr = datetime.strftime(datetime.now(), "%Y%m%dT%H%M%S%Z") key = round((now - datetime(2019, 1, 1, 0, 0, 0)).total_seconds()) logging.info("key: " + str(key)) # Add record to csv file. Notice that AppendBlob is not yet supported on ADLSgen2, see https://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-known-issues records = blob_service.get_blob_to_text( "raw", "testprivcmddataflow/WideWorldImporters-Sales/address/SalesLTAddress.txt" ).content records += "\n" + str( key ) + ",8713 Yosemite Ct.,,Bothell,Washington,United States,98011,268af621-76d7-4c78-9441-144fd139821a,2006-07-01 00:00:00.0000000" blob_service.create_blob_from_text( "raw", "testprivcmddataflow/WideWorldImporters-Sales/address/SalesLTAddress.txt", records) # Create event such that ADFv2 is triggered blob_service = BlockBlobService("testedlstorgen", token_credential=credentials) blob_service.create_blob_from_text("adftrigger", "adftrigger" + nowstr + ".txt", "")
def post_review(): data = request.get_json() if 'text' in data: data["analysis"] = get_emo(data["text"]) client = pymongo.MongoClient(os.getenv('CUSTOMCONNSTR_MONGOURL')) db = client['cloud-computing-homework-5-db'] collection = db['reviews'] collection.insert_one(data) elif 'audio' in data: block_blob_service = BlockBlobService( account_name='cch5blobstorage', account_key=os.getenv('BLOB_STORAGE_KEY')) container_name = 'review-audio-blobs' blob_id = uuid.uuid1() block_blob_service.create_blob_from_text( container_name, blob_id, data['audio']) # print(get_speech_to_text(blob_id)) #print("push") else: return '', 400 return '', 201
def store_contents(self, content_type: ContentType, container_name: str, content_name: str, content_data: str = None, content_path: str = None): """Function that stores file content. Based on the ContentType the function can either save a text or can read a file path (content_path) and copies it :param content_type: Content type, can be file or text :type content_type: enum :param container_name: Container name :type container_name: str :param content_name: File name :type content_name: str :param content_data: File content as text :type content_data: str :param content_path: File path to be copied into the storage :type content_path: str :raises: :class:`CustomException` """ try: self._logger.info('storing contents') if self._vdc_storage_account_key == None: self._vdc_storage_account_key = self.get_storage_account_key() block_blob_service = BlockBlobService( account_name=self._vdc_storage_account_name, account_key=self._vdc_storage_account_key) # Create container if does not exists if not block_blob_service.exists(container_name): self._logger.info( 'container {} does not exists, proceeding to create it'. format(container_name)) block_blob_service.create_container(container_name) self._logger.info('saving blob') if content_type == ContentType.TEXT: block_blob_service.create_blob_from_text( container_name=container_name, blob_name=content_name, text=content_data) elif content_type == ContentType.FILE: if Path(content_path).exists(): block_blob_service.create_blob_from_path( container_name=container_name, blob_name=content_name, file_path=content_path) else: raise CustomException('File does not exist.') except Exception as ex: raise CustomException( 'There was an unhandled exception: {}'.format(str(ex)))
def _save(self, data: pd.DataFrame) -> None: blob_service = BlockBlobService(**self._credentials) blob_service.create_blob_from_text( container_name=self._container_name, blob_name=self._filepath, text=data.to_csv(**self._save_args), **self._blob_from_text_args )
def put_file_in_azure(azure_emulator_coords: azure_utils.StorageCoordinates, azure_service: blob.BlockBlobService, sample_stream_content: str) -> typing.Generator: filename = 'test_put_file.txt' azure_service.create_blob_from_text(azure_emulator_coords.container, filename, sample_stream_content) yield filename azure_service.delete_blob(azure_emulator_coords.container, filename)
def save_in_blob(self, df, filename): data = df.to_csv(index=False, encoding='utf-8') try: block_blob_service = BlockBlobService( account_name=self.config['blob_account_name'], account_key=self.config['blob_account_key']) block_blob_service.create_blob_from_text( self.config['blob_image_container'], filename, data) except Exception as e: print(e)
def write_blob_from_text(self, sas_uri, blob_name, text): sas_service = BlockBlobService( account_name=self.get_account_from_uri(sas_uri), sas_token=self.get_sas_key_from_uri(sas_uri)) container_name = self.get_container_from_uri(sas_uri) sas_service.create_blob_from_text(container_name, blob_name, text, 'utf-8') return sas_service.make_blob_url(container_name, blob_name, sas_token=self.get_sas_key_from_uri(sas_uri))
class AzureCloudStorage(CloudStorageInterface): def __init__(self, account, key, container): self.block_blob_service = BlockBlobService(account_name=account, account_key=key) self.container_name = container self.block_blob_service.create_container(self.container_name) def storeBlob(self, path, data, content_type): logging.info("Storing to Azure [%s:%s] data: [%s]", self.container_name, path, data) self.block_blob_service.create_blob_from_text(self.container_name, blob_name=path, text=data)
def encrypt_and_store_aes_key(az_client, az_config, key_version, key): encrypted = az_client.encrypt(az_config['key_vault'], az_config['key_name'], key_version, 'RSA-OAEP', key) block_blob_service = BlockBlobService( account_name=az_config['account_name'], account_key=az_config['account_key']) block_blob_service.create_blob_from_text( az_config['container_name'], 'aes_key-' + az_config['uuid'], base64.b64encode(encrypted.result))
def combine_azure(self): from azure.storage.blob import BlockBlobService, ContentSettings feed_uri = self.settings.get("FEED_URI") feed_prefix = self.settings.get("CITY_SCRAPERS_DIFF_FEED_PREFIX", "%Y/%m/%d") account_name, account_key = feed_uri[8::].split("@")[0].split(":") container = feed_uri.split("@")[1].split("/")[0] blob_service = BlockBlobService(account_name=account_name, account_key=account_key) max_days_previous = 3 days_previous = 0 prefix_blobs = [] while days_previous <= max_days_previous: prefix_blobs = [ blob for blob in blob_service.list_blobs( container, prefix=( datetime.now() - timedelta(days=days_previous)).strftime(feed_prefix), ) ] if len(prefix_blobs) > 0: break days_previous += 1 spider_blob_names = self.get_spider_paths( [blob.name for blob in prefix_blobs]) meetings = [] for blob_name in spider_blob_names: feed_text = blob_service.get_blob_to_text(container, blob_name) meetings.extend([ json.loads(line) for line in feed_text.content.split("\n") if line ]) meetings = sorted(meetings, key=itemgetter("start")) yesterday_iso = (datetime.now() - timedelta(days=1)).isoformat()[:19] upcoming = [ meeting for meeting in meetings if meeting["start"] > yesterday_iso ] blob_service.create_blob_from_text( container, "latest.json", "\n".join([json.dumps(meeting) for meeting in meetings]), content_settings=ContentSettings(cache_control="no-cache"), ) blob_service.create_blob_from_text( container, "upcoming.json", "\n".join([json.dumps(meeting) for meeting in upcoming]), content_settings=ContentSettings(cache_control="no-cache"), )
def create_blob_sas_defintion(self, storage_account_name, vault_url): """ Creates a service SAS definition with access to a blob container. """ from azure.storage.blob import BlockBlobService, ContainerPermissions from azure.keyvault.models import SasTokenType, SasDefinitionAttributes from azure.keyvault import SecretId # create the blob sas definition template # the sas template uri for service sas definitions contains the storage entity url with the template token # this sample demonstrates constructing the template uri for a blob container, but a similar approach can # be used for all other storage service, i.e. File, Queue, Table # create a template sas token for the container service = BlockBlobService(account_name=storage_account_name, # don't sign the template with the storage account key use key 00000000 account_key='00000000') permissions = ContainerPermissions(read=True, write=True, delete=True, list=True) temp_token = service.generate_container_shared_access_signature(container_name='blobcontainer', permission=permissions, expiry='2020-01-01') # use the BlockBlobService to construct the template uri for the container sas definition blob_sas_template_uri = service.make_container_url(container_name='blobcontainer', protocol='https', sas_token=temp_token) # create the sas definition in the vault attributes = SasDefinitionAttributes(enabled=True) blob_sas_def = self.client.set_sas_definition(vault_base_url=vault_url, storage_account_name=storage_account_name, sas_definition_name='blobcontall', template_uri=blob_sas_template_uri, sas_type=SasTokenType.service, validity_period='PT2H', sas_definition_attributes=attributes) # use the sas definition to provision a sas token and use it to create a BlockBlobClient # which can interact with blobs in the container # get the secret_id of the container sas definition and get the token from the vault as a secret sas_secret_id = SecretId(uri=blob_sas_def.secret_id) blob_sas_token = self.client.get_secret(vault_base_url=sas_secret_id.vault, secret_name=sas_secret_id.name, secret_version=sas_secret_id.version).value service = BlockBlobService(account_name=storage_account_name, sas_token=blob_sas_token) service.create_blob_from_text(container_name='blobcontainer', blob_name='blob2', text=u'test blob2 data') blobs = list(service.list_blobs(container_name='blobcontainer')) for blob in blobs: service.delete_blob(container_name='blobcontainer', blob_name=blob.name)
class Blob(): def __init__(self, container_name='smartbin'): self.blob_service = BlockBlobService(account_name = 'iotinfstore', account_key = 'VxbwD/Cjvfi+ZObPIkqZ7AT8NKG3AqF6m0jYEUiwU12xtpiotyxIRNRyKu208P1+W+DdYKZd0SzFii2SrcsgWQ==') self.container_name = container_name def write_data_to_blob(self,data): self.blob_service.create_blob_from_text(self.container_name, str(uuid.uuid4()), data)
class AzureBlobWriter: def __init__(self, file_path="index.txt"): # Create a new file on every instantiation epoch_time = str(time.time()).split('.')[0] self._file_path = "{0}-{1}".format(file_path, epoch_time) self._blob_service = BlockBlobService( account_name=app.config['AZURE_BLOB_ACCOUNT'], account_key=app.config['AZURE_BLOB_KEY']) self._container_name = "ttds-indexes" def read(self): content = None try: # Get the most recently written file with the specified name highest_blob = 0 blob_to_load = "" blob_name = self._file_path.rsplit("-", 1)[0] blob_found = False container = self._blob_service.list_blobs(self._container_name) for blob in container: name_parts = blob.name.rsplit("-", 1) blob_prefix = name_parts[0] blob_order = int(name_parts[1]) if blob_prefix == blob_name and blob_order > highest_blob: highest_blob = blob_order blob_to_load = blob.name blob_found = True if blob_found: blob = self._blob_service.get_blob_to_text( self._container_name, blob_to_load) content = blob.content except: print("Unexpected error at {}.read: {}".format( __name__, sys.exc_info())) else: print("Read content from {} by {} at {}".format( self._file_path, __name__, datetime.now().strftime(app.config['DATETIME_FORMAT']))) return content def write(self, content): try: self._blob_service.create_blob_from_text(self._container_name, self._file_path, content) except: print("Unexpected error at {}.write: {}".format( __name__, sys.exc_info())) else: print( "Content of size {} has been written to {} by {} at {}".format( sizeof_fmt(content), self._file_path, __name__, datetime.now().strftime(app.config['DATETIME_FORMAT'])))
def upload_timestamp_file( azure_emulator_coords: azure_utils.StorageCoordinates, azure_service: blob.BlockBlobService, sample_timestamp: str) -> typing.Generator: azure_service.create_blob_from_text(azure_emulator_coords.container, endpoint.TIMESTAMP_FILENAME, sample_timestamp) yield azure_service.delete_blob(azure_emulator_coords.container, endpoint.TIMESTAMP_FILENAME)
def extract_transform(data): user=os.environ['STORAGE_USERNAME'] password=os.environ['STORAGE_PASSWORD'] container_name ='landingzone' block_blob_service = BlockBlobService(account_name=user, account_key=password) blobname =data["name"]+"_" + str(uuid.uuid4()) + ".csv" csv=data["name"]+","+str(data["count"])+","+data["meta"] block_blob_service.create_blob_from_text(container_name, blobname, csv) return "Uploaded blob "+blobname+" to landing zone\n"
def write_blob_data(file_name, content): try: block_blob_service = BlockBlobService(account_name=accountName, account_key=accountKey) block_blob_service.create_blob_from_text(postCallContainerName, file_name, content) tmp = tempfile.NamedTemporaryFile() block_blob_service.get_blob_to_stream(postCallContainerName, file_name, tmp) return 201 except: return 400
def main(): # Get credential parser = configparser.ConfigParser() parser.read('config.ini') STORAGE_ACCOUNT_NAME = parser.get('credential', 'STORAGE_ACCOUNT_NAME') STORAGE_ACCOUNT_KEY = parser.get('credential', 'STORAGE_ACCOUNT_KEY') CONTAINER_NAME_METADATA = parser.get('credential', 'CONTAINER_NAME_METADATA') # access to blob storage block_blob_service = BlockBlobService(account_name=STORAGE_ACCOUNT_NAME, account_key=STORAGE_ACCOUNT_KEY) # load structured data from blob blob_text = block_blob_service.get_blob_to_text(CONTAINER_NAME_METADATA, 'image-url.tsv') #print(blob_text.content) df = pd.DataFrame.from_csv(StringIO(blob_text.content), index_col=None, sep='\t') print(df.shape) # Frequency df_crosstab = pd.crosstab(index=df["Category"], columns="count") print(df_crosstab) print(df.shape) df_sub = pd.concat([ df.loc[df['Category'] == 'daily snack'], df.loc[df['Category'] == 'groceries'], df.loc[df['Category'] == 'dining out'], df.loc[df['Category'] == 'clothes and accessories'], df.loc[df['Category'] == 'fuel'], df.loc[df['Category'] == 'entertinement'] ]) print(df_sub.shape) print(df_sub.Category.unique()) # write cleaned dataframe to blob print("-----------write cleaned dataframe to blob------------") df_sub_str = df_sub.to_csv(sep='\t', index=False) #dfblobname = 'dataframe_6_classes.tsv' dfblobname = 'image-url-6-classes.tsv' settings = ContentSettings(content_type='text/tab-separated-values') block_blob_service.create_blob_from_text(CONTAINER_NAME_METADATA, dfblobname, df_sub_str, content_settings=settings) return
def write_blob(file_name, text): log("Writing blob: {}".format(file_name)) try: block_blob_service = BlockBlobService(account_name=account_name, account_key=account_key) block_blob_service.create_container(container_name) # Set the permission so the blobs are public. block_blob_service.set_container_acl( container_name, public_access=PublicAccess.Container) block_blob_service.create_blob_from_text(container_name, file_name, text) except Exception as e: log("Error while writing blob..{}".format(e))
def main(): # Get credential parser = configparser.ConfigParser() parser.read('config.ini') STORAGE_ACCOUNT_NAME = parser.get('credential', 'STORAGE_ACCOUNT_NAME') STORAGE_ACCOUNT_KEY = parser.get('credential', 'STORAGE_ACCOUNT_KEY') CONTAINER_NAME = parser.get('credential', 'CONTAINER_NAME') CONTAINER_NAME_METADATA = parser.get('credential', 'CONTAINER_NAME_METADATA') # access to blob storage block_blob_service = BlockBlobService(account_name=STORAGE_ACCOUNT_NAME, account_key=STORAGE_ACCOUNT_KEY) block_blob_service.set_container_acl(CONTAINER_NAME, public_access=PublicAccess.Container) generator = block_blob_service.list_blobs(CONTAINER_NAME) # empty dataframe df = pd.DataFrame({'Url': [], 'Category': []}) # get label from file blob_text = block_blob_service.get_blob_to_text( CONTAINER_NAME_METADATA, 'receipt_list_labelled.csv') #print(blob_text.content) df_label = pd.DataFrame.from_csv(StringIO(blob_text.content), index_col=None, sep=',') #print(df_label.shape); print(df_label) # index index = 0 for blob in generator: print(blob.name) # populate dataframe df.loc[index, 'Category'] = df_label.loc[index, 'category'] df.loc[ index, 'Url'] = imageurl = "https://" + STORAGE_ACCOUNT_NAME + ".blob.core.windows.net/" + CONTAINER_NAME + "/" + blob.name print(imageurl) index = index + 1 # write dataframe to blob print("-----------------------") df_str = df.to_csv(sep='\t', index=False) dfblobname = 'image-url.tsv' block_blob_service.create_blob_from_text(CONTAINER_NAME_METADATA, dfblobname, df_str) return
def process(data): user=os.environ['STORAGE_USERNAME'] password=os.environ['STORAGE_PASSWORD'] container_landing ='landingzone' container_staging = 'staging' block_blob_service = BlockBlobService(account_name=user, account_key=password) blobname = data["data"]["url"].split("/")[-1] content = block_blob_service.get_blob_to_text(container_landing, blobname).content block_blob_service.create_blob_from_text(container_staging,blobname,preprocessing(content)) block_blob_service.delete_blob(container_landing,blobname,delete_snapshots='include')
class azureObjectStorage: def __init__(self, account_name, account_key): self.blobService = BlockBlobService(account_name=account_name, account_key=account_key) def put(self, container, key, value, type='text/plain'): if isinstance(value, str): self.blobService.create_blob_from_text( container, key, value, content_settings=ContentSettings(type)) elif type(value) == file: self.blobService.create_blob_from_steam( container, key, value, content_settings=ContentSettings(type)) else: raise TypeError
def update_status_svg(self, spider, svg): from azure.storage.blob import BlockBlobService, ContentSettings blob_service = BlockBlobService( account_name=self.crawler.settings.get("AZURE_ACCOUNT_NAME"), account_key=self.crawler.settings.get("AZURE_ACCOUNT_KEY"), ) blob_service.create_blob_from_text( self.crawler.settings.get("CITY_SCRAPERS_STATUS_CONTAINER"), "{}.svg".format(spider.name), svg, content_settings=ContentSettings(content_type="image/svg+xml", cache_control="no-cache"), )
def nightly(build: str, account: str, container: str, sas: str, **_) -> None: client = BlockBlobService(account_name=account, sas_token=sas) modules_list = [] for wheel_file in glob.iglob(os.path.join(build, 'build/*.whl')): package_name = os.path.basename(wheel_file).split('-', maxsplit=1)[0].replace('_', '-') sdist_file = next(glob.iglob(os.path.join(build, 'source', f'{package_name}*.tar.gz'))) content_type, content_encoding = mimetypes.guess_type(os.path.basename(wheel_file)) content_settings = ContentSettings(content_type, content_encoding) client.create_blob_from_path(container_name=container, blob_name=f'{package_name}/{os.path.basename(wheel_file)}', file_path=wheel_file, content_settings=content_settings) content_type, content_encoding = mimetypes.guess_type(os.path.basename(sdist_file)) content_settings = ContentSettings(content_type, content_encoding) client.create_blob_from_path(container_name=container, blob_name=f'{package_name}/{os.path.basename(sdist_file)}', file_path=sdist_file, content_settings=content_settings) package_blobs = (os.path.basename(b.name) for b in client.list_blobs(container, prefix=package_name + '/') if b.name != f"{package_name}/") client.create_blob_from_text(container_name=container, blob_name=f'{package_name}/', text=generate_package_list_in_html(f'Links for {package_name}', package_blobs), content_settings=ContentSettings('text/html')) modules_list.append(f"{package_name}/") client.create_blob_from_text(container_name=container, blob_name='index.html', text=generate_package_list_in_html('Simple Index', modules_list), content_settings=ContentSettings('text/html'))