def enviar_aquivos_audio_blob(main_app, dir="audio_files/"): for file in glob.glob(dir + "*.wav"): try: print("Processando arquivo " + file + "...") meeting_code = file.split("_")[1].split("/")[1] blob = meeting_code + "/" + file print("Meeting code " + str(meeting_code)) blob_service = BlockBlobService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY) blob_service.create_blob_from_path(CONTAINER_NAME, blob, file) if os.path.exists(file): os.remove(file) queue_service = QueueService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY) queue_service.encode_function = QueueMessageFormat.text_base64encode payload = { "meeting-code": meeting_code, "blob": blob, "file-name": util.get_file_with_extension(file) } payload = json.dumps(payload, ensure_ascii=False) queue_service.put_message(QUEUE_NAME_AUDIO, payload) print("Arquivo " + file + " processado com sucesso.") main_app.mensagem["text"] = "File " + file + " synced successfully" except: traceback.format_exc()
def copy_blob_across_storage(self, sourceblobpath="", destinationblobpath="", sourcecontainer="", destinationcontainer="", destination_source_account_name="", destination_source_account_key=""): """ This method copies blob across different storage accounts. Args: sourceblobpath:source blob name destinationblobpath:destionation blob name sourcecontainer:source container name destinationcontainer:destination container destination_source_account_name: storage account name for destination storage account destination_source_account_key:storage account key for destination storage account """ destinationfileservice = BlockBlobService( account_name=destination_source_account_name, account_key=destination_source_account_key) local_path = self.download_blob(blobpath=sourceblobpath, container=sourcecontainer) if len(destinationblobpath) == 0: destinationblobpath = "/".join(local_path.split("/")[1:]) destinationfileservice.create_blob_from_path( container_name=destinationcontainer, blob_name=destinationblobpath, file_path=local_path) os.remove(local_path)
def __init__(self, storage_account_name="", storage_account_access_key=""): self.storage_account_name = storage_account_name self.storage_account_access_key = storage_account_access_key self.file_service = BlockBlobService( account_name=self.storage_account_name, account_key=self.storage_account_access_key)
def __init__(self, container_name, config: AzureStorageConfig): self._container_name = container_name self._blob_service = BlockBlobService( account_name=config.account_name, account_key=config.account_key ) self._blob_service.create_container(self._container_name)
def create(path: str, azure_info_path: Optional[str] = None): """This creates a RichPath object based on the input path. To create a remote path, just prefix it appropriately and pass in the path to the .json configuration. """ if path.startswith(AZURE_PATH_PREFIX): assert azure_info_path is not None, "An AzurePath cannot be created when azure_info_path is None." # Strip off the AZURE_PATH_PREFIX: path = path[len(AZURE_PATH_PREFIX):] account_name, container_name, path = path.split('/', 2) with open(azure_info_path, 'r') as azure_info_file: azure_info = json.load(azure_info_file) account_info = azure_info.get(account_name) if account_info is None: raise Exception( "Could not find access information for account '%s'!" % (account_name, )) sas_token = account_info.get('sas_token') account_key = account_info.get('account_key') if sas_token is not None: assert not sas_token.startswith( '?' ), 'SAS tokens should not start with "?". Just delete it.' # https://github.com/Azure/azure-storage-python/issues/301 blob_service = BlockBlobService(account_name=account_name, sas_token=sas_token) elif account_key is not None: blob_service = BlockBlobService(account_name=account_name, account_key=account_key) else: raise Exception( "Access to Azure storage account '%s' requires either account_key or sas_token!" % (account_name, )) # ERROR is too verbose, in particular when downloading based on etags an error is emitted when blob # download is aborted. logging.getLogger('azure.storage').setLevel(logging.CRITICAL) # Replace environment variables in the cache location cache_location = account_info.get('cache_location') if cache_location is not None: def replace_by_env_var(m) -> str: env_var_name = m.group(1) env_var_value = os.environ.get(env_var_name) if env_var_value is not None: return env_var_value else: return env_var_name cache_location = re.sub('\${([^}]+)}', replace_by_env_var, cache_location) return AzurePath(path, azure_container_name=container_name, azure_blob_service=blob_service, cache_location=cache_location) else: return LocalPath(path)
def upload_file_toblob(input_file, input_blob_name): blob_service = BlockBlobService( account_name=app.config['STORAGE_ACCOUNT_NAME'], account_key=app.config['STORAGE_ACCOUNT_KEY']) print('Uploading the input to blob storage...') blob_service.create_blob_from_path(app.config['STORAGE_CONTAINER_NAME'], input_blob_name, input_file)
class AzureStorageContainer(StorageContainer): """Azure implementation of Storage Container using BlockBlobService.""" def __init__(self, container_name, config: AzureStorageConfig): self._blob_service = None self._container_name = container_name self._config = config def _get_client(self): """ :return: BlockBlobService initialized with account name and key from config """ if self._blob_service is None: self._blob_service = BlockBlobService( account_name=self._config.account_name, account_key=self._config.account_key ) self._blob_service.create_container(self._container_name) return self._blob_service def upload_text(self, blob_name, text): """Uploads text to a new blob. :param blob_name: Name to give new blob :param text: Text to upload :return: None """ self._get_client().create_blob_from_text(self._container_name, blob_name, text) def list_blobs(self): """List all blobs in container. :return: List of blobs in container """ return self._get_client().list_blobs(self._container_name) def get_blob_to_text(self, file_name): """Get string from contents of blob. :param file_name: Name of blob file :return: Text from blob file """ return self._get_client().get_blob_to_text(self._container_name, file_name) @staticmethod def create(): """Initialize AzureStorageContainer with name and creds from config. :return: """ return AzureStorageContainer( ProcessConfig().config_container_name, AzureConfig().storage_config)
def uri_put_file(creds, uri, fp, content_type=None): assert fp.tell() == 0 assert uri.startswith('wabs://') def log_upload_failures_on_error(exc_tup, exc_processor_cxt): def standard_detail_message(prefix=''): return (prefix + ' There have been {n} attempts to upload ' 'file {url} so far.'.format(n=exc_processor_cxt, url=uri)) typ, value, tb = exc_tup del exc_tup # Screen for certain kinds of known-errors to retry from if issubclass(typ, socket.error): socketmsg = value[1] if isinstance(value, tuple) else value logger.info( msg='Retrying upload because of a socket error', detail=standard_detail_message( "The socket error's message is '{0}'.".format(socketmsg))) else: # For all otherwise untreated exceptions, report them as a # warning and retry anyway -- all exceptions that can be # justified should be treated and have error messages # listed. logger.warning( msg='retrying file upload from unexpected exception', detail=standard_detail_message( 'The exception type is {etype} and its value is ' '{evalue} and its traceback is {etraceback}'.format( etype=typ, evalue=value, etraceback=''.join(traceback.format_tb(tb))))) # Help Python GC by resolving possible cycles del tb url_tup = urlparse(uri) kwargs = dict(content_settings=ContentSettings(content_type), validate_content=True) conn = BlockBlobService(creds.account_name, creds.account_key, sas_token=creds.access_token, protocol='https') conn.create_blob_from_bytes(url_tup.netloc, url_tup.path.lstrip('/'), fp.read(), **kwargs) # To maintain consistency with the S3 version of this function we must # return an object with a certain set of attributes. Currently, that set # of attributes consists of only 'size' return _Key(size=fp.tell())
def uri_get_file(creds, uri, conn=None): assert uri.startswith('wabs://') url_tup = urlparse(uri) if conn is None: conn = BlockBlobService(creds.account_name, creds.account_key, sas_token=creds.access_token, protocol='https') data = io.BytesIO() conn.get_blob_to_stream(url_tup.netloc, url_tup.path.lstrip('/'), data) return data.getvalue()
def _get_client(self): """ :return: BlockBlobService initialized with account name and key from config """ if self._blob_service is None: self._blob_service = BlockBlobService( account_name=self._config.account_name, account_key=self._config.account_key ) self._blob_service.create_container(self._container_name) return self._blob_service
def uri_put_file(creds, uri, fp, content_type=None): assert fp.tell() == 0 assert uri.startswith('wabs://') def log_upload_failures_on_error(exc_tup, exc_processor_cxt): def standard_detail_message(prefix=''): return (prefix + ' There have been {n} attempts to upload ' 'file {url} so far.'.format(n=exc_processor_cxt, url=uri)) typ, value, tb = exc_tup del exc_tup # Screen for certain kinds of known-errors to retry from if issubclass(typ, socket.error): socketmsg = value[1] if isinstance(value, tuple) else value logger.info( msg='Retrying upload because of a socket error', detail=standard_detail_message( "The socket error's message is '{0}'." .format(socketmsg))) else: # For all otherwise untreated exceptions, report them as a # warning and retry anyway -- all exceptions that can be # justified should be treated and have error messages # listed. logger.warning( msg='retrying file upload from unexpected exception', detail=standard_detail_message( 'The exception type is {etype} and its value is ' '{evalue} and its traceback is {etraceback}' .format(etype=typ, evalue=value, etraceback=''.join(traceback.format_tb(tb))))) # Help Python GC by resolving possible cycles del tb url_tup = urlparse(uri) kwargs = dict( content_settings=ContentSettings(content_type), validate_content=True) conn = BlockBlobService(creds.account_name, creds.account_key, sas_token=creds.access_token, protocol='https') conn.create_blob_from_bytes(url_tup.netloc, url_tup.path.lstrip('/'), fp.read(), **kwargs) # To maintain consistency with the S3 version of this function we must # return an object with a certain set of attributes. Currently, that set # of attributes consists of only 'size' return _Key(size=fp.tell())
def delete_data_from_blob(prefix): from azure.storage.blob.blockblobservice import BlockBlobService ws = get_workspace() def_blob_store = ws.get_default_datastore() print("Deleting blobs from folder:", prefix) blob_service = BlockBlobService( def_blob_store.account_name, def_blob_store.account_key ) generator = blob_service.list_blobs( def_blob_store.container_name, prefix=prefix ) for blob in generator: if blob.name.endswith("mp4"): print("Deleting: " + blob.name) blob_service.delete_blob(def_blob_store.container_name, blob.name) generator = blob_service.list_blobs( def_blob_store.container_name, prefix=prefix ) for blob in generator: print("Deleting: " + blob.name) blob_service.delete_blob(def_blob_store.container_name, blob.name)
def upload_slide(): # TODO: If this config is not present, disable upload instead of failing to load accountName = app.config["AZURE_STORAGE_ACCOUNT_NAME"] containerName = app.config["AZURE_STORAGE_ACCOUNT_SVSUPLOAD_CONTAINER_NAME"] accountKey = app.config["AZURE_STORAGE_ACCOUNT_KEY"] blob_service = BlockBlobService(account_name=accountName, account_key=accountKey) permission = ContainerPermissions(write=True) now = datetime.now(timezone.utc) expiry = now + timedelta(hours=2) sasToken = blob_service.generate_container_shared_access_signature(container_name=containerName, permission=permission, protocol='https', start=now, expiry=expiry) container_url = f'https://{accountName}.blob.core.windows.net/{containerName}?{sasToken}' return render_template('slide/upload.html', container_url=container_url)
def uri_get_file(creds, uri, conn=None): assert uri.startswith('wabs://') url_tup = urlparse(uri) if conn is None: conn = BlockBlobService(creds.account_name, creds.account_key, sas_token=creds.access_token, protocol='https') data = io.BytesIO() conn.get_blob_to_stream(url_tup.netloc, url_tup.path.lstrip('/'), data) return data.getvalue()
def resize_and_convert(image, width=500, height=500, container="media"): if settings.DEBUG: img = Image.open(image) img = img.resize((width, height), Image.ANTIALIAS) img = img.convert("RGB") img.save(image.path, format="JPEG") else: temp = io.BytesIO() img = Image.open(image) img = img.resize((width, height), Image.ANTIALIAS) img = img.convert("RGB") img.save(temp, format="JPEG") bbs = BlockBlobService(account_name='liveportal2019', account_key=os.environ.get( 'LP_AZURE_STORAGE_KEY', '')) bbs.create_blob_from_bytes(container, image.name, temp.getvalue())
def test_no_retry_on_keyboadinterrupt(collect): """Ensure that KeyboardInterrupts are forwarded.""" key_name = 'test-key-name' b = B(name=key_name) # If vanilla KeyboardInterrupt is used, then sending SIGINT to the # test can cause it to pass improperly, so use a subtype instead. class MarkedKeyboardInterrupt(KeyboardInterrupt): pass collect.inject(MarkedKeyboardInterrupt('SIGINT, probably')) d = wabs_deleter.Deleter(BlockBlobService('test', 'ing'), 'test-container') with pytest.raises(MarkedKeyboardInterrupt): d.delete(b) # Exactly when coroutines are scheduled is non-deterministic, # so spin while yielding to provoke the # MarkedKeyboardInterrupt being processed within the # pytest.raises context manager. while True: gevent.sleep(0.1) # Only one key should have been aborted, since the purpose is to # *not* retry when processing KeyboardInterrupt. assert collect.aborted_keys == [key_name] # Turn off fault injection and flush/synchronize with close(). collect.inject(None) d.close() # Since there is no retrying, no keys should be deleted. assert not collect.deleted_keys
def connect(self, creds): """Return an azure BlockBlobService instance. """ return BlockBlobService(account_name=creds.account_name, account_key=creds.account_key, sas_token=creds.access_token, protocol='https')
def test_close_error(): """Ensure that attempts to use a closed Deleter results in an error.""" d = wabs_deleter.Deleter(BlockBlobService('test', 'ing'), 'test-container') d.close() with pytest.raises(exception.UserCritical): d.delete('no value should work')
def service(self): if self._service is None: self._service = BlockBlobService( self.account_name, self.account_key, is_emulated=self.is_emulated, custom_domain=self.custom_domain, ) return self._service
def test_processes_one_deletion(collect): key_name = 'test-key-name' b = B(name=key_name) d = wabs_deleter.Deleter(BlockBlobService('test', 'ing'), 'test-container') d.delete(b) d.close() assert collect.deleted_keys == [key_name]
def __init__(self, locator, jobChunkSize=maxAzureTablePropertySize): super(AzureJobStore, self).__init__() accountName, namePrefix = locator.split(':', 1) if '--' in namePrefix: raise ValueError( "Invalid name prefix '%s'. Name prefixes may not contain %s." % (namePrefix, self.nameSeparator)) if not self.containerNameRe.match(namePrefix): raise ValueError( "Invalid name prefix '%s'. Name prefixes must contain only digits, " "hyphens or lower-case letters and must not start or end in a " "hyphen." % namePrefix) # Reserve 13 for separator and suffix if len(namePrefix) > self.maxContainerNameLen - self.maxNameLen - len( self.nameSeparator): raise ValueError(( "Invalid name prefix '%s'. Name prefixes may not be longer than 50 " "characters." % namePrefix)) if '--' in namePrefix: raise ValueError( "Invalid name prefix '%s'. Name prefixes may not contain " "%s." % (namePrefix, self.nameSeparator)) self.locator = locator self.jobChunkSize = jobChunkSize self.accountKey = _fetchAzureAccountKey(accountName) self.accountName = accountName # Table names have strict requirements in Azure self.namePrefix = self._sanitizeTableName(namePrefix) # These are the main API entry points. self.tableService = TableService(account_key=self.accountKey, account_name=accountName) self.blobService = BlockBlobService(account_key=self.accountKey, account_name=accountName) # Serialized jobs table self.jobItems = None # Job<->file mapping table self.jobFileIDs = None # Container for all shared and unshared files self.files = None # Stats and logging strings self.statsFiles = None # File IDs that contain stats and logging strings self.statsFileIDs = None
def get_azure_storage_client(config): # Todo: Move away from global client. global azure_storage_client if azure_storage_client is not None: return azure_storage_client azure_storage_client = BlockBlobService( config.get("storage_account"), account_key=config.get("storage_key")) return azure_storage_client
def setUp(self): self.account_name = self.account_name or os.environ.get( "AZURE_BLOB_ACCOUNT_NAME") self.account_key = os.environ.get("AZURE_BLOB_ACCOUNT_KEY") try: warnings.simplefilter("ignore", ResourceWarning) except: pass self.connection = BlockBlobService(account_name=self.account_name, account_key=self.account_key)
def _blob_service(self, custom_domain=None, connection_string=None): # This won't open a connection or anything, # it's akin to a client return BlockBlobService(account_name=self.account_name, account_key=self.account_key, sas_token=self.sas_token, is_emulated=self.is_emulated, protocol=self.azure_protocol, custom_domain=custom_domain, connection_string=connection_string, token_credential=self.token_credential, endpoint_suffix=self.endpoint_suffix)
def timed_job(): config = configparser.ConfigParser() config.read('config/config.cfg') account = config.get('DEFAULT', 'ACCOUNT') key = config.get('DEFAULT', 'KEY') promi = config.get('DEFAULT', 'PROM') promup = promi.encode() container = config.get('DEFAULT', 'CONTAINER') url = config.get('DEFAULT', 'URL') blob_service = BlockBlobService(account_name=account, account_key=key) userAndPass = b64encode(promup).decode("ascii") headers = {'Authorization': 'Basic %s' % userAndPass} prom = PrometheusConnect(url=url, headers=headers, disable_ssl=False) metric_data = prom.all_metrics() time = datetime.now() metrics = [] values = [] for i in metric_data: metric = prom.get_metric_range_data(metric_name=i, start_time=time - timedelta(hours=1), end_time=time, chunk_size=timedelta(hours=1)) x = int(0) for d in metric: for name, dct in d.items(): dct = dict(dct) if name == 'metric': dct['id'] = x metrics.append(dct) else: for key in dct: va = {} va['time'] = key va['value'] = dct[key] va['id'] = x values.append(va) x = x + 1 df = pd.DataFrame(metrics) df1 = pd.DataFrame(values) df = pd.merge(df, df1, how='inner', left_on=['id'], right_on=['id']) df['time'] = pd.to_datetime(df['time'], unit='s') df = df.drop(['endpoint', 'service', 'id'], axis=1) write_pandas_dataframe_to_blob( blob_service, df, container, str((datetime.now()).date()) + '/' + str(datetime.now().time()).replace(':', '').replace(".", ''))
def __init__(self, account_name=None, account_key=None, sas_token=None, connection_string=None, **storage_options): account_name = account_name or os.environ.get( "AZURE_BLOB_ACCOUNT_NAME") account_key = account_key or os.environ.get("AZURE_BLOB_ACCOUNT_KEY") sas_token = sas_token or os.environ.get("AZURE_BLOB_SAS_TOKEN") connection_string = connection_string or os.environ.get( "AZURE_BLOB_CONNECTION_STRING") print(account_name, account_key) self.connection = BlockBlobService( account_name=account_name, account_key=account_key, sas_token=sas_token, connection_string=connection_string, protocol=storage_options.get("protocol") or "https", endpoint_suffix=storage_options.get("endpoint_suffix"), custom_domain=storage_options.get("custom_domain")) self.sep = "/"
def handler(blob): all_rows = [HEADER_ROW] key = blob.name.replace(container_name + "/", "") event_type = "Arrival" if "Depart" in key: event_type = "Departure" date = normalize_date(' '.join(key.split()[0:2])) outputfile = tempfile.NamedTemporaryFile(mode='r+', delete=False) excel2csv(blob, outputfile) rows = rowify(outputfile, date, event_type) print("Processed file {} with {} entries".format(key, len(rows))) all_rows.extend(rows) csv_rows = [",".join(row) for row in all_rows] file_extension = re.search('.+\.(.+?)$', key).group(1) new_file_name = 'translated/' + key.replace(file_extension, 'csv') block_blob_service = BlockBlobService(account_name=acct_name, account_key=acct_key) upload_data = '' for r in csv_rows: upload_data = upload_data + r + '\n' block_blob_service.create_blob_from_text(container_name=container_name, blob_name=new_file_name, text=str(upload_data))
def create_block_blob_service(self): ''' Creates a BlockBlobService object with the settings specified in the CloudStorageAccount. :return: A service object. :rtype: :class:`~azure.storage.blob.blockblobservice.BlockBlobService` ''' try: from azure.storage.blob.blockblobservice import BlockBlobService return BlockBlobService(self.account_name, self.account_key, sas_token=self.sas_token, is_emulated=self.is_emulated) except ImportError: raise Exception('The package azure-storage-blob is required. ' + 'Please install it using "pip install azure-storage-blob"')
def block_blob_service(self): ACCOUNT_NAME = os.environ['ACCOUNT_NAME'] ACCOUNT_KEY = os.environ['ACCOUNT_KEY'] block_blob_service = BlockBlobService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY) block_blob_service.create_container(self.container_name) block_blob_service.set_container_acl( self.container_name, public_access=PublicAccess.Container) return block_blob_service
def test_processes_many_deletions(collect): # Generate a target list of keys in a stable order target = sorted(['test-key-' + str(x) for x in range(20001)]) # Construct boto S3 Keys from the generated names and delete them # all. blobs = [B(name=key_name) for key_name in target] d = wabs_deleter.Deleter(BlockBlobService('test', 'ing'), 'test-container') for b in blobs: d.delete(b) d.close() # Sort the deleted key names to obtain another stable order and # then ensure that everything was passed for deletion # successfully. assert sorted(collect.deleted_keys) == target
class AzureStorageContainer(Common.Contracts.StorageContainer): def __init__(self, container_name, config: AzureStorageConfig): self._container_name = container_name self._blob_service = BlockBlobService( account_name=config.account_name, account_key=config.account_key ) self._blob_service.create_container(self._container_name) def upload_text(self, blob_name, text): self._blob_service.create_blob_from_text(self._container_name, blob_name, text) def list_blobs(self): return self._blob_service.list_blobs(self._container_name) def get_blob_to_text(self, file_name): return self._blob_service.get_blob_to_text(self._container_name, file_name)
def test_retry_on_normal_error(collect): """Ensure retries are processed for most errors.""" key_name = 'test-key-name' b = B(name=key_name) collect.inject(Exception('Normal error')) d = wabs_deleter.Deleter(BlockBlobService('test', 'ing'), 'test-container') d.delete(b) # Since delete_keys will fail over and over again, aborted_keys # should grow quickly. while len(collect.aborted_keys) < 2: gevent.sleep(0.1) # Since delete_keys has been failing repeatedly, no keys should be # successfully deleted. assert not collect.deleted_keys # Turn off fault injection and flush/synchronize with close(). collect.inject(None) d.close() # The one enqueued job should have been processed.n assert collect.deleted_keys == [key_name]
def apathetic_container_delete(container_name, *args, **kwargs): conn = BlockBlobService(*args, **kwargs) conn.delete_container(container_name) return conn