class AzureStorageContainer(Common.Contracts.StorageContainer): def __init__(self, container_name, config: AzureStorageConfig): self._container_name = container_name self._blob_service = BlockBlobService( account_name=config.account_name, account_key=config.account_key ) self._blob_service.create_container(self._container_name) def upload_text(self, blob_name, text): self._blob_service.create_blob_from_text(self._container_name, blob_name, text) def list_blobs(self): return self._blob_service.list_blobs(self._container_name) def get_blob_to_text(self, file_name): return self._blob_service.get_blob_to_text(self._container_name, file_name)
def handler(blob): all_rows = [HEADER_ROW] key = blob.name.replace(container_name + "/", "") event_type = "Arrival" if "Depart" in key: event_type = "Departure" date = normalize_date(' '.join(key.split()[0:2])) outputfile = tempfile.NamedTemporaryFile(mode='r+', delete=False) excel2csv(blob, outputfile) rows = rowify(outputfile, date, event_type) print("Processed file {} with {} entries".format(key, len(rows))) all_rows.extend(rows) csv_rows = [",".join(row) for row in all_rows] file_extension = re.search('.+\.(.+?)$', key).group(1) new_file_name = 'translated/' + key.replace(file_extension, 'csv') block_blob_service = BlockBlobService(account_name=acct_name, account_key=acct_key) upload_data = '' for r in csv_rows: upload_data = upload_data + r + '\n' block_blob_service.create_blob_from_text(container_name=container_name, blob_name=new_file_name, text=str(upload_data))
class AzureBlobFileSystem(object): def __init__(self, account_name=None, account_key=None, sas_token=None, connection_string=None, **storage_options): account_name = account_name or os.environ.get( "AZURE_BLOB_ACCOUNT_NAME") account_key = account_key or os.environ.get("AZURE_BLOB_ACCOUNT_KEY") sas_token = sas_token or os.environ.get("AZURE_BLOB_SAS_TOKEN") connection_string = connection_string or os.environ.get( "AZURE_BLOB_CONNECTION_STRING") print(account_name, account_key) self.connection = BlockBlobService( account_name=account_name, account_key=account_key, sas_token=sas_token, connection_string=connection_string, protocol=storage_options.get("protocol") or "https", endpoint_suffix=storage_options.get("endpoint_suffix"), custom_domain=storage_options.get("custom_domain")) self.sep = "/" def ls(self, container, pattern=None): return list( set( filter( lambda x: fnmatch.fnmatch(x, pattern) if pattern else x, map(lambda x: x.name, self.connection.list_blobs(container))))) def mkdir(self, container, dir_name): self.touch(container, "{dir_name}/".format(dir_name=dir_name)) def rm(self, container, full_path): if self.connection.exists(container, full_path): path_delete_lease = None try: path_delete_lease = self.connection.acquire_blob_lease( container, full_path) self.connection.delete_blob(container, full_path, lease_id=path_delete_lease) except: if path_delete_lease is not None: self.connection.release_blob_lease(container, full_path, path_delete_lease) else: raise IOError( "File '{file}' does not exist under container '{container}'". format(file=full_path, container=container)) def touch(self, container, full_path): container_lease = None try: container_lease = self.connection.acquire_container_lease( container) self.connection.create_blob_from_text(container, full_path, "") finally: if container_lease is not None: self.connection.release_container_lease( container, container_lease) return full_path def mv(self, container, src_path, dst_path): try: self.cp(container, src_path, dst_path) self.rm(container, src_path) return True except: self.rm(container, dst_path) return False def cp(self, container, full_src_path, full_dst_path): copy_container_lease = None try: copy_container_lease = self.connection.acquire_container_lease( container) self.connection.copy_blob( container, full_dst_path, self.connection.make_blob_url(container, full_src_path)) finally: if copy_container_lease is not None: self.connection.release_container_lease( container, copy_container_lease) def du(self, container): return { blob.name: blob.properties.content_length for blob in self.connection.list_blobs(container) } def last_modified(self, container, full_path): return self.connection.get_blob_properties( container, full_path).properties.last_modified def head(self, container, full_path, bytes_count): return self.connection.get_blob_to_bytes(container, full_path, start_range=0, end_range=bytes_count - 1).content def tail(self, container, full_path, bytes_count): size = self.connection.get_blob_properties( container, full_path).properties.content_length return self.connection.get_blob_to_bytes(container, full_path, start_range=size - bytes_count, end_range=size - 1).content def exists(self, container, full_path): return self.connection.exists(container, full_path)