class AzureStorageContainer(Common.Contracts.StorageContainer):
def __init__(self, container_name, config: AzureStorageConfig):
self._container_name = container_name
self._blob_service = BlockBlobService(
account_name=config.account_name,
account_key=config.account_key
)
self._blob_service.create_container(self._container_name)
def upload_text(self, blob_name, text):
self._blob_service.create_blob_from_text(self._container_name, blob_name, text)
def list_blobs(self):
return self._blob_service.list_blobs(self._container_name)
def get_blob_to_text(self, file_name):
return self._blob_service.get_blob_to_text(self._container_name, file_name)
def handler(blob):
all_rows = [HEADER_ROW]
key = blob.name.replace(container_name + "/", "")
event_type = "Arrival"
if "Depart" in key: event_type = "Departure"
date = normalize_date(' '.join(key.split()[0:2]))
outputfile = tempfile.NamedTemporaryFile(mode='r+', delete=False)
excel2csv(blob, outputfile)
rows = rowify(outputfile, date, event_type)
print("Processed file {} with {} entries".format(key, len(rows)))
all_rows.extend(rows)
csv_rows = [",".join(row) for row in all_rows]
file_extension = re.search('.+\.(.+?)$', key).group(1)
new_file_name = 'translated/' + key.replace(file_extension, 'csv')
block_blob_service = BlockBlobService(account_name=acct_name,
account_key=acct_key)
upload_data = ''
for r in csv_rows:
upload_data = upload_data + r + '\n'
block_blob_service.create_blob_from_text(container_name=container_name,
blob_name=new_file_name,
text=str(upload_data))
class AzureBlobFileSystem(object):
def __init__(self,
account_name=None,
account_key=None,
sas_token=None,
connection_string=None,
**storage_options):
account_name = account_name or os.environ.get(
"AZURE_BLOB_ACCOUNT_NAME")
account_key = account_key or os.environ.get("AZURE_BLOB_ACCOUNT_KEY")
sas_token = sas_token or os.environ.get("AZURE_BLOB_SAS_TOKEN")
connection_string = connection_string or os.environ.get(
"AZURE_BLOB_CONNECTION_STRING")
print(account_name, account_key)
self.connection = BlockBlobService(
account_name=account_name,
account_key=account_key,
sas_token=sas_token,
connection_string=connection_string,
protocol=storage_options.get("protocol") or "https",
endpoint_suffix=storage_options.get("endpoint_suffix"),
custom_domain=storage_options.get("custom_domain"))
self.sep = "/"
def ls(self, container, pattern=None):
return list(
set(
filter(
lambda x: fnmatch.fnmatch(x, pattern) if pattern else x,
map(lambda x: x.name,
self.connection.list_blobs(container)))))
def mkdir(self, container, dir_name):
self.touch(container, "{dir_name}/".format(dir_name=dir_name))
def rm(self, container, full_path):
if self.connection.exists(container, full_path):
path_delete_lease = None
try:
path_delete_lease = self.connection.acquire_blob_lease(
container, full_path)
self.connection.delete_blob(container,
full_path,
lease_id=path_delete_lease)
except:
if path_delete_lease is not None:
self.connection.release_blob_lease(container, full_path,
path_delete_lease)
else:
raise IOError(
"File '{file}' does not exist under container '{container}'".
format(file=full_path, container=container))
def touch(self, container, full_path):
container_lease = None
try:
container_lease = self.connection.acquire_container_lease(
container)
self.connection.create_blob_from_text(container, full_path, "")
finally:
if container_lease is not None:
self.connection.release_container_lease(
container, container_lease)
return full_path
def mv(self, container, src_path, dst_path):
try:
self.cp(container, src_path, dst_path)
self.rm(container, src_path)
return True
except:
self.rm(container, dst_path)
return False
def cp(self, container, full_src_path, full_dst_path):
copy_container_lease = None
try:
copy_container_lease = self.connection.acquire_container_lease(
container)
self.connection.copy_blob(
container, full_dst_path,
self.connection.make_blob_url(container, full_src_path))
finally:
if copy_container_lease is not None:
self.connection.release_container_lease(
container, copy_container_lease)
def du(self, container):
return {
blob.name: blob.properties.content_length
for blob in self.connection.list_blobs(container)
}
def last_modified(self, container, full_path):
return self.connection.get_blob_properties(
container, full_path).properties.last_modified
def head(self, container, full_path, bytes_count):
return self.connection.get_blob_to_bytes(container,
full_path,
start_range=0,
end_range=bytes_count -
1).content
def tail(self, container, full_path, bytes_count):
size = self.connection.get_blob_properties(
container, full_path).properties.content_length
return self.connection.get_blob_to_bytes(container,
full_path,
start_range=size -
bytes_count,
end_range=size - 1).content
def exists(self, container, full_path):
return self.connection.exists(container, full_path)