def __init__(self, storage_config: Dict[str, Any],
storage_paths: List[str], local_dir: str):
from azure.storage import blob
connection_string = storage_config.get("connection_string")
container = storage_config.get("container")
account_url = storage_config.get("account_url")
credential = storage_config.get("credential")
if storage_config.get("connection_string"):
self.client = blob.BlobServiceClient.from_connection_string(
connection_string)
elif account_url:
self.client = blob.BlobServiceClient(account_url, credential)
else:
raise ValueError(
"Either 'container_string' or 'account_url' must be specified."
)
if container is None:
raise ValueError("'container' must be specified.")
self.container_name = container if not container.endswith(
"/") else container[:-1]
self.local_dir = local_dir
self.storage_paths = storage_paths
self._file_records = {} # type: Dict[str, datetime.datetime]
def __init__(self, resource, credential = None):
self.resource = resource
self.acc = urlsplit(resource).netloc.split('.')[0]
if credential is None:
# only import azure.identity when it is certain we want to use it
# importing it takes several hundred milliseconds (and from a
# glance seems to make some http requests)
import azure.identity
credential = azure.identity.DefaultAzureCredential()
self.client = azblob.BlobServiceClient(
resource,
credential = credential,
)
def __init__(self,
storage_account_name,
client_id,
tenant_id,
secret_key,
storage_account_token=None,
mq_username=None,
mq_password=None,
mq_ip=None,
mq_vhost=None):
"""
abstraction around all azure blob related methods
:param storage_account_name:
:type storage_account_name:
:param client_id: Azure AD client id (application id)
:type client_id: str
:param tenant_id: Azure AD tenant id
:type tenant_id: str
:param secret_key: secret key for the app
:type secret_key: str
:param mq_username: rabbitmq username
:type mq_username: str
:param mq_password: rabbitmq password
:type mq_password: str
:param mq_ip: rabbitmq IP address
:type mq_ip: str
:param mq_vhost: rabbitmq vhost
:type mq_vhost: str
"""
self.account_name = storage_account_name
self.mq_username = mq_username
self.mq_password = mq_password
self.mq_ip = mq_ip
self.mq_vhost = mq_vhost
self.logger = self.__get_logger(add_azure_filter=True)
if not storage_account_token:
storage_account_token = self.__get_storage_account_token(
client_id, secret_key, tenant_id)
storage_account_url = self.__get_account_url(storage_account_name)
self.blob_client = azureblob.BlobServiceClient(storage_account_url,
storage_account_token)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
try:
now = datetime.datetime.now()
desired_hour = req.params.get('hour')
if desired_hour:
desired_hour = int(desired_hour)
else:
desired_hour = now.hour
blob_prefix = f'{os.environ["EVENT_HUB_NAME"]}/02/{now.year:02d}/{now.month:02d}/{now.day:02d}/{desired_hour:02d}'
service = blob.BlobServiceClient(account_url=os.environ["STORAGE_ACCOUNT_URL"])
container_client = service.get_container_client('readings')
blobs_iterator = container_client.list_blobs(name_starts_with = blob_prefix)
retreived_blobs = {}
for i in blobs_iterator:
index = int(i.name[-7:-5:])
retreived_blobs[index] = i
indexes = sorted(retreived_blobs, reverse=True)
blob_client = container_client.get_blob_client(retreived_blobs[indexes[0]])
stream_downloader = blob_client.download_blob()
blob_contents = stream_downloader.content_as_text()
blob_lines = blob_contents.split('\n')
readings = []
for i in blob_lines:
reading = json.loads(i)
body = base64.b64decode(reading['Body'])
jbody = json.loads(body)
if ('temperature' in jbody or 'humidity' in jbody):
jbody['EnqueuedTimeUtc'] = reading['EnqueuedTimeUtc']
readings.append(jbody)
return func.HttpResponse(json.dumps(readings))
except Exception as ex:
return func.HttpResponse("Something went wrong.", status_code=500)
def __init__(self, storage_account, storage_container, prefix):
self.storage_account = storage_account
self.storage_container = storage_container
self.prefix = prefix
client_id = os.environ["CLIENT_ID"]
secret_key = os.environ["SECRET_KEY"]
tenant_id = os.environ["TENANT_ID"]
storage_account_token = ClientSecretCredential(tenant_id, client_id,
secret_key)
storage_account_url = "https://{}.blob.core.windows.net".format(
self.storage_account)
self.blob_service = azureblob.BlobServiceClient(
storage_account_url, storage_account_token)
self.blob_service.MAX_BLOCK_SIZE = 64 * 1024 * 1024
finally:
output.close()
raise RuntimeError('could not write data to stream or decode bytes')
if __name__ == '__main__':
start_time = datetime.datetime.now().replace(microsecond=0)
print('Sample start: {}'.format(start_time))
print()
# Create the blob client, for use in obtaining references to
# blob storage containers and uploading files to containers.
blob_client = azureblob.BlobServiceClient(
account_url="https://{}.blob.core.windows.net".format(
config._STORAGE_ACCOUNT_NAME),
credential=config._STORAGE_ACCOUNT_KEY)
# Use the blob client to create the containers in Azure Storage if they
# don't yet exist.
input_container_name = 'input'
container_client = blob_client.create_container(input_container_name)
# The collection of data files that are to be processed by the tasks.
input_file_paths = [
os.path.join(sys.path[0], 'taskdata0.txt'),
os.path.join(sys.path[0], 'taskdata1.txt'),
os.path.join(sys.path[0], 'taskdata2.txt')
]
def CreateDiskFromSnapshotURI(
self,
snapshot: 'AZComputeSnapshot',
snapshot_uri: str,
region: Optional[str] = None,
disk_name: Optional[str] = None,
disk_name_prefix: Optional[str] = None,
disk_type: str = 'Standard_LRS') -> 'AZComputeDisk':
"""Create a new disk based on a SAS snapshot URI.
This is useful if e.g. one wants to make a copy of a disk in a separate
Azure account. This method will create a temporary Azure Storage account
within the destination account, import the snapshot from a downloadable
link (the source account needs to share the snapshot through a SAS link)
and then create a disk from the VHD file saved in storage. The Azure
storage account is then deleted.
Args:
snapshot (AZComputeSnapshot): Source snapshot to use.
snapshot_uri (str): The URI of the snapshot to copy.
region (str): Optional. The region in which to create the disk. If not
provided, the disk will be created in the default_region associated to
the AZAccount object.
disk_name (str): Optional. String to use as new disk name.
disk_name_prefix (str): Optional. String to prefix the disk name with.
disk_type (str): Optional. The sku name for the disk to create. Can be
Standard_LRS, Premium_LRS, StandardSSD_LRS, or UltraSSD_LRS.
Default is Standard_LRS.
Returns:
AZComputeDisk: Azure Compute Disk.
Raises:
RuntimeError: If the disk could not be created.
"""
if not region:
region = self.az_account.default_region
# Create a temporary Azure account storage to import the snapshot
storage_account_name = hashlib.sha1(
snapshot.resource_id.encode('utf-8')).hexdigest()[:23]
storage_account_url = 'https://{0:s}.blob.core.windows.net'.format(
storage_account_name)
# pylint: disable=line-too-long
storage_account_id, storage_account_access_key = self.az_account.storage.CreateStorageAccount(
storage_account_name, region=region)
# pylint: enable=line-too-long
blob_service_client = blob.BlobServiceClient(
account_url=storage_account_url,
credential=storage_account_access_key)
# Create a container within the Storage to receive the imported snapshot
container_name = storage_account_name + '-container'
snapshot_vhd_name = snapshot.name + '.vhd'
container_client = blob_service_client.get_container_client(
container_name)
try:
logger.info('Creating blob container {0:s}'.format(container_name))
container_client.create_container()
logger.info('Blob container {0:s} successfully created'.format(
container_name))
except exceptions.ResourceExistsError:
# The container already exists, so we can re-use it
logger.warning(
'Reusing existing container: {0:s}'.format(container_name))
# Download the snapshot from the URI to the storage
copied_blob = blob_service_client.get_blob_client(
container_name, snapshot_vhd_name)
logger.info(
'Importing snapshot to container from URI {0:s}. '
'Depending on the size of the snapshot, this process is going '
'to take a while.'.format(snapshot_uri))
copied_blob.start_copy_from_url(snapshot_uri)
copy_status = copied_blob.get_blob_properties().copy.status
while copy_status != 'success':
sleep(
5
) # Wait for the vhd to be imported in the Azure storage container
copy_status = copied_blob.get_blob_properties().copy.status
if copy_status in ('aborted', 'failed'):
raise RuntimeError('Could not import the snapshot from URI '
'{0:s}'.format(snapshot_uri))
logger.debug(
'Importing snapshot from URI {0:s}'.format(snapshot_uri))
logger.info('Snapshot successfully imported from URI {0:s}'.format(
snapshot_uri))
if not disk_name:
disk_name = common.GenerateDiskName(
snapshot, disk_name_prefix=disk_name_prefix)
# Create a new disk from the imported snapshot
creation_data = {
'location': region,
'creation_data': {
'source_uri': copied_blob.url,
'storage_account_id': storage_account_id,
'create_option': models.DiskCreateOption.import_enum
},
'sku': {
'name': disk_type
}
}
try:
logger.info('Creating disk: {0:s}'.format(disk_name))
request = self.compute_client.disks.create_or_update(
self.az_account.default_resource_group_name, disk_name,
creation_data)
while not request.done():
sleep(5) # Wait 5 seconds before checking disk status again
disk = request.result()
logger.info('Disk {0:s} successfully created'.format(disk_name))
except azure_exceptions.CloudError as exception:
raise RuntimeError(
'Could not create disk from URI {0:s}: {1:s}'.format(
snapshot_uri, str(exception)))
# Cleanup the temporary account storage
self.az_account.storage.DeleteStorageAccount(storage_account_name)
return AZComputeDisk(self.az_account, disk.id, disk.name,
disk.location, disk.zones)
def generate_tasks(job_id, dataset_list, production, flags):
"""
Adds a task for each input file in the collection to the specified job.
:param str job_id: The ID of the job to which to add the tasks.
:param list dataset_list: A collection of datasets names. One task will be
created for each dataset.
:param bool production: True if in production mode
:param str flags: extra single letter flags to pass in
"""
LOGGER.info('Adding {} tasks to job [{}]...'.format(
len(dataset_list), job_id))
tasks = list()
for dataset in dataset_list:
# Ensures that the container for the JSON config files exists
blob_service_client = blob.BlobServiceClient(
account_url=f"https://{STORAGE_ACCOUNT_NAME}.blob.core.windows.net",
credential=SAS_TOKEN)
try:
container = blob_service_client.create_container(
name=CONFIG_FOLDER)
except azure_exception.ResourceExistsError:
LOGGER.info(f"Container \"{CONFIG_FOLDER}\" already exists")
if production:
create_json_config(dataset, blob_service_client)
flag_list = [
f"--log ${{AZ_BATCH_NODE_ROOT_DIR}}/fsmounts/{FILE_SHARE_NAME}/"
f"logs/{DATETIME_NOWISH}/{dataset.name}/${{AZ_BATCH_TASK_ID}}.log"
]
for flag in flags:
# Stops the force flag getting sent to derivatives
if flag == "f" and isinstance(dataset, Derivative):
continue
else:
flag_list.append(f"-{flag}")
dependencies = []
if isinstance(dataset, Dataset):
flag_list.append(f"-t {dataset.timeout}")
flag_list.append(f"-i '{dataset.name}/*'")
elif isinstance(dataset, Derivative): # these are derivatives
# Create dependency
dependencies = [
generate_task_name(dependency)
for dependency in dataset.dependencies
]
flag_list.append(f"-d '{dataset.name}'")
elif isinstance(dataset, Publication) and dataset.is_derivative:
flag_list.append("-c")
flag_list.append(f"-i '{dataset.name}/*'")
command = generate_full_command(
f"{dataset.base_fn} {' '.join(flag_list)}", production, dataset)
tasks.append(
create_task(dataset, command, dependencies,
dataset.max_wall_clock_minutes, production))
return tasks