def create_target_container(self, data=None):
# for offline tests
if not self.is_live:
return "dummy_string"
# for actual live tests
self.target_container_name = "target" + str(uuid.uuid4())
container_client = ContainerClient(self.storage_endpoint,
self.target_container_name,
self.storage_key)
container_client.create_container()
if data:
self.upload_documents(data, container_client)
return self.generate_sas_url(self.target_container_name, "wl")
def test_create_client_for_emulator(self):
container_client = ContainerClient(
account_url='http://127.0.0.1:1000/devstoreaccount1',
container_name='newcontainer',
credential=
'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=='
)
self.assertEqual(container_client.container_name, "newcontainer")
self.assertEqual(container_client.account_name, "devstoreaccount1")
ContainerClient.from_container_url(
'http://127.0.0.1:1000/devstoreaccount1/newcontainer')
self.assertEqual(container_client.container_name, "newcontainer")
self.assertEqual(container_client.account_name, "devstoreaccount1")
def create_source_container(self, data, blob_prefix=""):
container_name = "src" + str(uuid.uuid4())
container_client = ContainerClient(self.storage_endpoint,
container_name, self.storage_key)
container_client.create_container()
if isinstance(data, list):
for blob in data:
container_client.upload_blob(name=blob_prefix +
str(uuid.uuid4()) + ".txt",
data=blob)
else:
container_client.upload_blob(name=blob_prefix + str(uuid.uuid4()) +
".txt",
data=data)
return self.generate_sas_url(container_name, "rl")
def update_status_svg(self, spider, svg):
from azure.storage.blob import ContainerClient, ContentSettings
container_client = ContainerClient(
"{}.blob.core.windows.net".format(
self.crawler.settings.get("AZURE_ACCOUNT_NAME")),
self.crawler.settings.get("CITY_SCRAPERS_STATUS_CONTAINER"),
credential=self.crawler.settings.get("AZURE_ACCOUNT_KEY"),
)
container_client.upload_blob(
"{}.svg".format(spider.name),
svg,
content_settings=ContentSettings(content_type="image/svg+xml",
cache_control="no-cache"),
overwrite=True,
)
def copy_output(step_id,
env):
account_url = f'https://{env.scoring_datastore_storage_name}.blob.core.windows.net'
src_blob_name = f'azureml/{step_id}/{env.scoring_datastore_storage_name}_out/parallel_run_step.txt'
src_blob_url = f'{account_url}/{env.scoring_datastore_output_container}/{src_blob_name}'
container_client = ContainerClient(account_url=account_url,
container_name=env.scoring_datastore_output_container,
credential=env.scoring_datastore_access_key)
src_blob_properties = container_client.get_blob_client(src_blob_name).get_blob_properties()
destfolder = src_blob_properties.last_modified.date().isoformat()
file_time = (src_blob_properties.last_modified.time()).isoformat('milliseconds').replace(':','_').replace('.','_')
filename_parts = env.scoring_datastore_output_filename.split('.')
dest_blob_name = f'{destfolder}/{filename_parts[0]}_{file_time}.{filename_parts[1]}'
dest_client = container_client.get_blob_client(dest_blob_name)
dest_client.start_copy_from_url(src_blob_url)
def list_blobs(account_name: str, account_key: str,
container_name: str) -> List[BlobProperties]:
""" List the blobs in an Azure Storage Blob Container.
:param account_name: Azure Storage account name.
:param account_key: Azure Storage account key.
:param container_name: the name of the container to list the blobs in.
:return: a list of the blobs that were found in the container.
"""
account_url = make_account_url(account_name)
client: ContainerClient = ContainerClient(account_url,
container_name,
credential=account_key)
blobs = client.list_blobs()
return [b for b in blobs]
def __init__(self, crawler, output_format):
from azure.storage.blob import ContainerClient
feed_uri = crawler.settings.get("FEED_URI")
account_name, account_key = feed_uri[8::].split("@")[0].split(":")
self.spider = crawler.spider
self.container = feed_uri.split("@")[1].split("/")[0]
self.container_client = ContainerClient(
"{}.blob.core.windows.net".format(account_name),
self.container,
credential=account_key,
)
self.feed_prefix = crawler.settings.get(
"CITY_SCRAPERS_DIFF_FEED_PREFIX", "%Y/%m/%d"
)
super().__init__(crawler, output_format)
def __init__(self, uri):
from azure.storage.blob import ContainerClient
container = uri.split("@")[1].split("/")[0]
filename = "/".join(uri.split("@")[1].split("/")[1::])
account_name, account_key = uri[8::].split("@")[0].split(":")
self.account_name = account_name
self.account_key = account_key
self.container = container
self.filename = filename
self.container_client = ContainerClient(
"{}.blob.core.windows.net".format(self.account_name),
self.container,
credential=self.account_key,
)
def copy_output(args):
print("Output : {}".format(args.output_path))
accounturl = "https://mlopsxebiaamlsa.blob.core.windows.net"
containerclient = ContainerClient(accounturl, args.score_container,
args.scoring_datastore_key)
destfolder = date.today().isoformat()
filetime = (
datetime.now(timezone.utc).time().isoformat("milliseconds").replace(
":", "_").replace(".", "_")) # noqa E501
destfilenameparts = args.scoring_output_filename.split(".")
destblobname = "{}/{}_{}.{}".format(destfolder, destfilenameparts[0],
filetime, destfilenameparts[1])
destblobclient = containerclient.get_blob_client(destblobname)
with open(os.path.join(args.output_path, "parallel_run_step.txt"),
"rb") as scorefile: # noqa E501
destblobclient.upload_blob(scorefile, blob_type="BlockBlob")
def _reinit_session(self):
"""
Create a new session
"""
if self.credential:
# Any supplied credential takes precedence over the environment
credential = self.credential
elif "AZURE_STORAGE_CONNECTION_STRING" in os.environ:
logging.info("Authenticating to Azure with connection string")
self.container_client = ContainerClient.from_connection_string(
conn_str=os.getenv("AZURE_STORAGE_CONNECTION_STRING"),
container_name=self.bucket_name,
)
return
else:
if "AZURE_STORAGE_SAS_TOKEN" in os.environ:
logging.info("Authenticating to Azure with SAS token")
credential = os.getenv("AZURE_STORAGE_SAS_TOKEN")
elif "AZURE_STORAGE_KEY" in os.environ:
logging.info("Authenticating to Azure with shared key")
credential = os.getenv("AZURE_STORAGE_KEY")
else:
logging.info(
"Authenticating to Azure with default credentials")
# azure-identity is not part of azure-storage-blob so only import
# it if needed
try:
from azure.identity import DefaultAzureCredential
except ImportError:
raise SystemExit(
"Missing required python module: azure-identity")
credential = DefaultAzureCredential()
session = requests.Session()
adapter = requests.adapters.HTTPAdapter(
pool_maxsize=self.REQUESTS_POOL_MAXSIZE)
session.mount("https://", adapter)
self.container_client = ContainerClient(
account_url=self.account_url,
container_name=self.bucket_name,
credential=credential,
max_single_put_size=self.max_single_put_size,
max_block_size=self.max_block_size,
session=session,
)
def _container_client(self, custom_domain=None, connection_string=None):
if custom_domain is None:
account_domain = "blob.core.windows.net"
else:
account_domain = custom_domain
if connection_string is None:
connection_string = "{}://{}.{}".format(self.azure_protocol,
self.account_name,
account_domain)
credential = None
if self.account_key:
credential = self.account_key
elif self.sas_token:
credential = self.sas_token
elif self.token_credential:
credential = self.token_credential
return ContainerClient(connection_string,
self.azure_container,
credential=credential)
def _azure_get_configs(layer: "Layer") -> List[str]:
providers = layer.gen_providers(0)
credentials = Azure.get_credentials()
storage_account_name = providers["terraform"]["backend"]["azurerm"][
"storage_account_name"
]
container_name = providers["terraform"]["backend"]["azurerm"]["container_name"]
storage_client = ContainerClient(
account_url=f"https://{storage_account_name}.blob.core.windows.net",
container_name=container_name,
credential=credentials,
)
prefix = "opta_config/"
blobs = storage_client.list_blobs(name_starts_with=prefix)
configs = [blob.name[len(prefix) :] for blob in blobs]
if layer.name in configs:
configs.remove(layer.name)
return configs
def delete_opta_config(self) -> None:
providers = self.layer.gen_providers(0)
credentials = self.get_credentials()
storage_account_name = providers["terraform"]["backend"]["azurerm"][
"storage_account_name"]
container_name = providers["terraform"]["backend"]["azurerm"][
"container_name"]
storage_client = ContainerClient(
account_url=f"https://{storage_account_name}.blob.core.windows.net",
container_name=container_name,
credential=credentials,
)
config_path = f"opta_config/{self.layer.name}"
try:
storage_client.delete_blob(config_path, delete_snapshots="include")
except ResourceNotFoundError:
logger.info("Remote opta config was already deleted")
def generate_writable_container_sas(account_name: str,
account_key: str,
container_name: str,
access_duration_hrs: float,
account_url: Optional[str] = None) -> str:
"""Creates a container and returns a SAS URI with read/write/list
permissions.
Args:
account_name: str, name of blob storage account
account_key: str, account SAS token or account shared access key
container_name: str, name of container to create, must not match an
existing container in the given storage account
access_duration_hrs: float
account_url: str, optional, defaults to default Azure Storage URL
Returns: str, URL to newly created container
Raises: azure.core.exceptions.ResourceExistsError, if container already
exists
NOTE: This method currently fails on non-default Azure Storage URLs. The
initializer for ContainerClient() assumes the default Azure Storage URL
format, which is a bug that has been reported here:
https://github.com/Azure/azure-sdk-for-python/issues/12568
"""
if account_url is None:
account_url = build_azure_storage_uri(account=account_name)
container_client = ContainerClient(account_url=account_url,
container_name=container_name,
credential=account_key)
container_client.create_container()
permissions = ContainerSasPermissions(read=True, write=True, list=True)
container_sas_token = generate_container_sas(
account_name=account_name,
container_name=container_name,
account_key=account_key,
permission=permissions,
expiry=datetime.utcnow() + timedelta(hours=access_duration_hrs))
return f'{account_url}/{container_name}?{container_sas_token}'
def upload_opta_config(self) -> None:
providers = self.layer.gen_providers(0)
credentials = self.get_credentials()
storage_account_name = providers["terraform"]["backend"]["azurerm"][
"storage_account_name"]
container_name = providers["terraform"]["backend"]["azurerm"][
"container_name"]
storage_client = ContainerClient(
account_url=f"https://{storage_account_name}.blob.core.windows.net",
container_name=container_name,
credential=credentials,
)
config_path = f"opta_config/{self.layer.name}"
storage_client.upload_blob(
name=config_path,
data=json.dumps(self.layer.structured_config()),
overwrite=True,
)
def get_preprocessed_file_from_azure():
service = ContainerClient(account_url=cfg.azure_details['account_url'],container_name=cfg.azure_details['storage_preprocessed_transcripts'], credential=cfg.azure_details['azure_storage_account_key'])
blob_list = service.list_blobs()
blob_name=''
for blob in blob_list:
blob_name = blob.name
# Create the BlobServiceClient object which will be used to create a container client
blob_service_client = BlobServiceClient.from_connection_string(cfg.azure_details['account_connection_string'])
local_file_name = blob_name
full_path_to_file = os.path.join(local_path, local_file_name)
# Create a blob client using the local file name as the name for the blob
container_name="preprocessed-transcripts"
blob_client = blob_service_client.get_blob_client(container=cfg.azure_details['storage_preprocessed_transcripts'], blob=local_file_name)
download_file_path = os.path.join(local_path, local_file_name)
with open(download_file_path, "wb") as download_file:
download_file.write(blob_client.download_blob().readall())
return local_file_name
def get_container_sas_url_service(
client: ContainerClient,
*,
read: bool = False,
write: bool = False,
delete: bool = False,
list_: bool = False,
delete_previous_version: bool = False,
tag: bool = False,
days: int = 30,
hours: int = 0,
minutes: int = 0,
) -> str:
account_name = client.account_name
container_name = client.container_name
account_key = get_storage_account_name_key_by_name(account_name)
start, expiry = sas_time_window(days=days, hours=hours, minutes=minutes)
sas = generate_container_sas(
account_name,
container_name,
account_key=account_key,
permission=ContainerSasPermissions(
read=read,
write=write,
delete=delete,
list=list_,
delete_previous_version=delete_previous_version,
tag=tag,
),
start=start,
expiry=expiry,
)
with_sas = ContainerClient(
get_url(account_name),
container_name=container_name,
credential=sas,
)
return cast(str, with_sas.url)
def __init__(self,
account_name,
account_key,
container,
listfiles,
dir_out,
nm_out,
extension,
previous_task,
*args,
**kwargs) :
super (ConsolidateServicoOperator, self).__init__ (*args, **kwargs)
self.client = ContainerClient (account_url = f"https://{account_name}.blob.core.windows.net/",
credential = account_key,
container_name = container)
self.listfiles = listfiles
self.extension = extension
self.dir_out = dir_out
self.nm_out = nm_out
self.previous_task = previous_task
self.temp_dir = _TEMP_FILE
def __init__(
self, account_url, # type: str
file_system_name, # type: str
credential=None, # type: Optional[Any]
**kwargs # type: Any
):
# type: (...) -> None
try:
if not account_url.lower().startswith('http'):
account_url = "https://" + account_url
except AttributeError:
raise ValueError("account URL must be a string.")
parsed_url = urlparse(account_url.rstrip('/'))
if not file_system_name:
raise ValueError("Please specify a file system name.")
if not parsed_url.netloc:
raise ValueError("Invalid URL: {}".format(account_url))
blob_account_url = convert_dfs_url_to_blob_url(account_url)
# TODO: add self.account_url to base_client and remove _blob_account_url
self._blob_account_url = blob_account_url
datalake_hosts = kwargs.pop('_hosts', None)
blob_hosts = None
if datalake_hosts:
blob_primary_account_url = convert_dfs_url_to_blob_url(datalake_hosts[LocationMode.PRIMARY])
blob_hosts = {LocationMode.PRIMARY: blob_primary_account_url, LocationMode.SECONDARY: ""}
self._container_client = ContainerClient(blob_account_url, file_system_name,
credential=credential, _hosts=blob_hosts, **kwargs)
_, sas_token = parse_query(parsed_url.query)
self.file_system_name = file_system_name
self._query_str, self._raw_credential = self._format_query_string(sas_token, credential)
super(FileSystemClient, self).__init__(parsed_url, service='dfs', credential=self._raw_credential,
_hosts=datalake_hosts, **kwargs)
# ADLS doesn't support secondary endpoint, make sure it's empty
self._hosts[LocationMode.SECONDARY] = ""
self._client = DataLakeStorageClient(self.url, file_system_name, None, pipeline=self._pipeline)
def get_container_sas_url_service(
client: ContainerClient,
*,
read: bool = False,
write: bool = False,
delete: bool = False,
list_: bool = False,
delete_previous_version: bool = False,
tag: bool = False,
duration: datetime.timedelta = CONTAINER_SAS_DEFAULT_DURATION,
) -> str:
account_name = client.account_name
container_name = client.container_name
account_key = get_storage_account_name_key_by_name(account_name)
start, expiry = sas_time_window(duration)
sas = generate_container_sas(
account_name,
container_name,
account_key=account_key,
permission=ContainerSasPermissions(
read=read,
write=write,
delete=delete,
list=list_,
delete_previous_version=delete_previous_version,
tag=tag,
),
start=start,
expiry=expiry,
)
with_sas = ContainerClient(
get_url(account_name),
container_name=container_name,
credential=sas,
)
return cast(str, with_sas.url)
def deleted_orders(request):
"""
Access all deleted orders from 'deleted-orders' container,
splits the order number and submission date from order's
name and lists them
:param request:
:return:
"""
container_client = ContainerClient(storage_url,
'deleted-orders',
credential=None)
deleted_order_list = container_client.list_blobs()
order_list = []
i = 1
for order in deleted_order_list:
order_no = order['name'].split('.')[0]
creation_date = order['name'].split('.')[1]
order_id_and_date_tuple = [order_no, creation_date]
order_list.append(order_id_and_date_tuple)
i += 1
order_list.sort(reverse=True)
n = 1
for order in order_list:
order.insert(0, n)
n += 1
page = request.GET.get('page', 1)
paginator = Paginator(order_list, 4)
orders = paginator.page(page)
# messages.info(request, 'yes man yes')
context = {
'deleted_orders': orders,
'redirect_url': 'deleted_orders',
}
print(request.COOKIES)
response = render(request, 'deleted_orders.html', context)
# response.delete_cookie('cookie_name1')
return response
def generate_writable_container_sas(account_name: str,
account_key: str,
container_name: str,
access_duration_hrs: float,
account_url: Optional[str] = None) -> str:
"""
Creates a container and returns a SAS URI with read/write/list
permissions.
Args:
account_name: str, name of blob storage account
account_key: str, account SAS token or account shared access key
container_name: str, name of container to create, must not match an
existing container in the given storage account
access_duration_hrs: float
account_url: str, optional, defaults to default Azure Storage URL
Returns: str, URL to newly created container
Raises: azure.core.exceptions.ResourceExistsError, if container already
exists
"""
if account_url is None:
account_url = build_azure_storage_uri(account=account_name)
with ContainerClient(account_url=account_url,
container_name=container_name,
credential=account_key) as container_client:
container_client.create_container()
permissions = ContainerSasPermissions(read=True, write=True, list=True)
container_sas_token = generate_container_sas(
account_name=account_name,
container_name=container_name,
account_key=account_key,
permission=permissions,
expiry=datetime.utcnow() + timedelta(hours=access_duration_hrs))
return f'{account_url}/{container_name}?{container_sas_token}'
def test_invalid_api_version(self):
with pytest.raises(ValueError) as error:
BlobServiceClient(
"https://foo.blob.core.windows.net/account",
credential="fake_key",
api_version="foo")
self.assertTrue(str(error.value).startswith("Unsupported API version 'foo'."))
with pytest.raises(ValueError) as error:
ContainerClient(
"https://foo.blob.core.windows.net/account",
self.container_name,
credential="fake_key",
api_version="foo")
self.assertTrue(str(error.value).startswith("Unsupported API version 'foo'."))
with pytest.raises(ValueError) as error:
BlobClient(
"https://foo.blob.core.windows.net/account",
self.container_name,
self._get_blob_reference(),
credential="fake_key",
api_version="foo")
self.assertTrue(str(error.value).startswith("Unsupported API version 'foo'."))
def main(args):
blob_account_url = f"https://{args.storageaccountname}.blob.core.windows.net"
input_blob = BlobClient.from_blob_url(args.url)
_, filename = os.path.split(input_blob.blob_name)
root, ext = os.path.splitext(filename)
print(f"root: {root}")
assert ext.lower() == ".mp4"
gps_output_container = ContainerClient(account_url=blob_account_url,
container_name="gpsdata",
credential=credential)
assert gps_output_container.exists()
video_upload_container = ContainerClient(
account_url=blob_account_url,
container_name="inputvideos",
credential=credential,
)
assert video_upload_container.exists()
# Download blob into "input.mp4"
print("Downloading video file")
with open(local_video_filename, "wb") as fh:
input_blob.download_blob().readinto(fh)
# GET CREATION DATE FROM FILE
print("Probing File")
probe = ffmpeg.probe(filename="input.mp4")
creation_time = probe["format"]["tags"]["creation_time"]
creation_datetime = datetime.datetime.strptime(creation_time,
"%Y-%m-%dT%H:%M:%S.%fZ")
print(f"Creation time: {creation_datetime}")
# STORE JSON TO YYYY/DD/MM/GOPROFILENAME.JSON
target_folder = os.path.join(
str(creation_datetime.year),
str(creation_datetime.month),
str(creation_datetime.day),
)
gps_json_output_filename = os.path.join(target_folder, f"{root}.json")
video_upload_filename = os.path.join(target_folder, f"{root}.MP4")
print(f"JSON FILENAME: {gps_json_output_filename}")
# Extract GPS data into gps_json_output_filename
print("Extracting GPS data")
result = subprocess.check_output("node process.js", shell=True)
print(result)
# Upload json file to gps data container
try:
with open("out.json", "rb") as fh:
extracted_gps = json.load(fh)
print("extraction successful")
gps_output_blob = gps_output_container.upload_blob(
name=gps_json_output_filename,
data=json.dumps(extracted_gps),
overwrite=True,
)
print(f"file '{gps_json_output_filename}' uploaded.")
print(gps_output_blob.get_blob_properties())
except ResourceExistsError:
print("File already exists")
# upload to inputvideos folder
with open(local_video_filename, "rb") as data:
length = os.path.getsize(local_video_filename)
gps_output_blob = video_upload_container.upload_blob(
name=video_upload_filename,
data=data,
length=length,
overwrite=True,
)
def check_subscription(tenant_id, tenant_name, sub_id, sub_name, creds):
print("\n\t[*] Checking subscription {}:".format(sub_name), flush=True)
storage_client = StorageManagementClient(creds, sub_id)
# Obtain the management object for resources
resource_client = ResourceManagementClient(creds, sub_id)
# Retrieve the list of resource groups
group_list = resource_client.resource_groups.list()
resource_groups = [group.name for group in list(group_list)]
print("\t\t[+] Found {} resource groups".format(len(resource_groups)),
flush=True)
group_to_names_dict = {group: dict() for group in resource_groups}
accounts_counter = 0
for group in resource_groups:
for item in storage_client.storage_accounts.list_by_resource_group(
group):
accounts_counter += 1
group_to_names_dict[group][item.name] = ''
print("\t\t[+] Found {} storage accounts".format(accounts_counter),
flush=True)
for group in resource_groups:
for account in group_to_names_dict[group].keys():
try:
storage_keys = storage_client.storage_accounts.list_keys(
group, account)
storage_keys = {v.key_name: v.value for v in storage_keys.keys}
group_to_names_dict[group][account] = storage_keys['key1']
except azure.core.exceptions.HttpResponseError:
print(
"\t\t[-] User do not have permissions to retrieve storage accounts keys in the given"
" subscription",
flush=True)
print("\t\t Can not scan storage accounts", flush=True)
return
output_list = list()
for group in resource_groups:
for account in group_to_names_dict[group].keys():
key = group_to_names_dict[group][account]
public_containers = check_storage_account(account, key)
for cont in public_containers:
access_level = cont.public_access
container_client = ContainerClient(
ENDPOINT_URL.format(account), cont.name, credential=key)
files = [f.name for f in container_client.list_blobs()]
ext_dict = count_files_extensions(files, EXTENSIONS)
row = [
tenant_id, tenant_name, sub_id, sub_name, group, account,
cont.name, access_level,
CONTAINER_URL.format(account, cont.name),
len(files)
]
for ext in ext_dict.keys():
row.append(ext_dict[ext])
output_list.append(row)
print("\t\t[+] Scanned all storage accounts successfully", flush=True)
if len(output_list) > 0:
print("\t\t[+] Found {} PUBLIC containers".format(len(output_list)),
flush=True)
else:
print("\t\t[+] No PUBLIC containers found")
header = [
"Tenant ID", "Tenant Name", "Subscription ID", "Subscription Name",
"Resource Group", "Storage Account", "Container",
"Public Access Level", "URL", "Total Files"
]
for ext in EXTENSIONS:
header.append(ext)
header.append("others")
write_csv('public-containers-{}.csv'.format(date.today()), header,
output_list)
def test_create_service_with_custom_account_endpoint_path(self):
account_name = "blobstorage"
account_key = "blobkey"
custom_account_url = "http://local-machine:11002/custom/account/path/" + self.sas_token
for service_type in SERVICES.items():
conn_string = 'DefaultEndpointsProtocol=http;AccountName={};AccountKey={};BlobEndpoint={};'.format(
account_name, account_key, custom_account_url)
# Act
service = service_type[0].from_connection_string(
conn_string, container_name="foo", blob_name="bar")
# Assert
self.assertEqual(service.account_name, account_name)
self.assertEqual(service.credential.account_name, account_name)
self.assertEqual(service.credential.account_key, account_key)
self.assertEqual(service.primary_hostname,
'local-machine:11002/custom/account/path')
service = BlobServiceClient(account_url=custom_account_url)
self.assertEqual(service.account_name, None)
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname,
'local-machine:11002/custom/account/path')
self.assertTrue(
service.url.startswith(
'http://local-machine:11002/custom/account/path/?'))
service = ContainerClient(account_url=custom_account_url,
container_name="foo")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname,
'local-machine:11002/custom/account/path')
self.assertTrue(
service.url.startswith(
'http://local-machine:11002/custom/account/path/foo?'))
service = ContainerClient.from_container_url(
"http://local-machine:11002/custom/account/path/foo?query=value")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname,
'local-machine:11002/custom/account/path')
self.assertEqual(service.url,
'http://local-machine:11002/custom/account/path/foo')
service = BlobClient(account_url=custom_account_url,
container_name="foo",
blob_name="bar",
snapshot="baz")
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.blob_name, "bar")
self.assertEqual(service.snapshot, "baz")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname,
'local-machine:11002/custom/account/path')
self.assertTrue(
service.url.startswith(
'http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz&'
))
service = BlobClient.from_blob_url(
"http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz&query=value"
)
self.assertEqual(service.account_name, None)
self.assertEqual(service.container_name, "foo")
self.assertEqual(service.blob_name, "bar")
self.assertEqual(service.snapshot, "baz")
self.assertEqual(service.credential, None)
self.assertEqual(service.primary_hostname,
'local-machine:11002/custom/account/path')
self.assertEqual(
service.url,
'http://local-machine:11002/custom/account/path/foo/bar?snapshot=baz'
)
import random
import numpy as np
import cv2
import imageio
from matplotlib import patches
from matplotlib.patches import Polygon
import matplotlib.pyplot as plt
from datetime import datetime
import zlib
import base64
from mrcnn.model import MaskRCNN
from mrcnn.utils import resize_image
account_url = "https://hecdf.blob.core.windows.net"
facts_blob_service = ContainerClient(account_url=account_url,
container_name=facts_container,
credential=facts_sas_token)
# ===============================
# INLINES
# ===============================
def get_people_local_path(sample_id: str, directory_path: str) -> str:
return os.path.join(directory_path, sample_id + '_people.json')
def get_poly_local_path(sample_id: str, directory_path: str) -> str:
return os.path.join(directory_path, sample_id + '_poly.json')
def main(myblob: func.InputStream):
try:
logging.info('Python blob trigger function processed a request.')
account_name = "###" # confidential
account_key = "###" # confidential
top_level_container_name = "###" # confidential
blob_service = ContainerClient(account_url=account_name, container_name=top_level_container_name, credential=account_key)
# Make connection with Azure SQL database
server = 'datatrust-ff.database.windows.net'
database = 'DataTrust'
username = '******'
password = '******'
cnxn = pyodbc.connect('DRIVER={ODBC Driver 17 for SQL Server};SERVER='+server+';PORT=1433;DATABASE='+database+';UID='+username+';PWD='+ password)
cursor = cnxn.cursor()
cursor.execute("SELECT @@version;")
row = cursor.fetchone()
while row:
print(row[0])
row = cursor.fetchone()
# Download xlsx-files from Azure blob storage
logging.info("\nList blobs in the container")
generator = blob_service.list_blobs()
for blob in generator:
if blob.name.endswith('.xlsx'):
logging.info("\t Blob name: " + blob.name)
file_name = re.sub('.*/', '', blob.name)
xlsx_file = open(file_name, 'wb')
b = blob_service.download_blob(blob)
b.readinto(xlsx_file)
#xlsx_file.write(b)
xlsx_file = open(file_name, 'rb')
data = xlsx_file.read()
data = pd.read_excel(data)
headers = list(data.columns.values)
list_values = data.values.tolist()
blob_name = blob.name
tableName = re.sub('.xlsx', '', blob_name)
tableName = re.sub('^.*/', '', tableName)
tableName = re.sub(' ', '_', tableName)
# If table exists: remove and rewrite
try:
cursor.execute("DROP TABLE dbo." + tableName)
except:
print('Table does not exist yet')
# Create new table
query_string = 'CREATE TABLE dbo.' + tableName + ' ('
# Add columns to table
columns = ''
for i in range(len(headers)):
headers[i] = re.sub('[ /-]', '_', str(headers[i]))
headers[i] = re.sub("[\(\)€'\.,]", '', str(headers[i]))
columns += headers[i] + ', '
if i == len(headers) - 1:
query_string += '\n' + headers[i] + ' VARCHAR(1000)'
else:
query_string += '\n' + headers[i] + ' VARCHAR(1000),'
query_string += '\n);'
query_string = re.sub('[/-]', '', query_string)
cursor.execute(query_string)
# Add rows to table
query_string = "INSERT INTO dbo." + tableName + "(" + columns[:-2] +") VALUES "
for row in range(len(list_values)):
list_values[row] = [str(i) for i in list_values[row]]
row_new = []
for item in list_values[row]:
item = re.sub('[\(\)\r\n\,\'\-]', '', item)
item = "'" + item + "'"
row_new.append(item)
row_new = ','.join(row_new)
if (row + 1) % 1000 == 0 or row + 1 == len(list_values):
query_string += '(' + row_new + ');'
print(query_string)
cursor.execute(query_string)
query_string = "INSERT INTO dbo." + tableName + "(" + columns[:-2] +") VALUES "
else:
query_string += '(' + row_new + '),'
cnxn.commit()
except Exception as e:
logging.exception(e)
from azure.storage.blob import ContainerClient
import numpy as np
import io
import cv2
import time
import matplotlib.pyplot as plt
# Dataset website: http://theairlab.org/tartanair-dataset/
account_url = 'https://tartanair.blob.core.windows.net/'
container_name = 'tartanair-release1'
container_client = ContainerClient(account_url=account_url,
container_name=container_name,
credential=None)
def get_environment_list():
'''
List all the environments shown in the root directory
'''
env_gen = container_client.walk_blobs()
envlist = []
for env in env_gen:
envlist.append(env.name)
return envlist
def get_trajectory_list(envname, easy_hard='Easy'):
'''
List all the trajectory folders, which is named as 'P0XX'
'''
def sample_batch_translation_with_storage():
import os
from azure.core.credentials import AzureKeyCredential
from azure.ai.documenttranslation import (DocumentTranslationClient,
DocumentTranslationInput,
TranslationTarget)
from azure.storage.blob import ContainerClient, generate_container_sas, ContainerSasPermissions
endpoint = os.environ["AZURE_DOCUMENT_TRANSLATION_ENDPOINT"]
key = os.environ["AZURE_DOCUMENT_TRANSLATION_KEY"]
source_storage_endpoint = os.environ["AZURE_STORAGE_SOURCE_ENDPOINT"]
source_storage_account_name = os.environ[
"AZURE_STORAGE_SOURCE_ACCOUNT_NAME"]
source_storage_container_name = os.environ[
"AZURE_STORAGE_SOURCE_CONTAINER_NAME"]
source_storage_key = os.environ["AZURE_STORAGE_SOURCE_KEY"]
target_storage_endpoint = os.environ["AZURE_STORAGE_TARGET_ENDPOINT"]
target_storage_account_name = os.environ[
"AZURE_STORAGE_TARGET_ACCOUNT_NAME"]
target_storage_container_name = os.environ[
"AZURE_STORAGE_TARGET_CONTAINER_NAME"]
target_storage_key = os.environ["AZURE_STORAGE_TARGET_KEY"]
translation_client = DocumentTranslationClient(endpoint,
AzureKeyCredential(key))
container_client = ContainerClient(
source_storage_endpoint,
container_name=source_storage_container_name,
credential=source_storage_key)
with open("document.txt", "rb") as doc:
container_client.upload_blob("document.txt", doc)
source_container_sas = generate_container_sas(
account_name=source_storage_account_name,
container_name=source_storage_container_name,
account_key=source_storage_key,
permission=ContainerSasPermissions.from_string("rl"))
target_container_sas = generate_container_sas(
account_name=target_storage_account_name,
container_name=target_storage_container_name,
account_key=target_storage_key,
permission=ContainerSasPermissions.from_string("rlwd"))
source_container_url = source_storage_endpoint + "/" + source_storage_container_name + "?" + source_container_sas
target_container_url = target_storage_endpoint + "/" + target_storage_container_name + "?" + target_container_sas
translation_inputs = [
DocumentTranslationInput(source_url=source_container_url,
targets=[
TranslationTarget(
target_url=target_container_url,
language_code="es")
],
prefix="document")
]
job_detail = translation_client.create_translation_job(translation_inputs)
job_result = translation_client.wait_until_done(job_detail.id)
if job_result.status == "Succeeded":
print("We translated our documents!")
if job_result.documents_failed_count > 0:
check_documents(translation_client, job_result.id)
elif job_result.status in ["Failed", "ValidationFailed"]:
if job_result.error:
print("Translation job failed: {}: {}".format(
job_result.error.code, job_result.error.message))
check_documents(translation_client, job_result.id)
exit(1)
container_client = ContainerClient(
target_storage_endpoint,
container_name=target_storage_container_name,
credential=target_storage_key)
target_container_client = container_client.from_container_url(
target_container_url)
with open("translated.txt", "wb") as my_blob:
download_stream = target_container_client.download_blob("document.txt")
my_blob.write(download_stream.readall())
