def create_storage_clients():
# type: (None) -> tuple
"""Create storage clients
:rtype: tuple
:return: blob_client, table_client, queue_client
"""
account_name = storage.get_storageaccount()
account_key = storage.get_storageaccount_key()
endpoint_suffix = storage.get_storageaccount_endpoint()
blob_client = azureblob.BlockBlobService(
account_name=account_name,
account_key=account_key,
endpoint_suffix=endpoint_suffix,
)
table_client = azuretable.TableService(
account_name=account_name,
account_key=account_key,
endpoint_suffix=endpoint_suffix,
)
queue_client = azurequeue.QueueService(
account_name=account_name,
account_key=account_key,
endpoint_suffix=endpoint_suffix,
)
return blob_client, table_client, queue_client
def create_storage_clients():
# type: (None) -> tuple
"""Create storage clients
:rtype: tuple
:return: blob_client, table_client, queue_client
"""
account_name = storage.get_storageaccount()
account_key = storage.get_storageaccount_key()
if account_key is None:
raise RuntimeError(
'No storage account key provided for storage account {}. If '
'accessing via AAD, ensure that a subscription id is specified '
'under management in the credentials configuration.'.format(
account_name))
endpoint_suffix = storage.get_storageaccount_endpoint()
blob_client = azureblob.BlockBlobService(
account_name=account_name,
account_key=account_key,
endpoint_suffix=endpoint_suffix,
)
table_client = azuretable.TableService(
account_name=account_name,
account_key=account_key,
endpoint_suffix=endpoint_suffix,
)
queue_client = azurequeue.QueueService(
account_name=account_name,
account_key=account_key,
endpoint_suffix=endpoint_suffix,
)
return blob_client, table_client, queue_client
def main():
# package, version = ('azure', '1.0.0')
# get a package to look at
# check that package and version.
# version data just gets filled in
# summary trickier.
# summary -> name,
# first_published (might be different than python2_start if
# not using trove classifier)
# python2_start (change if we find earlier),
# python2_end (change if we find earlier, remove if package
# after this come in and has python2),
# python3_start (change if we find earlier)
try:
qs = queue.QueueService(config.STORAGE_ACCOUNT_NAME,
config.STORAGE_ACCOUNT_KEY)
messages_in_batch = 5
while True:
messages = qs.get_messages(config.PACKAGE_QUEUE_NAME,
numofmessages=messages_in_batch,
visibilitytimeout=messages_in_batch *
60)
for message in messages:
entity = json.loads(message.message_text)
_process_one_package(entity["package"], entity["version"])
# once completed delete the message
qs.delete_message(config.PACKAGE_QUEUE_NAME,
message.message_id, message.pop_receipt)
except Exception as e:
# swallow exception here. we will just reprocess and delete the message.
# known failures:
# - connection aborted by get_messages sometimes. this happens with a connectionreseterror (10054)
# - Random json errors. Could add retry.
logger.error(traceback.format_exc())
def _create_credentials() -> tuple:
"""Create storage credentials
:rtype: tuple
:return: (blob_client, queue_client, table_client)
"""
sa, ep, sakey = os.environ['SHIPYARD_STORAGE_ENV'].split(':')
blob_client = azureblob.BlockBlobService(account_name=sa,
account_key=sakey,
endpoint_suffix=ep)
queue_client = azurequeue.QueueService(account_name=sa,
account_key=sakey,
endpoint_suffix=ep)
table_client = azuretable.TableService(account_name=sa,
account_key=sakey,
endpoint_suffix=ep)
return blob_client, queue_client, table_client
def create_clients():
# type: (None) -> tuple
"""Create storage clients
:rtype: tuple
:return: blob_client, queue_client, table_client
"""
blob_client = azureblob.BlockBlobService(account_name=_STORAGEACCOUNT,
account_key=_STORAGEACCOUNTKEY,
endpoint_suffix=_STORAGEACCOUNTEP)
queue_client = azurequeue.QueueService(account_name=_STORAGEACCOUNT,
account_key=_STORAGEACCOUNTKEY,
endpoint_suffix=_STORAGEACCOUNTEP)
table_client = azuretable.TableService(account_name=_STORAGEACCOUNT,
account_key=_STORAGEACCOUNTKEY,
endpoint_suffix=_STORAGEACCOUNTEP)
return blob_client, queue_client, table_client
def __init__(self, config):
self.config = config
self.config["_auto_confirm"] = False
self.batch_client = self._get_batch_client(
config['credentials']['batch'])
self.batch_client.config.add_user_agent(
'batch-shipyard/{}'.format('2.0.0rc2'))
config_storage_credentials = \
config['credentials']['storage']['__storage_account_name__']
parameters = {
'account_name': config_storage_credentials['account'],
'account_key': config_storage_credentials['account_key'],
'endpoint_suffix': config_storage_credentials['endpoint']
}
self.blob_client = azure_storage_blob.BlockBlobService(**parameters)
self.queue_client = azure_storage_queue.QueueService(**parameters)
self.table_client = azure_storage_table.TableService(**parameters)
def __init__(self):
self.qs = queue.QueueService(config.STORAGE_ACCOUNT_NAME, config.STORAGE_ACCOUNT_KEY)
self.qs.create_queue(config.PACKAGE_QUEUE_NAME)
self._background_queue_all_packages = None
self._needs_background_scan = False