def render_video(request):
template = loader.get_template('app/render_video.html')
vidstatus = 'No Video Found.'
queue_service = QueueService(account_name=os.environ['SVPD_STORAGE_ACCOUNT_NAME'], account_key=os.environ['SVPD_STORAGE_ACCOUNT_KEY'])
messages = queue_service.get_messages(os.environ['SVPD_STORAGE_ACCOUNT_READY_TO_ENCODE'], num_messages=1, visibility_timeout=1*60)
for message in messages:
vidstatus = 'Queued for Rendering: ' + message.content
message_obj = json.loads(message.content)
access_token = ams_authenticate()['access_token']
asset = ams_post_request(access_token, "Assets", {
'Name': message_obj['filename'],
'AlternateId': message_obj['folder']})
asset_container = urllib.parse.urlparse(asset['Uri']).path[1:]
asset_file = ams_post_request(access_token, "Files", {
'IsEncrypted': 'false',
'IsPrimary': 'false',
'MimeType': 'video/mp4',
'ContentFileSize': message_obj['size'],
'Name': message_obj['filename'],
'ParentAssetId': asset['Id']})
block_blob_service = BlockBlobService(account_name=os.environ['SVPD_STORAGE_ACCOUNT_NAME'], account_key=os.environ['SVPD_STORAGE_ACCOUNT_KEY'])
from_url = block_blob_service.make_blob_url(os.environ['SVPD_STORAGE_ACCOUNT_UPLOADED'], message_obj['folder'] + '/' + message_obj['filename'])
block_blob_service.copy_blob(asset_container, message_obj['filename'], from_url)
job = ams_verbose_post_request(access_token, "Jobs", {
'Name': message_obj['filename'],
'InputMediaAssets': [{
'__metadata': { 'uri': os.environ['AMS_API_ENDPOINT'] + 'Assets(\'' + asset['Id'] + '\')' }
}],
'Tasks': [{
'Name': 'Adaptive Streaming Task',
'Configuration': 'Adaptive Streaming',
'MediaProcessorId': 'nb:mpid:UUID:ff4df607-d419-42f0-bc17-a481b1331e56',
'TaskBody': '<?xml version="1.0" encoding="utf-16"?><taskBody><inputAsset>JobInputAsset(0)</inputAsset><outputAsset assetCreationOptions="0" assetFormatOption="0" assetName="' + message_obj['filename'] + ' - MES v1.1" storageAccountName="' + os.environ['SVPD_STORAGE_ACCOUNT_NAME'] + '">JobOutputAsset(0)</outputAsset></taskBody>'
},{
'Name': 'Indexing Task',
'Configuration': '<?xml version="1.0" encoding="utf-8"?><configuration version="2.0"><input><metadata key="title" value="blah" /></input><settings></settings><features><feature name="ASR"><settings><add key="Language" value="English" /><add key="GenerateAIB" value="False" /><add key="GenerateKeywords" value="True" /><add key="ForceFullCaption" value="False" /><add key="CaptionFormats" value="ttml;sami;webvtt" /></settings></feature></features></configuration>',
'MediaProcessorId': 'nb:mpid:UUID:233e57fc-36bb-4f6f-8f18-3b662747a9f8',
'TaskBody': '<?xml version="1.0" encoding="utf-16"?><taskBody><inputAsset>JobInputAsset(0)</inputAsset><outputAsset assetCreationOptions="0" assetFormatOption="0" assetName="' + message_obj['filename'] + ' - Indexed" storageAccountName="' + os.environ['SVPD_STORAGE_ACCOUNT_NAME'] + '">JobOutputAsset(1)</outputAsset></taskBody>'
}]
})
queue_service.put_message(os.environ['SVPD_STORAGE_ACCOUNT_ENCODING'], json.dumps({
'filename': message_obj['filename'],
'folder': message_obj['folder'],
'size': message_obj['size'],
'job': job['d']}))
queue_service.delete_message(os.environ['SVPD_STORAGE_ACCOUNT_READY_TO_ENCODE'], message.id, message.pop_receipt)
return HttpResponse(template.render({
'vidstatus': vidstatus,
}, request))
def consume(args):
# Get the service resources
queue_service = QueueService(
account_name=os.environ.get('QUEUE_STORAGE_ACCOUNT'),
account_key=os.environ.get('QUEUE_STORAGE_ACCESS_KEY'))
queue_service.decode_function = QueueMessageFormat.binary_base64decode
logger.debug('Listening for messages on {}'.format(args.azure_queue))
while True:
messages = queue_service.get_messages(
args.azure_queue,
num_messages=16,
visibility_timeout=args.wait_time)
for message in messages:
try:
process(args, message)
except AzureException as e:
logger.exception(e)
if not args.dry_run:
logger.debug('deleting message {}'.format(message.id))
queue_service.delete_message(args.azure_queue, message.id,
message.pop_receipt)
time.sleep(args.wait_time)
class AzureStorageQueue(Common.Contracts.Queue):
def __init__(self, queue_name, config: AzureStorageConfig):
self._queue_name = queue_name
self._queue_service = QueueService(account_name=config.account_name,
account_key=config.account_key)
self._queue_service.encode_function = QueueMessageFormat.text_base64encode
self._queue_service.decode_function = QueueMessageFormat.text_base64decode
def push(self, message):
self._queue_service.create_queue(self._queue_name)
self._queue_service.put_message(self._queue_name, message)
def pop(self):
self._queue_service.create_queue(self._queue_name)
messages = self._queue_service.peek_messages(self._queue_name)
for message in messages:
result = message.content
self._queue_service.delete_message(self._queue_name, message.id,
message.pop_receipt)
return result
def peek(self):
self._queue_service.create_queue(self._queue_name)
messages = self._queue_service.peek_messages(self._queue_name)
for message in messages:
return message.content
class AzureProvider(BaseProvider):
"""Queue Provider for the Microsoft Azure."""
def __init__(self, account_name, account_key):
self.queue_service = QueueService(
account_name=account_name, account_key=account_key
)
def put_message(self, queue_name: str, message: str):
self.queue_service.put_message(
queue_name=queue_name,
message_text=message,
)
def get_message(self, queue_name: str):
try:
queue_message = self.queue_service.get_messages(
queue_name=queue_name, numofmessages=1
).queue_messages[0]
except IndexError:
return {}
else:
return {
attr_name: getattr(queue_message, attr_name)
for attr_name in dir(queue_message)
if not attr_name.startswith('_')
}
def delete_message(self, queue_name: str, message: dict):
message_id = message.get('message_id')
pop_receipt = message.get('pop_receipt')
self.queue_service.delete_message(
queue_name, message_id=message_id, popreceipt=pop_receipt,
)
class QueueBase(object):
"""docstring for QueueBase"""
def __init__(self):
super(QueueBase, self).__init__()
self.queue_service = QueueService(
account_name='boburstorage',
account_key=
'wRgukLsyhLtnI7qEk8mSGnIBC+IsiTTXEDF1/xnmBGDudJLSeYdtyuVzuSN5/cplJz88AJPyoVyjCmL9N1ECXw=='
)
def save_message_to_queue(self, queue, data):
self.queue_service.put_message(queue, data)
print('url added into queue...\n')
def get_data_from_url(self):
print('reading data from blob using url in queue...\n')
data = self.queue_service.get_messages('taskqueue')
response = urllib2.urlopen(data[0].content)
numbers = response.read()
self.queue_service.delete_message('taskqueue', data[0].id,
data[0].pop_receipt)
return numbers.splitlines()
def get_messages_from_queue(self, queue):
return self.queue_service.get_messages(queue)
def delete_message_from_queue(self, queue, message):
self.queue_service.delete_message(queue, message.id,
message.pop_receipt)
class QueueBase(object):
"""docstring for QueueBase"""
def __init__(self):
super(QueueBase, self).__init__()
self.queue_service = QueueService(
account_name='bobur',
account_key=
'6e60FZapOXAmUbFBw0SpE1lHRP3RkXOMYRaalWmRBoz4+xI5tvjaJzxXuYyt+yfWxjPXpz5X3PmyIFiQmSkjbw=='
)
def save_message_to_queue(self, queue, data):
self.queue_service.put_message(queue, data)
print('url added into queue...\n')
def get_data_from_url(self):
print('reading data from blob using url in queue...\n')
data = self.queue_service.get_messages('taskqueue')
response = urllib2.urlopen(data[0].content)
numbers = response.read()
self.queue_service.delete_message('taskqueue', data[0].id,
data[0].pop_receipt)
return numbers.splitlines()
def get_messages_from_queue(self, queue):
return self.queue_service.get_messages(queue)
def delete_message_from_queue(self, queue, message):
self.queue_service.delete_message(queue, message.id,
message.pop_receipt)
class AzureQueue(object):
def __init__(self, queue_name):
self.conn = QueueService(account_name=os.getenv('AZURE_ACCOUNT_NAME'),
account_key=os.getenv('AZURE_ACCOUNT_KEY'))
self.queue_name = queue_name
self.conn.create_queue(queue_name)
self.conn.encode_function = QueueMessageFormat.binary_base64encode
self.conn.decode_function = QueueMessageFormat.binary_base64decode
def enqueue(self, func, *args, **kwargs):
task = SimpleTask(func, *args, **kwargs)
serialized_task = pickle.dumps(task, protocol=pickle.HIGHEST_PROTOCOL)
self.conn.put_message(self.queue_name, serialized_task)
return task.id
def dequeue(self):
messages = self.conn.get_messages(self.queue_name)
if len(messages) == 1:
serialized_task = messages[0]
task = pickle.loads(serialized_task.content)
self.conn.delete_message(self.queue_name, serialized_task.id,
serialized_task.pop_receipt)
return task
def get_length(self):
metadata = self.conn.get_queue_metadata(self.queue_name)
return metadata.approximate_message_count
def scrapping():
queue_service = QueueService(account_name=account_name, account_key=account_key)
while True:
messages = queue_service.get_messages('monitoring')
if messages:
for message in messages:
print(f"Receiving user {message.content} from monitoring service")
queue_service.delete_message('monitoring', message.id, message.pop_receipt)
time.sleep(30)
class AzureStorageQueue(Queue):
"""Interface for interacting with an Azure Storage Queue (through the Queue
contract)"""
def __init__(self, queue_name, config: AzureStorageConfig):
"""Initializes the storage queue.
:param queue_name: The name of the queue to access.
If a queue with this name doesn't already exist on the
storage account, the queue will be created on the first operation.
:param config: AzureStorageConfig with a valid account name and
account key
"""
self._queue_name = queue_name
self._queue_service = QueueService(account_name=config.account_name,
account_key=config.account_key)
self._queue_service.encode_function = \
QueueMessageFormat.text_base64encode
self._queue_service.decode_function = \
QueueMessageFormat.text_base64decode
def push(self, message):
"""Pushes a new message onto the queue."""
self._queue_service.create_queue(self._queue_name)
self._queue_service.put_message(self._queue_name, message)
def pop(self):
"""Pops the first message from the queue and returns it."""
self._queue_service.create_queue(self._queue_name)
# get_messages prevents another client from getting the message
# before we've had a chance to delete it. The visibility_timeout
# prevents the message from being seen by other clients
# for X seconds.
messages = self._queue_service.get_messages(self._queue_name,
visibility_timeout=30)
for message in messages:
result = message.content
self._queue_service.delete_message(self._queue_name, message.id,
message.pop_receipt)
return result
def peek(self):
"""Peeks the fist message from the queue and returns it."""
self._queue_service.create_queue(self._queue_name)
messages = self._queue_service.peek_messages(self._queue_name)
for message in messages:
return message.content
@staticmethod
def create(queue_name: str):
"""Helper function for creating a Azure Storage Queue from the
storage_config property defined inside of AzureConfig."""
azure_config = AzureConfig()
return AzureStorageQueue(queue_name, azure_config.storage_config)
def get_new_proxy():
queue_service = QueueService(account_name=account_name,
account_key=account_key)
messages = queue_service.get_messages('proxies')
if messages:
for message in messages:
new_proxy = message.content
queue_service.delete_message('proxies', message.id,
message.pop_receipt)
queue_service.put_message('proxies', new_proxy)
return new_proxy
class RegistrationQueueClient(object):
"""Obtains next registration object from queue"""
queueName = 'registrationsqueue'
def __init__(self, account_name, account_key):
self._queueService = QueueService(account_name, account_key)
print("RegistrationQueue Initialized")
def LookupTicket(self, processData):
message = self._queueService.get_messages(self.queueName, num_messages=1)
if(processData(message)):
self._queueService.delete_message(self.queueName, message.message_id, message.pop_receipt)
class SimulationListener(Thread):
def __init__(self):
Thread.__init__(self)
self._quit = Event()
self.daemon = True
self.log = logging.getLogger(__name__)
settings = Settings()
self.create_queue = 'create-simulated-machine'
self.destroy_queue = 'destroy-simulated-machine'
self.queue_service = QueueService(
account_name=settings.azure_queue_account,
account_key=settings.azure_queue_key
)
self.queue_service.create_queue(self.create_queue)
self.queue_service.create_queue(self.destroy_queue)
def run(self):
# dislike of unstoppable threads
while not self._quit.is_set():
try:
messages = self.queue_service.get_messages(self.create_queue, numofmessages=10)
for m in messages:
machine_json = b64decode(m.message_text)
machine = json.loads(machine_json)
self.log.info("Creating: " + machine["Name"] + " on " + machine["Provider"])
self.log.debug(machine_json)
requests.post("http://localhost:8080/machines", machine_json)
self.queue_service.delete_message(self.create_queue, m.message_id, m.pop_receipt)
except Exception as e:
self.log.error(e.message)
try:
messages = self.queue_service.get_messages(self.destroy_queue, numofmessages=10)
for m in messages:
machine_json = b64decode(m.message_text)
machine = json.loads(machine_json)
self.log.info("Deleting: " + machine["Name"] + " on " + machine["Provider"])
self.log.debug(machine_json)
requests.delete("http://localhost:8080/machines/" + machine["MachineId"])
self.queue_service.delete_message(self.destroy_queue, m.message_id, m.pop_receipt)
except Exception as e:
self.log.error(e.message)
sleep(1)
def quit(self):
self._quit.set()
def counting():
queue_service = QueueService(account_name=account_name,
account_key=account_key)
while True:
messages = queue_service.get_messages('newuser')
if messages:
for message in messages:
print(f"Receiving {message.content} from newuser channel...")
add_new_user(message.content)
print(f"Deleteing {message.content} from queue of users...")
queue_service.delete_message('newuser', message.id,
message.pop_receipt)
new_proxy = get_new_proxy()
print(f"Receiving proxy {new_proxy} from queue of proxies...")
time.sleep(30)
class RegistrationQueueClient(object):
"""Obtains next registration object from queue"""
queueName = 'registrationsqueue'
def __init__(self, account_name, account_key):
self._queueService = QueueService(account_name, account_key)
print("RegistrationQueue Initialized")
def LookupTicket(self, processData):
message = self._queueService.get_messages(self.queueName,
num_messages=1)
if (processData(message)):
self._queueService.delete_message(self.queueName,
message.message_id,
message.pop_receipt)
def main():
storage_account_name = str(sys.argv[1])
queue_name = str(sys.argv[2])
destination_folder = str(sys.argv[3])
if not storage_account_name or not queue_name or not destination_folder:
print("usage: get_message.py storage_account_name queue_name destination_folder")
sys.exit(1)
base64_regex = re.compile("^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?$")
key_file = "{0}/{1}/acc_key.txt".format(destination_folder, storage_account_name)
try :
with open(key_file, "r") as f:
account_key = f.read()
except IOError:
print("file not found or empty ", key_file)
queue_service = QueueService(storage_account_name, account_key)
metadata = queue_service.get_queue_metadata(queue_name)
approximate_message_count = metadata.approximate_message_count
print('fetching approximately {0} messages'.format(approximate_message_count))
messages = queue_service.get_messages(queue_name)
count = len(messages)
total = 0
while count > 0:
for message in messages:
if base64_regex.match(message.content):
msg = base64.b64decode(message.content) # Received message is encoded in base 64
data = json.loads(msg.decode('utf8'))
else:
msg = message.content
data = json.loads(msg)
filename, folder = get_file_name(storage_account_name, queue_name, destination_folder)
if not os.path.exists(folder):
os.makedirs(folder)
with open(r'{0}/{1}'.format(folder,filename), 'w') as f:
json.dump(data, f, indent=2)
queue_service.delete_message(queue_name, message.id, message.pop_receipt)
count -= 1
total += 1
print('successfully processed {0} messages of ~{1} from {2} queue {3}'.format(total,approximate_message_count,storage_account_name,queue_name))
messages = queue_service.get_messages(queue_name)
count = len(messages)
class AzureQueue(object):
def __init__(self, account_name, account_key, queue_name):
self.queue_name = queue_name
self.queue_service = QueueService(account_name=account_name, account_key=account_key)
self.queue_service.create_queue(self.queue_name)
def put_message_into_queue(self, content) -> QueueMessage:
"""
Publishes a message with `content`
:param content: The queue message
:returns: A QueueMessage that has the message as well as metadata
:rtype: QueueMessage
"""
return self.queue_service.put_message(self.queue_name, content)
def get_messages(self) -> list:
"""
Retrieves all of the messages that have been published into queue
:param content: The queue message
:returns: List of Queue messages
:rtype: list
"""
return self.queue_service.get_messages(self.queue_name)
def delete_message_from_queue(self, message_id, pop_receipt):
self.queue_service.delete_message(self.queue_name, message_id, pop_receipt)
def get_message_count(self):
queue_metadata = self.queue_service.get_queue_metadata(self.queue_name)
return queue_metadata.approximate_message_count
def delete(self):
return self.queue_service.delete_queue(self.queue_name)
def empty(self):
messages = queue_service.get_messages(self.queue_name,
num_messages=BATCH_NUMBER,
visibility_timeout=TIMEOUT_IN_SECONDS)
for message in messages:
self.queue_service.delete_message(self.queue_name, message.id, message.pop_receipt)
def queue():
account_name = config.STORAGE_ACCOUNT_NAME
account_key = config.STORAGE_ACCOUNT_KEY
queue_service = QueueService(account_name=account_name,
account_key=account_key)
print("Creating task queue")
task_queue_name = config.TASK_QUEUE_NAME
queue_service.create_queue(task_queue_name)
print("Task queue created")
queue_service.put_message(task_queue_name, u'message1')
messages = queue_service.get_messages(task_queue_name, num_messages=16)
for message in messages:
print(message.content)
queue_service.delete_message(task_queue_name, message.id,
message.pop_receipt)
class MoeHandler():
def __init__(self, api, generator, queue_name, conn_str):
self.api = api
self.gen_util = Generate_util(generator)
self.queue_name = queue_name
self.queue_service = QueueService(connection_string=conn_str)
self.queue_service.create_queue(queue_name)
# loops forever, reading messages and sending to gen util for parsing. Then sends the message
def read_messages(self):
while 1:
try:
messages = self.queue_service.get_messages(self.queue_name)
if not messages:
time.sleep(2) # We'll wait a few seconds and check again.
continue
for message in messages:
event = json.loads(message.content)
if event['msg_type'] == 'tweet':
self.handle_tweet(event)
elif event['msg_type'] == 'dm':
self.handle_dm(event)
self.queue_service.delete_message(
self.queue_name, message.id, message.pop_receipt)
except Exception as e:
print("Exception handling messages")
print(e)
def handle_dm(self, event):
imgpth = self.gen_util.parse_msg(event['text'])
media = self.api.media_upload(imgpth)
self.api.send_direct_message(
recipient_id=event['user_id'], text="Here's your generated anime girl!", attachment_type='media', attachment_media_id=media.media_id)
return
def handle_tweet(self, event):
imgpth = self.gen_util.parse_msg(event['text'])
self.api.update_with_media(imgpth, status="Here's your generated anime girl!",
in_reply_to_status_id=event['respond_id'], auto_populate_reply_metadata=True)
return
class TimeLineWorker:
QUEUE_KEY = ''
ACCOUNT_NAME = ''
QUEUE_NAME = ''
def __init__(self):
self.queue_service = QueueService(account_name=TimeLineWorker.ACCOUNT_NAME,
account_key=TimeLineWorker.QUEUE_KEY)
def insert_message(self):
obj = {
"message": "test message",
"other_key": 10
}
message = unicode(json.dumps(obj))
self.queue_service.put_message(TimeLineWorker.QUEUE_NAME, message)
def get_next_message(self):
messages = self.queue_service.get_messages(TimeLineWorker.QUEUE_NAME)
for message in messages:
print message.content
self.queue_service.delete_message(TimeLineWorker.QUEUE_NAME, message.id, message.pop_receipt)
if "chronos" in os.environ:
chronos = os.environ['chronos']
while True:
# set up azure queue
queue_service = QueueService(account_name=azurestoracct, account_key=azurequeuekey)
# get messages from azure queue
messages = queue_service.get_messages(azurequeue, num_messages=5)
# delete from queue, create container, start container
for message in messages:
# delete message from azure queue
queue_service.delete_message(azurequeue, message.id, message.pop_receipt)
if "docker" in os.environ:
# sample json
# {"Image": "neilpeterson/stock-report","Cmd": ["--symbols=msft;lnkd", "[email protected]"],"Env": ["gmuser = [email protected]", "gmpass = TempForDemo2016"]}
s = message.content.split(':')
data = json.loads('{"Image": "' + docker_image + '","Cmd": ["--symbols=' + s[0] +'", "--email=' + s[1] + '","--delay=' + str(delay) + '"]}')
print(data)
# create and start docker container
headers = {'Content-Type': 'application/json'}
r = requests.post(docker + "create", data=json.dumps(data), headers=headers)
b = json.loads(r.text)
x = requests.post(docker + b['Id'] + "/start")
'redis_server': os.environ['REDIS_SERVER'],
'sms_user': os.environ['SMS_USER'],
'sms_pass': os.environ['SMS_PASS']
}
stor_acc_name = service_keys['stor_acc_name']
stor_acc_key = service_keys['stor_acc_key']
redis_pass = service_keys['redis_pass']
redis_server = service_keys['redis_server']
sms_user = service_keys['sms_user']
sms_pass = service_keys['sms_pass']
# storage
queue_service = QueueService(account_name=stor_acc_name, account_key=stor_acc_key)
# redis
r = redis.StrictRedis(host=redis_server, port=6380, db=0, password=redis_pass, ssl=True)
while True:
messages = queue_service.get_messages('taskqueue', numofmessages=16, visibilitytimeout=5*60)
for message in messages:
d = json.loads(message.message_text)
suffix = d['suffix']
mobile = r.get(suffix)
image = d['image']
odp = str(image)
payload = {'username': sms_user, 'password': sms_pass, 'from': 'Alert', 'to': mobile, 'message': odp}
post = requests.post('https://api.smsapi.pl/sms.do', data=payload)
queue_service.delete_message('taskqueue', message.message_id, message.pop_receipt)
class AzureQueueStream:
def __init__(self, start_topic, bingKey, computerVisionApiKey, storageKey):
self.stop_event = threading.Event()
self.counter = 0
self.bingKey = bingKey
self.computerVisionApiKey = computerVisionApiKey
self.queue_service = None
if storageKey:
self.queue_service = QueueService(account_name='rpiimagedetectj34n5m', account_key=storageKey)
self.queue_service.create_queue('rpi-queue')
downloader = ImageDownloader(self.computerVisionApiKey)
self.source_thread = FrameStream(BingImageSource(start_topic, self.bingKey, downloader)).start()
def start(self):
t = threading.Thread(target=self.get_topic)
t.daemon = True
t.start()
return self
def get_topic(self):
while True:
if self.stop_event.is_set():
return
# drain queue, retrieve last message
lastMsg = None
if self.queue_service:
messages = self.queue_service.get_messages('rpi-queue')
for message in messages:
lastMsg = message
self.queue_service.delete_message('rpi-queue', message.id, message.pop_receipt)
if lastMsg and lastMsg.content:
jsonMsg = json.loads(base64.b64decode(lastMsg.content).decode('ascii'))
searchTopic = jsonMsg.get("Text", "dog")
self.source_thread.stop()
downloader = ImageDownloader(self.computerVisionApiKey, jsonMsg)
if searchTopic.startswith("http"):
self.source_thread = FrameStream(RemoteImageSource(searchTopic, downloader)).start()
else:
print("Bing Image Search topic:", searchTopic)
self.source_thread = FrameStream(BingImageSource(searchTopic, self.bingKey, downloader), queueSize = 16).start()
self.counter = 0
time.sleep(2)
def next_frame(self):
return self.source_thread.next_frame()
def stop(self):
self.source_thread.stop()
self.stop_event.set()
def send_to_azure(self, frameInfo, predictions):
msg = frameInfo.get("attrs", {}).get("template")
if self.queue_service and self.counter < 10 and msg and msg.get("RelatesTo", {}):
self.counter += 1
msg["Url"] = frameInfo.get("attrs", {}).get("url", "")
msg["Text"] = None
if predictions:
msg["Text"] = predictions[0][0]
msg["Label"] = frameInfo.get("attrs", {}).get("visionApiLabel", "")
self.queue_service.put_message('bot-queue', base64.b64encode(json.dumps(msg).encode('ascii')).decode('ascii'))
class Azure_Storage():
def __init__(self, create_new=False):
account_name = config.STORAGE_ACCOUNT_NAME
account_key = config.STORAGE_ACCOUNT_KEY
self.task_queue_name = config.TASK_QUEUE_NAME
self.table_name = config.TABLE_NAME
self.container_name = config.BLOB_CONTAINER_NAME
self.ImagePartitionKey = config.IMAGE_PARTITION_KEY
self.table_service = TableService(account_name=account_name,
account_key=account_key)
self.block_blob_service = BlockBlobService(account_name=account_name,
account_key=account_key)
self.queue_service = QueueService(account_name=account_name,
account_key=account_key)
if create_new:
queue_service.create_queue(task_queue_name)
block_blob_service.create_container(container_name)
table_service.create_table(table_name)
def put_image(self, image_uuid, image_bytes):
ret = self.block_blob_service.create_blob_from_bytes(
self.container_name, image_uuid, image_bytes)
return ret
def get_image(self, image_uuid):
ret = self.block_blob_service.get_blob_to_bytes(
self.container_name, image_uuid).content
return ret
def put_classification_result(self, image_uuid, results):
task = Entity()
task.PartitionKey = self.ImagePartitionKey
task.RowKey = image_uuid
task.results = str(results)
ret = self.table_service.insert_or_replace_entity(
self.table_name, task)
return ret
def get_classification_result(self, image_uuid):
try:
task = self.table_service.get_entity(self.table_name,
self.ImagePartitionKey,
image_uuid)
return task.results
except Exception as e:
return None
def put_task(self, taskmsg):
ret = self.queue_service.put_message(self.task_queue_name, taskmsg)
return ret
#payload is in message.content
def get_task(self, num_messages=16):
messages = self.queue_service.get_messages(self.task_queue_name,
num_messages=num_messages,
visibility_timeout=1 * 60)
return messages
def delete_task(self, message):
ret = self.queue_service.delete_message(self.task_queue_name,
message.id,
message.pop_receipt)
return ret
def rendered_video(request):
ism_uri = ''
vtt_uri = ''
template = loader.get_template('app/rendered_video.html')
vidstatus = 'No Running Job Found.'
# Get the next message from the queue
queue_service = QueueService(account_name=os.environ['SVPD_STORAGE_ACCOUNT_NAME'], account_key=os.environ['SVPD_STORAGE_ACCOUNT_KEY'])
messages = queue_service.get_messages(os.environ['SVPD_STORAGE_ACCOUNT_ENCODING'], num_messages=1, visibility_timeout=1*60)
for message in messages:
vidstatus = 'Rendering: ' + message.content
message_obj = json.loads(message.content)
access_token = ams_authenticate()['access_token']
# Get the details about the job
job = ams_get_request(access_token, message_obj['job']['__metadata']['uri'])
# is it done?
if job['State'] == 3:
vidstatus = 'Done Rendering: ' + message.content
#get a reference to our storage container
block_blob_service = BlockBlobService(account_name=os.environ['SVPD_STORAGE_ACCOUNT_NAME'], account_key=os.environ['SVPD_STORAGE_ACCOUNT_KEY'])
#get a list of all the input and output assets associated to our job
input_assets = ams_get_request(access_token, message_obj['job']['InputMediaAssets']['__deferred']['uri'])
output_assets = ams_get_request(access_token, message_obj['job']['OutputMediaAssets']['__deferred']['uri'])
#look through the input and output assets to figure out what one is for the indexer and for the Adaptive streaming files
index_asset = ''
stream_asset = ''
for output_asset in output_assets['value']:
if output_asset['Name'].endswith('- Indexed'):
index_asset = output_asset
elif output_asset['Name'].endswith('- MES v1.1'):
stream_asset = output_asset
#Get the storage container names for each
dest_container = urllib.parse.urlparse(stream_asset['Uri']).path[1:]
src_container = urllib.parse.urlparse(index_asset['Uri']).path[1:]
#loop over the indexer output files copying them to the adaptive streaming container
src_blobs = block_blob_service.list_blobs(src_container)
for src_blob in src_blobs:
block_blob_service.copy_blob(dest_container, src_blob.name, output_asset['Uri'] + '/' + src_blob.name)
#create the access policy if it doen't exist
access_policies = ams_get_request(access_token, os.environ['AMS_API_ENDPOINT'] + 'AccessPolicies')
access_policy_id = ''
for access_policy in access_policies['value']:
if access_policy['Name'] == 'StreamingAccessPolicy':
access_policy_id = access_policy['Id']
if access_policy_id == '':
access_policy = ams_verbose_post_request(access_token, 'AccessPolicies', {
'Name': 'StreamingAccessPolicy',
'DurationInMinutes': '52594560',
'Permissions': '9'
})
access_policy_id = access_policy['d']['Id']
#create the locator
locator = ams_verbose_post_request(access_token, 'Locators', {
'AccessPolicyId': access_policy_id,
'AssetId': stream_asset['Id'],
'Type': 2
})
#get the URLs to the streaming endpoint and the vtt file
locator_asset_files = ams_get_request(access_token, os.environ['AMS_API_ENDPOINT'] + 'Assets(\'' + locator['d']['AssetId'] + '\')/Files')
for locator_asset_file in locator_asset_files['value']:
if locator_asset_file['Name'].endswith('.ism'):
ism_uri = locator['d']['Path'] + locator_asset_file['Name'] + '/manifest'
vtt_uri = locator['d']['Path'] + message_obj['filename'] + '.vtt'
#delete the job
ams_delete_request(access_token, message_obj['job']['__metadata']['uri'])
#delete the unused assets
ams_delete_request(access_token, os.environ['AMS_API_ENDPOINT'] + 'Assets(\'' + index_asset['Id'] + '\')')
ams_delete_request(access_token, os.environ['AMS_API_ENDPOINT'] + 'Assets(\'' + input_assets['value'][0]['Id'] + '\')')
#add the video to the database
client = document_client.DocumentClient(os.environ['DOCUMENT_ENDPOINT'], {'masterKey': os.environ['DOCUMENT_KEY']})
db = docdb_CreateDatabaseIfNotExists(client, 'svpd')
collection = docdb_CreateCollectionIfNotExists(client, db, 'videos')
doc = client.CreateDocument(collection['_self'],
{
'id': message_obj['folder'].replace('/', '.'),
'filename': message_obj['filename'],
'vtt_uri': vtt_uri,
'ism_uri': ism_uri
})
#remove the message from the queue
queue_service.delete_message(os.environ['SVPD_STORAGE_ACCOUNT_ENCODING'], message.id, message.pop_receipt)
return HttpResponse(template.render({
'vidstatus': vidstatus,
'vtt_uri': vtt_uri,
'ism_uri': ism_uri
}, request))
class MainPawWorker:
"""
Main class to use for running a worker. call start_workers() to start.
"""
def __init__(self,
azure_storage_name,
azure_storage_private_key,
azure_queue_name,
azure_table_name,
tasks_module,
workers,
visibility_timeout=VISIBILITY_TIMEOUT):
"""
:param azure_storage_name: Name of Azure storage account
:param azure_storage_private_key: Private key of Azure storage account.
:param azure_queue_name: Name of the Azure queue to use.
:param azure_table_name: Name of the Azure table to use.
:param tasks_module: Module containing decorated functions to load from.
:param workers: Int of workers. Ex: 4
:param visibility_timeout: Seconds in int to keep message in Azure queue
"""
self.account_name = azure_storage_name
self.account_key = azure_storage_private_key
self.queue_name = azure_queue_name
self.table_name = azure_table_name
self.tasks_module = tasks_module
self.workers = workers
self.visibility_timeout = visibility_timeout
if self.visibility_timeout > MAXIMUM_VISIBILITY_TIMEOUT:
raise PawError('self.visibility_timeout bigger than allowed limit')
self.queue_service = QueueService(account_name=self.account_name,
account_key=self.account_key)
self.table_service = TableService(account_name=self.account_name,
account_key=self.account_key)
self.local_queue = Queue(self.workers)
# self.logger = logging.getLogger()
self.logger = LOGGER
self.logger.info(PAW_LOGO)
self.worker_process = Worker(
local_queue=self.local_queue,
queue_service=self.queue_service,
queue_name=self.queue_name,
table_service=self.table_service,
table_name=azure_table_name,
tasks=self._load_tasks(),
)
self.pool = Pool(self.workers, self.worker_process.run, ())
signal.signal(signal.SIGTERM, self.on_exit)
def on_exit(self, signum, frame):
self.pool.terminate()
sys.exit()
def _load_tasks(self):
"""
Loads and returns decorated functions from a given modules, as a dict
"""
tasks = dict([
o for o in getmembers(self.tasks_module)
if isfunction(o[1]) and hasattr(o[1], 'paw')
])
for t, f in tasks.items():
self.logger.info("REGISTERED '{}'".format(t))
if f.description:
self.logger.info("\tdescription: '{}'".format(f.description))
if not tasks:
self.logger.warning("No tasks found...")
return tasks
def start_workers(self, sleep_for=5):
"""
Starts workers and picks message from the Azure queue. On new
message, when the local queue has room, the message is placed for a
worker to pick-up
:param sleep_for: Seconds to sleep for after a loop end.
"""
self.queue_service.create_queue(self.queue_name)
create_table_if_missing(self.table_service, self.table_name)
try:
self.logger.info(
"Cleaning up dead jobs left in {}".format(STARTED))
dead_jobs = self.table_service.query_entities(
table_name=self.table_name,
filter="status eq '{}'".format(STARTED))
for job in dead_jobs.items:
log_to_table(table_service=self.table_service,
table_name=self.table_name,
message=job,
status=LOST_WORKER,
result="Lost worker, or task aborted.")
except AzureException as e:
self.logger.error("Cleaning dead tasks failed: {}".format(e))
while True:
if self.local_queue.full():
time.sleep(sleep_for)
try:
new_msg = self.queue_service.get_messages(
queue_name=self.queue_name,
num_messages=1,
visibility_timeout=self.visibility_timeout)
except AzureException:
self.logger.error("Error while getting message "
"from Azure queue. Trying to create "
"the queue")
self.queue_service.create_queue(self.queue_name)
time.sleep(sleep_for)
continue
if new_msg:
msg = new_msg[0]
try:
content = json.loads(msg.content)
except json.JSONDecodeError:
self.logger.critical('Json error {}'.format(
traceback.format_exc()))
try:
self.queue_service.delete_message(
queue_name=self.queue_name,
message_id=msg.id,
pop_receipt=msg.pop_receipt)
except AzureException:
self.logger.critical(
'Deleting invalid message from queue failed: '
'{}'.format(traceback.format_exc()))
continue
if msg.dequeue_count > MAXIMUM_DEQUEUE_COUNT:
log_to_table(
table_service=self.table_service,
table_name=self.table_name,
message=content,
status=FAILED,
result="PAW MESSAGE: Dequeue count exceeded.",
)
self.queue_service.delete_message(self.queue_name, msg.id,
msg.pop_receipt)
continue
content['msg'] = msg
while True:
try:
self.local_queue.put_nowait(content)
break
except Full:
self.logger.info('LOCAL QUEUE FULL: waiting...')
time.sleep(sleep_for)
self.logger.debug('ADDING: {}'.format(content['task_name']))
time.sleep(sleep_for)
def deletemessage(messageid, popreceipt):
queue_service = QueueService(account_name=azurestoracct,
account_key=azurequeuekey)
queue_service.delete_message(azurequeue, messageid, popreceipt)
from azure.storage.queue import QueueService
queue_service = QueueService(account_name='xxx', account_key='xxx')
messages = queue_service.get_messages('queue02',
num_messages=1,
visibility_timeout=5 * 60)
for message in messages:
print(message.content)
queue_service.delete_message('queue02', message.id, message.pop_receipt)
class StorageQueueContext():
"""Initializes the repository with the specified settings dict.
Required settings in config dict are:
- AZURE_STORAGE_NAME
- AZURE_STORAGE_KEY
"""
_models = []
_service = None
_storage_key = ''
_storage_name = ''
def __init__(self, **kwargs):
self._storage_name = kwargs.get('AZURE_STORAGE_NAME', '')
self._storage_key = kwargs.get('AZURE_STORAGE_KEY', '')
""" service init """
if self._storage_key != '' and self._storage_name != '':
self._service = QueueService(account_name = self._storage_name, account_key = self._storage_key, protocol='https')
""" registered models """
self._models = []
""" encrypt queue service """
if kwargs.get('AZURE_REQUIRE_ENCRYPTION', False):
# Create the KEK used for encryption.
# KeyWrapper is the provided sample implementation, but the user may use their own object as long as it implements the interface above.
kek = KeyWrapper(kwargs.get('AZURE_KEY_IDENTIFIER', 'otrrentapi'), kwargs.get('SECRET_KEY', 'super-duper-secret')) # Key identifier
# Create the key resolver used for decryption.
# KeyResolver is the provided sample implementation, but the user may use whatever implementation they choose so long as the function set on the service object behaves appropriately.
key_resolver = KeyResolver()
key_resolver.put_key(kek)
# Set the require Encryption, KEK and key resolver on the service object.
self._service.require_encryption = True
self._service.key_encryption_key = kek
self._service.key_resolver_funcion = key_resolver.resolve_key
pass
def __create__(self, queue) -> bool:
if (not self._service is None):
try:
self._service.create_queue(queue)
return True
except AzureException as e:
log.error('failed to create {} with error {}'.format(queue, e))
return False
else:
return True
pass
def register_model(self, storagemodel:object):
modelname = storagemodel.__class__.__name__
if isinstance(storagemodel, StorageQueueModel):
if (not modelname in self._models):
self.__create__(storagemodel._queuename)
self._models.append(modelname)
log.info('model {} registered successfully. Models are {!s}'.format(modelname, self._models))
pass
def put(self, storagemodel:object) -> StorageQueueModel:
""" insert queue message into storage """
modelname = storagemodel.__class__.__name__
if isinstance(storagemodel, StorageQueueModel):
if (modelname in self._models):
""" peek first message in queue """
try:
message = self._service.put_message(storagemodel._queuename, storagemodel.getmessage())
storagemodel.mergemessage(message)
except AzureException as e:
log.error('can not save queue message: queue {} with message {} because {!s}'.format(storagemodel._queuename, storagemodel.content, e))
storagemodel = None
else:
log.info('please register model {} first'.format(modelname))
storagemodel = None
else:
log.info('model {} is not a Queue Model'.format(modelname))
storagemodel = None
return storagemodel
def peek(self, storagemodel:object) -> StorageQueueModel:
""" lookup the next message in queue """
modelname = storagemodel.__class__.__name__
if isinstance(storagemodel, StorageQueueModel):
if (modelname in self._models):
""" peek first message in queue """
try:
messages = self._service.peek_messages(storagemodel._queuename, num_messages=1)
""" parse retrieved message """
for message in messages:
storagemodel.mergemessage(message)
""" no message retrieved ?"""
if storagemodel.id is None:
storagemodel = None
except AzureException as e:
log.error('can not peek queue message: queue {} with message {} because {!s}'.format(storagemodel._queuename, storagemodel.content, e))
storagemodel = None
else:
log.info('please register model {} first'.format(modelname))
storagemodel = None
else:
log.info('model {} is not a Queue Model'.format(modelname))
storagemodel = None
return storagemodel
def get(self, storagemodel:object, hide = 0) -> StorageQueueModel:
""" lookup the next message in queue """
modelname = storagemodel.__class__.__name__
if isinstance(storagemodel, StorageQueueModel):
if (modelname in self._models):
""" get first message in queue """
try:
if hide > 0:
messages = self._service.get_messages(storagemodel._queuename, num_messages=1, visibility_timeout = hide)
else:
messages = self._service.get_messages(storagemodel._queuename, num_messages=1)
""" parse retrieved message """
for message in messages:
storagemodel.mergemessage(message)
""" no message retrieved ?"""
if storagemodel.id is None:
storagemodel = None
except AzureException as e:
log.error('can not get queue message: queue {} with message {} because {!s}'.format(storagemodel._queuename, storagemodel.content, e))
storagemodel = None
else:
log.info('please register model {} first'.format(modelname))
storagemodel = None
else:
log.info('model {} is not a Queue Model'.format(modelname))
storagemodel = None
return storagemodel
def update(self, storagemodel:object, hide = 0) -> StorageQueueModel:
""" update the message in queue """
modelname = storagemodel.__class__.__name__
if isinstance(storagemodel, StorageQueueModel):
if (modelname in self._models):
""" check if message in queue """
if (storagemodel.id != '') and (storagemodel.pop_receipt != '') and (not storagemodel.id is None) and (not storagemodel.pop_receipt is None):
try:
content = storagemodel.getmessage()
message = self._service.update_message(storagemodel._queuename, storagemodel.id, storagemodel.pop_receipt, visibility_timeout = hide, content=content)
storagemodel.content = content
storagemodel.pop_receipt = message.pop_receipt
except AzureException as e:
log.error('can not update queue message: queue {} with message.id {!s} because {!s}'.format(storagemodel._queuename, storagemodel.id, e))
storagemodel = None
else:
log.info('cant update queuemessage {} due to missing id and pop_receipt'.format(modelname))
storagemodel = None
else:
log.info('please register model {} first'.format(modelname))
storagemodel = None
else:
log.info('model {} is not a Queue Model'.format(modelname))
storagemodel = None
return storagemodel
def delete(self, storagemodel:object) -> bool:
""" delete the message in queue """
modelname = storagemodel.__class__.__name__
deleted = False
if isinstance(storagemodel, StorageQueueModel):
if (modelname in self._models):
""" check if message in queue """
if (storagemodel.id != '') and (storagemodel.pop_receipt != '') and (not storagemodel.id is None) and (not storagemodel.pop_receipt is None):
try:
self._service.delete_message(storagemodel._queuename, storagemodel.id, storagemodel.pop_receipt)
deleted = True
except AzureException as e:
log.error('can not delete queue message: queue {} with message.id {!s} because {!s}'.format(storagemodel._queuename, storagemodel.id, e))
else:
log.info('cant update queuemessage {} due to missing id and pop_receipt'.format(modelname))
else:
log.info('please register model {} first'.format(modelname))
else:
log.info('model {} is not a Queue Model'.format(modelname))
return deleted
class Queue:
def __init__(self, account_name, account_key, queue_name="logqueue"):
"""Initialiaze a queue. The type is set by the
'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to
'AzureStorageQueue' then values must be provided for
'account_name' and 'account_key' which are values associated
with the Azure Storage account. 'queue_name' is optional and
defaults to 'logqueue'.
"""
self.log = Log()
self.queue_type = config.ACS_LOGGING_QUEUE_TYPE
self.queue_name = queue_name
# self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name)
if self.queue_type == "AzureStorageQueue":
self.createAzureQueues(account_name, account_key)
elif self.queue_type == "LocalFile":
self.file_queue = open(config.UNPROCESSED_LOG_FILE, 'w+')
else:
self.log.error("Unknown queue type: " + queue_type)
def getName(self):
return self.queue_name
def createAzureQueues(self, account_name, account_key):
"""
Create a queue for unprocessed log messages. Entries in the queue
will be dequeued, processed and deleted upon success.
"""
self.queue_service = QueueService(account_name, account_key)
self.queue_service.create_queue(self.queue_name)
def close(self):
"""Perform any necessary clearnup on the queue
at the end of a run.
"""
if self.queue_type == "AzureStorageQueue":
pass
elif self.queue_type == "LocalFile":
self.file_queue.close()
else:
self.log.error("Unknown queue type: " + queue_type)
def enqueue(self, msg):
if self.queue_type == "LocalFile":
file_queue.write(msg + '\n')
elif self.queue_type == "AzureStorageQueue":
self.queue_service.put_message(self.queue_name, msg)
else:
self.log.error("We don't know how to handle queues of type " + self.queue_type)
self.log.debug(msg)
def dequeue(self):
messages = []
if self.queue_type == "LocalFile":
with open(config.UNPROCESSED_LOG_FILE, 'r') as f:
messages = f.readlines()[1]
elif self.queue_type == "AzureStorageQueue":
messages = self.queue_service.get_messages(self.queue_name)
return messages
def delete(self, message):
self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt)
# with open(config.PROCESSED_LOG_FILE, 'a') as processed:
# processed.write(log)
# os.remove(config.UNPROCESSED_LOG_FILE)
def delete_queue(self, queue_name):
self.queue_service.delete_queue(queue_name, False)
def getLength(self):
"""
Get the approximate length of the queue
"""
queue_metadata = self.queue_service.get_queue_metadata(self.queue_name)
count = queue_metadata['x-ms-approximate-messages-count']
return int(count)
def peek_messages(self, num_messages):
"""
Peek at the top messages in the queue. This method does not remove the
messages from the queue.
"""
return self.queue_service.peek_messages(self.queue_name, num_messages)
class TweetSender(object):
run = True
def __init__(self):
signal.signal(signal.SIGINT, self.signal_handler)
self.fps = int(os.environ.get('FPS', '40'))
self.yd = float(os.environ.get('YEAR_DURATION', '0.5'))
self.queue_name = os.environ['AZURE_QUEUE_NAME']
self.fail_queue_name = '%sfail' % self.queue_name
self.logger = logging.getLogger('main')
twitter_auth = tweepy.OAuthHandler(
os.environ['TWITTER_CONSUMER_KEY'],
os.environ['TWITTER_CONSUMER_SECRET'])
twitter_auth.set_access_token(os.environ['TWITTER_APP_TOKEN_KEY'],
os.environ['TWITTER_APP_TOKEN_SECRET'])
self.twitter_api = tweepy.API(twitter_auth)
twitter_user = self.twitter_api.me()
self.print_stdout('Logged in as @%s' % twitter_user.screen_name)
self.vid_tweet = VideoTweet(self.twitter_api)
self.queue_service = QueueService(
account_name=os.environ['AZURE_STORAGE_ACCOUNT_NAME'],
account_key=os.environ['AZURE_STORAGE_ACCOUNT_KEY'])
self.queue_service.create_queue(self.queue_name)
self.queue_service.create_queue(self.fail_queue_name)
self.print_stdout('Created queue %s' % self.queue_name)
def print_stdout(self, message):
self.logger.info(message)
print(message)
sys.stdout.flush()
def print_exception(self, message):
self.logger.exception(message)
traceback.print_exc()
print(message, file=sys.stderr)
sys.stderr.flush()
def status_url(self, status):
return 'https://twitter.com/%s/status/%s/' % (status.user.screen_name,
status.id)
def signal_handler(self, signal, frame):
self.print_stdout('Exiting...')
self.run = False
def generate_mp4_file(self, data):
if 'percentiles' in data:
return write_mp4(data, self.yd, self.fps)
return None
def process_messages(self):
for message in self.queue_service.get_messages(self.queue_name,
num_messages=1):
try:
content = base64.b64decode(message.content).decode('utf-8')
data = json.loads(content)
filename = self.generate_mp4_file(data)
if filename is not None:
status = self.vid_tweet.tweet(filename, **data['tweet'])
self.print_stdout(
'Sent status with media: %s - %s' %
(data['tweet']['status'], self.status_url(status)))
else:
status = self.twitter_api.update_status(**data['tweet'])
self.print_stdout(
'Sent status: %s - %s' %
(data['tweet']['status'], self.status_url(status)))
except Exception:
self.print_exception('Cannot send status: %s' % content)
self.queue_service.put_message(self.fail_queue_name,
message.content)
self.queue_service.delete_message(self.queue_name, message.id,
message.pop_receipt)
def run(self):
self.print_stdout('Start poll loop')
while self.run:
self.process_messages()
time.sleep(1)
class Queue:
def __init__(self, account_name, account_key, queue_name="logqueue"):
"""Initialiaze a queue. The type is set by the
'ACS_LOGGING_QUEUE_TYPE' environment variable. If it is set to
'AzureStorageQueue' then values must be provided for
'account_name' and 'account_key' which are values associated
with the Azure Storage account. 'queue_name' is optional and
defaults to 'logqueue'.
"""
self.log = Log()
self.queue_type = config.ACS_LOGGING_QUEUE_TYPE
self.queue_name = queue_name
# self.log.debug("Queue type: " + self.queue_type + " / " + self.queue_name)
if self.queue_type == "AzureStorageQueue":
self.createAzureQueues(account_name, account_key)
elif self.queue_type == "LocalFile":
self.file_queue = open(config.UNPROCESSED_LOG_FILE, "w+")
else:
self.log.error("Unknown queue type: " + queue_type)
def createAzureQueues(self, account_name, account_key):
"""
Create a queue for unprocessed log messages. Entries in the queue
will be dequeued, processed and deleted upon success.
"""
self.queue_service = QueueService(account_name, account_key)
self.queue_service.create_queue(self.queue_name)
def close(self):
"""Perform any necessary clearnup on the queue
at the end of a run.
"""
if self.queue_type == "AzureStorageQueue":
pass
elif self.queue_type == "LocalFile":
self.file_queue.close()
else:
self.log.error("Unknown queue type: " + queue_type)
def enqueue(self, msg, level="INFO"):
msg = level + " - " + msg
if self.queue_type == "LocalFile":
file_queue.write(msg + "\n")
elif self.queue_type == "AzureStorageQueue":
self.queue_service.put_message(self.queue_name, msg)
self.log.debug(msg)
def dequeue(self):
messages = []
if self.queue_type == "LocalFile":
with open(config.UNPROCESSED_LOG_FILE, "r") as f:
messages = f.readlines()[1]
elif self.queue_type == "AzureStorageQueue":
messages = self.queue_service.get_messages(self.queue_name)
return messages
def delete(self, message):
self.queue_service.delete_message(self.queue_name, message.message_id, message.pop_receipt)
# with open(config.PROCESSED_LOG_FILE, 'a') as processed:
# processed.write(log)
# os.remove(config.UNPROCESSED_LOG_FILE)
def delete_queue(self, queue_name):
self.queue_service.delete_queue(queue_name, False)
def getLength(self):
"""
Get the approximate length of the queue
"""
queue_metadata = self.queue_service.get_queue_metadata(self.queue_name)
count = queue_metadata["x-ms-approximate-messages-count"]
return count
import os
from azure.storage.queue import QueueService
import time
# Grab environment variables.
AZURE_STORAGE_ACCT = os.environ['AZURE_STORAGE_ACCT']
AZURE_QUEUE = os.environ['AZURE_QUEUE']
AZURE_QUEUE_KEY = os.environ['AZURE_QUEUE_KEY']
# Build queue object
queue_service = QueueService(account_name=AZURE_STORAGE_ACCT,
account_key=AZURE_QUEUE_KEY)
while True:
# Get queue count
metadata = queue_service.get_queue_metadata(AZURE_QUEUE)
queue_length = metadata.approximate_message_count
print(queue_length)
messages = queue_service.get_messages(AZURE_QUEUE, num_messages=32)
for message in messages:
queue_service.delete_message(AZURE_QUEUE, message.id,
message.pop_receipt)
print("Message deleted")
class Workload(object):
def __init__(self, config):
self.config = config
self.log_queue_service = QueueService(account_name=self.config.storage_account_name,
sas_token=self.config.workload_tracker_sas_token)
self.workload_complete_event = None
self.scheduler_start_event = None
self.jobs_queued_done = None
self.processor_events = []
self.processor_fork_events = []
self.job_consolidation_status_events = []
self.job_processing_status_events = []
def get_events(self):
print "Getting Relevant Workload Events"
while True:
# Setting message visibility to 2 days, so that the message won't get processed again
messages = self.log_queue_service.get_messages(self.config.workload_tracker_queue_name, 32)
for msg in messages:
parsed = json.loads(msg.content)
event = WorkloadEvent(WorkloadEventType(parsed["event_type"]), msg.insertion_time, parsed["content"])
# Workload Completed
if event.event_type == WorkloadEventType.WORKLOAD_DONE:
self.workload_complete_event = event
# Scheduler - Main Start
if event.event_type == WorkloadEventType.SCHEDULER_START:
self.scheduler_start_event = event
# Processor - Main Start
if event.event_type == WorkloadEventType.PROCESSOR_START:
self.processor_events.append(event)
# Processor - Fork Start
if event.event_type == WorkloadEventType.PROCESSOR_FORK_START:
self.processor_fork_events.append(event)
# Job Consolidation Status
if event.event_type == WorkloadEventType.WORKLOAD_CONSOLIDATION_STATUS:
self.job_consolidation_status_events.append(event)
# Job Processing Status
if event.event_type == WorkloadEventType.WORKLOAD_PROCESSING_STATUS:
self.job_processing_status_events.append(event)
# All Jobs Queued
if event.event_type == WorkloadEventType.JOBS_QUEUE_DONE:
self.jobs_queued_done = event
if event is not None:
print str(event.timestamp) + " " + event.contents
# Delete the message
self.log_queue_service.delete_message(self.config.workload_tracker_queue_name, msg.id, msg.pop_receipt)
# Stop when the workload is completed
if self.workload_complete_event is not None and self.scheduler_start_event is not None:
break
# Sleeping to avoid spamming if the queue is empty
if not messages:
time.sleep(10)
def time_elapse(self, evt1, evt2):
return divmod((evt2.timestamp - evt1.timestamp).total_seconds(), 60)
def print_summary(self):
print "\nSummary: "
print "Scheduler Started: " + str(self.scheduler_start_event.timestamp)
print "Workload Completed: " + str(self.workload_complete_event.timestamp)
# Jobs Queued
print "All Jobs Queued: " + str(self.jobs_queued_done.timestamp)
elapse = self.time_elapse(self.scheduler_start_event, self.jobs_queued_done)
print "Jobs Queued Elapsed Time: " + str(elapse[0]) + " mins, " + str(elapse[1]) + " secs"
# Workload Completion
elapse = self.time_elapse(self.scheduler_start_event, self.workload_complete_event)
print "Workload Elapsed Time: " + str(elapse[0]) + " mins, " + str(elapse[1]) + " secs"
# Processor Information
self.processor_events.sort(key=lambda evt: evt.timestamp)
print "Number of Processors: " + str(len(self.processor_events))
print "\tNumber of Forked Processors Instances: " + str(len(self.processor_fork_events))
print "\tFirst Processor Up: " + str(self.processor_events[0].timestamp)
print "\tLast Processor Up: " + str(self.processor_events[-1].timestamp)
# Job Processor Completion
self.job_processing_status_events.sort(key=lambda evt: evt.timestamp)
completed_processing_event = None
for job_proc_evt in self.job_processing_status_events:
status = job_proc_evt.contents.split()[-1].split('/')
# Find when 100% of the jobs are reported to be completed processing
if status[0] == status[1]:
completed_processing_event = job_proc_evt
break
print "Final Processing Time: " + str(completed_processing_event.timestamp)
elapse = self.time_elapse(self.scheduler_start_event, completed_processing_event)
print "Processing Elapsed Time: " + str(elapse[0]) + " mins, " + str(elapse[1]) + " secs"
###
# Use the Azure Storage Storage SDK for Python to read each message from the Queue
###
print(
'\nWith some messages in our Azure Storage Queue, let\'s read the first message in the Queue to signal we start to process that customer\'s order.'
)
input('Press Enter to continue...')
# When you get each message, they become hidden from other parts of the applications being able to see it.
# Once you have successfully processed the message, you then delete the message from the Queue.
# This behavior makes sure that if something goes wrong in the processing of the message, it is then dropped back in the Queue for processing in the next cycle.
messages = queue_service.get_messages('pizzaqueue')
for message in messages:
print(('\n' + message.content))
queue_service.delete_message('pizzaqueue', message.id, message.pop_receipt)
input('\nPress Enter to continue...')
metadata = queue_service.get_queue_metadata('pizzaqueue')
print(
'If we look at the Queue again, we have one less message to show we have processed that order and a yummy pizza will be on it\'s way to the customer soon.'
)
print(('Number of messages in the queue: ' +
str(metadata.approximate_message_count)))
input('\nPress Enter to continue...')
###
# This was a quick demo to see Queues in action.
# Although the actual cost is minimal since we deleted all the messages from the Queue, it's good to clean up resources when you're done
###
class AzureService(object):
VISIBILITY_TIMEOUT = 5*60
def __init__(self, connection_string, container_name, queue_get, queue_push, logger=None):
self.ctnname = container_name
self.getname = queue_get
self.pushname = queue_push
self.qs = QueueService(connection_string=connection_string,
protocol='https',
# endpoint_suffix='core.windows.net'
)
self.bs = BlockBlobService(connection_string=connection_string)
self.qs.create_queue(self.getname, timeout=1)
self.qs.create_queue(self.pushname, timeout=1)
self.bs.create_container(self.ctnname, timeout=1)
if logger: logger.info('Init Azure success')
def pushMessage(self, message, qname=None, logger=None):
if qname is None:
qname = self.pushname
try:
self.qs.put_message(self.pushname, message)
except Exception as e:
if logger:
logger.exception('ERROR PUSH MESSAGE ')
else:
print 'ERROR PUSH MESSAGE '
print e
def getMessage(self, qname=None, num=1, logger=None):
if qname is None:
qname = self.getname
try:
message = self.qs.get_messages(qname, num, visibility_timeout=self.VISIBILITY_TIMEOUT)
except Exception as e:
if logger:
logger.exception('ERROR GET MESSAGE ')
else:
print 'ERROR GET MESSAGE '
print e
return []
return message
def getReceiptInfo(self, logger=None):
message = self.getMessage(logger=logger)
if len(message) > 0:
rinfo = ReceiptSerialize.fromjson(message[0].content)
return message[0], rinfo
else:
return None, None
def count(self):
metadata_get = self.qs.get_queue_metadata(self.getname)
metadata_push = self.qs.get_queue_metadata(self.pushname)
generator = self.bs.list_blobs(self.ctnname)
bc = 0
for blob in generator:
bc += 1
return {'get_count' : metadata_get.approximate_message_count,
'push_count': metadata_push.approximate_message_count,
'blob_count': bc
}
def uploadFolder(self, folderpath, logger):
for filename in os.listdir(folderpath):
if len(filename) > 4:
suffix = filename[-4:].upper()
else:
continue
if '.JPG' == suffix or 'JPEG' == suffix:
receipt_metadata = ReceiptSerialize()
receipt_metadata.receiptBlobName = unicode(filename, 'utf-8')
self.qs.put_message(self.getname, b64encode(receipt_metadata.toString()).decode('utf-8'))
self.bs.create_blob_from_path(self.ctnname, receipt_metadata.receiptBlobName, os.path.join(folderpath, filename), max_connections=2, timeout=None)
logger.info('upload %s', filename)
def getImage(self, imgname, logger=None):
localpath= os.path.join(args.download_dir, imgname)
try:
self.bs.get_blob_to_path(self.ctnname, imgname, localpath)
except AzureMissingResourceHttpError as e:
if logger:
logger.error('Blob named ' + imgname + ' doesnot exist.' , exc_info=True)
else:
print 'Blob named ' + imgname + ' doesnot exist.'
print e
return ''
except Exception as e:
if logger:
logger.error('Exception while getting blob.', exc_info=True)
else:
print 'Exception while getting blob.'
print e
return None
return localpath
def deleteMessage(self, message, qname=None, logger=None):
if qname is None:
qname = self.getname
try:
self.qs.delete_message(qname, message.id, message.pop_receipt)
except Exception as e:
if logger:
logger.exception('ERROR DELETE MESSAGE ')
else:
print 'ERROR DELETE MESSAGE '
print e
def deleteImage(self, imgname, logger=None):
try:
self.bs.delete_blob(self.ctnname, imgname)
except Exception as e:
if logger:
logger.exception('ERROR DELETE IMAGE ')
else:
print 'ERROR DELETE IMAGE '
print e
def cleanUp(self):
count = 0
print('deleted: ')
while True:
messages = self.qs.get_messages(self.getname)
for message in messages:
count += 1
self.qs.delete_message(self.getname, message.id, message.pop_receipt)
if len(messages) == 0: break
print(str(count) + ' from queue-get')
count = 0
while True:
messages = self.qs.get_messages(self.pushname)
for message in messages:
count += 1
self.qs.delete_message(self.pushname, message.id, message.pop_receipt)
if len(messages) == 0: break
print(str(count) + ' from queue-push')
count = 0
generator = self.bs.list_blobs(self.ctnname)
for blob in generator:
count += 1
self.bs.delete_blob(self.ctnname, blob.name)
print(str(count) + ' from container')
class AzureQueueStorageIn(InputModule):
'''Consumes messages from Azure Queue Storage
Consume messages from the Azure Queue Storage service.
Parameters::
- account_name(str)("wishbone")
| The account name to authenticate to
- account_key(str)("wishbone")
| The account key to authenticate to the queue
- auto_message_delete(bool)(True)
| Once the message is consumed from the queue delete it immediately.
- b64decode(bool)(True)
| Decode the message payload.
- destination(str)("data")
| The location write the payload to
- endpoint_suffix(str)("core.windows.net")
| The endpoint suffix of the service
- native_events(bool)(False)
| Whether to expect incoming events to be native Wishbone events
- payload(str/dict/int/float)("test")
| The content of the test message.
- queue_name(str)("wishbone")
| The name of the queue to consume
- visibility_timeout(int)(None)
| The amount of time the consumed message remains invisible to
| other consumers before it gets deleted.
Queues::
- outbox
| Outgoing events.
- delete
| Events to delete from Queue storage.
'''
def __init__(self,
actor_config,
destination="data",
payload=None,
native_events=False,
account_name="wishbone",
account_key="wishbone",
queue_name="wishbone",
endpoint_suffix='core.windows.net',
auto_message_delete=True,
visibility_timeout=None,
b64decode=True):
InputModule.__init__(self, actor_config)
self.pool.createQueue("outbox")
self.pool.createQueue("delete")
self.decode = Plain().handler
def getMessages(self):
try:
self.queue_service = QueueService(
account_name=self.kwargs.account_name,
account_key=self.kwargs.account_key,
endpoint_suffix=self.kwargs.endpoint_suffix,
)
self.queue_service.create_queue(self.kwargs.queue_name)
except Exception as err:
message = "Failed to connect to Azure Queue Service https://%s.queue.%s/%s Reason: " % (
self.kwargs.account_name, self.kwargs.endpoint_suffix,
self.kwargs.queue_name)
raise Exception(message + str(err).partition("\n")[0])
else:
self.logging.info(
"Connected to Azure Queue Service https://%s.queue.%s/%s" %
(self.kwargs.account_name, self.kwargs.endpoint_suffix,
self.kwargs.queue_name))
while self.loop():
for message in self.queue_service.get_messages(
self.kwargs.queue_name,
visibility_timeout=self.kwargs.visibility_timeout):
for event in self.processIncomingMessage(message):
self.submit(event, "outbox")
if self.kwargs.auto_message_delete:
self.queue_service.delete_message(self.kwargs.queue_name,
message.id,
message.pop_receipt)
def preHook(self):
logger = logging.getLogger('azure.storage')
logger.setLevel(logging.CRITICAL)
self.sendToBackground(self.getMessages)
self.sendToBackground(self.processDeleteMessage)
def processIncomingMessage(self, message):
if self.kwargs.b64decode:
data = b64decode(message.content)
else:
data = message.content
for chunk in [data, None]:
for payload in self.decode(chunk):
event = self.generateEvent(payload, self.kwargs.destination)
event = self.__setMetaData(event, message)
yield event
def processDeleteMessage(self):
while self.loop():
event = self.pool.queue.delete.get()
self.queue_service.delete_message(
self.kwargs.queue_name, event.get('tmp.%s.id' % (self.name)),
event.get('tmp.%s.pop_receipt' % (self.name)))
def __setMetaData(self, event, message):
metadata = {
"id": message.id,
"insertion_time": message.insertion_time.strftime('%s'),
"expiration_time": message.expiration_time.strftime('%s'),
"dequeue_count": message.dequeue_count,
"pop_receipt": message.pop_receipt,
"time_next_visible": message.time_next_visible.strftime('%s')
}
event.set(metadata, "tmp.%s" % (self.name))
return event
# This is our initial state: the target image.
# Note that `scipy.optimize.fmin_l_bfgs_b` can only process flat vectors.
x = preprocess_image(target_image_path)
x = x.flatten()
for i in range(iterations):
print('Start of iteration', i)
start_time = time.time()
x, min_val, info = fmin_l_bfgs_b(evaluator.loss,
x,
fprime=evaluator.grads,
maxfun=20)
print('Current loss value:', min_val)
# Save current generated image
img = x.copy().reshape((img_height, img_width, 3))
img = deprocess_image(img)
fname = result_prefix + '_at_iteration_%d.png' % i
#print(fname)
imsave(fname, img)
end_time = time.time()
print('Image saved as', fname)
print('Iteration %d completed in %ds' % (i, end_time - start_time))
# Generate image
imsave(output_image_path + output_image_name, img)
saveToBlob(output_image_name, output_image_path + output_image_name)
blob_image_url = 'https://imprinter.blob.core.windows.net/imprinted/' + output_image_name
queue_service.put_message('imprintresults', blob_image_url)
queue_service.delete_message(queue_name, message.id,
message.pop_receipt)
# Pedro Perez - 2015 # # ----------------------------------------------------------------------------- import sys import os import subprocess from azure.storage.queue import QueueService account_name = "mcdockerqueue" account_key = sys.argv[1] queuename = "servers" queue_service = QueueService(account_name, account_key) queue_metadata = queue_service.get_queue_metadata(queuename) count = queue_metadata['x-ms-approximate-messages-count'] print "There are %s messages in the queue" % count if count > 0: messages = queue_service.get_messages(queuename) for message in messages: print(message.message_text) # Remove message from the queue queue_service.delete_message(queuename, message.message_id, message.pop_receipt) else: print "There are no messages to process"
