def callback(message):
# pull() blocks until a message is received
print("in callback")
data = message.data
print(data)
#msg_string = base64.b64decode(data)
#print("msg_string",msg_string)
msg_data = json.loads(data)
#print("msg_data",msg_data)
content_type = msg_data["contentType"]
print("content_type", content_type)
attributes = message.attributes
print("attributes type: ", type(attributes))
event_type = attributes['eventType']
bucket_id = attributes['bucketId']
object_id = attributes['objectId']
generation = attributes['objectGeneration']
#[END msg_format]
print("{0} process starts".format(object_id))
start_process = datetime.datetime.now()
# <Your custom process>
print("checking event_type: ", event_type)
if event_type == 'OBJECT_FINALIZE':
print("Instantiating Mediator")
m = Mediator(bucket_id, object_id, content_type, PROJECT_ID,
dataset_id, table_id)
print("Calling Speech to text")
m.speech_to_text()
# <End of your custom process>
end_process = datetime.datetime.now()
print("{0} process stops".format(object_id))
# Write logs only if needed for analytics or debugging
print(
"{media_url} processed by instance {instance_hostname} in {amount_time}"
.format(media_url=data,
instance_hostname=INSTANCE_NAME,
amount_time=str(end_process - start_process)))
message.ack()
def main(toprocess, subscription, refresh, dataset_id, table_id):
"""
"""
subscription_id = "projects/{0}/subscriptions/{1}".format(PROJECT_ID, subscription)
subscription = pubsub.subscription.Subscription(subscription_id, client=pubsub_client)
if not subscription.exists():
sys.stderr.write('Cannot find subscription {0}\n'.format(sys.argv[1]))
return
r = Recurror(refresh - 10, postpone_ack)
# pull() blocks until a message is received
while True:
#[START sub_pull]
resp = subscriptions.pull("maxMessages": toprocess)
#[END sub_pull]
for ack_id, message in resp:
# We need to do this to get contentType. The rest is in attributes
#[START msg_format]
data = message.data
msg_string = base64.b64decode(data)
msg_data = json.loads(msg_string)
content_type = msg_data["contentType"]
attributes = message.attributes
event_type = attributes['eventType']
bucket_id = attributes['bucketId']
object_id = attributes['objectId']
generation = attributes['objectGeneration']
#[END msg_format]
# Start refreshing the acknowledge deadline.
r.start(ack_ids=[ack_id], refresh=refresh, sub=sub)
Logger.log_writer("{0} process starts".format(object_id))
start_process = datetime.datetime.now()
# <Your custom process>
if event_type == 'OBJECT_FINALIZE':
m = Mediator(bucket_id, object_id, content_type, PROJECT_ID, dataset_id, table_id)
m.speech_to_text()
# <End of your custom process>
end_process = datetime.datetime.now()
Logger.log_writer("{0} process stops".format(object_id))
#[START ack_msg]
# Delete the message in the queue by acknowledging it.
subscription.acknowledge([ack_id])
#[END ack_msg]
# Write logs only if needed for analytics or debugging
Logger.log_writer(
"{media_url} processed by instance {instance_hostname} in {amount_time}"
.format(
media_url=msg_string,
instance_hostname=INSTANCE_NAME,
amount_time=str(end_process - start_process)
)
)
# Stop the ackDeadLine refresh until next message.
r.stop()
def main(toprocess, subscription, refresh, dataset_id, table_id):
sub = "projects/{0}/subscriptions/{1}".format(PROJECT_ID, subscription)
r = Recurror(refresh - 10, postpone_ack)
# pull() blocks until a message is received
while True:
#[START sub_pull]
# Pull the data when available.
resp = pubsub_client.projects().subscriptions().pull(
subscription=sub,
body={
"maxMessages": toprocess
}
).execute()
#[END sub_pull]
if resp:
# Get the amount of media that one instance can processed
# For this demo, we keep it to one per instance.
m = resp.get('receivedMessages')[0]
if m:
message = m.get('message')
ack_id = m.get('ackId')
msg_string = base64.b64decode(message.get('data'))
msg_data = json.loads(msg_string)
bucket = msg_data["bucket"]
filename = msg_data["name"]
filetype = msg_data["type"]
fn = filename.split('.')[0]
# Start refreshing the acknowledge deadline.
r.start(ack_ids=[ack_id], refresh=refresh, sub=sub)
Logger.log_writer("{0} process starts".format(filename))
start_process = datetime.datetime.now()
# <Your custom process>
m = Mediator(bucket, filename, filetype, PROJECT_ID, dataset_id, table_id)
m.speech_to_text()
# <End of your custom process>
end_process = datetime.datetime.now()
Logger.log_writer("{0} process stops".format(filename))
#[START ack_msg]
# Delete the message in the queue by acknowledging it.
pubsub_client.projects().subscriptions().acknowledge(
subscription=sub,
body={
'ackIds': [ack_id]
}
).execute()
#[END ack_msg]
# Logs to see what's going on.
Logger.log_writer(
"{media_url} processed by instance {instance_hostname} in {amount_time}"
.format(
media_url=msg_string,
instance_hostname=INSTANCE_NAME,
amount_time=str(end_process - start_process)
)
)
# Stop the ackDeadLine refresh until next message.
r.stop()
