def __init__(self,
config_path: str = configPath,
project_id: str = None,
client_id: str = None,
client_secret: str = None,
topic_id: str = None,
gcp_project_id: str = None,
sub_id: str = None):
"""
Load or get an authorization config. If a new authorization is being
made, it needs the project_id, client_id, and client_secret.
"""
configKeys = [
'clientID', 'clientSecret', 'projectID', 'topicID', 'subID',
'gcpProjectID', 'accessToken', 'expiresAt', 'refreshToken'
]
self.configPath = config_path
self.streamInfo = None
if os.path.isfile(self.configPath):
with open(self.configPath) as configFile:
config = json.load(configFile)
# Check if all config keys are here
for key in config.keys():
if key not in configKeys:
raise Exception('Config file missing key.')
print('Config loaded.')
self.config = config
self.baseURL = f'https://smartdevicemanagement.googleapis.com/' \
f'v1/enterprises/{self.config["projectID"]}/'
# Create authorization headers
self.authHeaders = {
'Content-Type': 'application/json',
'Authorization': f'Bearer {self.config["accessToken"]}'
}
self.devices = self._getDevices()['devices']
self.structures = self._getStructures()['structures']
else:
if project_id is None or client_id is None or client_secret is None:
raise Exception('Missing authorization args')
# Create OAuth2 link
authLink = f'https://nestservices.google.com/partnerconnections/' \
f'{project_id}/auth?' \
f'redirect_uri=https://www.google.com&' \
f'access_type=offline&prompt=consent&' \
f'client_id={client_id}' \
f'&response_type=code&' \
f'scope=https://www.googleapis.com/auth/sdm.service'
print('Follow the link below and come back with the code')
print(authLink)
authCode = input('Auth Code: ')
while len(authCode) == 0:
authCode = input('Auth Code: ')
# Do authorization
params = {
'client_id': client_id,
'client_secret': client_secret,
'code': authCode,
'grant_type': 'authorization_code',
'redirect_uri': 'https://www.google.com'
}
r = rq.post('https://www.googleapis.com/oauth2/v4/token',
data=params)
if not r.status_code == 200:
print(r.text)
raise Exception('Bad response.')
# Parse response
response = r.json()
accessToken = response['access_token']
refreshToken = response['refresh_token']
expiresAt = response['expires_in'] + time()
self.baseURL = f'https://smartdevicemanagement.googleapis.com/' \
f'v1/enterprises/{project_id}/'
# Send device request to finish authorization
url = f'{self.baseURL}devices'
self.authHeaders = {
'Content-Type': 'application/json',
'Authorization': f'Bearer {accessToken}'
}
r = rq.get(url=url, headers=self.authHeaders)
if not r.status_code == 200:
print(r.text)
raise Exception('Bad response.')
self.devices = r.json()['devices']
config = {
'clientID': client_id,
'clientSecret': client_secret,
'projectID': project_id,
'topicID': topic_id,
'subID': sub_id,
'gcpProjectID': gcp_project_id,
'accessToken': accessToken,
'expiresAt': expiresAt,
'refreshToken': refreshToken
}
# Save json
with open(self.configPath, 'w') as out:
json.dump(config, out)
print('Authorization complete.')
self.config = config
self.structures = self._getStructures()['structures']
# Check if a subscription is available for pubsub
subber = pubsub_v1.SubscriberClient()
subPath = f'projects/{self.config["gcpProjectID"]}'
response = subber.list_subscriptions(subPath)
self.eventsReady = False
for sub in response:
if sub.name.split('/')[-1] == sub_id:
self.eventsReady = True
if not self.eventsReady:
print('No valid subscriptions!')
print('Create subscription online. Events will not work.')
print(
'More Info: https://developers.google.com/nest/device-access/subscribe-to-events'
)
def synchronous_pull_with_lease_management(project_id, subscription_id):
"""Pulling messages synchronously with lease management"""
# [START pubsub_subscriber_sync_pull_with_lease]
import logging
import multiprocessing
import sys
import time
from google.api_core import retry
from google.cloud import pubsub_v1
multiprocessing.log_to_stderr()
logger = multiprocessing.get_logger()
logger.setLevel(logging.INFO)
processes = dict()
# TODO(developer)
# project_id = "your-project-id"
# subscription_id = "your-subscription-id"
subscriber = pubsub_v1.SubscriberClient()
subscription_path = subscriber.subscription_path(project_id,
subscription_id)
response = subscriber.pull(
request={
"subscription": subscription_path,
"max_messages": 3
},
retry=retry.Retry(deadline=300),
)
# Start a process for each message based on its size modulo 10.
for message in response.received_messages:
process = multiprocessing.Process(target=time.sleep,
args=(sys.getsizeof(message) % 10, ))
processes[process] = (message.ack_id, message.message.data)
process.start()
while processes:
# Take a break every second.
if processes:
time.sleep(1)
for process in list(processes):
ack_id, msg_data = processes[process]
# If the process is running, reset the ack deadline.
if process.is_alive():
subscriber.modify_ack_deadline(
request={
"subscription": subscription_path,
"ack_ids": [ack_id],
# Must be between 10 and 600.
"ack_deadline_seconds": 15,
})
logger.info(f"Reset ack deadline for {msg_data}.")
# If the process is complete, acknowledge the message.
else:
subscriber.acknowledge(request={
"subscription": subscription_path,
"ack_ids": [ack_id]
})
logger.info(f"Acknowledged {msg_data}.")
processes.pop(process)
print(
f"Received and acknowledged {len(response.received_messages)} messages from {subscription_path}."
)
# Close the underlying gPRC channel. Alternatively, wrap subscriber in
# a 'with' block to automatically call close() when done.
subscriber.close()
import os, uuid, time, json, random, datetime
from google.cloud import pubsub_v1
from google.api_core.exceptions import AlreadyExists
CHANNEL_MAP = {22: "Cartoon", 23: "YoYo TV", 52: "News"}
project_id = os.environ.get('GCP_PROJECT_ID') # Google Project Id
topic_id = "channel" # Topic Id
topic_path = f"projects/{project_id}/topics/{topic_id}"
container_name = os.environ.get('HOSTNAME')
sub = f"sub-{container_name}-{uuid.uuid4().hex}"
sub_path = f"projects/{project_id}/subscriptions/{sub}"
subscriber = pubsub_v1.SubscriberClient()
# 新建一個 Subscription
try:
subscriber.create_subscription(name=sub_path, topic=topic_path)
except AlreadyExists:
print(f"{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
f' [WARNING] Subscription already exists, sub_path: {sub_path}')
def callback(message):
data = json.loads(message.data.decode())
if type := data.get('type'):
if type == 'show_channel_info':
print(
f"{datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
f" [INFO] Message received, the channel is {CHANNEL_MAP.get(data.get('id'))}"
)
message.ack()
def run_main(event, context):
project_id = google.auth.default()[1]
logger = alert_handler.AlertHandler(project_id)
# Retrieve pubsub messages for all the tests that have been kicked off by
# the test runner.
subscriber = pubsub_v1.SubscriberClient()
project = subscriber.project_path(project_id)
subscription = None
for s in subscriber.list_subscriptions(project):
if s.topic.split('/')[-1] == METRICS_WRITTEN_TOPIC:
subscription = s.name
break
if not subscription:
subscription_id = subscriber.subscription_path(
project_id, 'metrics-handler-subscription')
topic = subscriber.topic_path(project_id, METRICS_WRITTEN_TOPIC)
subscription = subscriber.create_subscription(
subscription_id, topic, ack_deadline_seconds=300).name
try:
all_msgs = subscriber.pull(subscription, 100).received_messages
except google.api_core.exceptions.DeadlineExceeded:
logger.info(
'No messages found for subscription: {}'.format(subscription))
return
# Group messages by test. Each test might have made multiple attempts and
# therefore could have multiple messages.
test_name_to_msgs = collections.defaultdict(list)
ids_to_ack = []
for msg in all_msgs:
data_str = msg.message.data
try:
data = json.loads(data_str)
data['publish_time'] = msg.message.publish_time.seconds
data['ack_id'] = msg.ack_id
test_name_to_msgs[data['test_name']].append(data)
except Exception as e:
logger.error(
'Metrics handler encountered an invalid message in pubsub queue '
'for topic `{}` which led to Exception: {}. This message will '
'be acknowledged and ignored. The message was: {}'.format(
METRICS_WRITTEN_TOPIC, e, msg))
ids_to_ack.append(msg.ack_id)
# Grab the latest message for each test. We will process only that message
# and all other messages for that test will be ack'ed without being processed.
msgs_to_process = []
for test_name, msgs in test_name_to_msgs.items():
sorted_msgs = sorted(msgs, key = lambda x: x['publish_time'])
ids_to_ack.extend([msg['ack_id'] for msg in msgs[:-1]])
msgs_to_process.append(msgs[-1])
logger.info('Finished deduplicating messages from test runs.')
# Note: it's good to ack early and often since pubsub will resend messages
# that are not ack'ed within the queue's deadline.
if ids_to_ack:
logger.info('Will ack these ids: {}'.format(ids_to_ack))
subscriber.acknowledge(subscription, ids_to_ack)
logger.info('Successful ack for ids: {}'.format(ids_to_ack))
if not msgs_to_process:
logger.info('No messages to process. Stopping early.')
return
# TODO: Add support for multi-zone and/or multi-cluster setups.
zone = msgs_to_process[0].get('zone')
cluster = msgs_to_process[0].get('cluster_name')
status_handler = job_status_handler.JobStatusHandler(
project_id, zone, cluster, logger)
# Handle the metrics for each test. Ack if the process was successful or if
# the message is permanently invalid. Do not ack if the test is still running
# so that we will retry again later once that test has finished running.
for msg in msgs_to_process:
try:
logger.info('Pubsub message to process: {}'.format(msg))
should_ack = _process_pubsub_message(msg, status_handler, logger)
except Exception:
logger.error(
'Encountered exception while attempting to process message {}. '
'The message will be acknowledged to prevent more crashes. '
'Exception: {}'.format(msg, traceback.format_exc()))
should_ack = True
if should_ack:
logger.info('Finished processing message. Will ack')
subscriber.acknowledge(subscription, [msg['ack_id']])
logger.info('Acknowledged ack_id: {}'.format(msg['ack_id']))
else:
logger.info('Finished processing message. Will not ack')
logger.info('Processed a message for each of the following tests: '
'{}'.format([x['test_name'] for x in msgs_to_process]))
logger.send_email()
credentials = service_account.Credentials.from_service_account_file(
GOOGLE_APPLICATION_CREDENTIALS_PATH)
# Callback function, called when a message arrives
def callback(message):
# Do somwething with the message
print(f"Message received:")
print(f"{message}.")
# Need to acknowledge receipt, if not will be resent
message.ack()
# Create a subscriber client to access pub/sub service
subscriber = pubsub_v1.SubscriberClient(credentials=credentials)
# Create subscription object
subscription_path = subscriber.subscription_path(PROJECT_ID, SUBSCRIPTION_ID)
# Subscribe (start receiving message) and give it the callback function name that will process incoming message
streaming_pull_future = subscriber.subscribe(subscription_path,
callback=callback)
try:
# Blocking event to wait, need to do this so it won't immediately end example program without receiving message
streaming_pull_future.result(timeout=5)
except TimeoutError as err:
# Exception when wait has timeout
streaming_pull_future.cancel()
print(f"Exception: {err}.")
import picamera import base64 import subprocess import sys import face_recognition import numpy import re import glob from PIL import Image from google.cloud import pubsub_v1 from google.cloud import firestore from google.cloud import storage subscriber_pi = pubsub_v1.SubscriberClient() camera = picamera.PiCamera() authorized_encodings = [] pir = 8 # Pin 8 : PIR yellow = 10 # Pin 10 : yellow LED lock = 12 # Pin 12 : door lock green = 16 # Pin 16 : green LED red = 18 # Pin 18 : red LED blue = 22 # Pin 22 : blue LED pair_switch = 24 # Pin 24 : user pairing button lock_switch = 26 # Pin 26 : toggles door lock GPIO.setmode(GPIO.BOARD) # set GPIO mode to correct physical numbering GPIO.setup(pir, GPIO.IN) # setup GPIO pin PIR as input GPIO.setup(yellow, GPIO.OUT) # setup led outputs
def subscriber_client():
subscriber_client = pubsub_v1.SubscriberClient()
yield subscriber_client
# Close the subscriber client properly during teardown.
subscriber_client.close()
def poll_notifications(project, subscription_name):
"""Polls a Cloud Pub/Sub subscription for new GCS events for display."""
# [BEGIN poll_notifications]
service_account_info = json.load(open("service_account.json"))
audience = "https://pubsub.googleapis.com/google.pubsub.v1.Subscriber"
credentials = jwt.Credentials.from_service_account_info(
service_account_info, audience=audience)
subscriber = pubsub_v1.SubscriberClient(credentials=credentials)
subscription_path = subscriber.subscription_path(project,
subscription_name)
def download_image(name, bucketName="dynartwork-277815.appspot.com"):
storage_client = storage.Client.from_service_account_json(
'service_account.json')
bucket = storage_client.get_bucket(bucketName)
# get bucket data as blob
print(f'Opening {bucketName}/{name}')
blob = bucket.get_blob(name)
json_data = blob.download_as_string()
text_file = open(name, "wb")
n = text_file.write(json_data)
text_file.close()
def callback(message):
#show_image(message.attributes['objectId'])
#print("Received message:\n{}".format(summarize(message)))
data = message.data.decode("utf-8")
attributes = message.attributes
event_type = attributes["eventType"]
bucket_id = attributes["bucketId"]
object_id = attributes["objectId"]
print(object_id)
message.ack()
var = "Message not important"
if "data" in object_id:
originalName = object_id[5:-4]
imageName = f'{originalName}_showed.jpg'
download_image(imageName, 'processed_artworks')
# display the image
print("Showing image:{}".format(imageName))
global image
previousImage = image
image = subprocess.Popen([
"feh", "--hide-pointer", "-x", "-q", "-B", "black",
f"/home/pi/Documents/{imageName}"
])
time.sleep(2)
previousImage.kill()
var = "Image correctly showed"
print("Result of callback:{}".format(var))
subscriber.subscribe(subscription_path, callback=callback)
# The subscriber is non-blocking, so we must keep the main thread from
# exiting to allow it to process messages in the background.
print("Listening for messages on {}".format(subscription_path))
global image
image = subprocess.Popen([
"feh", "--hide-pointer", "-x", "-F",
f"/home/pi/Documents/dynartwork.png"
])
while True:
time.sleep(60)
import google.auth
from google.cloud import pubsub_v1
from xialib_gcp import PubsubSubscriber
project_id = google.auth.default()[1]
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
header, data, id = s.unpack_message(message)
print("{}: {}".format(subscription_id, header))
s.ack(project_id, subscription_id, id)
loop = asyncio.get_event_loop()
task_backlog = PubsubSubscriber(
sub_client=pubsub_v1.SubscriberClient()).stream('x-i-a-test',
'insight-backlog-debug',
callback=callback)
task_cockpit = PubsubSubscriber(
sub_client=pubsub_v1.SubscriberClient()).stream('x-i-a-test',
'insight-cockpit-debug',
callback=callback)
task_cleaner = PubsubSubscriber(
sub_client=pubsub_v1.SubscriberClient()).stream('x-i-a-test',
'insight-cleaner-debug',
callback=callback)
task_loader = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient()).stream(
'x-i-a-test', 'insight-loader-debug', callback=callback)
task_merger = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient()).stream(
'x-i-a-test', 'insight-merger-debug', callback=callback)
task_packager = PubsubSubscriber(
def __init__(self, project, topic_name, subscription_name):
logger.debug("Subscriber client created")
self.subscriber = pubsub_v1.SubscriberClient()
self.subscription_path = self.subscriber.subscription_path(
project, subscription_name)
def subscribe(subscription_name, worker, give_up=False):
"""Receives and spawns threads to handle jobs received in Pub/Sub"""
global shutdown_requested
message = None # The current active message
lock = threading.Lock()
client = pubsub_v1.SubscriberClient()
subscription_path = client.subscription_path(GCLOUD_PROJECT_ID,
subscription_name)
def renew_deadline():
"""Repeatedly give the active message more time to be processed to prevent it being resent"""
while not (message == None and shutdown_requested):
if message != None:
try:
with lock:
client.modify_ack_deadline(subscription_path,
[str(message.ack_id)],
SUB_ACK_DEADLINE)
logging.debug(
'Reset ack deadline for {} for {}s'.format(
message.message.data.decode(),
SUB_ACK_DEADLINE))
time.sleep(SUB_SLEEP_TIME)
except Exception as e:
logging.warning('Could not reset ack deadline', exc_info=e)
watcher = threading.Thread(target=renew_deadline)
watcher.start()
# Repeatedly check for new jobs until SIGINT/SIGTERM received
logging.info('Listening for jobs')
try:
while not shutdown_requested:
response = client.pull(subscription_path,
max_messages=1,
return_immediately=True)
if not response.received_messages:
logging.info('Job queue is empty')
time.sleep(SUB_SLEEP_TIME)
continue
if len(response.received_messages) > 1:
logging.warning(
'Received more than one job when only one expected')
with lock:
message = response.received_messages[0]
logging.info('Beginning: {}'.format(message.message.data.decode()))
process = multiprocessing.Process(
target=worker, args=(message.message.data.decode(), ))
process.start()
process.join()
if process.exitcode == 0:
# Success; acknowledge and return
try:
client.acknowledge(subscription_path, [message.ack_id])
logging.info('Ending and acknowledged: {}'.format(
message.message.data.decode()))
except Exception as e:
logging.error('Could not end and acknowledge: {}'.format(
message.message.data.decode()),
exc_info=e)
elif give_up and (int(time.time()) -
message.message.publish_time.seconds) > 600:
# Failure; give up and acknowledge
try:
client.acknowledge(subscription_path, [message.ack_id])
logging.error('Failed but acknowledged: {}'.format(
message.message.data.decode()))
except Exception as e:
logging.error(
'Failed but could not acknowledge: {}'.format(
message.message.data.decode()),
exc_info=e)
else:
# Failure; refuse to acknowledge
logging.error('Failed, not acknowledged: {}'.format(
message.message.data.decode()))
# Stop extending this message's deadline in the "watcher" thread
with lock:
message = None
except Exception as e:
logging.critical('Exception encountered. ', exc_info=e)
finally:
# If there is an exception, make sure the "watcher" thread shuts down
shutdown_requested = True
def subscriber_client(self):
self.log_debug("initialize subscriber client")
return pubsub_v1.SubscriberClient(credentials=self.credentials)
def pubsub_interface(subscription_path,
input_topic,
initial_data=None,
delay_seconds=1.0,
callback=None):
import functools
from google.cloud import pubsub_v1
from google.cloud.pubsub_v1.subscriber.message import Message
from rillbeam import tapp
if callback is None:
def callback(pane, message):
# type: (tapp.Pane, Message) -> None
message.ack()
with pane.batch:
pane.write(
'{} : {}'.format(message.publish_time,
message.data.decode()), 'cyan')
subscriber = pubsub_v1.SubscriberClient()
publisher = pubsub_v1.PublisherClient()
with tapp.App() as app:
app.write('Beginning interactive pubsub session.',
'yellow',
attrs=['bold'])
app.write()
app.write('Subscriber {!r}...'.format(subscription_path), 'yellow')
app.write('Publisher {!r}...'.format(input_topic), 'yellow')
app.write()
app.write(
'Send messages to pubsub. Output messages will print '
'when they are received.', 'green')
app.write('Type \'exit\' to stop.', 'green', attrs=['bold'])
prompt = app.pane(3, app.width, app.line, 0)
y = app.line + prompt.height
height = app.height - 1 - y
ypos = y
col1 = int(app.width * 0.4)
col2 = int(app.width * 0.6)
app.write('stdout', y=ypos, attrs=['bold'])
app.write('subscriber', x=col1, y=y, attrs=['bold'])
logpane = app.pane(height, col1, ypos + 1, 0)
logging.getLogger().addHandler(CursesHandler(logpane))
streampane = app.pane(height, col2, ypos + 1, col1)
sub_future = subscriber.subscribe(subscription_path,
callback=functools.partial(
callback, streampane))
if initial_data:
_logger.info('Sending {} initial packages...'.format(
len(initial_data)))
with streampane.batch:
for msg in initial_data:
_logger.info('Sending {!r}...'.format(msg))
time.sleep(delay_seconds)
publisher.publish(input_topic, data=msg)
try:
while True:
prompt.win.clear()
prompt.win.border()
try:
msg = prompt.prompt()
except KeyboardInterrupt:
continue
if not msg:
continue
elif msg.lower() == 'exit':
break
else:
_logger.info('Sending {!r}...'.format(msg))
publisher.publish(input_topic, data=msg)
finally:
sub_future.cancel()
os.environ.setdefault('GOOGLE_CLOUD_PROJECT', 'project-id')
os.environ.setdefault('MY_TOPIC_NAME', 'test_memory_leak-sub')
topic_name = 'projects/{project_id}/topics/{topic}'.format(
project_id=os.getenv('GOOGLE_CLOUD_PROJECT'),
topic=os.getenv('MY_TOPIC_NAME'), # Set this to something appropriate.
)
subscription_name = 'projects/{project_id}/subscriptions/{sub}'.format(
project_id=os.getenv('GOOGLE_CLOUD_PROJECT'),
sub=os.getenv('MY_TOPIC_NAME'), # Set this to something appropriate.
)
@profile
def callback(message):
# Memory intensive operation
# x = [n for n in range(int(1e5))]
message.ack()
print("ack")
with pubsub_v1.SubscriberClient() as subscriber:
future = subscriber.subscribe(subscription_name, callback)
print("Starting subscriber")
try:
print("Listening...")
future.result(timeout=30)
except KeyboardInterrupt:
future.cancel()
def test_check_messages():
sub = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
for message in sub.pull('x-i-a-test', 'xialib-sub-01'):
header, data, id = sub.unpack_message(message)
assert len(json.loads(gzip.decompress(data).decode())) == 2
sub.ack('x-i-a-test', 'xialib-sub-01', id)
found = False # Check if topic exists in project
for topic in publisher.list_topics(
project_path): # topic is a fully qualified topic path
if topic.name == topic_name:
found = True
if not found: # If not found, create it
publisher.create_topic(topic_name)
# Publish message. Please note since subscriber does not exist yet, this message is not going to be delivered
#future = publisher.publish (topic_name, b'My first message!', spam='this is spam') # Publish a message
#if future._completed: # Check if successful
# print ("Message sent successfully!")
# Create subscriber
subscriber = pubsub_v1.SubscriberClient() # Creates a subscriber client
subscription_name = 'projects/{project_id}/subscriptions/{sub}'.format(
project_id="famous-store-237108", sub="mysubscription")
subscription_path = subscriber.subscription_path(
"famous-store-237108",
"mysubscription") # Creates a fully qualified subscriber path
found = False # Check if subscription exists in project
for subscription in subscriber.list_subscriptions(
project_path): # subscription is a fully qualified subscription path
if subscription.name == subscription_name:
found = True
if not found: # If not found, create it
sub_instance = subscriber.create_subscription(name=subscription_name,
topic=topic_name)
def synchronous_pull_with_lease_management(project_id, subscription_name):
"""Pulling messages synchronously with lease management"""
# [START pubsub_subscriber_sync_pull_with_lease]
import logging
import multiprocessing
import random
import time
from google.cloud import pubsub_v1
# TODO project_id = "Your Google Cloud Project ID"
# TODO subscription_name = "Your Pub/Sub subscription name"
subscriber = pubsub_v1.SubscriberClient()
subscription_path = subscriber.subscription_path(project_id,
subscription_name)
NUM_MESSAGES = 2
ACK_DEADLINE = 30
SLEEP_TIME = 10
# The subscriber pulls a specific number of messages.
response = subscriber.pull(subscription_path, max_messages=NUM_MESSAGES)
multiprocessing.log_to_stderr()
logger = multiprocessing.get_logger()
logger.setLevel(logging.INFO)
def worker(msg):
"""Simulates a long-running process."""
RUN_TIME = random.randint(1, 60)
logger.info("{}: Running {} for {}s".format(
time.strftime("%X", time.gmtime()), msg.message.data, RUN_TIME))
time.sleep(RUN_TIME)
# `processes` stores process as key and ack id and message as values.
processes = dict()
for message in response.received_messages:
process = multiprocessing.Process(target=worker, args=(message, ))
processes[process] = (message.ack_id, message.message.data)
process.start()
while processes:
for process in list(processes):
ack_id, msg_data = processes[process]
# If the process is still running, reset the ack deadline as
# specified by ACK_DEADLINE once every while as specified
# by SLEEP_TIME.
if process.is_alive():
# `ack_deadline_seconds` must be between 10 to 600.
subscriber.modify_ack_deadline(
subscription_path,
[ack_id],
ack_deadline_seconds=ACK_DEADLINE,
)
logger.info("{}: Reset ack deadline for {} for {}s".format(
time.strftime("%X", time.gmtime()),
msg_data,
ACK_DEADLINE,
))
# If the processs is finished, acknowledges using `ack_id`.
else:
subscriber.acknowledge(subscription_path, [ack_id])
logger.info("{}: Acknowledged {}".format(
time.strftime("%X", time.gmtime()), msg_data))
processes.pop(process)
# If there are still processes running, sleeps the thread.
if processes:
time.sleep(SLEEP_TIME)
print("Received and acknowledged {} messages. Done.".format(
len(response.received_messages)))
subscriber.close()
def __init__(self, project_id, client_identifier='default'):
self.publisher = pubsub_v1.PublisherClient()
self.subscriber = pubsub_v1.SubscriberClient()
self.client_identifier = client_identifier
self.project_id = project_id
cur = connectionsql.cursor(pymysql.cursors.DictCursor) # creating a cursor to read and write from the database
#setting for controlling our recieveing email
SCOPES = 'https://mail.google.com/' #scopes are used to set permissons that the app wants, current scope can perform all actions on an gmail id
CLIENT_SECRET_FILE = './storage/secret.json' #secret file is generated by the api manager and is used to identify the email you want to connect to
APPLICATION_NAME = name #name of the app you want to connect to the email id with
project_id = 'locationbasedimages' #name of your project on google cloud platform
#creating workers for pubsub system
pubsubclient = pubsub_v1.PublisherClient() #creating a publisher worker so create and manage topics
subscriber = pubsub_v1.SubscriberClient() #creating a subsciber worker so create and manage subscriptions for the topics
project = pubsubclient.project_path(project_id) #setting the project you want the pubsub to work on
gmailHistoryId = '' #creating an empty varibale to hold the last history id, to make sure the process only checks the newer mails from this historyID
checkedEmails = [] #creating a empty list to add the id of all the checked mails , so as it to not check them again by mistake
#Used to get credentials for the recieveing emailid , checks if the authentication process is already performed, if not performs it.
def Getcredentials():
store = Storage('./storage/myauth.json')
credentials = store.get()
if not credentials or credentials.invalid: # fetches and checks if the credentials exist and if the credentials are still valid
flow = oauth2client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES) # starts the authentication process
flow.user_agent = APPLICATION_NAME # gives the auth process your application name
credentials = tools.run_flow(flow,store) # gets the new credentials and saves them to the myauth.json file
print('fetched new credentials')
return credentials #return the new credentials to the main program
def get_subscriber(self):
return pubsub.SubscriberClient()
def read_from_topic(self, topic_name):
# Get kafka properties from config table
pubsub_config_pandas_df = self.get_pubsub_config(
topic_name, self.tenant)
project_id = pubsub_config_pandas_df.iloc[0]['project']
subscriber_id = pubsub_config_pandas_df.iloc[0]['subscription_name']
self.logger.info(f"Project_id ==> {project_id}")
self.logger.info(f"subscriber_id ==> {subscriber_id}")
# initialize subscriber client
subscriber = pubsub_v1.SubscriberClient()
# get subscriber path from subscriber id
subscription_path = subscriber.subscription_path(
project_id, subscriber_id)
self.logger.info(f"Subscription Path ==> {subscription_path}")
# subscribe to the subscription which listens from its registered topic
future = subscriber.subscribe(subscription_path, self.callback)
self.logger.info(
f"Subscribed and listening for messages on {subscription_path} ..."
)
# with clause handles auto closing of action
with subscriber:
try:
# When `timeout` is not set, result() will block indefinitely,
# unless an exception is encountered first.
future.result(timeout=20)
except TimeoutError:
future.cancel() # Trigger the shutdown.
future.result() # Block until the shutdown is complete.
self.logger.info(f"Total messages consumed : {self.message_count}")
# Create spark dataframe
if len(self.consumed_message_list) > 0:
self.logger.info("found records")
df = self.spark_session.createDataFrame(self.consumed_message_list,
["key", "value"])
df = df.withColumn('run_date', f.lit(self.run_date)) \
.withColumn('run_time', f.to_timestamp(f.lit(self.ADAPTER_RUN_START))) \
.withColumn('topic', f.lit(topic_name))
df = self.spark_session.createDataFrame(df.rdd,
self.pubsub_df_schema)
# archive to consolidated disposition table
if df.rdd.isEmpty():
self.logger.info(
f"No records consumed from {topic_name} on {subscription_path} to archive"
)
else:
self.logger.info(f"Archiving to : {self.pubsub_disposition_table}")
self.bq_api.save_as_bigquery_table(df,
self.pubsub_disposition_table,
'append')
self.logger.info(
f"Archiving completed with {self.run_date} and {self.ADAPTER_RUN_START}"
)
schema = self.spark_session.read.json(
df.rdd.map(lambda row: row.value)).schema
return df, schema # df with schema as key,value,run_date,run_time,topic name
def create_patron(self, patron):
if self._subscriber is None:
self._subscriber = pubsub_v1.SubscriberClient()
async def new_subscriber(
self, creds, subscription_name,
callback) -> pubsub_v1.subscriber.futures.StreamingPullFuture:
subscriber = pubsub_v1.SubscriberClient(credentials=creds)
return subscriber.subscribe(subscription_name, callback)
import uuid
from google.api_core.exceptions import AlreadyExists
from google.cloud import pubsub_v1
import mock
import pytest
import sub # noqa
UUID = uuid.uuid4().hex
PROJECT = os.environ["GCLOUD_PROJECT"]
TOPIC = "quickstart-sub-test-topic-" + UUID
SUBSCRIPTION = "quickstart-sub-test-topic-sub-" + UUID
publisher_client = pubsub_v1.PublisherClient()
subscriber_client = pubsub_v1.SubscriberClient()
@pytest.fixture(scope="module")
def topic_path():
topic_path = publisher_client.topic_path(PROJECT, TOPIC)
try:
topic = publisher_client.create_topic(topic_path)
yield topic.name
except AlreadyExists:
yield topic_path
publisher_client.delete_topic(topic_path)
def update_subscription_with_dead_letter_policy(project_id,
topic_id,
subscription_id,
dead_letter_topic_id,
max_delivery_attempts=5):
"""Update a subscription's dead letter policy."""
# [START pubsub_dead_letter_update_subscription]
from google.cloud import pubsub_v1
from google.cloud.pubsub_v1.types import DeadLetterPolicy, FieldMask
# TODO(developer)
# project_id = "your-project-id"
# TODO(developer): This is an existing topic that the subscription
# with dead letter policy is attached to.
# topic_id = "your-topic-id"
# TODO(developer): This is an existing subscription with a dead letter policy.
# subscription_id = "your-subscription-id"
# TODO(developer): This is an existing dead letter topic that the subscription
# with dead letter policy will forward dead letter messages to.
# dead_letter_topic_id = "your-dead-letter-topic-id"
# TODO(developer): This is the maximum number of delivery attempts allowed
# for a message before it gets delivered to a dead letter topic.
# max_delivery_attempts = 5
publisher = pubsub_v1.PublisherClient()
subscriber = pubsub_v1.SubscriberClient()
topic_path = publisher.topic_path(project_id, topic_id)
subscription_path = subscriber.subscription_path(project_id,
subscription_id)
dead_letter_topic_path = publisher.topic_path(project_id,
dead_letter_topic_id)
subscription_before_update = subscriber.get_subscription(
request={"subscription": subscription_path})
print(f"Before the update: {subscription_before_update}.")
# Indicates which fields in the provided subscription to update.
update_mask = FieldMask(paths=["dead_letter_policy"])
# Construct a dead letter policy you expect to have after the update.
dead_letter_policy = DeadLetterPolicy(
dead_letter_topic=dead_letter_topic_path,
max_delivery_attempts=max_delivery_attempts,
)
# Construct the subscription with the dead letter policy you expect to have
# after the update. Here, values in the required fields (name, topic) help
# identify the subscription.
subscription = pubsub_v1.types.Subscription(
name=subscription_path,
topic=topic_path,
dead_letter_policy=dead_letter_policy,
)
with subscriber:
subscription_after_update = subscriber.update_subscription(
request={
"subscription": subscription,
"update_mask": update_mask
})
print(f"After the update: {subscription_after_update}.")
# [END pubsub_dead_letter_update_subscription]
return subscription_after_update
def worker(msg):
start_time = time.time()
print(msg.message.data)
task_id = int(msg.message.data)
client = datastore.Client(project_id)
key = client.key(task_kind, task_id)
params = client.get(key)
# Setting the status to 'InProgress'
mark_in_progress(client, task_id)
cancertype = params['cancertype']
category = params['category']
shard_length = int(params['shard_length'])
shard_index = int(params['shard_index'])
gcs_output_path = params['gcs_output_path']
print('Loading metadata...')
image_file_metadata_filename = 'data/caches_basic_annotations.txt'
util.gsutil_cp('{}/{}/caches_basic_annotations.txt'.format(gcs_ann_path, cancertype), 'data/', make_dir=True)
image_files_metadata = pd.read_csv(image_file_metadata_filename, skiprows=range(1, shard_index*shard_length+1), nrows=shard_length)
shard_length_tiles = len(image_files_metadata.index)
label_names = ['is_tumor']
print('Downloading cache files...')
image_files_metadata['cache_values'] = choose_input_list.load_cache_values(image_files_metadata,
bucket_name = tiles_input_bucket,
notebook = False)
# print('Downloading tiles...')
# bucket = handle_google_cloud_apis.gcsbucket(project_id, tiles_input_bucket)
# def download_tile(df_row, bucket):
# gcs_rel_path = df_row['GCSurl'][len('gs://' + bucket.bucket_name)+1:]
# bucket.download_from_gcs(gcs_rel_path, output_dir=df_row['rel_path'])
# tqdm.pandas()
# image_files_metadata.progress_apply(lambda df_row: download_tile(df_row, bucket), axis=1)
crossval_groups = ['training','testing','validation']
if category not in crossval_groups+['all']:
raise Exception('Unknown cross validation category.')
# Create tfrecords for each category
if category != 'all': # keyword 'all' will loop through all three categories
crossval_groups = [category]
tfrecords_folder = 'tfrecords_{}'.format(cancertype)
util.mkdir_if_not_exist(tfrecords_folder)
for category in crossval_groups:
print('Creating TFRecord for {:s}...'.format(category))
handle_tfrecords.create_tfrecords_per_category_for_caches(image_files_metadata, label_names, category,
tfrecord_file_name_prefix = tfrecords_folder + '/tfrecord{:d}'.format(shard_index))
tfrecords_bucket = re.search('gs://(.+?)/', gcs_output_path).group(1)
prefix = 'gs://' + tfrecords_bucket + '/'
gcs_directory = "".join(gcs_output_path.rsplit(prefix))
bucket = handle_google_cloud_apis.gcsbucket(project_name=project_id, bucket_name=tfrecords_bucket)
bucket.copy_files_to_gcs(tfrecords_folder, gcs_directory, verbose=True)
command = 'du -s ' + tfrecords_folder + '/'
tfrecord_size_MBi = round(int(os.popen(command).read().split()[0])/1000,1) # in MB
# Removing local files
command = "rm -rf " + tfrecords_folder
os.popen(command)
os.popen("rm -rf tcga_tiles")
elapsed_time_s = round((time.time() - start_time), 1) # in seconds
completed_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# We now can comfirm the job
client = datastore.Client(project_id)
mark_done(client=client, task_id=task_id, completed_time=completed_time,
elapsed_time_s=elapsed_time_s, shard_length_tiles=shard_length_tiles,
tfrecord_size_MBi=tfrecord_size_MBi)
print('Finish Timestamp: {} - Time elapsed: {} seconds.'.format(completed_time, elapsed_time_s))
subscriber = pubsub.SubscriberClient()
subscription_path = subscriber.subscription_path(project_id, subscription_name)
# Acknowledging the message
subscriber.acknowledge(subscription_path, [msg.ack_id])
print("{}: Acknowledged {}".format(
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), msg.message.data))
def subscriber():
yield pubsub_v1.SubscriberClient()
import sys
import re
from tqdm import tqdm
import json
import glob
project_id = PROJECT_ID
subscription_name = SUBSCRIPTION_NAME
input_bucket = INPUT_BUCKET
task_kind = TASK_KIND
annotations_path = ANNOTATIONS_PATH
results_path = RESULTS_PATH
pancancer_tfrecords_path = PANCANCER_TFRECORDS_PATH
gcs_output_path = GCS_OUTPUT_PATH
subscriber = pubsub.SubscriberClient()
subscription_path = subscriber.subscription_path(project_id, subscription_name)
NUM_MESSAGES = 1
ACK_DEADLINE = 60
SLEEP_TIME = 30
def mark_done(client, task_id, completed_time, elapsed_time_s):
with client.transaction():
key = client.key(task_kind, task_id)
task = client.get(key)
if not task:
raise ValueError('{} {} does not exist.'.format(
task_kind, task_id))
def subscriber_client():
subscriber_client = pubsub_v1.SubscriberClient()
yield subscriber_client
subscriber_client.close()
def predict():
print("I was here 1")
if request.method == 'POST':
print(request.form.get('age'))
try:
age = float(request.form['age'])
income = float(request.form['income'])
Gender = float(request.form['Gender'])
MaritalStatus = float(request.form['MaritalStatus'])
HaveKids = float(request.form['HaveKids'])
isVeg = float(request.form['isVeg'])
IsStudent = float(request.form['IsStudent'])
email= request.form['email']
offer_received = 0
pred_args = [age, income, Gender,MaritalStatus,HaveKids,isVeg,IsStudent]
pred_args_arr = np.array(pred_args)
pred_args_arr = pred_args_arr.reshape(1, -1)
# mul_reg = open("multiple_regression_model.pkl", "rb")
# ml_model = joblib.load(mul_reg)
model_prediction = ml_model.predict(pred_args_arr)
model_prediction = round(float(model_prediction), 2)
cred = service_account.Credentials.from_service_account_file(
'subway_cred.json')
# Setting up the Configuration Variables:
project_id = "subwayoffers"
bucket_name = "bucket_subway"
topic_name = "subwaytopic"
subscription_name = "subwayoffers"
dataset_name = "data"
table_name = "Profile"
publisher = pubsub_v1.PublisherClient(credentials=cred)
client = pubsub_v1.PublisherClient(credentials=cred)
topic_path = publisher.topic_path(project_id, topic_name)
print(topic_path)
subscriber = pubsub_v1.SubscriberClient(credentials=cred)
topic_path = subscriber.topic_path(project_id, topic_name)
subscription_path = subscriber.subscription_path(project_id, subscription_name)
model_prediction_dict='dict'
if isVeg==0:
model_prediction_dict = "Free Subway Chicken Sandwich"
elif HaveKids==1:
model_prediction_dict = "40% on Happy Meal"
else:
model_prediction_dict = "30 % of on any Subway Sandwich"
if model_prediction==1 :
offer_received='1'
model_prediction = 0
msg = Message(subject="Subway offers",
sender=app.config.get("MAIL_USERNAME"),
recipients=[email], # replace with your email for testing
body="Thank you for taking this survey. Your offer is "+str(model_prediction_dict))
print(str(model_prediction))
mail.send(msg)
else:
msg = Message(subject="Subway offers",
sender=app.config.get("MAIL_USERNAME"),
recipients=[email], # replace with your email for testing
body="Thank you for taking this survey. You have earned a cookie")
mail.send(msg)
#print(str(model_prediction))
model_prediction=0
data_row = {"age": age, "income": income, "Gender": Gender, "MaritalStatus": MaritalStatus,
"HaveKids": HaveKids, "isVeg": isVeg, "IsStudent": IsStudent, "offer_received": offer_received,
"offer_completed": model_prediction, "email": email}
# print(data_row)
message_data = json.dumps(data_row)
message_data = message_data.encode('utf-8')
print(message_data)
# Publishing a message on the PubSub Topic Created:
response = publisher.publish(topic_path, message_data, origin='python-sample')
print(response)
except ValueError:
return "Please check if the values are entered correctly"
return render_template('predict.html', prediction = model_prediction)
