def create_publisher():
return pubsub_v1.PublisherClient()
def publisher_client():
yield pubsub_v1.PublisherClient()
import json
import logging
import os
project_id = os.getenv('GCLOUD_PROJECT')
# TODO: Load the Cloud Pub/Sub module
from google.cloud import pubsub_v1
# END TODO
from flask import current_app
# TODO: Create a Pub/Sub Publisher Client
publisher = pubsub_v1.PublisherClient()
# END TODO
# TODO: Create Topic Object to reference feedback topic
topic_path = publisher.topic_path(project_id, 'feedback')
# END TODO
# TODO: Create a Pub/Sub Subscriber Client
sub_client = pubsub_v1.SubscriberClient()
# END TODO
def ingest():
topic_name = os.getenv('TOPIC')
url = os.getenv('URL')
publisher = pubsub_v1.PublisherClient(
batch_settings=pubsub_v1.types.BatchSettings(max_latency=5))
topic_path = publisher.topic_path(project_id, topic_name)
feed = feedparser.parse(url)
pages = []
today_date = f"{datetime.datetime.now():%Y-%m-%d}"
for post in feed.entries:
post_date = "%d-%02d-%02d" % (post.published_parsed.tm_year,\
post.published_parsed.tm_mon, \
post.published_parsed.tm_mday)
#if post_date == today_date:
if post_date:
print("post date: " + post_date)
print("post title: " + post.title)
print("post link: " + post.link)
page = rq.get(post.link).text
pages.append(page)
chunk_size = 50
count = 0
message_count = 0
chunk = []
print('Publishing data to {} ...'.format(topic_path))
for page in pages:
text = ""
flag = 0
soup = BeautifulSoup(page, "lxml")
for s in soup.findAll('p'):
if '<em>− Climate News Network</em>' in str(s):
text = text + s.text.encode("utf-8").decode("utf-8").replace(
'− Climate News Network', '') + "\n"
flag = 1
elif not '<p><' in str(s) and flag == 0:
text = text + s.text.encode("utf-8").decode("utf-8") + "\n"
if count < chunk_size:
chunk.append(text)
count += 1
if count == chunk_size:
bytes_chunk = bytes("@@".join(chunk).encode('utf-8'))
publisher.publish(topic_path, data=bytes_chunk)
chunk = []
count = 0
message_count = message_count + 1
if count > 0:
bytes_chunk = bytes("@@".join(chunk).encode('utf-8'))
publisher.publish(topic_path, data=bytes_chunk)
print('Published {} rows in {} messages'.format(
(message_count * chunk_size) + count,
message_count + math.ceil(count / chunk_size)))
subscribe()
return 'ok'
def publish_active_elections(event, context):
"""
Publishes elections to Pub/Sub topic with an error handler.
Data included in attributes of messsage:
- election_id=election['id'],
- name=election['name'],
- electionDay=election['electionDay'],
- ocdDivisionId=election['ocdDivisionId']
"""
def get_callback(f, data):
def callback(f):
try:
logging.info(f.result())
futures.pop(data)
except: # noqa
logging.info("Please handle {} for {}.".format(
f.exception(), data))
return callback
# Job status
logging.info("Starting job to publish elections.")
logging.info("""Trigger: messageId {} published at {}""".format(
context.event_id, context.timestamp))
# Initiate job
civic = VoterInfo()
date = dt.datetime.now().date()
# Load elections data
logging.info("Load list of current elections.")
elections = civic.load_current_elections("current_elections",
"current_elections.json")
project_id = "election-tracker-268319"
topic_name = "active-elections"
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project_id, topic_name)
futures = dict()
for election in elections:
# Set message attributes
data = str(election['id'])
logging.info(f"Publishing message to {topic_path}")
# When you publish a message, the client returns a future. Data must be a bytestring.
futures.update({data: None})
future = publisher.publish(topic_path,
data=data.encode("utf-8"),
election_id=election['id'],
name=election['name'],
electionDay=election['electionDay'],
ocdDivisionId=election['ocdDivisionId'])
futures[data] = future
# Publish failures shall be handled in the callback function.
future.add_done_callback(get_callback(future, data))
# Wait for all the publish futures to resolve before exiting.
while futures:
time.sleep(5)
logging.info(
f"Published active elections for current elections as of {str(date)}")
def download_html(message, context):
"""Receives a message object from pubsub, on the key 'data' it retrieves an url
Download this url into the _IN_BUCKET or republish if it fails
If same name let the gcloud trigger update handle
Arguments:
data {[base64 encoded string]} -- object json encoded with data:{file_path}
context {[object]} -- [description]
"""
# The bucket to store this html page
_OUT_BUCKET = os.environ["OUTPUT_HTML_BUCKET"]
# The topic of this function
_THIS_FUNCTION_TOPIC = os.environ["THIS_TOPIC"]
# The topic that will be passed the json path
_PARSE_FUNCTION_TOPIC = os.environ["OUTPUT_JSON_TOPIC"]
# Instantiating log client
LOG_CLIENT = cloud_logging.Client()
HANDLER = LOG_CLIENT.get_default_handler()
LOGGER = logging.getLogger('cloudLogger')
LOGGER.setLevel(logging.INFO)
LOGGER.addHandler(HANDLER)
def __error_path(publisher, pub_obj_encoded, tries, url, error):
"""Function to handle possible errors on pagination
Args:
pub_obj_encoded ([dict]): [pubsub dict witn infos of the page and tries]
tries ([int]): [number of tries that this page was tried]
url ([str]): [url to be parsed for pagination]
"""
if tries < 5:
publisher.publish(_THIS_FUNCTION_TOPIC, pub_obj_encoded)
else:
raise Exception(
"%s was already parsed 5 times, ended with %s page" % (url, error))
try:
error_client = error_reporting.Client()
# Getting the url of the pagination page
data = base64.b64decode(message['data']).decode('utf-8')
json_decoded = json.loads(data)
url = json_decoded['url']
# Out file name to gsbucket
file_name = url.split('/')[-1].replace(':','_').replace(';','_')
storage_client = storage.Client()
bucket = storage_client.get_bucket(_OUT_BUCKET)
blob = bucket.blob(file_name)
blob.metadata = blob.metadata or {}
blob.metadata['url'] = url
# If blob exists let gcloud trigger update handle
response = requests.get(url, headers=HEADERS)
publisher = pubsub_v1.PublisherClient()
# Adding number o tries
tries = 0
if 'tries' in json_decoded:
tries = int(json_decoded['tries']) + 1
# Object for failure maximum of tries
pub_obj_encoded = json.dumps(
{'url': url, 'tries': tries}).encode("utf-8")
# If the status is not 200 the requestor was blocked send back
if response.status_code != 200:
__error_path(publisher, pub_obj_encoded, tries,
url, error=response.status_code)
else:
soup = BeautifulSoup(response.content, 'lxml')
# Special case where this website bad implemented http errors
if soup.select('title')[0].text == 'Error 500':
__error_path(publisher, pub_obj_encoded, tries, url, error=500)
publisher.publish(_THIS_FUNCTION_TOPIC, url.encode('utf-8'))
else:
# Saving the html by the url name
pub_obj_encoded = json.dumps(
{'file_path': file_name, 'url': url}).encode("utf-8")
# Storing the blob
blob.upload_from_string(response.text)
# Publish path to be parsed and transformed to json if new
publisher.publish(_PARSE_FUNCTION_TOPIC, pub_obj_encoded)
except Exception as error:
error_client.report_exception()
def main():
args = parse_command_line_args()
global image_dict
# Create the MQTT client and connect to Cloud IoT.
client = mqtt.Client(
client_id='projects/{}/locations/{}/registries/{}/devices/{}'.format(
args.project_id, args.cloud_region, args.registry_id,
args.device_id))
client.username_pw_set(username='******',
password=create_jwt(args.project_id,
args.private_key_file,
args.algorithm))
client.tls_set(ca_certs=args.ca_certs, tls_version=ssl.PROTOCOL_TLSv1_2)
device = Device(args.device_id, args.service_account_json)
dapp_id = args.dapp_id
dapp_key = args.dapp_key
dapp_addr = args.dapp_addr
check_authentication(dapp_id, dapp_key, dapp_addr, device.get_id(),
args.project_id, args.registry_id, args.cloud_region,
device)
os.system("rm -rf ../" + device.get_id() + "/sounds")
os.system("mkdir ../" + device.get_id() + "/sounds")
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(args.project_id,
args.pubsub_subscription)
client.on_connect = device.on_connect
client.on_publish = device.on_publish
client.on_disconnect = device.on_disconnect
client.on_subscribe = device.on_subscribe
client.on_message = device.on_message
client.connect(args.mqtt_bridge_hostname, args.mqtt_bridge_port)
client.loop_start()
# mqtt_service_topic = 'projects/project2-277316/topics/my-topic' #central
# This is the topic that the device will receive configuration updates on.
mqtt_config_topic = '/devices/{}/config'.format(args.device_id)
# Wait up to 5 seconds for the device to connect.
device.wait_for_connection(5)
# Subscribe to the config topic.
client.subscribe(mqtt_config_topic, qos=1)
path = args.images_path + "*.jpg"
global send_rek
global send_rek_ack
global ack_sent
global send_pol_ack
global send_pol
global authorized
for filename in glob.glob(path): #assuming gif
im = Image.open(filename)
send_rek.append(im.filename)
def get_callback(f, data):
def callback(f):
try:
print(f.result())
futures.pop(data)
except: # noqa
# print("Please handle {} for {}.".format(f.exception(), data))
print("\n")
return callback
i = 0
while True:
if authorized:
while (send_rek):
image_name = send_rek.pop()
payload_json = {
'type': 'REK',
'img_name': image_name,
'dev_id': device.get_id()
}
payload = json.dumps(payload_json)
print(
"Publishing initial request for rekognition service for image "
+ image_name + "\n\n\n\n\n")
future = publisher.publish(topic_path, payload)
future.add_done_callback(get_callback(future, payload))
now = datetime.datetime.utcnow()
# device.get_mutex().acquire()
# ack_sent[image_name] = now
# device.get_mutex().release()
time.sleep(1)
while (send_rek_ack):
image_name, node_id = send_rek_ack.pop()
image_data = convertImageToByteArray(image_name)
payload_json = {
'type': 'REKACK',
'img_name': image_name,
'node_id': node_id,
'dev_id': device.get_id(),
'img_data': image_data
}
payload = json.dumps(payload_json)
print(
"Publishing acknowledgement for rekognition service for image "
+ image_name + " to rekognition device " + node_id +
"\n\n\n\n\n")
future = publisher.publish(topic_path, payload)
future.add_done_callback(get_callback(future, payload))
# now = datetime.datetime.utcnow()
# device.get_mutex().acquire()
# ack_sent[image_name] = now
# device.get_mutex().release()
time.sleep(1)
while (send_pol):
image_name, labels = send_pol.pop()
payload_json = {
'type': 'POL',
'img_name': image_name,
'dev_id': device.get_id(),
'labels': labels
}
payload = json.dumps(payload_json)
print(
"Publishing initial request for polly service for image " +
image_name + "\n\n\n\n\n")
future = publisher.publish(topic_path, payload)
future.add_done_callback(get_callback(future, payload))
now = datetime.datetime.utcnow()
# device.get_mutex().acquire()
# ack_sent[image_name] = now
# device.get_mutex().release()
time.sleep(1)
while (send_pol_ack):
image_name, second = send_pol_ack.pop()
node_id, labels = second
payload_json = {
'type': 'POLACK',
'img_name': image_name,
'node_id': node_id,
'dev_id': device.get_id(),
'img_data': labels
}
payload = json.dumps(payload_json)
print(
"Publishing acknowledgement for polly service for image " +
image_name + " to polly device " + node_id + "\n\n\n\n\n")
future = publisher.publish(topic_path, payload)
future.add_done_callback(get_callback(future, payload))
# now = datetime.datetime.utcnow()
# device.get_mutex().acquire()
# ack_sent[image_name] = now
# device.get_mutex().release()
time.sleep(1)
# keys_to_rem = list()
# device.get_mutex().acquire()
# for key in ack_sent.keys():
# now = datetime.datetime.utcnow()
# if key in ack_sent.keys():
# diff = now-ack_sent[key]
# if diff.total_seconds() > 60:
# keys_to_rem.append(key)
# if key in image_dict:
# image_dict.pop(key)
# send_rek.append(key)
# for key in keys_to_rem:
# ack_sent.pop(key)
# device.get_mutex().release()
time.sleep(1000)
client.disconnect()
client.loop_stop()
print('Finished loop successfully. Goodbye!')
def main():
"""Executes Steaming pipeline to store dataset into BigQuery table."""
parser = argparse.ArgumentParser(
description=('Send session data to Cloud Pub/Sub' +
' simulating real-time messaging'))
parser.add_argument('--project',
help='Example: --project $DEVSHELL_PROJECT_ID',
required=True)
parser.add_argument(
'--topic',
help='Topic name to publish messages. Example: --topic $TOPIC_NAME',
required=True)
parser.add_argument('--subscription',
help='''Subscription name to receive messages.
Example: --subscription $SUBSCRIPTION_NAME''',
required=True)
parser.add_argument(
'--input',
help='Path to file in GCS bucket. Example: --input gs://$PROJECT/$FILE',
required=True)
parser.add_argument('--speedFactor',
type=int,
default=5,
help=('Hours of data (<6) to publish in 1 minute.' +
' Example: --speedfactor=1'),
choices=[1, 2, 3, 4, 5],
required=True)
args = parser.parse_args()
# create Pub/Sub notification topic
logging.basicConfig(format='%(levelname)s: %(message)s',
level=logging.INFO)
publisher = pubsub_v1.PublisherClient()
subscriber = pubsub_v1.SubscriberClient()
topic_path = publisher.topic_path(args.project, args.topic)
subscription_path = subscriber.subscription_path(args.project,
args.subscription)
# create topic to publish and subscription to receive messages
topic = publisher.create_topic(request={"name": topic_path})
logging.info('Created pub/sub topic {0}'.format(topic))
with subscriber:
subscription = subscriber.create_subscription(request={
"name": subscription_path,
"topic": topic_path
})
logging.info('Created pub/sub subscription {0}'.format(subscription))
# Read dataset 1 chunk at once
logging.info('Reading Data from CSV File')
user_session_chunks = pd.read_csv(args.input,
chunksize=int(args.speedFactor *
(10**5)))
logging.info('Pre-Processing CSV Data')
for user_sessions in user_session_chunks:
# Preprocess data in a chunk
user_sessions = preprocess_data(user_sessions)
# Transform Data into Messages and Simulate Streaming
stream_data_chunk(user_sessions, publisher, topic_path)
logging.info(
'Successfully Published all Messages from Dataset as a Stream')
def publisher():
return pubsub_v1.PublisherClient()
logging.info("User location could not be found | {}".format(query))
return None
wkt_loc = "POINT({} {})".format(
loc['candidates'][0]['geometry']['location']['lng'],
loc['candidates'][0]['geometry']['location']['lat'])
logging.info("User location successfully found | {}".format(query))
return wkt_loc
config = configparser.ConfigParser()
config.read("config.conf")
PROJECT_NAME = config["Config"]["PROJECT_NAME"]
in_cloud = os.environ.get('AM_I_IN_A_DOCKER_CONTAINER', False)
if in_cloud:
pubsub_client = pubsub_v1.PublisherClient()
lang_client = language.LanguageServiceClient()
secrets = CloudSecretsHelper(PROJECT_NAME)
logging.debug('Detected running on Google Cloud')
client = gcl.Client()
client.setup_logging()
handler = CloudLoggingHandler(client)
cloud_logger = logging.getLogger('cloudLogger')
cloud_logger.setLevel(logging.DEBUG)
cloud_logger.addHandler(handler)
else:
credentials = service_account.Credentials.from_service_account_file(
config["Auth"]["AUTH_FILE_PATH"])
pubsub_client = pubsub_v1.PublisherClient(credentials=credentials)
lang_client = language.LanguageServiceClient(credentials=credentials)
dsongcp.flights_simevents
WHERE
EVENT_TIME >= @startTime
AND EVENT_TIME < @endTime
ORDER BY
EVENT_TIME ASC
"""
job_config = bq.QueryJobConfig(query_parameters=[
bq.ScalarQueryParameter("jitter", "INT64", jitter),
bq.ScalarQueryParameter("startTime", "TIMESTAMP", args.startTime),
bq.ScalarQueryParameter("endTime", "TIMESTAMP", args.endTime),
])
rows = bqclient.query(querystr, job_config=job_config)
# create one Pub/Sub notification topic for each type of event
publisher = pubsub.PublisherClient()
topics = {}
for event_type in ['wheelsoff', 'arrived', 'departed']:
topics[event_type] = publisher.topic_path(args.project, event_type)
try:
publisher.get_topic(topic=topics[event_type])
logging.info("Already exists: {}".format(topics[event_type]))
except:
logging.info("Creating {}".format(topics[event_type]))
publisher.create_topic(name=topics[event_type])
# notify about each row in the dataset
programStartTime = datetime.datetime.utcnow()
simStartTime = datetime.datetime.strptime(
args.startTime, TIME_FORMAT).replace(tzinfo=pytz.UTC)
logging.info('Simulation start time is {}'.format(simStartTime))
def __init__(self, project_id):
self.project_id = project_id
self.publisher = pubsub_v1.PublisherClient()
self.topic_path = None
def publish(project_id, topic_id, data):
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project_id, topic_id)
future = publisher.publish(topic_path, data=data.encode("utf-8"))
print(future.result())
def publish_message(topic_name, data):
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project_id, topic_name)
publisher.publish(topic_path, data.encode('utf-8'))
def get_publisher(self):
return pubsub.PublisherClient()
def send_message(message):
publisher = pubsub_v1.PublisherClient()
publisher.publish(topic=TOPIC, data=bytes(str(message), 'utf-8'))
logging.debug(str(message))
def make_publisher_subscription(topic, subscription_name):
""" Make a publisher and subscriber client, and create the necessary resources """
publisher = pubsub.PublisherClient()
try:p
publisher.create_topic(topic)
print('xxx')
class PubSub:
publisher = pubsub_v1.PublisherClient()
# The `topic_path` method creates a fully qualified identifier
# in the form `projects/{project_id}/topics/{topic_name}`
topic_path = publisher.topic_path(PROJECT, TOPIC)
subscriber = pubsub_v1.SubscriberClient()
subscription_path = subscriber.subscription_path(PROJECT, SUB)
def send(self):
for n in range(1, 2):
data = u"Message number {}".format(n)
# Data must be a bytestring
data = data.encode("utf-8")
# When you publish a message, the client returns a future.
future = self.publisher.publish(self.topic_path, data=data)
print(future.result())
print("Published messages.")
def read(self, timeout):
# The `subscription_path` method creates a fully qualified identifier
# in the form `projects/{project_id}/subscriptions/{subscription_name}`
def callback(message):
print("Received message: {}".format(message))
message.ack()
streaming_pull_future = self.subscriber.subscribe(
self.subscription_path, callback=callback)
print("Listening for messages on {}..\n".format(
self.subscription_path))
# Wrap subscriber in a 'with' block to automatically call close()
# when done.
with self.subscriber:
try:
# When `timeout` is not set, result() will block indefinitely,
# unless an exception is encountered first.
streaming_pull_future.result(timeout=timeout)
except: # noqa
streaming_pull_future.cancel()
def readMsgProcess(self, timeout):
def callback(message):
m = Mail()
q = BigQ()
m.populateSnippet()
for i in m.data:
q.insert(i[0], i[1], i[2], i[4])
print("Received message: {}".format(message))
message.ack()
streaming_pull_future = self.subscriber.subscribe(
self.subscription_path, callback=callback)
print("Listening for messages on {}..\n".format(
self.subscription_path))
# Wrap subscriber in a 'with' block to automatically call close()
# when done.
with self.subscriber:
try:
# When `timeout` is not set, result() will block indefinitely,
# unless an exception is encountered first.
streaming_pull_future.result(timeout=timeout)
except: # noqa
streaming_pull_future.cancel()
import json
from google.auth import jwt
from google.cloud import pubsub_v1
service_account_info = json.load(open("./credentials.json"))
audience = "https://pubsub.googleapis.com/google.pubsub.v1.Subscriber"
credentials = jwt.Credentials.from_service_account_info(
service_account_info, audience=audience
)
subscriber = pubsub_v1.SubscriberClient(credentials=credentials)
publisher_audience = "https://pubsub.googleapis.com/google.pubsub.v1.Publisher"
credentials_pub = credentials.with_claims(audience=publisher_audience)
publisher = pubsub_v1.PublisherClient(credentials=credentials_pub)
RFC3339_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S-00:00'
colors = [ "red", "orange", "yellow", "green", "blue", "indigo", "violet" ]
source = [ "site", "mobile" ]
def get_event():
event = json.dumps({
'color': random.choice(colors),
'source': random.choice(source),
'created': datetime.datetime.utcnow().strftime(RFC3339_TIME_FORMAT),
'user_id': random.randint(1001,1099)
})
return event
ap.add_argument("-p", "--print", required=True, help="local or gcp env")
args = vars(ap.parse_args())
if args['env'] != 'gcp':
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = './sa.json'
topic_name = "store_events"
project_id = "yonis-sandbox-20180926"
batch_settings = pubsub_v1.types.BatchSettings(
max_bytes=512000,
max_latency=5, # One second
max_messages=500)
pubsub_client = pubsub_v1.PublisherClient(batch_settings)
topic_path = pubsub_client.topic_path(project_id, topic_name)
while True:
bask = basket_orders()
basket_rows = bask.basket_orders()
for r in bask.basket:
if args['print'] == "print":
print r
message_future = pubsub_client.publish(topic_path,
data=r.encode('utf-8'))
print('sent {} items'.format(bask.qty))
def callback_data(request):
# [START functions_callback_data]
"""HTTP Cloud Function.
Args:
data (dict): The dictionary with data specific to the given event.
context (google.cloud.functions.Context): The Cloud Functions event
metadata.
"""
import os
import json
import base64
import datetime
from flask import abort
from rfc3339 import rfc3339
from google.cloud import datastore, pubsub_v1
if request.method == 'POST':
http_user = os.environ.get('HTTP_USER')
http_passwd = os.environ.get('HTTP_PASSWD')
request_user = request.authorization["username"]
request_passwd = request.authorization["password"]
if request_user == http_user and request_passwd == http_passwd:
request_dict = request.get_json()
print('Received Sigfox message: {}'.format(request_dict))
publisher = pubsub_v1.PublisherClient()
topic = os.environ.get('PUBSUB_TOPIC_DATA')
project_id = os.environ.get('GCP_PROJECT')
if not topic or not project_id:
print('Error reading Function environment variables')
return abort(500)
topic_name = 'projects/{project_id}/topics/{topic}'.format(
project_id=project_id, topic=topic)
try:
time_int = int(request_dict['time'])
except ValueError:
time_int = float(request_dict['time'])
time_int = int(time_int)
event_time = rfc3339(time_int)
request_json = json.dumps(request_dict)
message_id = publisher.publish(topic_name,
request_json.encode('utf-8'),
ts=event_time.encode('utf-8'))
print('Message published to Pub/Sub topic: {}'.format(topic_name))
if 'ack' in request_dict:
if (request_dict['ack'] == 'true' and request_dict['device']
and request_dict['deviceType']):
device_type = request_dict['deviceType']
ds_kind = os.environ.get('DATASTORE_KIND')
ds_property = os.environ.get('DATASTORE_PROPERTY')
ds_client = datastore.Client()
ds_key = ds_client.key(ds_kind, device_type)
e = ds_client.get(ds_key)
if not e:
print('Datastore entity not found for deviceType: {}'.
format(device_type))
return '', 204
elif ds_property not in e:
print(
'Datastore property: {} not found for deviceType: {}'
.format(ds_property, device_type))
return '', 204
elif len(e[ds_property]) > 16:
print(
'Datastore config: {} wrong length for deviceType: {}. Should be max 16 HEX characters (8 bytes)'
.format(e[ds_property], device_type))
return '', 204
else:
device = request_dict['device']
response_dict = {}
response_dict[device] = {
'downlinkData': e[ds_property]
}
response_json = json.dumps(response_dict)
print('Sending downlink message: {}'.format(
response_json))
return response_json, 200
else:
return '', 204
else:
return '', 204
else:
print('Invalid HTTP Basic Authentication: '
'{}'.format(request.authorization))
return abort(401)
else:
print('Invalid HTTP Method to invoke Cloud Function. '
'Only POST supported')
return abort(405)
def main():
# [START main]
import configparser
from google.cloud import pubsub_v1
import RPi.GPIO as GPIO
#import gpiozero as io
import time
import os
#import wget
configParser = configparser.RawConfigParser()
#print('Beginning file download with wget module')
#url = 'http://iot-labs.no/img/iotlabs_roundmesh.png'
#wget.download(url, '/home/haakonsbakken/iot-labs.png')
if os.environ.get('SNAP_DATA') == 'True':
configFilePath = r'$SNAP_DATA/parameters.conf'
try:
configParser.read(configFilePath)
credentialsPath = configParser.get('telemetry', 'credentials_path')
print('Read credentials path from {}: {}'.format(
configFilePath, credentialsPath))
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentialsPath
except:
print('Exception when reading parameters from {}'.format(
configFilePath))
# logging.config.fileConfig('$SNAP_DATA/logging.conf')
#else:
# logging.config.fileConfig('logging.conf')
#sanity check
if 'GOOGLE_APPLICATION_CREDENTIALS' in os.environ:
print('$GOOGLE_APPLICATION_CREDENTIALS: {}'.format(
os.environ['GOOGLE_APPLICATION_CREDENTIALS']))
else:
print('Could not find $GOOGLE_APPLICATION_CREDENTIALS')
#should exit!
os.environ[
'GOOGLE_APPLICATION_CREDENTIALS'] = '/home/haakonsbakken/pubsubcredentials.json'
# Set up PIN 21 as input
GPIO.setmode(GPIO.BCM)
GPIO.setwarnings(False)
GPIO.setup(21, GPIO.IN)
#pin = io.Button("GPIO21")
project_id = "appliedautonomybackend"
topic_name = "test-iot-topic"
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project_id, topic_name)
def callback(message_future):
# When timeout is unspecified, the exception method waits indefinitely.
if message_future.exception(timeout=30):
print('Publishing message on {} threw an Exception {}.'.format(
topic_name, message_future.exception()))
else:
print(message_future.result())
# Every second read input
while True:
val = GPIO.input(21)
#val = pin.is_pressed
print('GPIO21 = {}'.format(val))
# Publish messages.
data = 'GPIO21: {}'.format(val)
# Data must be a bytestring
data = data.encode('utf-8')
# When you publish a message, the client returns a future.
message_future = publisher.publish(
topic_path,
data=data,
timestamp=str(time.gmtime()).encode('utf-8'),
description='IoT test GPIO telemetry')
message_future.add_done_callback(callback)
print('Published {} of message ID {}.'.format(data,
message_future.result()))
time.sleep(1)
def publish_active_divisions(event, context):
"""
Publishes parsed election data by division to a Pub/Sub topic with an error handler.
For each division associated with an election, message includes:
- election_id=election_id, # As returned by Civic Information API
- address=address, # Address of geo division associated with election parsed from locales data.
- geo_id=geo_id # Fips code or similar geodivision identifier as parsed from locales data
"""
def get_callback(f, data):
def callback(f):
try:
logging.info(f.result())
futures.pop(data)
except: # noqa
logging.info("Please handle {} for {}.".format(
f.exception(), data))
return callback
# Job status
logging.info("Starting job to parse election.")
logging.info("""Trigger: messageId {} published at {}""".format(
context.event_id, context.timestamp))
# Initiate job
civic = VoterInfo()
logging.info("Load addreses by locale")
locales = civic.load_address_locales("address_locales",
"addresses_county.csv")
project_id = "election-tracker-268319"
topic_name = "active-divisions"
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(project_id, topic_name)
futures = dict()
# Parse election
try:
election = event['attributes']
except Exception as error:
logging.error("Error: Message does not contain event attributes.")
logging.error(error)
raise
election_id = election['election_id'] #renamed to avoid conflict
election_name = election['name']
election_ocdid = election['ocdDivisionId']
# Get state abbr from OCDid
election_ocdid = election_ocdid.split("/")[-1].split(":")[-1].upper()
logging.debug(f"election_id: {election_id}")
logging.debug(f"election_ocdid: {election_ocdid}")
logging.debug(f"election_name: {election_name}")
# Subset data by OCDid
# Except test election
if election_name == 'VIP Test Election':
logging.debug("Election name 'VIP Test Election' excluded.")
return
# If election is national, return data for all records
elif election_ocdid == 'US':
active = locales.copy()
# If election is statewide, return data for all records in state.
else:
active = locales.loc[locales['state_abbr'] == election_ocdid, :].copy()
# Ensure active elections not null
try:
assert (active.empty == False)
except Exception as e:
logging.error("Unable to subset data by OCDid.")
raise
# publish active division
for index, row in active.iterrows():
data = str(row["fips"])
logging.info(f"Publishing message to {topic_path}")
futures.update({data: None})
# When you publish a message, the client returns a future. Data must be a bytestring.
future = publisher.publish(topic_path,
data=data.encode("utf-8"),
election_id=election_id,
address=row['address'],
geo_id=str(row["fips"]))
futures[data] = future
# Publish failures shall be handled in the callback function.
future.add_done_callback(get_callback(future, data))
# Wait for all the publish futures to resolve before exiting.
while futures:
time.sleep(5)
logging.info(
f"Published active divisions for election {election_id} to {topic_name}"
)
def __init__(self, project_id: str, topic_id: str) -> None:
"""Init."""
self.publisher = pubsub_v1.PublisherClient()
self.topic_path = self.publisher.topic_path(project_id, topic_id)
def _publish(self, topic, event_dict, **kwargs):
message = json.dumps(event_dict).encode("utf-8")
if self.publisher_client is None:
self.publisher_client = pubsub_v1.PublisherClient(credentials=self.credentials)
self.publisher_client.publish(topic, message, **kwargs)
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
import hmac
import json
from google.cloud import pubsub_v1
publisher_client = pubsub_v1.PublisherClient()
project_id = "<PROJECT_ID>"
topic_name = "<PUBSUB_TOPIC_NAME>"
topic_path = publisher_client.topic_path(project_id, topic_name)
futures = dict()
with open("config.json", "r") as f:
data = f.read()
config = json.loads(data)
# Python 3+ version of https://github.com/slackapi/python-slack-events-api/blob/master/slackeventsapi/server.py
def verify_signature(request):
""" Takes a Slack request and determines
def end_to_end(project_id, topic_name, subscription_name, num_messages):
# [START pubsub_end_to_end]
import time
from google.cloud import pubsub_v1
# TODO project_id = "Your Google Cloud Project ID"
# TODO topic_name = "Your Pub/Sub topic name"
# TODO num_messages = number of messages to test end-to-end
# Instantiates a publisher and subscriber client
publisher = pubsub_v1.PublisherClient()
subscriber = pubsub_v1.SubscriberClient()
# The `topic_path` method creates a fully qualified identifier
# in the form `projects/{project_id}/topics/{topic_name}`
topic_path = subscriber.topic_path(project_id, topic_name)
# The `subscription_path` method creates a fully qualified identifier
# in the form `projects/{project_id}/subscriptions/{subscription_name}`
subscription_path = subscriber.subscription_path(project_id,
subscription_name)
# Create the topic.
topic = publisher.create_topic(topic_path)
print("\nTopic created: {}".format(topic.name))
# Create a subscription.
subscription = subscriber.create_subscription(subscription_path,
topic_path)
print("\nSubscription created: {}\n".format(subscription.name))
publish_begin = time.time()
# Publish messages.
for n in range(num_messages):
data = u"Message number {}".format(n)
# Data must be a bytestring
data = data.encode("utf-8")
# When you publish a message, the client returns a future.
future = publisher.publish(topic_path, data=data)
print("Published {} of message ID {}.".format(data, future.result()))
publish_time = time.time() - publish_begin
messages = set()
def callback(message):
print("Received message: {}".format(message))
# Unacknowledged messages will be sent again.
message.ack()
messages.add(message)
subscribe_begin = time.time()
# Receive messages. The subscriber is nonblocking.
subscriber.subscribe(subscription_path, callback=callback)
print("\nListening for messages on {}...\n".format(subscription_path))
while True:
if len(messages) == num_messages:
subscribe_time = time.time() - subscribe_begin
print("\nReceived all messages.")
print("Publish time lapsed: {:.2f}s.".format(publish_time))
print("Subscribe time lapsed: {:.2f}s.".format(subscribe_time))
break
else:
# Sleeps the thread at 50Hz to save on resources.
time.sleep(1.0 / 50)
# Release subscriber's underlying resources.
subscriber.close()
def task_manager_scope(session):
print("[Training task manager] Started. Retrain_flag:", re_train, file=sys.stderr)
session = sessionMaker.scoppedSession() # Threadsafe
# Maybe better to have this somewhere else
version = get_current_version(session=session)
if version.machine_learning_settings_id is None:
ml_settings.machine_learning_settings_new(session=session)
# Advance one for training if not retraining
if re_train == 0:
ml_settings.machine_learning_settings_edit(session=session, next_id=True)
project = get_current_project(session=session)
machine_learning_settings = get_ml_settings(session=session, version=version)
JOB_NAME = "__projectID_" + str(project.id) + "__versionID_" + str(version.id) + "__ml_compute_id_" + str(machine_learning_settings.ml_compute_engine_id)
if re_train == 1:
machine_learning_settings.re_train_id += 1
JOB_NAME += "__retrainID_" + str(machine_learning_settings.re_train_id)
machine_learning_settings.JOB_NAME = JOB_NAME
session.add(machine_learning_settings)
session.commit()
# Do YAML for retraining
# TODO way to detect if this is needed or not...
yamlNew(hold_thread=True)
labelMapNew()
fasterRcnnResnetNew(re_train=re_train) # Config file
tfrecordsNew(hold_thread=True)
### TRAINING
runTraining(session)
config = {}
config['PUBSSUB_TOPIC'] = settings.PUB_SUB_TOPIC
config['PROJECT'] = settings.GOOGLE_PROJECT_NAME
publisher = pubsub_v1.PublisherClient()
topic_path = publisher.topic_path(config['PROJECT'], config['PUBSSUB_TOPIC'])
JOB_NAME = "train_" + machine_learning_settings.JOB_NAME
JOB_NAME_FORMATTED = projectID + "/jobs/" + JOB_NAME
training_flag = True
while training_flag is True:
request = ml.projects().jobs().get(name=JOB_NAME_FORMATTED)
# TODO error handling
response = request.execute()
data = json.dumps(response)
print(data, file=sys.stderr)
data = data.encode()
publisher.publish(topic_path, data=data)
a = response['state']
if a == "SUCCEEDED" or a == "FAILED" or a =="CANCELLED":
training_flag = False
else:
time.sleep(30)
#### END TRAINING
# Now need to run new model on re training
if re_train == 0:
runNewModel(session)
##### FROZEN
trainingFrozenRun(session)
JOB_NAME = "frozen_user_" + machine_learning_settings.JOB_NAME
JOB_NAME_FORMATTED = projectID + "/jobs/" + JOB_NAME
frozen_flag = True
while frozen_flag is True:
request = ml.projects().jobs().get(name=JOB_NAME_FORMATTED)
# TODO error handling
response = request.execute()
data = json.dumps(response)
print(data, file=sys.stderr)
data = data.encode()
publisher.publish(topic_path, data=data)
a = response['state']
if a == "SUCCEEDED" or a == "FAILED" or a =="CANCELLED":
frozen_flag = False
else:
time.sleep(30)
#####
runNewVersion(session)
time.sleep(60*8) # Sleep while long running operation
runInferenceSingle()
print("[Training task manager] SUCCESS", file=sys.stderr)
t.cancel()
def parse_and_paginate(message, context):
"""Function that parses the page from pagination and sends the next page to pubsub
Arguments:
data {[base64.encoded]} -- object of pubsub with hashed url on data['data']
context {[]} -- context of the pubsub element
Raises:
Exception: [Error, page is invalid or has no data]
"""
# pubsub topic of this function
_THIS_FUNCTION_TOPIC = os.environ["THIS_TOPIC"]
# pubsub topic that the function will be passed
_DOWNLOAD_HTML_TOPIC = os.environ["DOWNLOAD_HTML_TOPIC"]
# base url to be aggregated to parsed url
_BASE_URL = os.environ['BASE_URL']
# selector of the css paging button
_PAGINATION_CSS_SELECTOR = os.environ['PAGINATION_SELECTOR']
# selector of the parsed objects
_CHILD_CSS_SELECTOR = os.environ['PARSE_SELECTOR']
# Instantiating log client
LOG_CLIENT = cloud_logging.Client()
HANDLER = LOG_CLIENT.get_default_handler()
LOGGER = logging.getLogger("PAGINATION")
LOGGER.setLevel(logging.INFO)
LOGGER.addHandler(HANDLER)
error_client = error_reporting.Client()
def __error_path(publisher, pub_obj_encoded, tries, url, error):
"""Function to handle possible errors on pagination
Args:
pub_obj_encoded ([dict]): [pubsub dict with infos of the page and tries]
tries ([int]): [number of tries that this page was tried]
url ([str]): [url to be parsed for pagination]
"""
if tries < 5:
publisher.publish(_THIS_FUNCTION_TOPIC, pub_obj_encoded)
else:
raise ConnectionError(
"%s pagination already parsed 5 times, ended with %s page",
url, error)
data = base64.b64decode(message['data']).decode('utf-8')
json_decoded = json.loads(data)
url_decode = json_decoded['url']
# Adding number o tries
tries = 0
if 'tries' in json_decoded:
tries = int(json_decoded['tries']) + 1
# Creating element and getting response
publisher = pubsub_v1.PublisherClient()
# Object for failure
pub_obj_encoded = json.dumps({
'url': url_decode,
'tries': tries
}).encode("utf-8")
response = requests.get(url_decode, headers=HEADERS)
try:
response.raise_for_status()
soup = BeautifulSoup(response.content, 'lxml')
# If the request has error page 500 follow error path
if soup.select('title')[0].text == 'Error 500':
raise HTTPError("500 Server Error: PAGE OFFLINE")
else:
# Next url soup object
next_url = soup.select(_PAGINATION_CSS_SELECTOR)
if next_url:
# Loging if there is no next url and publish
next_url = next_url[0].select('a')[0]['href']
if not (next_url.startswith("http://")
or next_url.startswith("https://")):
next_url = _BASE_URL + next_url
pub_next_obj = json.dumps({"url": next_url})
publisher.publish(_THIS_FUNCTION_TOPIC,
pub_next_obj.encode('utf-8'))
else:
logging.info("Last url %s", url_decode)
# Products <a/> attributes to be parsed
products_soups = soup.select(_CHILD_CSS_SELECTOR)
if not products_soups:
raise Exception('Invalid value of products')
products_url = [attribute['href'] for attribute in products_soups]
# Publishing urls to the products topic
for url in products_url:
if not (url.startswith("http://")
or url.startswith("https://")):
url = _BASE_URL + url
product_obj = json.dumps({"url": url})
publisher.publish(_DOWNLOAD_HTML_TOPIC,
product_obj.encode('utf-8'))
except HTTPError as error:
if error.response.status_code == 403:
__error_path(publisher,
pub_obj_encoded,
tries,
url_decode,
error=403)
else:
logging.error(error)
error_client.report_exception()
except ConnectionError as error:
logging.error("PAGE MAX TRIES: %s", str(error))
error_client.report_exception()
except Exception as error:
logging.error(error)
error_client.report_exception()
def __init__(self, project_id, topic_name):
from google.cloud import pubsub_v1
self.publisher = pubsub_v1.PublisherClient()
self.topic_path = self.publisher.topic_path(project_id, topic_name)
self._sent_counter = 0
self._published_counter = 0
