class HubSender(object): sender = "" client = "" def __init__(self, devenv='doh'): """ Create Event Hub Sender """ if devenv == 'prod': ADDRESS = "amqps://doh-airqual-eventhub.servicebus.windows.net/doh-airqual-event-hub-prod" USER = "******" KEY = "1p1aHCJc5IbamvnzlnvUa2wlvXsaJpSAbAORGlPRaQ4=" elif devenv == "dev": ADDRESS = "amqps://az-doh-airqual-eventhub.servicebus.windows.net/az-doh-airqual-eventhub" USER = "******" KEY = "pfWGtB6obtiUCwwAobAAuaq7B9SZSsnpHY6ArEbeS1A=" self.client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) self.sender = self.client.add_sender(partition="1") self.client.run() def senddata(self, readings): self.sender.send(EventData(readings)) def stop(self): self.client.stop() def __del__(self): #self.sender.close self.client.stop()
def to_azure(events): client = EventHubClient(ADDRESS, debug=True) sender = client.add_sender(partition="0") client.run() try: for event in events: sender.send(EventData(event)) except: raise finally: client.stop()
def main(req: func.HttpRequest) -> func.HttpResponse: logger = logging.getLogger(__name__) formatter = logging.Formatter( '%(asctime)s %(name)s %(levelname)s: %(message)s') func_context = os.environ['FUNCTION_CONTEXT'] logger.debug(f"Function context --> {func_context}") credentials = None subscription_id = None kv_credentials = None kv_subscription_id = None if func_context == 'local': filehandler = logging.FileHandler('func.log') filehandler.setFormatter(formatter) logger.addHandler(filehandler) logger.setLevel(logging.DEBUG) credentials, subscription_id = get_local_credentials() else: console = logging.StreamHandler() console.setLevel(logging.INFO) console.setFormatter(formatter) credentials, subscription_id = get_azure_credentials() logger.debug('Python HTTP trigger function processed a request.') logger.debug(f"method={req.method}, url={req.url}, params={req.params}") logger.debug(f"body={req.get_json()}") # Handle WebHook webhook = req.get_json() #Create an empty dict within webhook for motsID webhook['additionalData'] = {} addl_data = webhook['additionalData'] addl_data['motsID'] = "0000" # Key Vault stuff. kv_mgmt_client = KeyVaultManagementClient(credentials, subscription_id) kv_client = KeyVaultClient(credentials) namespace = get_kv_secret(kv_client, 'EventHubNamespace') event_hub = get_kv_secret(kv_client, 'EventHub') user = get_kv_secret(kv_client, 'EventHubKeyName') key = get_kv_secret(kv_client, 'EventHubKey') amqp_uri = f"https://{namespace}.servicebus.windows.net/{event_hub}" eh_client = EventHubClient(amqp_uri, debug=False, username=user, password=key) eh_sender = eh_client.add_sender(partition="0") eh_client.run() eh_sender.send(EventData(json.dumps(webhook))) logger.info(f"sending event to {amqp_uri}, {json.dumps(webhook)}") date = datetime.datetime.now() return func.HttpResponse(json.dumps({'date': date, 'status': 'SUCCESS'}))
def __init__(self): #load config with open('config.json', 'r') as json_file: self.config = json.load(json_file) # Create Event Hubs client client = EventHubClient(self.config["EH_ADDRESS"], debug=False, username=self.config["EH_USER"], password=self.config["EH_KEY"]) Worker.event_hub_sender = client.add_sender(partition="0") client.run()
def isNewLabeledData(eh_url, eh_offset_url, eh_account, eh_key): ''' Examines the EventHub to identify if sufficient quantities of new training data is available to trigger a re-train ''' CONSUMER_GROUP = "$default" PARTITION = "0" offset_client = EventHubClient(eh_offset_url, debug=False, username=eh_account, password=eh_key) offset_receiver = offset_client.add_receiver(CONSUMER_GROUP, PARTITION, prefetch=5000) offset_sender = offset_client.add_sender(partition="0") offset_client.run() #Retrieves the current offset/sequence number for the write event queue from the dedicated offset queue offsets = offset_receiver.receive(timeout=50) current_offset = -1 #Default to -1 or reading the entire feed if another offset is not retrieved logging.info("{0} write messages recieved".format(len(offsets))) for offset in offsets: offset_event = json.loads(offset.body_as_str()) current_offset = offset_event['CURRENT_OFFSET'] logging.info("Retrieved previous offset event {0}".format(offset_event)) current_offset = -1 #Use the retrieved offset/sequence number to retrieve new writes event_client = EventHubClient(eh_url, debug=False, username=eh_account, password=eh_key) receiver = event_client.add_receiver(CONSUMER_GROUP, PARTITION, prefetch=5000, offset=Offset(current_offset)) event_client.run() batch = receiver.receive(timeout=50) new_label_count = len(batch) for stuff in batch: logging.info("Offset {0}".format(stuff.sequence_number)) current_offset = int(stuff.sequence_number) if int(stuff.sequence_number) > current_offset else current_offset logging.info("Message {0}".format(stuff.body_as_str())) logging.info("Processed {0} new label writes".format(new_label_count)) #Write the last retrieved offset/sequence number to the offset message queue to be used in the next read offset_sender.send(EventData(json.dumps({"TIMESTAMP": datetime.datetime.now().timestamp(), "CURRENT_OFFSET": current_offset}))) logging.info("Stored current offset event {0}".format(current_offset)) #sender.send(EventData(json.dumps({"EVENT_TYPE": "LABEL_WRITE", "LABEL_INDEX":face_hash, "WRITE_TIMESTAMP": datetime.datetime.now().timestamp()}))) #Close queue clients offset_client.stop() event_client.stop() #Return true if more then results found to execute retrain return True if new_label_count > 5 else False
class Producer: client = None sender = None def __init__(self, address, user, key): self.client = EventHubClient(address, debug=False, username=user, password=key) self.sender = self.client.add_sender(partition="0") self.client.run() def send(self, msg): self.sender.send(EventData(str(msg))) def stop(self): self.client.stop()
def send_msg(addr, user, password): from azure.eventhub import EventHubClient, Sender, EventData client = EventHubClient(addr, username=user, password=password) sender = client.add_sender(partition='1') client.run() try: from datetime import datetime current_time = datetime.now().isoformat() message = "hi from event hub {}".format(current_time) event_data = EventData(message) sender.send(event_data) except: raise # return client finally: client.stop()
def execute(self): dataset_id = '{}_{}'.format(self.dataset_url, self.dataset_filename) latest_status = self.redis_client.get(dataset_id) if latest_status is not None: latest_status = latest_status.decode(ENCODING) client = EventHubClient( read_config.cfg.get('nulplabs', 'hub_address'), debug=False, username=read_config.cfg.get('nulplabs', 'hub_user'), password=read_config.cfg.get('nulplabs', 'hub_passwd')) sender = client.add_sender(partition="0") client.run() if latest_status == 'COMPLETED' or latest_status == 'ATTEMPT TO REFILL': self.redis_client.set(dataset_id, 'ATTEMPT TO REFILL') print('ATTEMPT TO REFILL') return False client = Socrata(self.dataset_url, None) self.redis_client.set(dataset_id, 'STARTED') for i in range(int(NUMBER_OF_MESSAGES / MESSAGES_PER_FETCH)): results = client.get(self.dataset_filename, limit=MESSAGES_PER_FETCH, offset=MESSAGES_PER_FETCH * i) results_df = pd.DataFrame.from_records(results) # print(json.dumps(results)) current_progress = '{} - {}'.format( str(i * MESSAGES_PER_FETCH + 1), str((i + 1) * MESSAGES_PER_FETCH)) message = EventData(batch=self.data_generator(results)) sender.send(message) self.redis_client.set(dataset_id, current_progress) # print('Progress {}'.format(current_progress)) # print(results_df) self.redis_client.set(self.dataset_url + "_" + self.dataset_filename, 'COMPLETED') print('COMPLETED')
def run_job(): # Create Event Hubs client client = EventHubClient(config.ADDRESS, debug=False, username=config.USER, password=config.KEY) sender = client.add_sender(partition="0") client.run() while True: try: messages = [ create_advert_event(brands), create_impression_event(urls) ] for message in messages: print('Sending message to Event Hubs: ' + str(message)) sender.send(EventData(message)) time.sleep(1) except: raise
def main(): """ Main method of sender """ namespace = os.environ['EVENT_HUB_NAMESPACE'] ehname = os.environ['EVENT_HUB_NAME'] address = "amqps://" + namespace + ".servicebus.windows.net/" + ehname # SAS policy and key are not required if they are encoded in the URL user = os.environ.get('EVENT_HUB_SAS_POLICY') key = os.environ.get('EVENT_HUB_SAS_KEY') try: if not address: raise ValueError("No EventHubs URL supplied.") client = EventHubClient(address, debug=False, username=user, password=key) sender = client.add_sender(partition="0") client.run() try: start_time = time.time() for i in range(100): message = '{ "intValue": ' + str(i) + ' }' LOGGER.info("Sending message: %s", message) sender.send(EventData(message)) except: raise finally: end_time = time.time() client.stop() run_time = end_time - start_time LOGGER.info("Runtime: %s seconds", run_time) except KeyboardInterrupt: pass
class QueueEventHub(QueueInterface): def __init__(self, address, user, key): self.address = address self.user = user self.key = key self.counter = 0 self.client_batch = EventHubClient(self.address, debug=False, username=self.user, password=self.key) self.sender = self.client_batch.add_sender() self.client_batch.run() def batch_send(self, events: List[TimeserieRecord]): self.counter += len(events) data = EventData( batch=[json.dumps(dataclasses.asdict(e)) for e in events if e]) self.sender.transfer(data) self.sender.wait() def send(self, events: List[TimeserieRecord]): return self.batch_send(events)
def sendToEventHub(data): try: client = EventHubClient(cfg['EventHubURL'], debug=False, username=cfg['EventHubPolicyName'], password=cfg['EventHubPrimaryKey']) sender = client.add_sender(partition="0") client.run() try: count = 0 for payload in data: sender.send(EventData(json.dumps(payload))) #logPrint("Payload sent: " + json.dumps(payload)) count += 1 except: logPrint("Send to Eventhub Failed") raise finally: logPrint(str(count) + " payloads sent") data.clear() client.stop() except KeyboardInterrupt: pass
class EventHubOutputFormatter(OutputFormatterBaseClass): def __init__(self): NAMESPACE = os.environ['EVENT_HUB_NAMESPACE'] EHNAME = os.environ['EVENT_HUB_NAME'] ADDRESS = "amqps://" + NAMESPACE + ".servicebus.windows.net/" + EHNAME # SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') self.client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) self.sender = self.client.add_sender(partition="0") self.client.run() def send(self, message): self.sender.send(EventData(message)) pass def close(self): self.client.stop() pass
class EventHubStreamingClient(AbstractStreamingClient): """Event Hub streaming client.""" def __init__(self, config): # pragma: no cover """ Azure EventHub streaming client implementation. Configuration keys: AZURE_STORAGE_ACCESS_KEY AZURE_STORAGE_ACCOUNT EVENT_HUB_CONSUMER_GROUP EVENT_HUB_NAME EVENT_HUB_NAMESPACE EVENT_HUB_SAS_KEY EVENT_HUB_SAS_POLICY LEASE_CONTAINER_NAME TIMEOUT """ storage_account_name = config.get("AZURE_STORAGE_ACCOUNT") storage_key = config.get("AZURE_STORAGE_ACCESS_KEY") lease_container_name = config.get("LEASE_CONTAINER_NAME") namespace = config.get("EVENT_HUB_NAMESPACE") eventhub = config.get("EVENT_HUB_NAME") consumer_group = config.get("EVENT_HUB_CONSUMER_GROUP", '$default') user = config.get("EVENT_HUB_SAS_POLICY") key = config.get("EVENT_HUB_SAS_KEY") try: self.timeout = int(config['TIMEOUT']) except (KeyError, ValueError): self.timeout = None self.logger = Logger() # Create EPH Client if storage_account_name is not None and storage_key is not None: self.eph_client = EventHubConfig( sb_name=namespace, eh_name=eventhub, policy=user, sas_key=key, consumer_group=consumer_group) self.eh_options = EPHOptions() self.eh_options.release_pump_on_timeout = True self.eh_options.auto_reconnect_on_error = False self.eh_options.debug_trace = False self.storage_manager = AzureStorageCheckpointLeaseManager( storage_account_name, storage_key, lease_container_name) # Create Send client else: address = "amqps://" + namespace + \ ".servicebus.windows.net/" + eventhub try: self.send_client = EventHubClient( address, debug=False, username=user, password=key) self.sender = self.send_client.add_sender() self.send_client.run() except Exception as ex: self.logger.error('Failed to init EH send client: %s', ex) raise def start_receiving(self, on_message_received_callback): # pragma: no cover loop = asyncio.get_event_loop() try: host = EventProcessorHost( EventProcessor, self.eph_client, self.storage_manager, ep_params=[on_message_received_callback], eph_options=self.eh_options, loop=loop) tasks = asyncio.gather(host.open_async(), self.wait_and_close(host, self.timeout)) loop.run_until_complete(tasks) finally: loop.stop() def send(self, message): # pragma: no cover try: self.sender.send(EventData(body=message)) self.logger.info('Sent message: %s', message) return True except Exception as ex: self.logger.error('Failed to send message to EH: %s', ex) return False def stop(self): # pragma: no cover try: self.send_client.stop() except Exception as ex: self.logger.error('Failed to close send client: %s', ex) @staticmethod async def wait_and_close(host: EventProcessorHost, timeout: Optional[float] = None): # pragma: no cover """Run a host indefinitely or until the timeout is reached.""" if timeout is None: while True: await asyncio.sleep(1) else: await asyncio.sleep(timeout) await host.close_async()
def processEntityImages(choice_blobs, db_account_name, db_account_key, ca_file_uri, db_config, cs_account, cs_key, eh_url, eh_account, eh_key): ''' Tests entity images for the presence of a face using Azure Cognitive Services, extracts the face based on the provided bounding box, applies the facial classifier if available and then writes the raw image, face to CosmosDB ''' #Initialize the cognitive services account to perform facial detection BASE_CS_URL = 'https://virginia.api.cognitive.microsoft.us/face/v1.0/' # Replace with your regional Base URL CF.Key.set(cs_key) CF.BaseUrl.set(BASE_CS_URL) def extractFace(image_bytes): ''' ''' face_list = CF.face.detect(image_bytes) if len(face_list) == 1: face_rectangle = face_list[0]['faceRectangle'] nparr = np.fromstring(image_bytes, np.uint8) img_byte = Image.open(io.BytesIO(image_bytes)) face_iso_image = img_byte.crop( (face_rectangle['left'], face_rectangle['top'], face_rectangle['left'] + face_rectangle['width'], face_rectangle['top'] + face_rectangle['height'])) return face_iso_image else: return None #Convert the base image to PIL object, then detect and extract the face component of the image blob_image_faces = list( map( lambda blob_tuple: (blob_tuple[0], Image.open(io.BytesIO(blob_tuple[1])), extractFace(blob_tuple[1])), choice_blobs)) logging.debug("Face detection run on {0} images".format( len(blob_image_faces))) #Connect to the database ssl_opts = { 'ca_certs': ca_file_uri, 'ssl_version': PROTOCOL_TLSv1_2, 'cert_reqs': CERT_REQUIRED # Certificates are required and validated } auth_provider = PlainTextAuthProvider(username=db_account_name, password=db_account_key) endpoint_uri = db_account_name + '.cassandra.cosmosdb.azure.com' cluster = Cluster([endpoint_uri], port=10350, auth_provider=auth_provider, ssl_options=ssl_opts) #If no db config file is passed, look for the container environment variables if db_config is None: keyspace = os.environ['DB_KEYSPACE'] personaTableName = os.environ['DB_PERSONA_TABLE'] subPersonaTableName = os.environ['DB_SUB_PERSONA_TABLE'] subPersonaFaceEdgeTableName = os.environ['DB_SUB_PERSONA_EDGE_TABLE'] faceSubPersonaEdgeTableName = os.environ['DB_SUB_PERSONA_EDGE_TABLE'] rawImageTableName = os.environ['DB_RAW_IMAGE_TABLE'] faceTableName = os.environ['DB_REFINED_IMAGE_TABLE'] #Otherwise load db config else: keyspace = db_config['db-keyspace'] personaTableName = db_config['db-persona-table'] subPersonaTableName = db_config['db-sub-persona-table'] subPersonaFaceEdgeTableName = db_config[ 'db-sub-persona-face-edge-table'] faceSubPersonaEdgeTableName = db_config[ 'db-face-sub-persona-edge-table'] rawImageTableName = db_config['db-raw-image-table'] faceTableName = db_config['db-face-image-table'] #Prepare Cosmos DB session and insertion queries session = cluster.connect(keyspace) personaInsertQuery = session.prepare("INSERT INTO " + personaTableName + " (persona_name) VALUES (?)") subPersonaInsertQuery = session.prepare( "INSERT INTO " + subPersonaTableName + " (sub_persona_name, persona_name) VALUES (?, ?)") subPersonaFaceEdgeInsertQuery = session.prepare( "INSERT INTO " + subPersonaFaceEdgeTableName + " (sub_persona_name, assoc_face_id, label_v_predict_assoc_flag) VALUES (?,?,?)" ) faceSubPersonaEdgeInsertQuery = session.prepare( "INSERT INTO " + faceSubPersonaEdgeTableName + " (sub_persona_name, assoc_face_id, label_v_predict_assoc_flag) VALUES (?,?,?)" ) rawInsertQuery = session.prepare( "INSERT INTO " + rawImageTableName + " (image_id, file_uri, image_bytes) VALUES (?,?,?)") refinedInsertQuery = session.prepare( "INSERT INTO " + faceTableName + " (face_id, raw_image_edge_id, face_bytes, feature_bytes) VALUES (?,?,?,?)" ) client = EventHubClient(eh_url, debug=False, username=eh_account, password=eh_key) sender = client.add_sender(partition="0") client.run() face_write_count = 0 face_label_write_count = 0 face_unlabeled_write_count = 0 for (blob, image_bytes, face_bytes) in blob_image_faces: if face_bytes is not None: file_name = blob.name (entity, usage, number) = file_name.split('-') #Generate the face classifier features from the face using VGG-face on Keras if face_bytes.mode != "RGB": face_bytes = face_bytes.convert("RGB") face_bytes = face_bytes.resize((img_width, img_height)) image = img_to_array(face_bytes) image = np.expand_dims(image, axis=0) image = imagenet_utils.preprocess_input(image) stuff = vgg_face_feature_gen.predict(image, batch_size=1).flatten() writer = avro.io.DatumWriter(schema) bytes_writer = io.BytesIO() encoder = avro.io.BinaryEncoder(bytes_writer) writer.write({"features": stuff.tolist()}, encoder) face_feature_bytes = bytes_writer.getvalue() #For each image extract the label and use to generate a persona, redundant writes will cancel out #Writes the entity label to the persona table #For the time being also write the entity label to the subpersona table and associate with the persona table session.execute(personaInsertQuery, (entity, )) session.execute(subPersonaInsertQuery, (entity, entity)) logging.info( "Writing persona, sub-persona {0} to DB table {1}".format( entity, subPersonaTableName)) #Resizes the image to ensure the write query does not exceed maximum size width, height = image_bytes.size if width > height: transform_factor = 256 / width else: transform_factor = 256 / height compact_image_bytes = image_bytes.resize( (round(height * transform_factor), round(width * transform_factor))) #Writes the raw image to its table imgByteArr = io.BytesIO() compact_image_bytes.save(imgByteArr, format='PNG') compact_image_bytes = imgByteArr.getvalue() hashed_bytes = hashlib.md5(compact_image_bytes).digest() hashed_bytes_int = int.from_bytes( hashed_bytes, byteorder='big') #Good identifier, sadly un image_hash = str( hashed_bytes_int ) #Stupid workaround to Python high precision int incompatability session.execute(rawInsertQuery, (image_hash, file_name, compact_image_bytes)) logging.info("Writing raw image to DB {0}".format(image_hash)) #Resizes the image to ensure the write query does not exceed maximum size width, height = face_bytes.size if width > height: transform_factor = 256 / width else: transform_factor = 256 / height compact_face_bytes = face_bytes.resize( (round(height * transform_factor), round(width * transform_factor))) #Write each face extracted from the image to the DB as a refined image imgByteArr = io.BytesIO() compact_face_bytes.save(imgByteArr, format='PNG') compact_face_bytes = imgByteArr.getvalue() face_hash_bytes = hashlib.md5(compact_face_bytes).digest() face_hashed_bytes_int = int.from_bytes( face_hash_bytes, byteorder='big') #Good identifier, sadly un face_hash = str( face_hashed_bytes_int ) #Stupid workaround to Python high precision int incompatability session.execute(refinedInsertQuery, (face_hash, image_hash, compact_face_bytes, face_feature_bytes)) logging.info("Writing face image to DB {0}".format(face_hash)) face_write_count += 1 #If the data is part of the training set, write edges between the sub-personas, face images if usage == "Train": session.execute(subPersonaFaceEdgeInsertQuery, (entity, face_hash, True)) session.execute(faceSubPersonaEdgeInsertQuery, (entity, face_hash, True)) sender.send( EventData( json.dumps({ "EVENT_TYPE": "LABEL_WRITE", "LABEL_INDEX": face_hash, "WRITE_TIMESTAMP": datetime.datetime.now().timestamp() }))) logging.info("Writing face label to DB {0}".format(face_hash)) face_label_write_count += 1 else: #Silly engineering workaround to make it easier to find the unlabeled images later for re-prediction session.execute(subPersonaFaceEdgeInsertQuery, ("TEMPORARY", face_hash, False)) session.execute(faceSubPersonaEdgeInsertQuery, ("TEMPORARY", face_hash, False)) logging.info( "Writing unlabeled face {0} to DB".format(face_hash)) face_unlabeled_write_count += 1 #Otherwise do not write an edge, these will be predicted later by the training service #session.close() client.stop() logging.info("Wrote {0} faces to DB".format(face_write_count)) logging.info("Wrote {0} face labels to DB".format(face_label_write_count))
# Un arreglo con las palabras claves a buscar KeyPhrases = [ '#AvengersEndGame, EndGame, #Avengers, Avengers, #Vengadores, @GameOfThrones, #ForTheThrone, #GameofThrones, @Marvel, @Avengers, @Wendy' ] # KeyPhrases = ['@TecdeMonterrey', '#OrgulloTec', '#EXATEC', '#SomosTec'] print("====== Running App ======") try: # La clase EventHubClient define una conexión para enviar y recibir eventos EH_Client = EventHubClient(Credentials.EH_Address, Credentials.EH_SASName, Credentials.EH_PrimaryKey, debug=False) # Agrega un sender para enviar los eventos EH_Sender = EH_Client.add_sender() # Abre las conecciones y corre los clientes Sender/Receiver EH_Client.run() # Obtiene la autenticación de las credenciales Auth = Get_Authentication() # Crea el objeto listener de tweets myStreamListener = MyStreamListener() # Crea el objeto para empezar a recibir tweets myStream = Stream(Auth, myStreamListener) print(">> Listening tweets") # Filtra tweets por las palabras clave myStream.filter(track=KeyPhrases, stall_warnings=True)
class EventHubStreamingClient(AbstractStreamingClient): # pylint: disable=too-many-instance-attributes """Event Hub streaming client.""" def __init__(self, config): # pragma: no cover """ Azure EventHub streaming client implementation. Configuration keys: AZURE_STORAGE_ACCESS_KEY AZURE_STORAGE_ACCOUNT EVENT_HUB_CONSUMER_GROUP EVENT_HUB_NAME EVENT_HUB_NAMESPACE EVENT_HUB_SAS_KEY EVENT_HUB_SAS_POLICY LEASE_CONTAINER_NAME TIMEOUT """ storage_account_name = config.get("AZURE_STORAGE_ACCOUNT") storage_key = config.get("AZURE_STORAGE_ACCESS_KEY") lease_container_name = config.get("LEASE_CONTAINER_NAME") namespace = config.get("EVENT_HUB_NAMESPACE") eventhub = config.get("EVENT_HUB_NAME") consumer_group = config.get("EVENT_HUB_CONSUMER_GROUP", '$Default') user = config.get("EVENT_HUB_SAS_POLICY") key = config.get("EVENT_HUB_SAS_KEY") try: self.timeout = int(config['TIMEOUT']) except (KeyError, ValueError): self.timeout = None self.logger = Logger() self.loop = None # Create EPH Client if storage_account_name is not None and storage_key is not None: self.eph_client = EventHubConfig(sb_name=namespace, eh_name=eventhub, policy=user, sas_key=key, consumer_group=consumer_group) self.eh_options = EPHOptions() self.eh_options.release_pump_on_timeout = True self.eh_options.auto_reconnect_on_error = False self.eh_options.debug_trace = False self.storage_manager = AzureStorageCheckpointLeaseManager( storage_account_name, storage_key, lease_container_name) self.tasks = None signal.signal(signal.SIGTERM, self.exit_gracefully) # Create Send client else: address = "amqps://" + namespace + \ ".servicebus.windows.net/" + eventhub try: self.send_client = EventHubClient(address, debug=False, username=user, password=key) self.sender = self.send_client.add_sender() self.send_client.run() except Exception as ex: self.logger.error('Failed to init EH send client: %s', ex) raise def start_receiving(self, on_message_received_callback): # pragma: no cover self.loop = asyncio.get_event_loop() try: host = EventProcessorHost(EventProcessor, self.eph_client, self.storage_manager, ep_params=[on_message_received_callback], eph_options=self.eh_options, loop=self.loop) self.tasks = asyncio.gather( host.open_async(), self.wait_and_close(host, self.timeout)) self.loop.run_until_complete(self.tasks) except KeyboardInterrupt: self.logger.info( "Handling keyboard interrupt or SIGINT gracefully.") # Canceling pending tasks and stopping the loop for task in asyncio.Task.all_tasks(): task.cancel() self.loop.run_forever() self.tasks.exception() raise finally: if self.loop.is_running(): self.loop.stop() def exit_gracefully(self, signum, frame): # pylint: disable=unused-argument """Handle signal interrupt (SIGTERM) gracefully.""" self.logger.info("Handling signal interrupt %s gracefully." % signum) # Canceling pending tasks and stopping the loop self.stop() def send(self, message): # pragma: no cover try: self.sender.send(EventData(body=message)) self.logger.info('Sent message: %s', message) return True except Exception as ex: self.logger.error('Failed to send message to EH: %s', ex) return False def stop(self): # pragma: no cover if self.loop: # Stop consumer for task in asyncio.Task.all_tasks(): task.cancel() self.loop.run_forever() if self.tasks: self.tasks.exception() if self.loop.is_running(): self.loop.stop() else: # Stop producer try: self.send_client.stop() except Exception as ex: self.logger.error('Failed to close send client: %s', ex) @staticmethod async def wait_and_close( host: EventProcessorHost, timeout: Optional[float] = None): # pragma: no cover """Run a host indefinitely or until the timeout is reached.""" if timeout is None: while True: await asyncio.sleep(1) else: await asyncio.sleep(timeout) await host.close_async()
def main(): log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() #STEP-2 model_xml = '/opt/intel/computer_vision_sdk/deployment_tools/intel_models/person-vehicle-bike-detection-crossroad-0078/FP16/person-vehicle-bike-detection-crossroad-0078.xml' model_bin = '/opt/intel/computer_vision_sdk/deployment_tools/intel_models/person-vehicle-bike-detection-crossroad-0078/FP16/person-vehicle-bike-detection-crossroad-0078.bin' net = IENetwork.from_ir(model=model_xml, weights=model_bin) model_reid_xml = '/opt/intel/computer_vision_sdk/deployment_tools/intel_models/person-reidentification-retail-0079/FP16/person-reidentification-retail-0079.xml' model_reid_bin = '/opt/intel/computer_vision_sdk/deployment_tools/intel_models/person-reidentification-retail-0079/FP16/person-reidentification-retail-0079.bin' net_reid = IENetwork.from_ir(model=model_reid_xml, weights=model_reid_bin) log.info("Initializing plugin for {} device...".format(args.device)) plugin = IEPlugin(device=args.device, plugin_dirs=args.plugin_dir) if args.cpu_extension and 'CPU' in args.device: plugin.add_cpu_extension( '/opt/intel/computer_vision_sdk/deployment_tools/inference_engine/lib/ubuntu_16.04/intel64/libcpu_extension.so' ) if "CPU" in plugin.device: supported_layers = plugin.get_supported_layers(net) not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(plugin.device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) #STEP-3 exec_net = plugin.load(network=net, num_requests=1) exec_net_reid = plugin.load(network=net_reid, num_requests=1) #STEP-4 input_blob = next(iter(net.inputs)) #input_blob = 'data' out_blob = next(iter(net.outputs)) #out_blob = 'detection_out' model_n, model_c, model_h, model_w = net.inputs[ input_blob] #model_n, model_c, model_h, model_w = 1, 3, 1024, 1024 input_blob_reid = next(iter(net_reid.inputs)) #input_blob = 'data' out_blob_reid = next(iter(net_reid.outputs)) #out_blob = 'detection_out' model_reid_n, model_reid_c, model_reid_h, model_reid_w = net_reid.inputs[ input_blob_reid] #model_n, model_c, model_h, model_w = 1, 3, 160, 64 # print(net_reid.inputs[input_blob_reid]) del net del net_reid if args.input == 'cam': input_stream = 0 else: input_stream = args.input #STEP-5 # url = "http://192.168.1.16:8081/?action=stream" # videof = "video.mp4" cap = cv2.VideoCapture(input_stream) personData = {} ADDRESS = args.address USER = args.user KEY = args.key client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) sender = client.add_sender(partition="0") client.run() while cap.isOpened(): ret, frame = cap.read() if not ret: break #STEP-6 cap_w = cap.get(3) cap_h = cap.get(4) in_frame = cv2.resize(frame, (model_w, model_h)) in_frame = in_frame.transpose( (2, 0, 1)) # Change data layout from HWC to CHW in_frame = in_frame.reshape((model_n, model_c, model_h, model_w)) #STEP-7 exec_net.start_async(request_id=0, inputs={input_blob: in_frame}) if exec_net.requests[0].wait(-1) == 0: res = exec_net.requests[0].outputs[out_blob] #print(res.shape) #STEP-8 for obj in res[0][0]: class_id = int(obj[1]) if class_id == 1: if obj[2] > args.prob_threshold: xmin = int(obj[3] * cap_w) ymin = int(obj[4] * cap_h) xmax = int(obj[5] * cap_w) ymax = int(obj[6] * cap_h) # Draw box and label\class_id frame_org = frame.copy() person = frame_org[ymin:ymax, xmin:xmax] in_frame_reid = cv2.resize( person, (model_reid_w, model_reid_h)) in_frame_reid = in_frame_reid.transpose( (2, 0, 1)) # Change data layout from HWC to CHW in_frame_reid = in_frame_reid.reshape( (model_reid_n, model_reid_c, model_reid_h, model_reid_w)) cv2.imshow("Reid", person) exec_net_reid.start_async( request_id=0, inputs={input_blob: in_frame_reid}) if exec_net_reid.requests[0].wait(-1) == 0: res_reid = exec_net_reid.requests[0].outputs[ out_blob_reid] reIdVector = res_reid[0].reshape(-1, ) # print(reIdVector) foundId = findMatchingPerson(reIdVector) # print("REID:" + str(foundId)) color = (255, 0, 0) cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), color, 2) # cv2.putText(frame, str(class_id) + ' ' + str(round(obj[2] * 100, 1)) + ' %', (xmin, ymin - 7), cv2.FONT_HERSHEY_COMPLEX, 0.6, color, 1) idColor = (0, 0, 255) cv2.putText(frame, "ID_{0:4d}".format(foundId), (xmin, ymin - 7), cv2.FONT_HERSHEY_COMPLEX, 0.8, idColor, 1) try: personData["detecttime"] = datetime.datetime.now( ).isoformat() personData["No"] = str(1) personData["faceId"] = str(foundId) message = json.dumps(personData) sender.send(EventData(message)) except: raise #STEP-9 cv2.imshow("Detection Results", frame) key = cv2.waitKey(1) if key == 27: break # print("MAX_REID={0}".format(len(globalReIdVec))) #STEP-10 cv2.destroyAllWindows() del exec_net del exec_net_reid del plugin client.stop()
# Address can be in either of these formats: # "amqps://<URL-encoded-SAS-policy>:<URL-encoded-SAS-key>@<mynamespace>.servicebus.windows.net/myeventhub" # "amqps://<mynamespace>.servicebus.windows.net/myeventhub" # SAS policy and key are not required if they are encoded in the URL ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') PARTITION = os.environ.get('PARTITION') if len(sys.argv) < 6: print("using enviornment variables") else: INTERVAL_IN_SECONDS = sys.argv[1] STREAM_ID = sys.argv[2] ADDRESS = sys.argv[3] USER = sys.argv[4] KEY = sys.argv[5] PARTITION = sys.argv[6] if not ADDRESS: raise ValueError("No EventHubs URL supplied.") print("INTERVAL_IN_SECONDS:", INTERVAL_IN_SECONDS, "STREAM_ID:", STREAM_ID, "ADDRESS:", ADDRESS) client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) sender = client.add_sender(partition=PARTITION) client.run() thread = ProducerThread(listener, STREAM_ID, float(INTERVAL_IN_SECONDS))
def main(): log.basicConfig(format="[ %(levelname)s ] %(message)s", level=log.INFO, stream=sys.stdout) args = build_argparser().parse_args() #STEP-2 model_xml = '/opt/intel/computer_vision_sdk/deployment_tools/intel_models/face-detection-retail-0004/FP16/face-detection-retail-0004.xml' model_bin = '/opt/intel/computer_vision_sdk/deployment_tools/intel_models/face-detection-retail-0004/FP16/face-detection-retail-0004.bin' net = IENetwork.from_ir(model=model_xml, weights=model_bin) model_age_xml = '/opt/intel/computer_vision_sdk/deployment_tools/intel_models/age-gender-recognition-retail-0013/FP16/age-gender-recognition-retail-0013.xml' model_age_bin = '/opt/intel/computer_vision_sdk/deployment_tools/intel_models/age-gender-recognition-retail-0013/FP16/age-gender-recognition-retail-0013.bin' net_age = IENetwork.from_ir(model=model_age_xml, weights=model_age_bin) model_reid_xml = '/opt/intel/computer_vision_sdk/deployment_tools/intel_models/face-reidentification-retail-0001/FP16/face-reidentification-retail-0001.xml' model_reid_bin = '/opt/intel/computer_vision_sdk/deployment_tools/intel_models/face-reidentification-retail-0001/FP16/face-reidentification-retail-0001.bin' net_reid = IENetwork.from_ir(model=model_reid_xml, weights=model_reid_bin) # Plugin initialization for specified device and load extensions library if specified log.info("Initializing plugin for {} device...".format(args.device)) plugin = IEPlugin(device=args.device, plugin_dirs=args.plugin_dir) if args.cpu_extension and 'CPU' in args.device: # plugin.add_cpu_extension(args.cpu_extension) plugin.add_cpu_extension( '/home/tsunamac/inference_engine_samples/intel64/Release/lib/libcpu_extension.so' ) # plugin = IEPlugin(device='CPU', plugin_dirs=None) # plugin.add_cpu_extension('C:/Intel/computer_vision_sdk_2018.3.343/deployment_tools/inference_engine/bin/intel64/Release/cpu_extension.dll') # plugin.add_cpu_extension('/opt/intel/computer_vision_sdk/inference_engine/lib/ubuntu_16.04/intel64/libcpu_extension.so') if "CPU" in plugin.device: supported_layers = plugin.get_supported_layers(net) not_supported_layers = [ l for l in net.layers.keys() if l not in supported_layers ] if len(not_supported_layers) != 0: log.error( "Following layers are not supported by the plugin for specified device {}:\n {}" .format(plugin.device, ', '.join(not_supported_layers))) log.error( "Please try to specify cpu extensions library path in sample's command line parameters using -l " "or --cpu_extension command line argument") sys.exit(1) #STEP-3 exec_net = plugin.load(network=net, num_requests=1) exec_net_age = plugin.load(network=net_age, num_requests=1) exec_net_reid = plugin.load(network=net_reid, num_requests=1) #STEP-4 input_blob = next(iter(net.inputs)) #input_blob = 'data' out_blob = next(iter(net.outputs)) #out_blob = 'detection_out' model_n, model_c, model_h, model_w = net.inputs[ input_blob] #model_n, model_c, model_h, model_w = 1, 3, 300, 300 input_blob_age = next(iter(net_age.inputs)) #input_blob = 'data' out_blob_age = next(iter(net_age.outputs)) #out_blob = '' out_blob_prob = "prob" model_age_n, model_age_c, model_age_h, model_age_w = net_age.inputs[ input_blob_age] #model_n, model_c, model_h, model_w = 1, 3, 62, 62 input_blob_reid = next(iter(net_reid.inputs)) #input_blob = 'data' out_blob_reid = next(iter(net_reid.outputs)) #out_blob = '' model_reid_n, model_reid_c, model_reid_h, model_reid_w = net_reid.inputs[ input_blob_reid] #model_n, model_c, model_h, model_w = 1, 3, 62, 62 print("B={},C={},H={},W={}".format(model_reid_n, model_reid_c, model_reid_h, model_reid_w)) del net del net_age del net_reid if args.input == 'cam': input_stream = 0 else: input_stream = args.input # assert os.path.isfile(args.input), "Specified input file doesn't exist" #STEP-5 url = "http://192.168.1.16:8081/?action=stream" video = "video.mp4" cap = cv2.VideoCapture(input_stream) personData = {} ADDRESS = args.address USER = args.user KEY = args.key client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) sender = client.add_sender(partition="0") client.run() while cap.isOpened(): ret, frame = cap.read() if not ret: break #STEP-6 cap_w = cap.get(3) cap_h = cap.get(4) in_frame = cv2.resize(frame, (model_w, model_h)) in_frame = in_frame.transpose( (2, 0, 1)) # Change data layout from HWC to CHW in_frame = in_frame.reshape((model_n, model_c, model_h, model_w)) #STEP-7 exec_net.start_async(request_id=0, inputs={input_blob: in_frame}) if exec_net.requests[0].wait(-1) == 0: res = exec_net.requests[0].outputs[out_blob] #STEP-8 for obj in res[0][0]: if obj[2] > args.prob_threshold: xmin = int(obj[3] * cap_w) ymin = int(obj[4] * cap_h) xmax = int(obj[5] * cap_w) ymax = int(obj[6] * cap_h) frame_org = frame.copy() face = frame_org[ymin:ymax, xmin:xmax] in_frame_age = cv2.resize(face, (model_age_w, model_age_h)) in_frame_age = in_frame_age.transpose( (2, 0, 1)) # Change data layout from HWC to CHW in_frame_age = in_frame_age.reshape( (model_age_n, model_age_c, model_age_h, model_age_w)) # cv2.rectangle(dst,(xmin, ymin), (xmax, ymax), (255, 0 ,0), 1) cv2.imshow("age", face) exec_net_age.start_async(request_id=0, inputs={input_blob: in_frame_age}) if exec_net_age.requests[0].wait(-1) == 0: res_gender = exec_net_age.requests[0].outputs[ out_blob_prob] # res_age = exec_net_age.requests[0].outputs[out_blob_age] # print(res_age[0].reshape(-1,)) AgeGender = res_gender[0].reshape(-1, ) # print(AgeGender) in_frame_reid = cv2.resize(face, (model_reid_w, model_reid_h)) in_frame_reid = in_frame_reid.transpose( (2, 0, 1)) # Change data layout from HWC to CHW in_frame_reid = in_frame_reid.reshape( (model_reid_n, model_reid_c, model_reid_h, model_reid_w)) exec_net_reid.start_async( request_id=0, inputs={input_blob: in_frame_reid}) if exec_net_reid.requests[0].wait(-1) == 0: res_reid = exec_net_reid.requests[0].outputs[ out_blob_reid] reIdVector = res_reid[0].reshape(-1, ) # print(reIdVector) foundId = findMatchingPerson(reIdVector) print("ID:" + str(foundId)) class_id = int(obj[1]) # Draw box and label\class_id color = (0, 0, 255) genderColor = (147, 20, 255) if AgeGender[1] < 0.5 else (255, 0, 0) cv2.rectangle(frame, (xmin, ymin), (xmax, ymax), genderColor, 2) # cv2.putText(frame, str(class_id) + ' ' + str(round(obj[2] * 100, 1)) + ' %', (xmin, ymin - 7), cv2.FONT_HERSHEY_COMPLEX, 0.8, color, 1) cv2.putText(frame, "ID_{0:4d}".format(foundId), (xmin, ymin - 7), cv2.FONT_HERSHEY_COMPLEX, 0.8, color, 1) try: personData["detecttime"] = datetime.datetime.now( ).isoformat() personData["No"] = str(1) personData["faceId"] = str(foundId) message = json.dumps(personData) sender.send(EventData(message)) except: raise #STEP-9 cv2.imshow("Detection Results", frame) key = cv2.waitKey(1) if key == 27: break #STEP-10 cv2.destroyAllWindows() del exec_net del plugin client.stop()
# En caso de no tener un URL valido if not ehAddress: # Notifica el error y termina el programa raise ValueError("No EventHubs URL supplied.") """ La clase EventHubClient define una conexión para enviar y recibir eventos Párametros: address = El URL para conectarse al Event Hub username = El nompre del SAS Policy password = La contraseña del SAS Policy debug = Si se quiere hacer debug de la conexión """ ehClient = EventHubClient(ehAddress, SASName, PrimaryKey, debug=False) # Agrega un sender para enviar los eventos ehSender = ehClient.add_sender() """ Abre las conecciones y corre los clientes Sender/Receiver En caso de iniciar con exito no regresa nada y en caso contrario va a mostrar una excepción """ ehClient.run() # Ejecuta el envío y cacha los posibles errores try: iN = 100 # Número de eventos a enviar iniTime = time.time() # Tiempo en el que comenzó el programa # Loop para enviar iN eventos for iI in range(iN): # Crea el JSON String a enviar como eventos
def run(eh_address, eh_user, eh_key, gh_endpoint, gh_token): info("STARTED github to eventhub") headers = {"Authorization": "token {}".format(gh_token)} monitor = Monitor(info) eh_client = EventHubClient(eh_address, debug=False, username=eh_user, password=eh_key) sender = None try: sender = BufferedEventHubSender( eh_client.add_sender(), flush_cb=lambda took, count: debug( "EVENTHUB REQUEST | took: {} sec, sent {} records to {}.".format(took, count, eh_client.eh_name) ), ) failed = eh_client.run() if failed: raise EventHubError("Couldn't connect to EH {}".format(eh_client.eh_name)) except EventHubError as e: error("EventHubError", e.message) seconds_per_request = round( 1.0 / (5000 / 60 / 60), 2 ) # requests / minutes / seconds = requests per sec, ^-1=secs per request cache = SlidingCache() events = 0 loop = True while loop: loop_start_time = time.time() monitor.report() try: resp = requests.get(gh_endpoint, headers=headers) resp.raise_for_status() monitor.requests_issued += 1 data = sorted(resp.json(), key=lambda x: x["id"]) payload = "" debug("GITHUB REQUEST | took {} sec, got {} events.".format(resp.elapsed.total_seconds(), len(data))) for d in data: if d["id"] not in cache: try: sender.push(d) except EventHubError as e: error("EventHubError", e.message) monitor.events_sent += 1 cache.add(d.get("id")) cycle_took = time.time() - loop_start_time delay = seconds_per_request - cycle_took debug("CYCLE DONE | took {}, waiting for {}".format(cycle_took, max(delay, 0))) if delay > 0: time.sleep(delay) except requests.HTTPError as e: if resp.status_code in [429, 403]: time_to_wait = int( float(resp.headers.get("X-RateLimit-Reset", 60)) - datetime.datetime.utcnow().timestamp() ) info("waiting for {}".format(time_to_wait)) if time_to_wait > 0: time.sleep(time_to_wait) error("HTTP EXCEPTION", repr(e)) except EventHubError as e: error("Failed to send events to eventhub, skipping", repr(e)) except Exception as e: error("UNEXPECTED ERROR", repr(e)) traceback.print_exc() os.kill(os.getpid(), 9)
# "amqps://<URL-encoded-SAS-policy>:<URL-encoded-SAS-key>@<mynamespace>.servicebus.windows.net/myeventhub" # "amqps://<mynamespace>.servicebus.windows.net/myeventhub" # For example: ADDRESS = "amqps://ehubns-uat-hk-peak-di.servicebus.windows.net/clickstreamperformancetesting" # SAS policy and key are not required if they are encoded in the URL USER = "******" KEY = "cJffnCOjNePr6igC5F/56hhC5NZjF93RSv8QTVf2zSc=" try: if not ADDRESS: raise ValueError("No EventHubs URL supplied.") # Create Event Hubs client client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) sender = client.add_sender() client.run() message1 = '{"schema":"iglu:com.snowplowanalytics.snowplow\/payload_data\/jsonschema\/1-0-3","data":[{"vp":"750x1334","se_la":"Verify","res":"750x1334","p":"mob","uid":"1b5acb24-76c8-404a-838e-d6708731bc07","co":"{\\"schema\\":\\\"iglu:com.snowplowanalytics.snowplow\\\/contexts\\\/jsonschema\\\/1-0-1\\",\\"data\\":[{\\"schema\\":\\"iglu:com.hsbc\\\/payme_globaldata\\\/jsonschema\\\/1-0-0\\",\\"data\\":{\\"page_category\\":\\"onb\\",\\"page_business_line\\":\\"cmb\\",\\"page_type\\":\\"verification\\",\\"page_security_level\\":\\"0\\",\\"page_customer_group\\":\\"svf\\"}},{\\"schema\\":\\"iglu:com.snowplowanalytics.snowplow\\\/mobile_context\\\/jsonschema\\\/1-0-1\\",\\"data\\":{\\"osType\\":\\"ios\\",\\"networkType\\":\\"wifi\\",\\"osVersion\\":\\"11.3.1\\",\\"appleIdfv\\":\\"095679CF-4921-4215-8890-3D5A999FB4BA\\",\\"carrier\\":\\"CMHK\\",\\"deviceManufacturer\\":\\"Apple Inc.\\",\\"appleIdfa\\":\\"B73CFCA5-20B7-4C6B-B0F7-78BB4FD13EA1\\",\\"deviceModel\\":\\"iPhone\\"}},{\\"schema\\":\\"iglu:com.snowplowanalytics.snowplow\\\/client_session\\\/jsonschema\\\/1-0-1\\",\\"data\\":{\\"previousSessionId\\":null,\\"firstEventId\\":\\"3ec033f6-c504-4fca-b34d-b0a057dd09a3\\",\\"sessionId\\":\\"5ced1f1a-e4ed-464d-882c-e8dc3b812ebd\\",\\"userId\\":\\"6a0a2ba6-9e5e-42b9-933a-6f83ef80a4dd\\",\\"sessionIndex\\":1,\\"storageMechanism\\":\\"SQLITE\\"}}]}","stm":"1548642167092","se_pr":"content","dtm":"1548642166530","tv":"ios-0.8.0","tna":"EventHubTracker_UAT","se_ca":"mobile:payme:b:onb:verify_BIB:landing_withoutVerify","se_va":"0","e":"se","lang":"en-US","se_ac":"button click","duid":"74b91465-4100-4bde-8eca-5703f5629d07","aid":"PM4B","eid":"d63c095b-dcc6-4bd4-8615-f53c9ba3c9b6"}]}' message2 = '{"schema":"iglu:com.snowplowanalytics.snowplow/payload_data/jsonschema/1-0-4","data":[{"se_la":"28D","eid":"7ac02907-c077-4629-aa80-1d0fab00adfe","tv":"andr-0.7.0","duid":"2e0d206b-f8e6-4c24-a523-e49caefb98af","e":"se","tna":"EventHubTracker_UAT","tz":"Asia/Hong_Kong","se_ca":"mobile:payme:b:pay:tran_bank:transac_hist_bank","se_ac":"button click","se_pr":"content","co":"{\"schema\":\"iglu:com.snowplowanalytics.snowplow/contexts/jsonschema/1-0-1\",\"data\":[{\"schema\":\"iglu:com.hsbc/payme_globaldata/jsonschema/1-0-0\",\"data\":{\"page_customer_group\":\"svf\",\"page_type\":\"transaction\",\"page_category\":\"pay\",\"page_security_level\":\"30\",\"page_business_line\":\"cmb\"}},{\"schema\":\"iglu:com.snowplowanalytics.snowplow/client_session/jsonschema/1-0-1\",\"data\":{\"sessionIndex\":4,\"storageMechanism\":\"SQLITE\",\"firstEventId\":\"c9168f71-86c0-4e99-9585-5eed64e5f87e\",\"sessionId\":\"225a9364-c399-4694-a021-09c26bcd9c0a\",\"previousSessionId\":\"e37d9ed4-c694-4326-907c-47b86a1ebdb8\",\"userId\":\"21e47fde-a2ee-484b-b569-00e5c88ff73b\"}},{\"schema\":\"iglu:com.snowplowanalytics.snowplow/mobile_context/jsonschema/1-0-1\",\"data\":{\"osVersion\":\"8.0.0\",\"osType\":\"android\",\"androidIdfa\":\"fe680b52-bf06-4feb-ace5-128d93dd0e43\",\"deviceModel\":\"SM-G9650\",\"deviceManufacturer\":\"samsung\",\"networkType\":\"wifi\"}}]}","stm":"1545190901404","p":"mob","uid":"d0b95748-9c2b-44c4-902e-d46d973c62bf","dtm":"1545190900547","lang":"zh-Hant-HK","aid":"PM4B"}]}' # message_err="""{""schema"":""iglu:com.snowplowanalytics.snowplow\/payload_data\/jsonschema\/1-0-3"",""data"":[{""ue_pr"":""{\""schema\"":\""iglu:com.snowplowanalytics.snowplow\\\/unstruct_event\\\/jsonschema\\\/1-0-0\"",\""data\"":{\""schema\"":\""iglu:com.com.hsbc\\\/payme_globaldata\\\/jsonschema\\\/1-0-0\"",\""data\"":{\""tag\"":\""mobile:payme:c:pay:pay_scan:scanner_disabled\"",\""values\"":{\""page_customer_group\"":\""svf\"",\""page_name\"":\""mobile:payme:c:pay:pay_scan:scanner_disabled\"",\""page_business_line\"":\""rbwm\"",\""page_category\"":\""pay\"",\""site_section\"":\""mobile\"",\""page_security_level\"":\""30\"",\""page_type\"":\""transaction\""}}}}"",""vp"":""1125x2436"",""eid"":""a0bd77e3-946c-4901-95b3-53c991238905"",""p"":""mob"",""co"":""{\""schema\"":\""iglu:com.snowplowanalytics.snowplow\\\/contexts\\\/jsonschema\\\/1-0-1\"",\""data\"":[{\""schema\"":\""iglu:com.snowplowanalytics.snowplow\\\/mobile_context\\\/jsonschema\\\/1-0-1\"",\""data\"":{\""osType\"":\""ios\"",\""osVersion\"":\""12.2\"",\""appleIdfv\"":\""AC138160-FBCE-4556-8AC6-47CF65536370\"",\""carrier\"":\""CMHK\"",\""deviceManufacturer\"":\""Apple Inc.\"",\""networkType\"":\""wifi\"",\""deviceModel\"":\""iPhone\""}},{\""schema\"":\""iglu:com.snowplowanalytics.snowplow\\\/client_session\\\/jsonschema\\\/1-0-1\"",\""data\"":{\""previousSessionId\"":\""3d52bc8d-b93a-4a48-b18f-6fe39da4e45d\"",\""firstEventId\"":\""7944cd75-5404-4edf-a1e6-b0af467c8cb8\"",\""sessionId\"":\""c626ad36-51a4-4792-95b4-a228bd722c36\"",\""userId\"":\""2620450a-8900-43dc-aedb-be5098d73652\"",\""sessionIndex\"":2,\""storageMechanism\"":\""SQLITE\""}}]}"",""stm"":""1559100469001"",""dtm"":""1559100468992"",""tv"":""ios-0.8.0"",""tna"":""EventHubTracker_"",""e"":""ue"",""lang"":""en-HK"",""aid"":""PM4C"",""res"":""1125x2436""}]}""" # message_err2="""{""schema"":""iglu:com.snowplowanalytics.snowplow\/payload_data\/jsonschema\/1-0-3"",""data"":[{""ue_pr"":""{\""schema\"":\""iglu:com.snowplowanalytics.snowplow\\\/unstruct_event\\\/jsonschema\\\/1-0-0\"",\""data\"":{\""schema\"":\""iglu:com.acme_company\\\/demo_ios_event\\\/jsonschema\\\/1-0-0\"",\""data\"":{\""tag\"":\""mobile:payme:c:mai:notifications:pending\"",\""values\"":{\""page_customer_group\"":\""svf\"",\""page_type\"":\""transaction\"",\""site_section\"":\""mobile\"",\""page_name\"":\""mobile:payme:c:mai:notifications:pending\"",\""page_category\"":\""pay\"",\""page_business_line\"":\""rbwm\"",\""page_security_level\"":\""30\""}}}}"",""vp"":""1242x2208"",""eid"":""ec70027d-6adb-470b-98f8-200763aed8fb"",""p"":""mob"",""co"":""{\""schema\"":\""iglu:com.snowplowanalytics.snowplow\\\/contexts\\\/jsonschema\\\/1-0-1\"",\""data\"":[{\""schema\"":\""iglu:com.snowplowanalytics.snowplow\\\/mobile_context\\\/jsonschema\\\/1-0-1\"",\""data\"":{\""osType\"":\""ios\"",\""osVersion\"":\""12.2\"",\""appleIdfv\"":\""150A4927-8C66-4AA0-B8A8-A6DD594857AB\"",\""carrier\"":\""1O1O \\\/ csl\"",\""deviceManufacturer\"":\""Apple Inc.\"",\""networkType\"":\""wifi\"",\""deviceModel\"":\""iPhone\""}},{\""schema\"":\""iglu:com.snowplowanalytics.snowplow\\\/client_session\\\/jsonschema\\\/1-0-1\"",\""data\"":{\""previousSessionId\"":\""365551cc-3077-4a9a-936e-1799c80e10c3\"",\""firstEventId\"":\""ec70027d-6adb-470b-98f8-200763aed8fb\"",\""sessionId\"":\""a1f1f8f8-a82a-41ad-a30d-f89bcae17828\"",\""userId\"":\""165774d4-6cc5-4e32-b30d-507f2afd913d\"",\""sessionIndex\"":38,\""storageMechanism\"":\""SQLITE\""}}]}"",""stm"":""1558593758831"",""dtm"":""1558593758818"",""tv"":""ios-0.8.0"",""tna"":""EventHubTracker_"",""e"":""ue"",""lang"":""en"",""aid"":""PM4C"",""res"":""1242x2208""}]}""" # message_pm4b="""{"schema":"iglu:com.snowplowanalytics.snowplow\/payload_data\/jsonschema\/1-0-3","data":[{"vp":"1242x2208","se_la":"1D","res":"1242x2208","p":"mob","uid":"7a88f763-6214-4d23-a995-f41e260d802d","co":"{\"schema\":\"iglu:com.snowplowanalytics.snowplow\\\/contexts\\\/jsonschema\\\/1-0-1\",\"data\":[{\"schema\":\"iglu:com.hsbc\\\/payme_globaldata\\\/jsonschema\\\/1-0-0\",\"data\":{\"page_category\":\"pay\",\"page_business_line\":\"cmb\",\"page_type\":\"transaction\",\"page_security_level\":\"30\",\"page_customer_group\":\"svf\"}},{\"schema\":\"iglu:com.snowplowanalytics.snowplow\\\/mobile_context\\\/jsonschema\\\/1-0-1\",\"data\":{\"osType\":\"ios\",\"osVersion\":\"12.3.1\",\"networkTechnology\":\"CTRadioAccessTechnologyLTE\",\"appleIdfv\":\"BFE0861D-A255-4BA5-B0B9-7288C3109020\",\"carrier\":\"3\",\"deviceManufacturer\":\"Apple Inc.\",\"networkType\":\"wifi\",\"deviceModel\":\"iPhone\"}},{\"schema\":\"iglu:com.snowplowanalytics.snowplow\\\/client_session\\\/jsonschema\\\/1-0-1\",\"data\":{\"previousSessionId\":\"55677fdb-341e-428b-a489-2f2049d4560c\",\"firstEventId\":\"ff514540-8fb8-4581-b97b-cac297ae1754\",\"sessionId\":\"8de38280-5835-4329-98ad-df48375b8ddd\",\"userId\":\"2d2572cb-24ac-4c23-bd0a-0f55df4ef1d8\",\"sessionIndex\":3,\"storageMechanism\":\"SQLITE\"}}]}","stm":"1559916325388","se_pr":"content","dtm":"1559916325381","tv":"ios-0.8.0","tna":"EventHubTracker_PREPROD","se_ca":"mobile:payme:b:pay:tran_bank:transac_hist_bank","se_va":"0","e":"se","lang":"en-US","se_ac":"button click","duid":"ee7e00ba-a85d-45b3-a1be-f6fc512b5633","aid":"PM4B","eid":"00b49267-37d7-4d77-a023-acace6fe3e8f"}]}""" # message_pm4b2="""{"schema":"iglu:com.snowplowanalytics.snowplow\/payload_data\/jsonschema\/1-0-3","data":[{"ue_pr":"{\"schema\":\"iglu:com.snowplowanalytics.snowplow\\\/unstruct_event\\\/jsonschema\\\/1-0-0\",\"data\":{\"schema\":\"iglu:com.snowplowanalytics.snowplow\\\/screen_view\\\/jsonschema\\\/1-0-0\",\"data\":{\"id\":\"mobile:payme:b:pay:tran_bank:home_keypad\",\"name\":\"mobile:payme:b:pay:tran_bank:home_keypad\"}}}","vp":"1242x2208","eid":"c90b1a55-dd44-4543-b4d0-d65cc251e0b8","p":"mob","uid":"7a88f763-6214-4d23-a995-f41e260d802d","co":"{\"schema\":\"iglu:com.snowplowanalytics.snowplow\\\/contexts\\\/jsonschema\\\/1-0-1\",\"data\":[{\"schema\":\"iglu:com.hsbc\\\/payme_globaldata\\\/jsonschema\\\/1-0-0\",\"data\":{\"page_category\":\"pay\",\"page_business_line\":\"cmb\",\"page_type\":\"transaction\",\"page_security_level\":\"30\",\"page_customer_group\":\"svf\"}},{\"schema\":\"iglu:com.snowplowanalytics.snowplow\\\/mobile_context\\\/jsonschema\\\/1-0-1\",\"data\":{\"osType\":\"ios\",\"osVersion\":\"12.3.1\",\"networkTechnology\":\"CTRadioAccessTechnologyLTE\",\"appleIdfv\":\"BFE0861D-A255-4BA5-B0B9-7288C3109020\",\"carrier\":\"3\",\"deviceManufacturer\":\"Apple Inc.\",\"networkType\":\"wifi\",\"deviceModel\":\"iPhone\"}},{\"schema\":\"iglu:com.snowplowanalytics.snowplow\\\/client_session\\\/jsonschema\\\/1-0-1\",\"data\":{\"previousSessionId\":\"55677fdb-341e-428b-a489-2f2049d4560c\",\"firstEventId\":\"ff514540-8fb8-4581-b97b-cac297ae1754\",\"sessionId\":\"8de38280-5835-4329-98ad-df48375b8ddd\",\"userId\":\"2d2572cb-24ac-4c23-bd0a-0f55df4ef1d8\",\"sessionIndex\":3,\"storageMechanism\":\"SQLITE\"}}]}","stm":"1559916323206","dtm":"1559916323197","tv":"ios-0.8.0","tna":"EventHubTracker_PREPROD","e":"ue","lang":"en-US","duid":"ee7e00ba-a85d-45b3-a1be-f6fc512b5633","aid":"PM4B","res":"1242x2208"}]}""" # message_pm4b3="""{"schema":"iglu:com.snowplowanalytics.snowplow\/payload_data\/jsonschema\/1-0-3","data":[{"vp":"1242x2208","se_la":"Back","res":"1242x2208","p":"mob","uid":"7a88f763-6214-4d23-a995-f41e260d802d","co":"{\"schema\":\"iglu:com.snowplowanalytics.snowplow\\\/contexts\\\/jsonschema\\\/1-0-1\",\"data\":[{\"schema\":\"iglu:com.hsbc\\\/payme_globaldata\\\/jsonschema\\\/1-0-0\",\"data\":{\"page_category\":\"pay\",\"page_business_line\":\"cmb\",\"page_type\":\"transaction\",\"page_security_level\":\"30\",\"page_customer_group\":\"svf\"}},{\"schema\":\"iglu:com.snowplowanalytics.snowplow\\\/mobile_context\\\/jsonschema\\\/1-0-1\",\"data\":{\"osType\":\"ios\",\"osVersion\":\"12.3.1\",\"networkTechnology\":\"CTRadioAccessTechnologyLTE\",\"appleIdfv\":\"BFE0861D-A255-4BA5-B0B9-7288C3109020\",\"carrier\":\"3\",\"deviceManufacturer\":\"Apple Inc.\",\"networkType\":\"wifi\",\"deviceModel\":\"iPhone\"}},{\"schema\":\"iglu:com.snowplowanalytics.snowplow\\\/client_session\\\/jsonschema\\\/1-0-1\",\"data\":{\"previousSessionId\":\"55677fdb-341e-428b-a489-2f2049d4560c\",\"firstEventId\":\"ff514540-8fb8-4581-b97b-cac297ae1754\",\"sessionId\":\"8de38280-5835-4329-98ad-df48375b8ddd\",\"userId\":\"2d2572cb-24ac-4c23-bd0a-0f55df4ef1d8\",\"sessionIndex\":3,\"storageMechanism\":\"SQLITE\"}}]}","stm":"1559916463113","se_pr":"content","dtm":"1559916463102","tv":"ios-0.8.0","tna":"EventHubTracker_PREPROD","se_ca":"mobile:payme:b:pay:dyn_qr:transac_detail_payment","se_va":"0","e":"se","lang":"en-US","se_ac":"button click","duid":"ee7e00ba-a85d-45b3-a1be-f6fc512b5633","aid":"PM4B","eid":"fa07a434-b9bc-4a01-a2c0-93b121e385a5"}]}"""
def main(req: func.HttpRequest) -> func.HttpResponse: logger = logging.getLogger(__name__) formatter = logging.Formatter( '%(asctime)s %(name)s %(levelname)s: %(message)s') func_context = os.environ['FUNCTION_CONTEXT'] logger.debug(f"Function context --> {func_context}") credentials = None subscription_id = None kv_credentials = None kv_subscription_id = None if func_context == 'local': filehandler = logging.FileHandler('func.log') filehandler.setFormatter(formatter) logger.addHandler(filehandler) logger.setLevel(logging.DEBUG) credentials, subscription_id = get_local_credentials() else: console = logging.StreamHandler() console.setLevel(logging.INFO) console.setFormatter(formatter) credentials, subscription_id = get_azure_credentials() logger.debug('Python HTTP trigger function processed a request.') logger.debug(f"method={req.method}, url={req.url}, params={req.params}") logger.debug(f"body={req.get_json()}") # Handle WebHook webhook = req.get_json() # Get resource information specifically tags if this is an alert resource_id = None if check_keys(webhook, 'data', 'context', 'resourceId'): resource_id = webhook['data']['context']['resourceId'] elif check_keys('data', 'context', 'activityLog', 'resourceId'): resource_id = webhook['data']['context']['activityLog']['resourceId'] elif check_keys('data', 'context', 'scope'): resource_id = webhook['data']['context']['scope'] elif check_keys('data', 'context', 'activityLog', 'authorization', 'scope'): resource_id = webhook['data']['context']['activityLog'][ 'authorization']['scope'] if resource_id: resource_client = ResourceManagementClient(credentials, subscription_id) try: resource = resource_client.resources.get_by_id( resource_id, api_version='2018-06-01') if resource.tags: webhook['tags'] = resource.tags logger.info(f"adding tags {resource.tags}") else: logger.info(f"no tags found in resource {resource_id}") except: logger.error( f"received exception from ResourceManagementClient for {resource_id}" ) else: logger.info("no resource_id found in webhook") # Key Vault stuff kv_mgmt_client = KeyVaultManagementClient(credentials, subscription_id) kv_client = KeyVaultClient(credentials) namespace = get_kv_secret(kv_client, 'EventHubNamespace') event_hub = get_kv_secret(kv_client, 'EventHub') user = get_kv_secret(kv_client, 'EventHubKeyName') key = get_kv_secret(kv_client, 'EventHubKey') amqp_uri = f"https://{namespace}.servicebus.windows.net/{event_hub}" eh_client = EventHubClient(amqp_uri, debug=False, username=user, password=key) eh_sender = eh_client.add_sender(partition="0") eh_client.run() eh_sender.send(EventData(json.dumps(webhook))) logger.info(f"sending event to {amqp_uri}, {json.dumps(webhook)}") date = datetime.datetime.now() return func.HttpResponse(json.dumps({'status': 'SUCCESS'}))
# SAS policy and key are not required if they are encoded in the URL USER = os.environ.get('EVENT_HUB_SAS_POLICY') KEY = os.environ.get('EVENT_HUB_SAS_KEY') def callback(outcome, condition): logger.info("Message sent. Outcome: {}, Condition: {}".format( outcome, condition)) try: if not ADDRESS: raise ValueError("No EventHubs URL supplied.") client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) sender = client.add_sender(partition="1") client.run() try: start_time = time.time() for i in range(100): sender.transfer(EventData(str(i)), callback=callback) logger.info("Queued 100 messages.") sender.wait() logger.info("Finished processing queue.") except: raise finally: end_time = time.time() client.stop() run_time = end_time - start_time logger.info("Runtime: {} seconds".format(run_time))
# Author: Mark Moore # Created: 6/10/2020 # Last updated: 6/11/2020 # Simulate a sine wave to send to an event hub and plot in real time. # Reference: https://en.wikipedia.org/wiki/Sine_wave to get the math to plot a sine wave import time # get time for the delay from math import * # get math for the sine function from azure.eventhub import EventHubClient, Sender, EventData # get eventhub functions Frequency=80 # Frequency is the number of cycles over time. Frequencydelay = .1 # Frequencydelay is a delay of time clock time for each loop to smooth the plot in PowerBI. pi=3.1415926535897932384626433832795028841971693993751057 # The value of pi ADDRESS = "amqps://yournamespace.servicebus.windows.net/youehname" # Event hub and namespace Policy = "YourPolicyName" # Event hub Username KEY = "YourPolicyKey" # Event hub Key client = EventHubClient(ADDRESS, debug=False, username=Policy, password=KEY) # Create the client object sender = client.add_sender(partition="0") # Create the sender object client.run() # Start the client while True: # Infinite loop for n in range(Frequency): # inner loop that repeats 'Frequency' times p=sin(2*pi*n/Frequency) # fudged sine wave math since this is plotted in PowerBI in real time not on a graph where time is plotted as static. p=p*100 # Increase the value of p. Left alone, p will be between 1 and -1 and will plot as 0 for all points. msg = '{"p":%i}' % (p) # construct json message with plot point to send to event hub sender.send(EventData(msg)) # send json to event hub time.sleep(Frequencydelay) # Added to smooth out plot. Increse to have fewer cycle per second decrease to have more
"type": "record", "name": "User", "fields": [ {"name": "name", "type": "string"}, {"name": "favorite_number", "type": ["int", "null"]}, {"name": "favorite_color", "type": ["string", "null"]} ] } try: if not ADDRESS: raise ValueError("No EventHubs URL supplied.") # Create Event Hubs client client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) sender = client.add_sender(partition="0") client.run() try: start_time = time.time() for i in range(100): print("Sending message: {}".format(i)) writer = DatumWriter(SCHEMA) bytes_writer = io.BytesIO() encoder = avro.io.BinaryEncoder(bytes_writer) writer = DataFileWriter(open("users.avro", "wb"), DatumWriter(), SCHEMA) writer.append({"name": "Alyssa", "favorite_number": 256}) writer.append({"name": "Ben", "favorite_number": 7, "favorite_color": "red"}) writer.close() writer.write({"name": "123", "favorite_number": 10}, encoder) raw_bytes = bytes_writer.getvalue() sender.send(EventData(raw_bytes))
def main(argv): FILE_DATA = '' ADDRESS = '' try: opts, args = getopt.getopt(argv,"hi:o:",["iSym="]) except getopt.GetoptError: print ('pushtsdata.py -i <FILE_DATA>') sys.exit(2) for opt, arg in opts: if opt == '-h': print ('pushtsdata.py -i <FILE_DATA>') sys.exit() elif opt in ("-i", "--iSym"): FILE_DATA = arg from azure.eventhub import EventHubClient, Sender, EventData logger = logging.getLogger("azure") # Address can be in either of these formats: # "amqps://<URL-encoded-SAS-policy>:<URL-encoded-SAS-key>@<mynamespace>.serviceb us.windows.net/myeventhub" # "amqps://<mynamespace>.servicebus.windows.net/myeventhub" # For example: #ADDRESS = "amqps://mynamespace.servicebus.windows.net/myeventhub" #ADDRESS = "sb://timeseriesns.servicebus.windows.net/timeserieseh" # Get pricing data from static file created using gettsdata.py script if FILE_DATA == "price_data": #ADDRESS="sb://tseventhubsns.servicebus.windows.net/tseventhubs" #USER = "******" #KEY = "" ADDRESS = sb://<Your event hub namespace>.servicebus.windows.net/<Your event hub name for price data> USER = <Your shared access policy name for price data> KEY = <generated key for price data> # Get sentiment from static file created using gettsdata.py script elif FILE_DATA == "senti_data": #ADDRESS="sb://tseventhubsns.servicebus.windows.net/sentieventhub" #USER = "******" #KEY = "" ADDRESS = sb://<Your event hub namespace>.servicebus.windows.net/<Your event hub name for sentiment data> USER = <Your shared access policy name for sentiment data> KEY = <generated key for sentiment data> try: if not ADDRESS: raise ValueError("No EventHubs URL supplied.") # Create Event Hubs client on partition 1 client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY) sender = client.add_sender(partition="1") client.run() try: start_time = time.time() f = open(FILE_DATA, "r") for line in f: words = line.split() # Push data to Azure eventhub sender.send(EventData(line)) print (line) except: raise finally: end_time = time.time() client.stop() run_time = end_time - start_time f.close() logger.info("Runtime: {} seconds".format(run_time)) except KeyboardInterrupt: pass
class PgCDCPluginToEventHub(CDCBase): """ Postgres CDC plugin to send logs to event hub. :param poll_frequency: frequency at which Postresql database should be polled. Default is `1` seconds. :type poll_frequency: int """ # plugin type and plugin name plugin_type = PlugInType.CDC plugin_name = "PgCDCPluginToEventHub" plugin_parameters = { "poll_frequency": { "type": float }, "event_hub_address": { "type": str }, "event_hub_user": { "type": str }, "event_hub_key": { "type": str } } def __init__(self, poll_frequency: int = 1, event_hub_address: str = None, event_hub_user: str = None, event_hub_key: str = None, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.poll_frequency = poll_frequency self.event_hub_address = event_hub_address self.event_hub_user = event_hub_user self.event_hub_key = event_hub_key self._validate_parameters() self.client = EventHubClient(self.event_hub_address, debug=False, username=event_hub_user, password=event_hub_key) self.sender = self.client.add_sender(partition="0") def _validate_parameters(self): """ Validate the parameters :return: """ if self.event_hub_address is None \ or len(self.event_hub_address.strip()) == 0 or \ self.event_hub_user is None or \ len(self.event_hub_user.strip()) == 0 or \ self.event_hub_key is None or \ len(self.event_hub_key.strip()) == 0: raise ValueError("Check event hub configuration") def _set_status(self, status: str): """ Set the status of the running plugin :param status: status """ status = f"CDC - {status}" self.status = status self.logger.info(status) def execute(self): self.client.run() self.logger.info("in progress - started") connection = psycopg2.connect(self.connection_string) cursor = connection.cursor() slot_name = "siirto_slot" # create the slot, if doesn't already exists cursor.execute(f"SELECT 1 FROM pg_replication_slots WHERE slot_name = '{slot_name}';") rows = cursor.fetchall() cursor_exists = False for row in rows: cursor_exists = True if not cursor_exists: cursor.execute(f"SELECT 'init' FROM " f"pg_create_logical_replication_slot('{slot_name}', 'wal2json');") current_table_cdc_file_name = {} for table_name in self.table_names: table_name_in_folder = table_name.replace(".", "_") cdc_folder_for_table = os.path.join(self.output_folder_location, table_name_in_folder) file_indexes = [] if os.path.exists(cdc_folder_for_table): file_indexes = [int(file_name.replace(f"{table_name}_cdc_", "").replace(".csv", "")) for file_name in list(os.listdir(cdc_folder_for_table)) if re.search(f"^{table_name}_cdc_.*.csv$", file_name)] file_index = 1 if len(file_indexes) > 0: file_index = max(file_indexes) + 1 if not os.path.exists(cdc_folder_for_table): os.mkdir(cdc_folder_for_table) file_to_write = os.path.join(self.output_folder_location, table_name_in_folder, f"{table_name}_cdc_{file_index}.csv") current_table_cdc_file_name[table_name] = { 'file_to_write': file_to_write, 'index': file_index } tables_string = ",".join(self.table_names) while self.is_running: self.logger.info("running cdc pull iteration") cursor.execute(f"SELECT lsn, data FROM pg_logical_slot_peek_changes('{slot_name}', " f"NULL, NULL, 'pretty-print', '1', " f"'add-tables', '{tables_string}');") rows = cursor.fetchall() rows_collected = {} max_lsn = None # read the WALs for row in rows: max_lsn = row[0] change_set = json.loads(row[1]) change_set_entries = change_set["change"] if 'change' in change_set else [] for change_set_entry in change_set_entries: table_name = f"{change_set_entry['schema']}.{change_set_entry['table']}" \ if 'table' in change_set_entry else None if table_name: self.sender.send(EventData(json.dumps(change_set_entry))) # remove the WALs if max_lsn: cursor.execute(f"SELECT 1 FROM pg_logical_slot_get_changes('{slot_name}', " f"'{max_lsn}', NULL, 'pretty-print', '1', " f"'add-tables', '{tables_string}');") # sleep for one second, before next pool time.sleep(self.poll_frequency) print(f'cleaning the {slot_name}') cursor.execute(f"SELECT 'stop' FROM pg_drop_replication_slot('{slot_name}');") print(f'cleared the {slot_name}') self.logger.info("stopped") def setup_graceful_shutdown(self) -> None: """ Handles the graceful shutdown of the process. Cleans the slot from pg, if already created :return: """ def signal_handler(sig, frame): print("cleaning the slot") self.is_running = False signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler)
class EventHubStreamingClient(AbstractStreamingClient): def __init__(self, config): """ Class to create an EventHubStreamingClient instance. :param config: Dictionary file with all the relevant parameters. """ super().__init__() self.message_callback = None self.config = config self.storage_account_name = self.config.get("AZURE_STORAGE_ACCOUNT") self.storage_key = self.config.get("AZURE_STORAGE_ACCESS_KEY") self.lease_container_name = self.config.get("LEASE_CONTAINER_NAME") self.namespace = self.config.get("EVENT_HUB_NAMESPACE") self.eventhub = self.config.get("EVENT_HUB_NAME") self.consumer_group = self.config.get("EVENT_HUB_CONSUMER_GROUP") if self.consumer_group is None: self.consumer_group = '$default' self.user = self.config.get("EVENT_HUB_SAS_POLICY") self.key = self.config.get("EVENT_HUB_SAS_KEY") if self.config.get("TIMEOUT"): try: self.timeout = int(self.config.get("TIMEOUT")) except ValueError: self.timeout = None else: self.timeout = None # Create EPH Client if self.storage_account_name is not None and self.storage_key is not None: self.eph_client = EventHubConfig( sb_name=self.namespace, eh_name=self.eventhub, policy=self.user, sas_key=self.key, consumer_group=self.consumer_group) self.eh_options = EPHOptions() self.eh_options.release_pump_on_timeout = True self.eh_options.auto_reconnect_on_error = False self.eh_options.debug_trace = False self.storage_manager = AzureStorageCheckpointLeaseManager( self.storage_account_name, self.storage_key, self.lease_container_name) # Create Send client else: address = "amqps://" + self.namespace + \ ".servicebus.windows.net/" + self.eventhub try: self.send_client = EventHubClient(address, debug=False, username=self.user, password=self.key) self.sender = self.send_client.add_sender() self.send_client.run() except Exception as e: logger.error('Failed to init EH send client: ' + str(e)) raise def start_receiving(self, on_message_received_callback): """ Receive messages from an EventHubStreamingClient. :param on_message_received_callback: Callback function. """ loop = asyncio.get_event_loop() try: host = EventProcessorHost(EventProcessor, self.eph_client, self.storage_manager, ep_params=[on_message_received_callback], eph_options=self.eh_options, loop=loop) tasks = asyncio.gather(host.open_async(), self.wait_and_close(host, self.timeout)) loop.run_until_complete(tasks) finally: loop.stop() def send(self, message): """ Send a message to an EventHubStreamingClient. :param message: A string input to upload to event hub. """ try: self.sender.send(EventData(body=message)) logger.info('Sent message: {}'.format(message)) except Exception as e: logger.error('Failed to send message to EH: ' + str(e)) def stop(self): try: self.send_client.stop() except Exception as e: logger.error('Failed to close send client: ' + str(e)) @staticmethod async def wait_and_close(host, timeout): """ Run EventProcessorHost indefinitely. :param host: An EventProcessorHost instance. :param timeout: Length of time to run the EventProcessorHost for. """ if timeout is None: while True: await asyncio.sleep(1) else: await asyncio.sleep(timeout) await host.close_async()