def __init__(self, filename, schema):
self.conf = Configuration(filename)
self.host, self.username, self.password, self.dbName = self.conf.getMySQLDetails(
)
self.kafka_host, self.kafka_port = self.conf.getBrokerDetails()
self.topic, self.consumergroup = self.conf.getConsumerDetails()
self.schemaAvro = schema
# Open database connection
self.db = MySQLdb.connect(self.host, self.username, self.password,
self.dbName)
self.cursor = self.db.cursor()
class kafkamysql:
def __init__(self,filename,schema):
self.conf = Configuration(filename)
self.host,self.username,self.password,self.dbName = self.conf.getMySQLDetails()
self.kafka_host,self.kafka_port = self.conf.getBrokerDetails()
self.topic,self.consumergroup = self.conf.getConsumerDetails()
self.schemaAvro = schema
# Open database connection
self.db = MySQLdb.connect(self.host,self.username,self.password,self.dbName)
self.cursor = self.db.cursor()
# Insert Into Database
def insertIntoDatabase(self,d):
keys=[]
values=[]
for k, v in d.iteritems():
keys.append(k)
values.append(v)
columns=','.join(str(k) for k in keys)
data=','.join("'"+str(v)+"'" for v in values)
sql="insert into raw_clicks ("+columns+") values("+data+")"
try:
self.cursor.execute(sql)
self.db.commit()
except:
self.db.rollback()
def divoltecall(self):
# get configuration parameters from confguration file
# Kafka Broker Configuration
broker_config=self.kafka_host+":"+str(self.kafka_port)
# To consume messages
consumer = KafkaConsumer(self.topic,
group_id=self.consumergroup,
bootstrap_servers=[broker_config])
# read Avro schema
schema = avro.schema.parse(open(self.schemaAvro).read())
for msg in consumer:
bytes_reader = io.BytesIO(msg.value)
decoder = avro.io.BinaryDecoder(bytes_reader)
reader = avro.io.DatumReader(schema)
user1 = reader.read(decoder)
self.insertIntoDatabase(user1)
# disconnect from server
self.db.close()
class kafkamysql:
def __init__(self, filename, schema):
self.conf = Configuration(filename)
self.host, self.username, self.password, self.dbName = self.conf.getMySQLDetails(
)
self.kafka_host, self.kafka_port = self.conf.getBrokerDetails()
self.topic, self.consumergroup = self.conf.getConsumerDetails()
self.schemaAvro = schema
# Open database connection
self.db = MySQLdb.connect(self.host, self.username, self.password,
self.dbName)
self.cursor = self.db.cursor()
# Insert Into Database
def insertIntoDatabase(self, d):
keys = []
values = []
for k, v in d.iteritems():
keys.append(k)
values.append(v)
columns = ','.join(str(k) for k in keys)
data = ','.join("'" + str(v) + "'" for v in values)
sql = "insert into raw_clicks (" + columns + ") values(" + data + ")"
try:
self.cursor.execute(sql)
self.db.commit()
except:
self.db.rollback()
def divoltecall(self):
# get configuration parameters from confguration file
# Kafka Broker Configuration
broker_config = self.kafka_host + ":" + str(self.kafka_port)
# To consume messages
consumer = KafkaConsumer(self.topic,
group_id=self.consumergroup,
bootstrap_servers=[broker_config])
# read Avro schema
schema = avro.schema.parse(open(self.schemaAvro).read())
for msg in consumer:
bytes_reader = io.BytesIO(msg.value)
decoder = avro.io.BinaryDecoder(bytes_reader)
reader = avro.io.DatumReader(schema)
user1 = reader.read(decoder)
self.insertIntoDatabase(user1)
# disconnect from server
self.db.close()
def __init__(self,filename,schema): self.conf = Configuration(filename) self.host,self.username,self.password,self.dbName = self.conf.getMySQLDetails() self.kafka_host,self.kafka_port = self.conf.getBrokerDetails() self.topic,self.consumergroup = self.conf.getConsumerDetails() self.schemaAvro = schema # Open database connection self.db = MySQLdb.connect(self.host,self.username,self.password,self.dbName) self.cursor = self.db.cursor()
def divoltecall():
# get configuration parameters from confguration file
conf = Configuration("default.yml")
host,username,password,dbName = conf.getMySQLDetails()
kafka_host,kafka_port = conf.getBrokerDetails()
topic,consumergroup = conf.getConsumerDetails()
schemaAvro = conf.getAvroSchema()
# Kafka Broker Configuration
broker_config=kafka_host+":"+str(kafka_port)
# To consume messages
consumer = KafkaConsumer(topic,
group_id=consumergroup,
bootstrap_servers=[broker_config])
# read Avro schema
schema = avro.schema.parse(open(schemaAvro).read())
# Open database connection
db = MySQLdb.connect(host,username,password,dbName)
# prepare a cursor object using cursor() method
cursor = db.cursor()
for msg in consumer:
bytes_reader = io.BytesIO(msg.value)
decoder = avro.io.BinaryDecoder(bytes_reader)
reader = avro.io.DatumReader(schema)
user1 = reader.read(decoder)
insertIntoDatabase(user1)
# disconnect from server
db.close()
values=[]
for k, v in d.iteritems():
keys.append(k)
values.append(v)
columns=','.join(str(k) for k in keys)
data=','.join("'"+str(v)+"'" for v in values)
sql="insert into raw_clicks ("+columns+") values("+data+")"
try:
cursor.execute(sql)
db.commit()
except:
db.rollback()
# get configuration parameters from confguration file
conf = Configuration("default.yml")
host,username,password,dbName = conf.getMySQLDetails()
kafka_host,kafka_port = conf.getBrokerDetails()
topic,consumergroup = conf.getConsumerDetails()
schemaAvro = conf.getAvroSchema()
# Kafka Broker Configuration
broker_config=kafka_host+":"+str(kafka_port)
# To consume messages
consumer = KafkaConsumer(topic,
group_id=consumergroup,
bootstrap_servers=[broker_config])
# read Avro schema
schema = avro.schema.parse(open(schemaAvro).read())
values = []
for k, v in d.iteritems():
keys.append(k)
values.append(v)
columns = ','.join(str(k) for k in keys)
data = ','.join("'" + str(v) + "'" for v in values)
sql = "insert into raw_clicks (" + columns + ") values(" + data + ")"
try:
cursor.execute(sql)
db.commit()
except:
db.rollback()
# get configuration parameters from confguration file
conf = Configuration("default.yml")
host, username, password, dbName = conf.getMySQLDetails()
kafka_host, kafka_port = conf.getBrokerDetails()
topic, consumergroup = conf.getConsumerDetails()
schemaAvro = conf.getAvroSchema()
# Kafka Broker Configuration
broker_config = kafka_host + ":" + str(kafka_port)
# To consume messages
consumer = KafkaConsumer(topic,
group_id=consumergroup,
bootstrap_servers=[broker_config])
# read Avro schema
schema = avro.schema.parse(open(schemaAvro).read())
