def metadata(cls, database_url) -> dict:
meta = sqlalchemy.MetaData()
engine = sqlalchemy.create_engine(database_url)
config = parse_url(database_url)
meta.reflect(bind=engine)
metatable = {}
for sql_table_name, sql_table in meta.tables.items():
table = Table.read_from_sqltable(sql_table, config['name'])
key = "{db}/{table}".format(db=table.db_name, table=table.name)
metatable[key] = table
return metatable
def __init__(self, database_url):
Sink.__init__(self)
settings = parse_url(database_url)
loop = asyncio.get_event_loop()
bootstrap = '{host}:{port}'.format(host=settings['host'],port=settings['port'])
logger.info(bootstrap)
self.producer = AIOKafkaProducer(
loop=loop,
bootstrap_servers = bootstrap
)
self.topic = settings['name']
debug(self.producer)
def __init__(self,source:str, sink:str, meta:str):
mconf = parse_url(meta)
if mconf['scheme'] == 'unimetad':
self.metaserver = MetaServer(meta)
else:
raise Exception("unregister meta")
sconf = parse_url(source)
if sconf['scheme'] == 'mysql':
self.source = MysqlSource(source, meta=self.metaserver)
elif sconf['scheme'] == 'kafka':
self.source = KafkaSource(source, meta=self.metaserver)
else:
raise Exception("unregister source")
dconf = parse_url(sink)
if dconf['scheme'] == 'clickhouse':
self.sink = ClickHouseSink(sink)
elif dconf['scheme'] == 'kafka':
self.sink = KafkaSink(sink)
else:
raise Exception("unregister sink")
def __init__(self,database_url, meta:MetaServer, server_id=None):
if server_id is None:
server_id = random.randint(1,1000)
Source.__init__(self)
settings = parse_url(database_url)
settings['db'] = settings['name']
del settings['name']
del settings['scheme']
self.metatable = Table.metadata(database_url)
self.stream = BinLogStreamReader(connection_settings = settings,
server_id=server_id,
blocking=True,
only_events=[DeleteRowsEvent, WriteRowsEvent, UpdateRowsEvent])
def __init__(self, database_url, meta:MetaServer):
Source.__init__(self)
settings = parse_url(database_url)
self.name = settings['name']
user = settings['user']
if user is None:
user = "******"
loop = asyncio.get_event_loop()
bootstrap = '{host}:{port}'.format(host=settings['host'],port=settings['port'])
self.consumer = AIOKafkaConsumer(
self.name, loop=loop,
bootstrap_servers = bootstrap,
group_id = user
)
self.meta = meta
def __init__(self,metaserver):
self.meta = parse_url(metaserver)
self.tables = {}
def test_parse() -> None:
database_url = config['source'].get("url")
db = parse_url(database_url)
debug(db)