def metadata(cls, database_url) -> dict: meta = sqlalchemy.MetaData() engine = sqlalchemy.create_engine(database_url) config = parse_url(database_url) meta.reflect(bind=engine) metatable = {} for sql_table_name, sql_table in meta.tables.items(): table = Table.read_from_sqltable(sql_table, config['name']) key = "{db}/{table}".format(db=table.db_name, table=table.name) metatable[key] = table return metatable
def __init__(self, database_url): Sink.__init__(self) settings = parse_url(database_url) loop = asyncio.get_event_loop() bootstrap = '{host}:{port}'.format(host=settings['host'],port=settings['port']) logger.info(bootstrap) self.producer = AIOKafkaProducer( loop=loop, bootstrap_servers = bootstrap ) self.topic = settings['name'] debug(self.producer)
def __init__(self,source:str, sink:str, meta:str): mconf = parse_url(meta) if mconf['scheme'] == 'unimetad': self.metaserver = MetaServer(meta) else: raise Exception("unregister meta") sconf = parse_url(source) if sconf['scheme'] == 'mysql': self.source = MysqlSource(source, meta=self.metaserver) elif sconf['scheme'] == 'kafka': self.source = KafkaSource(source, meta=self.metaserver) else: raise Exception("unregister source") dconf = parse_url(sink) if dconf['scheme'] == 'clickhouse': self.sink = ClickHouseSink(sink) elif dconf['scheme'] == 'kafka': self.sink = KafkaSink(sink) else: raise Exception("unregister sink")
def __init__(self,database_url, meta:MetaServer, server_id=None): if server_id is None: server_id = random.randint(1,1000) Source.__init__(self) settings = parse_url(database_url) settings['db'] = settings['name'] del settings['name'] del settings['scheme'] self.metatable = Table.metadata(database_url) self.stream = BinLogStreamReader(connection_settings = settings, server_id=server_id, blocking=True, only_events=[DeleteRowsEvent, WriteRowsEvent, UpdateRowsEvent])
def __init__(self, database_url, meta:MetaServer): Source.__init__(self) settings = parse_url(database_url) self.name = settings['name'] user = settings['user'] if user is None: user = "******" loop = asyncio.get_event_loop() bootstrap = '{host}:{port}'.format(host=settings['host'],port=settings['port']) self.consumer = AIOKafkaConsumer( self.name, loop=loop, bootstrap_servers = bootstrap, group_id = user ) self.meta = meta
def __init__(self,metaserver): self.meta = parse_url(metaserver) self.tables = {}
def test_parse() -> None: database_url = config['source'].get("url") db = parse_url(database_url) debug(db)