def merge(source, destination):
"""Merge two historian databases by copying from source to destination.
"""
meta = {
'channel_announcements': None,
'channel_updates': None,
'node_announcements': None,
}
with db_session(source) as source, db_session(destination) as target:
# Not strictly necessary, but I like progress indicators and ETAs.
for table in meta.keys():
rows = source.execute(f"SELECT count(*) FROM {table}")
count, = rows.next()
meta[table] = count
for r, in tqdm(
source.execute("SELECT raw FROM channel_announcements"),
total=meta['channel_announcements'],
):
msg = parse(r)
if isinstance(r, memoryview):
r = bytes(r)
target.merge(ChannelAnnouncement.from_gossip(msg, r))
for r, in tqdm(
source.execute(
"SELECT raw FROM channel_updates ORDER BY timestamp ASC"),
total=meta['channel_updates'],
):
msg = parse(r)
if isinstance(r, memoryview):
r = bytes(r)
target.merge(ChannelUpdate.from_gossip(msg, r))
for r, in tqdm(
source.execute(
"SELECT raw FROM node_announcements ORDER BY timestamp ASC"
),
total=meta['node_announcements'],
):
msg = parse(r)
if isinstance(r, memoryview):
r = bytes(r)
target.merge(NodeAnnouncement.from_gossip(msg, r))
target.commit()
def __next__(self):
pos = self.stream.tell()
length = varint_decode(self.stream)
if length is None:
raise StopIteration
msg = self.stream.read(length)
if len(msg) != length:
raise ValueError(
"Error reading snapshot at {pos}: incomplete read of {length} bytes, only got {lmsg} bytes"
.format(pos=pos, length=length, lmsg=len(msg)))
if not self.decode:
return msg
return parse(msg)
def store(self, raw: bytes) -> None:
try:
msg = gossipd.parse(raw)
cls = None
if isinstance(msg, parser.ChannelUpdate):
cls = ChannelUpdate
elif isinstance(msg, parser.ChannelAnnouncement):
cls = ChannelAnnouncement
elif isinstance(msg, parser.NodeAnnouncement):
cls = NodeAnnouncement
self.session.merge(cls.from_gossip(msg, raw))
except Exception as e:
print("Exception parsing gossip message:", e)
def store(self, raw: bytes) -> None:
try:
msg = gossipd.parse(raw)
cls = None
if isinstance(msg, gossipd.ChannelUpdate):
cls = ChannelUpdate
elif isinstance(msg, gossipd.ChannelAnnouncement):
cls = ChannelAnnouncement
elif isinstance(msg, gossipd.NodeAnnouncement):
cls = NodeAnnouncement
else:
return;
self.session.merge(cls.from_gossip(msg, raw))
except Exception as e:
logging.warn(f"Exception parsing gossip message: {e}")