def test_apply_reader_with_location(mock):
mock.set_script(("""\
sequenceNumber=100
timestamp=2017-08-26T11\:04\:02Z
""", """
n1 x10.0 y23.0
w1 Nn1,n2
"""))
svr = rserv.ReplicationServer("http://test.io", "opl")
class Handler(CountingHandler):
def way(self, w):
self.counts[1] += 1
assert_equals(2, len(w.nodes))
assert_equals(1, w.nodes[0].ref)
assert_equals(10, w.nodes[0].location.lon)
assert_equals(23, w.nodes[0].location.lat)
assert_equals(2, w.nodes[1].ref)
assert_false(w.nodes[1].location.valid())
h = Handler()
diffs = svr.collect_diffs(100, 100000)
assert_is_not_none(diffs)
diffs.reader.apply(h, idx="flex_mem")
assert_equals(h.counts, [1, 1, 0, 0])
def test_get_state_too_many_retries(mock):
mock.set_script(("""\
#Sat Aug 26 11:04:04 UTC 2017
txnMaxQueried=1219304113
sequenceNumber=2594669
timestamp=""",
"""\
#Sat Aug 26 11:04:04 UTC 2017
txnMaxQueried=1219304113
sequenceNumber=2594669
timestamp=""",
"""\
#Sat Aug 26 11:04:04 UTC 2017
txnMaxQueried=1219304113
sequenceNumber=2594669
timestamp=""",
"""\
#Sat Aug 26 11:04:04 UTC 2017
txnMaxQueried=1219304113
sequenceNumber=2594669
timestamp=2017-08-26T11\\:04\\:02Z"""))
res = rserv.ReplicationServer("https://test.io").get_state_info()
assert res is None
assert mock.call_count == 3
def test_apply_reader_with_location(mock):
mock.set_script(("""\
sequenceNumber=100
timestamp=2017-08-26T11\\:04\\:02Z
""", """
n1 x10.0 y23.0
w1 Nn1,n2
"""))
svr = rserv.ReplicationServer("https://test.io", "opl")
class Handler(CountingHandler):
def way(self, w):
self.counts[1] += 1
assert 2 == len(w.nodes)
assert 1 == w.nodes[0].ref
assert 10 == w.nodes[0].location.lon
assert 23 == w.nodes[0].location.lat
assert 2 == w.nodes[1].ref
assert not w.nodes[1].location.valid()
h = Handler()
diffs = svr.collect_diffs(100, 100000)
assert diffs is not None
diffs.reader.apply(h, idx="flex_mem")
assert h.counts == [1, 1, 0, 0]
def test_get_diff_url():
svr = rserv.ReplicationServer("https://who.is/me/")
data = [
(1, 'https://who.is/me//000/000/001.osc.gz'),
(500, 'https://who.is/me//000/000/500.osc.gz'),
(83750, 'https://who.is/me//000/083/750.osc.gz'),
(999999999, 'https://who.is/me//999/999/999.osc.gz'),
]
for i, o in data:
assert_equals(o, svr.get_diff_url(i))
def test_get_state_url():
svr = rserv.ReplicationServer("http://text.org")
data = [
(None, 'http://text.org/state.txt'),
(1, 'http://text.org/000/000/001.state.txt'),
(999, 'http://text.org/000/000/999.state.txt'),
(1000, 'http://text.org/000/001/000.state.txt'),
(573923, 'http://text.org/000/573/923.state.txt'),
(3290012, 'http://text.org/003/290/012.state.txt'),
]
for i, o in data:
assert_equals(o, svr.get_state_url(i))
def test_apply_diffs_with_simplify(mock):
mock.set_script(("""\
sequenceNumber=100
timestamp=2017-08-26T11\\:04\\:02Z
""", """
n1 v23
n1 v24
w1
r1
"""))
svr = rserv.ReplicationServer("https://test.io", "opl")
h = CountingHandler()
assert 100 == svr.apply_diffs(h, 100, 10000, simplify=True)
assert [1, 1, 1, 0] == h.counts
def test_apply_diffs_without_simplify(mock):
mock.set_script(("""\
sequenceNumber=100
timestamp=2017-08-26T11\:04\:02Z
""", """
n1 v23
n1 v24
w1
r1
"""))
svr = rserv.ReplicationServer("http://test.io", "opl")
h = CountingHandler()
assert_equals(100, svr.apply_diffs(h, 100, 10000, simplify=False))
assert_equals([2, 1, 1, 0], h.counts)
def test_apply_diffs_count(mock):
mock.set_script(("""\
sequenceNumber=100
timestamp=2017-08-26T11\\:04\\:02Z
""", """
n1
w1
r1
"""))
svr = rserv.ReplicationServer("https://test.io", "opl")
h = CountingHandler()
assert 100 == svr.apply_diffs(h, 100, 10000)
assert h.counts == [1, 1, 1, 0]
def test_get_state_sequence_cut(mock):
mock.set_script(("""\
#Sat Aug 26 11:04:04 UTC 2017
txnMaxQueried=1219304113
sequenceNumber=259""", """\
#Sat Aug 26 11:04:04 UTC 2017
txnMaxQueried=1219304113
sequenceNumber=2594669
timestamp=2017-08-26T11\:04\:02Z"""))
res = rserv.ReplicationServer("https://test.io").get_state_info()
assert_is_not_none(res)
assert_equals(res.timestamp, mkdate(2017, 8, 26, 11, 4, 2))
assert_equals(res.sequence, 2594669)
assert_equal(mock.call_count, 2)
def test_get_state_valid(mock):
mock.set_result("""\
#Sat Aug 26 11:04:04 UTC 2017
txnMaxQueried=1219304113
sequenceNumber=2594669
timestamp=2017-08-26T11\\:04\\:02Z
txnReadyList=
txnMax=1219304113
txnActiveList=1219303583,1219304054,1219304104""")
res = rserv.ReplicationServer("https://test.io").get_state_info()
assert res is not None
assert res.timestamp == mkdate(2017, 8, 26, 11, 4, 2)
assert res.sequence == 2594669
assert mock.call_count == 1
def test_get_state_valid(mock):
mock.set_result("""\
#Sat Aug 26 11:04:04 UTC 2017
txnMaxQueried=1219304113
sequenceNumber=2594669
timestamp=2017-08-26T11\:04\:02Z
txnReadyList=
txnMax=1219304113
txnActiveList=1219303583,1219304054,1219304104""")
res = rserv.ReplicationServer("http://test.io").get_state_info()
assert_is_not_none(res)
assert_equals(res.timestamp, mkdate(2017, 8, 26, 11, 4, 2))
assert_equals(res.sequence, 2594669)
assert_equal(mock.call_count, 1)
def test_apply_reader_with_simplify(mock):
mock.set_script(("""\
sequenceNumber=100
timestamp=2017-08-26T11\:04\:02Z
""", """
n1 v23
n1 v24
w1
r1
"""))
svr = rserv.ReplicationServer("http://test.io", "opl")
h = CountingHandler()
diffs = svr.collect_diffs(100, 100000)
assert_is_not_none(diffs)
diffs.reader.apply(h, simplify=True)
assert_equals([1, 1, 1, 0], h.counts)
def test_get_state_sequence_cut(mock):
mock.set_script(("""\
#Sat Aug 26 11:04:04 UTC 2017
txnMaxQueried=1219304113
sequenceNumber=259""",
"""\
#Sat Aug 26 11:04:04 UTC 2017
txnMaxQueried=1219304113
sequenceNumber=2594669
timestamp=2017-08-26T11\\:04\\:02Z"""))
res = rserv.ReplicationServer("https://test.io").get_state_info()
assert res is not None
assert res.timestamp == mkdate(2017, 8, 26, 11, 4, 2)
assert res.sequence == 2594669
assert mock.call_count == 2
def test_apply_reader_without_simplify(mock):
mock.set_script(("""\
sequenceNumber=100
timestamp=2017-08-26T11\\:04\\:02Z
""", """
n1 v23
n1 v24
w1
r1
"""))
svr = rserv.ReplicationServer("https://test.io", "opl")
h = CountingHandler()
diffs = svr.collect_diffs(100, 100000)
assert diffs is not None
diffs.reader.apply(h, simplify=False)
assert [2, 1, 1, 0] == h.counts
def test_get_state_server_timeout(mock):
mock.side_effect = URLError(reason='Mock')
svr = rserv.ReplicationServer("https://test.io")
assert svr.get_state_info() is None
def relation(self, r):
self.rels.add(r)
if __name__ == '__main__':
if len(sys.argv) != 4:
print(
"Usage: python osm_replication_stats.py <server_url> <start_time> <max kB>"
)
sys.exit(-1)
server_url = sys.argv[1]
start = dt.datetime.strptime(sys.argv[2], "%Y-%m-%dT%H:%M:%SZ")
if sys.version_info >= (3, 0):
start = start.replace(tzinfo=dt.timezone.utc)
maxkb = min(int(sys.argv[3]), 10 * 1024)
repserv = rserv.ReplicationServer(server_url)
seqid = repserv.timestamp_to_sequence(start)
print("Initial sequence id:", seqid)
h = FileStatsHandler()
seqid = repserv.apply_diffs(h, seqid, maxkb)
print("Final sequence id:", seqid)
h.nodes.outstats("Nodes")
h.ways.outstats("Ways")
h.rels.outstats("Relations")
def test_get_diff_url(inp, outp):
svr = rserv.ReplicationServer("https://who.is/me/")
assert outp, svr.get_diff_url(inp)
def test_get_state_url(inp, outp):
svr = rserv.ReplicationServer("https://text.org")
assert outp == svr.get_state_url(inp)
#!/usr/bin/python3
import sys
from osmium.replication import server
if __name__ == '__main__':
if len(sys.argv) != 3:
print(
"Usage: python check_server_for_updates.py <server url> <sequence id>"
)
sys.exit(254)
seqid = int(sys.argv[2])
state = server.ReplicationServer(sys.argv[1]).get_state_info()
if state is None:
print("ERROR: Cannot get state from URL %s." % (sys.argv[1], ))
sys.exit(253)
if state.sequence <= seqid:
print("Database up to date.")
sys.exit(1)
print("New data available (%i => %i)." % (seqid, state.sequence))
sys.exit(0)
parser = argparse.ArgumentParser(description='osmium-tool based pipeline')
parser.add_argument('directory', help='Working directory - needs a lot of space')
parsed = parser.parse_args()
workdir = parsed.directory
planet = os.path.join(workdir,'planet.osm.pbf')
PLANET_OSM_PBF = 'https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf'
if not os.path.isfile(planet):
logging.warning('Downloading planet.osm.pbf')
subprocess.call(['wget','-O',planet,PLANET_OSM_PBF])
fileinfo = json.loads(subprocess.check_output(['osmium','fileinfo','-j',planet]))
option = fileinfo['header']['option']
daily = server.ReplicationServer('https://planet.openstreetmap.org/replication/day')
if 'osmosis_replication_sequence_number' in option:
seqnum = int(option['osmosis_replication_sequence_number'])
else:
timestamp = fileinfo['header']['option']['osmosis_replication_timestamp']
timestamp = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ")
timestamp = timestamp.replace(tzinfo=timezone.utc)
logging.warning("Timestamp is {0}".format(timestamp))
seqnum = daily.timestamp_to_sequence(timestamp)
logging.warning("Seqnum is {0}".format(seqnum))
latest = daily.get_state_info().sequence
logging.warning("Latest is {0}".format(latest))
if seqnum == latest:
exit(0)
