def test_apply_reader_with_location(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\:04\:02Z """, """ n1 x10.0 y23.0 w1 Nn1,n2 """)) svr = rserv.ReplicationServer("http://test.io", "opl") class Handler(CountingHandler): def way(self, w): self.counts[1] += 1 assert_equals(2, len(w.nodes)) assert_equals(1, w.nodes[0].ref) assert_equals(10, w.nodes[0].location.lon) assert_equals(23, w.nodes[0].location.lat) assert_equals(2, w.nodes[1].ref) assert_false(w.nodes[1].location.valid()) h = Handler() diffs = svr.collect_diffs(100, 100000) assert_is_not_none(diffs) diffs.reader.apply(h, idx="flex_mem") assert_equals(h.counts, [1, 1, 0, 0])
def test_get_state_too_many_retries(mock): mock.set_script(("""\ #Sat Aug 26 11:04:04 UTC 2017 txnMaxQueried=1219304113 sequenceNumber=2594669 timestamp=""", """\ #Sat Aug 26 11:04:04 UTC 2017 txnMaxQueried=1219304113 sequenceNumber=2594669 timestamp=""", """\ #Sat Aug 26 11:04:04 UTC 2017 txnMaxQueried=1219304113 sequenceNumber=2594669 timestamp=""", """\ #Sat Aug 26 11:04:04 UTC 2017 txnMaxQueried=1219304113 sequenceNumber=2594669 timestamp=2017-08-26T11\\:04\\:02Z""")) res = rserv.ReplicationServer("https://test.io").get_state_info() assert res is None assert mock.call_count == 3
def test_apply_reader_with_location(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\\:04\\:02Z """, """ n1 x10.0 y23.0 w1 Nn1,n2 """)) svr = rserv.ReplicationServer("https://test.io", "opl") class Handler(CountingHandler): def way(self, w): self.counts[1] += 1 assert 2 == len(w.nodes) assert 1 == w.nodes[0].ref assert 10 == w.nodes[0].location.lon assert 23 == w.nodes[0].location.lat assert 2 == w.nodes[1].ref assert not w.nodes[1].location.valid() h = Handler() diffs = svr.collect_diffs(100, 100000) assert diffs is not None diffs.reader.apply(h, idx="flex_mem") assert h.counts == [1, 1, 0, 0]
def test_get_diff_url(): svr = rserv.ReplicationServer("https://who.is/me/") data = [ (1, 'https://who.is/me//000/000/001.osc.gz'), (500, 'https://who.is/me//000/000/500.osc.gz'), (83750, 'https://who.is/me//000/083/750.osc.gz'), (999999999, 'https://who.is/me//999/999/999.osc.gz'), ] for i, o in data: assert_equals(o, svr.get_diff_url(i))
def test_get_state_url(): svr = rserv.ReplicationServer("http://text.org") data = [ (None, 'http://text.org/state.txt'), (1, 'http://text.org/000/000/001.state.txt'), (999, 'http://text.org/000/000/999.state.txt'), (1000, 'http://text.org/000/001/000.state.txt'), (573923, 'http://text.org/000/573/923.state.txt'), (3290012, 'http://text.org/003/290/012.state.txt'), ] for i, o in data: assert_equals(o, svr.get_state_url(i))
def test_apply_diffs_with_simplify(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\\:04\\:02Z """, """ n1 v23 n1 v24 w1 r1 """)) svr = rserv.ReplicationServer("https://test.io", "opl") h = CountingHandler() assert 100 == svr.apply_diffs(h, 100, 10000, simplify=True) assert [1, 1, 1, 0] == h.counts
def test_apply_diffs_without_simplify(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\:04\:02Z """, """ n1 v23 n1 v24 w1 r1 """)) svr = rserv.ReplicationServer("http://test.io", "opl") h = CountingHandler() assert_equals(100, svr.apply_diffs(h, 100, 10000, simplify=False)) assert_equals([2, 1, 1, 0], h.counts)
def test_apply_diffs_count(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\\:04\\:02Z """, """ n1 w1 r1 """)) svr = rserv.ReplicationServer("https://test.io", "opl") h = CountingHandler() assert 100 == svr.apply_diffs(h, 100, 10000) assert h.counts == [1, 1, 1, 0]
def test_get_state_sequence_cut(mock): mock.set_script(("""\ #Sat Aug 26 11:04:04 UTC 2017 txnMaxQueried=1219304113 sequenceNumber=259""", """\ #Sat Aug 26 11:04:04 UTC 2017 txnMaxQueried=1219304113 sequenceNumber=2594669 timestamp=2017-08-26T11\:04\:02Z""")) res = rserv.ReplicationServer("https://test.io").get_state_info() assert_is_not_none(res) assert_equals(res.timestamp, mkdate(2017, 8, 26, 11, 4, 2)) assert_equals(res.sequence, 2594669) assert_equal(mock.call_count, 2)
def test_get_state_valid(mock): mock.set_result("""\ #Sat Aug 26 11:04:04 UTC 2017 txnMaxQueried=1219304113 sequenceNumber=2594669 timestamp=2017-08-26T11\\:04\\:02Z txnReadyList= txnMax=1219304113 txnActiveList=1219303583,1219304054,1219304104""") res = rserv.ReplicationServer("https://test.io").get_state_info() assert res is not None assert res.timestamp == mkdate(2017, 8, 26, 11, 4, 2) assert res.sequence == 2594669 assert mock.call_count == 1
def test_get_state_valid(mock): mock.set_result("""\ #Sat Aug 26 11:04:04 UTC 2017 txnMaxQueried=1219304113 sequenceNumber=2594669 timestamp=2017-08-26T11\:04\:02Z txnReadyList= txnMax=1219304113 txnActiveList=1219303583,1219304054,1219304104""") res = rserv.ReplicationServer("http://test.io").get_state_info() assert_is_not_none(res) assert_equals(res.timestamp, mkdate(2017, 8, 26, 11, 4, 2)) assert_equals(res.sequence, 2594669) assert_equal(mock.call_count, 1)
def test_apply_reader_with_simplify(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\:04\:02Z """, """ n1 v23 n1 v24 w1 r1 """)) svr = rserv.ReplicationServer("http://test.io", "opl") h = CountingHandler() diffs = svr.collect_diffs(100, 100000) assert_is_not_none(diffs) diffs.reader.apply(h, simplify=True) assert_equals([1, 1, 1, 0], h.counts)
def test_get_state_sequence_cut(mock): mock.set_script(("""\ #Sat Aug 26 11:04:04 UTC 2017 txnMaxQueried=1219304113 sequenceNumber=259""", """\ #Sat Aug 26 11:04:04 UTC 2017 txnMaxQueried=1219304113 sequenceNumber=2594669 timestamp=2017-08-26T11\\:04\\:02Z""")) res = rserv.ReplicationServer("https://test.io").get_state_info() assert res is not None assert res.timestamp == mkdate(2017, 8, 26, 11, 4, 2) assert res.sequence == 2594669 assert mock.call_count == 2
def test_apply_reader_without_simplify(mock): mock.set_script(("""\ sequenceNumber=100 timestamp=2017-08-26T11\\:04\\:02Z """, """ n1 v23 n1 v24 w1 r1 """)) svr = rserv.ReplicationServer("https://test.io", "opl") h = CountingHandler() diffs = svr.collect_diffs(100, 100000) assert diffs is not None diffs.reader.apply(h, simplify=False) assert [2, 1, 1, 0] == h.counts
def test_get_state_server_timeout(mock): mock.side_effect = URLError(reason='Mock') svr = rserv.ReplicationServer("https://test.io") assert svr.get_state_info() is None
def relation(self, r): self.rels.add(r) if __name__ == '__main__': if len(sys.argv) != 4: print( "Usage: python osm_replication_stats.py <server_url> <start_time> <max kB>" ) sys.exit(-1) server_url = sys.argv[1] start = dt.datetime.strptime(sys.argv[2], "%Y-%m-%dT%H:%M:%SZ") if sys.version_info >= (3, 0): start = start.replace(tzinfo=dt.timezone.utc) maxkb = min(int(sys.argv[3]), 10 * 1024) repserv = rserv.ReplicationServer(server_url) seqid = repserv.timestamp_to_sequence(start) print("Initial sequence id:", seqid) h = FileStatsHandler() seqid = repserv.apply_diffs(h, seqid, maxkb) print("Final sequence id:", seqid) h.nodes.outstats("Nodes") h.ways.outstats("Ways") h.rels.outstats("Relations")
def test_get_diff_url(inp, outp): svr = rserv.ReplicationServer("https://who.is/me/") assert outp, svr.get_diff_url(inp)
def test_get_state_url(inp, outp): svr = rserv.ReplicationServer("https://text.org") assert outp == svr.get_state_url(inp)
#!/usr/bin/python3 import sys from osmium.replication import server if __name__ == '__main__': if len(sys.argv) != 3: print( "Usage: python check_server_for_updates.py <server url> <sequence id>" ) sys.exit(254) seqid = int(sys.argv[2]) state = server.ReplicationServer(sys.argv[1]).get_state_info() if state is None: print("ERROR: Cannot get state from URL %s." % (sys.argv[1], )) sys.exit(253) if state.sequence <= seqid: print("Database up to date.") sys.exit(1) print("New data available (%i => %i)." % (seqid, state.sequence)) sys.exit(0)
parser = argparse.ArgumentParser(description='osmium-tool based pipeline') parser.add_argument('directory', help='Working directory - needs a lot of space') parsed = parser.parse_args() workdir = parsed.directory planet = os.path.join(workdir,'planet.osm.pbf') PLANET_OSM_PBF = 'https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf' if not os.path.isfile(planet): logging.warning('Downloading planet.osm.pbf') subprocess.call(['wget','-O',planet,PLANET_OSM_PBF]) fileinfo = json.loads(subprocess.check_output(['osmium','fileinfo','-j',planet])) option = fileinfo['header']['option'] daily = server.ReplicationServer('https://planet.openstreetmap.org/replication/day') if 'osmosis_replication_sequence_number' in option: seqnum = int(option['osmosis_replication_sequence_number']) else: timestamp = fileinfo['header']['option']['osmosis_replication_timestamp'] timestamp = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%SZ") timestamp = timestamp.replace(tzinfo=timezone.utc) logging.warning("Timestamp is {0}".format(timestamp)) seqnum = daily.timestamp_to_sequence(timestamp) logging.warning("Seqnum is {0}".format(seqnum)) latest = daily.get_state_info().sequence logging.warning("Latest is {0}".format(latest)) if seqnum == latest: exit(0)