def setUp(self):
import shutil
shutil.rmtree("tmp-osmbin/", True)
OsmBin.InitFolder("tmp-osmbin/")
self.o = OsmBin.OsmBin("tmp-osmbin/", "w")
self.o.Import("tests/osmbin_open_relations.osm")
del self.o
self.xml_res_file = "tests/out/osmbin_open_relations.test.xml"
(self.conf,
self.analyser_config) = self.init_config(dst=self.xml_res_file)
def analyser(self, osmbin_path="/data/work/osmbin/data/"):
timestamp = datetime.datetime.now()
self.error_file.analyser(timestamp, self.analyser_version())
doc6010 = dict(
detail = T_(
'''A relation that should be a closed polygon and it is not. An issue is
reported at each end of open part.'''))
self.error_file.classs(id = 1, item = 6010, level = 3, tags = ['geom', 'boundary'],
title = T_('Open relation type=boundary'),
**doc6010)
self.error_file.classs(id = 5, item = 1170, level = 2, tags = ['geom'],
title = T_('Open relation type=multipolygon'))
for admin_level in range(0, 15):
if admin_level <= 6:
level = 1
elif admin_level <= 8:
level = 2
else:
level = 3
self.error_file.classs(id = 100 + admin_level, item = 6010, level = level, tags = ['geom', 'boundary'],
title = T_f('Open relation type=boundary admin_level={0}', admin_level),
**doc6010)
self.classs = {"boundary": 1, "multipolygon": 5}
try:
self.bin = OsmBin.OsmBin(osmbin_path)
self.bin.CopyRelationTo(self)
del self.bin
finally:
self.error_file.analyser_end()
def analyser(self, osmbin_path="/data/work/osmbin/data/"):
timestamp = datetime.datetime.now()
self.error_file.analyser(timestamp, self.analyser_version())
self.error_file.classs(1, 6010, 3, ["geom", "boundary"],
T_(u"Open relation type=boundary"))
self.error_file.classs(5, 1170, 2, ["geom"],
T_(u"Open relation type=multipolygon"))
for admin_level in range(0, 15):
if admin_level <= 6:
level = 1
elif admin_level <= 8:
level = 2
else:
level = 3
self.error_file.classs(
100 + admin_level, 6010, level, ["geom", "boundary"],
T_(u"Open relation type=boundary admin_level=%d", admin_level))
self.classs = {"boundary": 1, "multipolygon": 5}
try:
self.bin = OsmBin.OsmBin(osmbin_path)
self.bin.CopyRelationTo(self)
del self.bin
finally:
self.error_file.analyser_end()
def setUp(self):
import os
import shutil
shutil.rmtree("tmp-osmbin/", True)
OsmBin.InitFolder("tmp-osmbin/")
self.o = OsmBin.OsmBin("tmp-osmbin/", "w")
self.o.Import("tests/osmbin_open_relations.osm")
del self.o
dirname = "tests/out/"
try:
os.makedirs(dirname)
except OSError:
if os.path.isdir(dirname):
pass
else:
raise
self.xml_res_file = os.path.join(dirname, "osmbin_open_relations.test.xml")
(self.conf, self.analyser_config) = self.init_config(dst=self.xml_res_file)
def setUp(self):
import os
import shutil
from modules import config
self.test_dir = config.dir_tmp + "/tests/osmbin/"
shutil.rmtree(self.test_dir, True)
OsmBin.InitFolder(self.test_dir)
self.o = OsmBin.OsmBin(self.test_dir, "w")
self.o.Import("tests/osmbin_open_relations.osm")
del self.o
dirname = config.dir_tmp + "/tests/"
try:
os.makedirs(dirname)
except OSError:
if os.path.isdir(dirname):
pass
else:
raise
self.xml_res_file = os.path.join(dirname, "osmbin_open_relations.test.xml")
(self.conf, self.analyser_config) = self.init_config(dst=self.xml_res_file)
def _load_reader(self):
if hasattr(self.config, 'osmosis_manager') and self.config.osmosis_manager:
self._reader = self.config.osmosis_manager.osmosis()
return
try:
from modules import OsmBin
self._reader = OsmBin.OsmBin("/data/work/osmbin/data")
return
except IOError:
pass
if hasattr(self.config, "reader"):
self._reader = self.config.reader
else:
from modules import OsmSaxAlea
self._reader = OsmSaxAlea.OsmSaxReader(self.config.src, self.config.src_state)
def _load_reader(self):
if hasattr(self.config, 'db_string') and self.config.db_string:
from modules import OsmOsis
self._reader = OsmOsis.OsmOsis(self.config.db_string, self.config.db_schema)
return
try:
from modules import OsmBin
self._reader = OsmBin.OsmBin("/data/work/osmbin/data")
return
except IOError:
pass
if hasattr(self.config, "reader"):
self._reader = self.config.reader
else:
from modules import OsmSaxAlea
self._reader = OsmSaxAlea.OsmSaxReader(self.config.src)
def analyser(self, osmbin_path="/data/work/osmbin/data/"):
timestamp = datetime.datetime.now()
self.error_file.analyser(timestamp)
self.error_file.classs(
1, 6010, 3, ["geom", "boundary"], {
"fr": u"Relation type=boundary ouverte",
"en": u"Open relation type=boundary",
"es": u"Relación abierta type=boundary"
})
self.error_file.classs(
2, 6010, 3, ["geom"], {
"fr": u"Relation type=multipolygon ouverte",
"en": u"Open relation type=multipolygon",
"es": u"Relación abierta type=multipolygon"
})
for admin_level in xrange(0, 15):
if admin_level <= 6:
level = 1
elif admin_level <= 8:
level = 2
else:
level = 3
self.error_file.classs(
100 + admin_level, 6010, level, ["geom", "boundary"], {
"fr":
u"Relation ouverte type=boundary admin_level=%d" %
admin_level,
"en":
u"Open relation type=boundary admin_level=%d" % admin_level
})
self.classs = {"boundary": 1, "multipolygon": 2}
self.bin = OsmBin.OsmBin(osmbin_path)
self.bin.CopyRelationTo(self)
del self.bin
self.error_file.analyser_end()
def update(wanted_end_sequence=None):
global pool
global pool_jobs
global lock_num_launched
global num_launched
# get lock
if not os.path.exists(work_path):
os.makedirs(work_path)
lock = lockfile.FileLock(lock_file)
lock.acquire(timeout=0)
# get local sequence number
def get_sequence_num(s):
for line in s.split("\n"):
(key, sep, value) = line.partition("=")
if key.strip() == "sequenceNumber":
return int(value)
try:
print(os.path.join(orig_diff_path, "state.txt"))
f = open(os.path.join(orig_diff_path, "state.txt"), "r")
begin_sequence = get_sequence_num(f.read())
f.close()
except IOError:
lock.release()
raise
# get remote sequence number
try:
f = urllib.request.urlopen(os.path.join(remote_diff_url, "state.txt"))
server_state = f.read().decode("utf-8")
except IOError:
lock.release()
raise
end_sequence = min(begin_sequence + 10000, get_sequence_num(server_state))
if wanted_end_sequence:
end_sequence = min(end_sequence, wanted_end_sequence)
f.close()
try:
begin_sequence = int(begin_sequence)
end_sequence = int(end_sequence)
except TypeError:
lock.release()
raise
# download diffs, and apply the polygon on them
for i in range(begin_sequence + 1, end_sequence + 1):
print(time.strftime("%Y-%m-%d %H:%M:%S"), i)
for path in [orig_diff_path] + modif_diff_path + [bbox_diff_path]:
tmp_path = os.path.join(
path, "%03d/%03d" % (i // (1000 * 1000), (i // 1000) % 1000))
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
file_location = "%03d/%03d/%03d" % (i // (1000 * 1000),
(i // 1000) % 1000, i % 1000)
# download diff file
print(time.strftime("%Y-%m-%d %H:%M:%S"), " download diff")
orig_diff_file = os.path.join(orig_diff_path, file_location)
for ext in (".osc.gz", ".state.txt"):
try:
(filename, headers) = urllib.request.urlretrieve(
os.path.join(remote_diff_url, file_location) + ext,
orig_diff_file + ext)
except IOError:
lock.release()
raise
file_date = time.mktime(
dateutil.parser.parse(headers["Last-Modified"]).astimezone(
dateutil.tz.tzlocal()).timetuple())
os.utime(orig_diff_file + ext, (file_date, file_date))
if not skip_diff_generation:
generate_bbox_diff(orig_diff_path, file_location, file_date,
bbox_diff_path)
if multiproc_enabled:
lock_num_launched.acquire()
for country in top_countries:
country_param = countries_param[country]
num_launched += 1
if multiproc_enabled:
pool_jobs.append(
pool.apply_async(
generate_diff,
(bbox_diff_path, file_location, file_date,
country_param[0], country_param[1], country),
callback=launch_dep_countries))
else:
pool_jobs = generate_diff(bbox_diff_path, file_location,
file_date, country_param[0],
country_param[1], country)
launch_dep_countries(pool_jobs)
if multiproc_enabled:
lock_num_launched.release()
while True:
lock_num_launched.acquire()
local_num_launched = num_launched
lock_num_launched.release()
if local_num_launched == 0 and len(pool_jobs) == 0:
break
for r in pool_jobs:
r.get()
pool_jobs.remove(r)
assert num_launched == 0
# update osmbin
print(time.strftime("%Y-%m-%d %H:%M:%S"), " update osmbin")
diff_read = OsmSax.OscSaxReader(orig_diff_file + ".osc.gz")
o = OsmBin.OsmBin("/data/work/osmbin/data", "w")
diff_read.CopyTo(o)
del o
del diff_read
# update symbolic links to state.txt
print(time.strftime("%Y-%m-%d %H:%M:%S"),
" update links to state.txt")
update_symlink(orig_diff_file + ".state.txt",
os.path.join(orig_diff_path, "state.txt"))
os.utime(os.path.join(orig_diff_path, "state.txt"),
(file_date, file_date))
sys.stdout.flush()
if multiproc_enabled:
pool.close()
pool.join()
# free lock
sys.stdout.flush()
lock.release()