def osc_modif(config, options):
if options.poly:
from modules import OsmGeom
f = open(options.poly, "r")
name = f.readline().strip()
poly = OsmGeom.read_multipolygon(f)
poly_buffered = poly.buffer(0.1, 8)
f.close()
else:
poly = None
try:
from modules.OsmBin import OsmBin
if not hasattr(options, "osmbin_path"):
options.osmbin_path = "/data/work/osmbin/data/"
reader = OsmBin(options.osmbin_path)
except IOError:
from modules import OsmOsis
reader = OsmOsis.OsmOsis(config.dbs, config.dbp)
in_osc = OsmSax.OscSaxReader(options.source)
if options.position_only:
out_osc = OsmSax.OscPositionSaxWriter(options.dest, "UTF-8", reader)
elif poly:
out_osc = OsmSax.OscFilterSaxWriter(options.dest, "UTF-8", reader,
OsmGeom.check_intersection, poly,
poly_buffered)
elif options.bbox:
out_osc = OsmSax.OscBBoxSaxWriter(options.dest, "UTF-8", reader)
else:
# out_osc = OsmSax.OscSaxWriter(options.dest, "UTF-8", reader)
out_osc = OsmSax.OscSaxWriter(options.dest, "UTF-8")
in_osc.CopyTo(out_osc)
del in_osc
del out_osc
del reader
def update(wanted_end_sequence=None):
global pool
global pool_jobs
global lock_num_launched
global num_launched
# get lock
if not os.path.exists(work_path):
os.makedirs(work_path)
lock = lockfile.FileLock(lock_file)
lock.acquire(timeout=0)
# get local sequence number
def get_sequence_num(s):
for line in s.split("\n"):
(key, sep, value) = line.partition("=")
if key.strip() == "sequenceNumber":
return int(value)
try:
print(os.path.join(orig_diff_path, "state.txt"))
f = open(os.path.join(orig_diff_path, "state.txt"), "r")
begin_sequence = get_sequence_num(f.read())
f.close()
except IOError:
lock.release()
raise
# get remote sequence number
try:
f = urllib.request.urlopen(os.path.join(remote_diff_url, "state.txt"))
server_state = f.read().decode("utf-8")
except IOError:
lock.release()
raise
end_sequence = min(begin_sequence + 10000, get_sequence_num(server_state))
if wanted_end_sequence:
end_sequence = min(end_sequence, wanted_end_sequence)
f.close()
try:
begin_sequence = int(begin_sequence)
end_sequence = int(end_sequence)
except TypeError:
lock.release()
raise
# download diffs, and apply the polygon on them
for i in range(begin_sequence + 1, end_sequence + 1):
print(time.strftime("%Y-%m-%d %H:%M:%S"), i)
for path in [orig_diff_path] + modif_diff_path + [bbox_diff_path]:
tmp_path = os.path.join(
path, "%03d/%03d" % (i // (1000 * 1000), (i // 1000) % 1000))
if not os.path.exists(tmp_path):
os.makedirs(tmp_path)
file_location = "%03d/%03d/%03d" % (i // (1000 * 1000),
(i // 1000) % 1000, i % 1000)
# download diff file
print(time.strftime("%Y-%m-%d %H:%M:%S"), " download diff")
orig_diff_file = os.path.join(orig_diff_path, file_location)
for ext in (".osc.gz", ".state.txt"):
try:
(filename, headers) = urllib.request.urlretrieve(
os.path.join(remote_diff_url, file_location) + ext,
orig_diff_file + ext)
except IOError:
lock.release()
raise
file_date = time.mktime(
dateutil.parser.parse(headers["Last-Modified"]).astimezone(
dateutil.tz.tzlocal()).timetuple())
os.utime(orig_diff_file + ext, (file_date, file_date))
if not skip_diff_generation:
generate_bbox_diff(orig_diff_path, file_location, file_date,
bbox_diff_path)
if multiproc_enabled:
lock_num_launched.acquire()
for country in top_countries:
country_param = countries_param[country]
num_launched += 1
if multiproc_enabled:
pool_jobs.append(
pool.apply_async(
generate_diff,
(bbox_diff_path, file_location, file_date,
country_param[0], country_param[1], country),
callback=launch_dep_countries))
else:
pool_jobs = generate_diff(bbox_diff_path, file_location,
file_date, country_param[0],
country_param[1], country)
launch_dep_countries(pool_jobs)
if multiproc_enabled:
lock_num_launched.release()
while True:
lock_num_launched.acquire()
local_num_launched = num_launched
lock_num_launched.release()
if local_num_launched == 0 and len(pool_jobs) == 0:
break
for r in pool_jobs:
r.get()
pool_jobs.remove(r)
assert num_launched == 0
# update osmbin
print(time.strftime("%Y-%m-%d %H:%M:%S"), " update osmbin")
diff_read = OsmSax.OscSaxReader(orig_diff_file + ".osc.gz")
o = OsmBin.OsmBin("/data/work/osmbin/data", "w")
diff_read.CopyTo(o)
del o
del diff_read
# update symbolic links to state.txt
print(time.strftime("%Y-%m-%d %H:%M:%S"),
" update links to state.txt")
update_symlink(orig_diff_file + ".state.txt",
os.path.join(orig_diff_path, "state.txt"))
os.utime(os.path.join(orig_diff_path, "state.txt"),
(file_date, file_date))
sys.stdout.flush()
if multiproc_enabled:
pool.close()
pool.join()
# free lock
sys.stdout.flush()
lock.release()