def run_change(self, conf):
self.logger.log(self.logger.log_av_r+"run osmosis replication"+self.logger.log_ap)
diff_path = conf.download["diff_path"]
xml_change = os.path.join(diff_path, "change.osc.gz")
shutil.copyfile(os.path.join(diff_path, "state.txt"),
os.path.join(diff_path, "state.txt.old"))
try:
osmosis_lock = self.lock_database()
self.set_pgsql_schema()
cmd = [conf.bin_osmosis]
cmd += ["--read-replication-interval", "workingDirectory=%s" % diff_path]
cmd += ["--simplify-change", "--write-xml-change", "file=%s" % xml_change]
cmd += ["-quiet"]
self.logger.execute_err(cmd)
self.psql_c("TRUNCATE TABLE actions")
cmd = [conf.bin_osmosis]
cmd += ["--read-xml-change", xml_change]
cmd += ["--write-pgsql-change", "database=%s"%conf.db_base, "user=%s"%conf.db_user, "password=%s"%conf.db_password]
cmd += ["-quiet"]
self.logger.execute_err(cmd)
self.logger.log(self.logger.log_av_r+"import osmosis change post scripts"+self.logger.log_ap)
for script in conf.osmosis_change_post_scripts:
self.logger.log(script)
self.psql_f(script)
self.set_pgsql_schema(reset=True)
del osmosis_lock
# Fill metainfo table
gisconn = psycopg2.connect(conf.db_string)
giscurs = gisconn.cursor()
osm_state = OsmState(os.path.join(diff_path, "state.txt"))
osm_state_old = OsmState(os.path.join(diff_path, "state.txt.old"))
giscurs.execute("UPDATE metainfo SET tstamp = %s, tstamp_action = %s", [osm_state.timestamp(), osm_state_old.timestamp()])
gisconn.commit()
giscurs.close()
gisconn.close()
return xml_change
except:
self.logger.log(self.logger.log_av_r+"got error, aborting"+self.logger.log_ap)
shutil.copyfile(os.path.join(diff_path, "state.txt.old"),
os.path.join(diff_path, "state.txt"))
raise
def run_change(self, conf):
self.logger.log(self.logger.log_av_r+"run osmosis replication"+self.logger.log_ap)
diff_path = conf.download["diff_path"]
xml_change = os.path.join(diff_path, "change.osc.gz")
shutil.copyfile(os.path.join(diff_path, "state.txt"),
os.path.join(diff_path, "state.txt.old"))
try:
osmosis_lock = self.lock_database()
self.set_pgsql_schema()
cmd = [conf.bin_osmosis]
cmd += ["--read-replication-interval", "workingDirectory=%s" % diff_path]
cmd += ["--simplify-change", "--write-xml-change", "file=%s" % xml_change]
cmd += ["-quiet"]
self.logger.execute_err(cmd)
self.psql_c("TRUNCATE TABLE actions")
cmd = [conf.bin_osmosis]
cmd += ["--read-xml-change", xml_change]
cmd += ["--write-pgsql-change", "database=%s"%conf.db_base, "user=%s"%conf.db_user, "password=%s"%conf.db_password]
cmd += ["-quiet"]
self.logger.execute_err(cmd)
self.logger.log(self.logger.log_av_r+"import osmosis change post scripts"+self.logger.log_ap)
for script in conf.osmosis_change_post_scripts:
self.logger.log(script)
self.psql_f(script)
self.set_pgsql_schema(reset=True)
del osmosis_lock
# Fill metainfo table
gisconn = self.osmosis().conn()
giscurs = gisconn.cursor()
osm_state = OsmState(os.path.join(diff_path, "state.txt"))
osm_state_old = OsmState(os.path.join(diff_path, "state.txt.old"))
giscurs.execute("UPDATE metainfo SET tstamp = %s, tstamp_action = %s", [osm_state.timestamp(), osm_state_old.timestamp()])
gisconn.commit()
giscurs.close()
self.osmosis_close()
return xml_change
except:
self.logger.err("got error, aborting")
shutil.copyfile(os.path.join(diff_path, "state.txt.old"),
os.path.join(diff_path, "state.txt"))
raise
def update_metainfo(self, conf):
# Fill metainfo table
gisconn = psycopg2.connect(self.db_string)
giscurs = gisconn.cursor()
try:
diff_path = conf.download["diff_path"]
osm_state = OsmState(os.path.join(diff_path, "state.txt")).timestamp()
except:
from modules.OsmPbf import OsmPbfReader
osm_state = OsmPbfReader(conf.download["dst"], None).timestamp()
giscurs.execute("UPDATE metainfo SET tstamp = %s", [osm_state])
gisconn.commit()
giscurs.close()
gisconn.close()
def run_osmosis_diff(self, conf):
self.logger.log(self.logger.log_av_r + "run osmosis replication" +
self.logger.log_ap)
diff_path = conf.download["diff_path"]
xml_change = os.path.join(diff_path, "change.osc.gz")
tmp_pbf_file = conf.download["dst"] + ".tmp"
shutil.copyfile(os.path.join(diff_path, "state.txt"),
os.path.join(diff_path, "state.txt.old"))
dir_country_tmp = os.path.join(self.conf.dir_tmp, self.db_schema)
shutil.rmtree(dir_country_tmp, ignore_errors=True)
os.makedirs(dir_country_tmp)
try:
is_uptodate = False
nb_iter = 0
osm_state = OsmState(os.path.join(diff_path, "state.txt"))
prev_state_ts = osm_state.timestamp()
cur_ts = datetime.datetime.today()
print("state: ", osm_state.timestamp(), end=' ')
if osm_state.timestamp() < (cur_ts - datetime.timedelta(days=10)):
# Skip updates, and directly download .pbf file if extract is too old
self.logger.log(self.logger.log_av_r +
"stop updates, to download full extract" +
self.logger.log_ap)
return (False, None)
while not is_uptodate and nb_iter < 10:
nb_iter += 1
self.logger.log("iteration=%d" % nb_iter)
try:
cmd = [conf.bin_osmosis]
cmd += [
"--read-replication-interval",
"workingDirectory=%s" % diff_path
]
cmd += [
"--simplify-change", "--write-xml-change",
"file=%s" % xml_change
]
cmd += ["-quiet"]
self.logger.execute_err(cmd)
except:
self.logger.log("waiting 2 minutes")
time.sleep(2 * 60)
continue
cmd = [conf.bin_osmosis]
cmd += ["--read-xml-change", "file=%s" % xml_change]
cmd += ["--read-pbf", "file=%s" % conf.download["dst"]]
cmd += ["--apply-change", "--buffer"]
cmd += ["--write-pbf", "file=%s" % tmp_pbf_file]
cmd += ["-quiet"]
self.logger.execute_err(cmd)
shutil.move(tmp_pbf_file, conf.download["dst"])
# find if state.txt is more recent than one day
osm_state = OsmState(os.path.join(diff_path, "state.txt"))
cur_ts = datetime.datetime.today()
print("state: ",
nb_iter,
" - ",
osm_state.timestamp(),
end=' ')
if prev_state_ts is not None:
print(" ", prev_state_ts - osm_state.timestamp())
if osm_state.timestamp() > (cur_ts -
datetime.timedelta(days=1)):
is_uptodate = True
elif prev_state_ts == osm_state.timestamp():
is_uptodate = True
else:
prev_state_ts = osm_state.timestamp()
if not is_uptodate:
# we didn't get the latest version of the pbf file
self.logger.log(self.logger.log_av_r +
"didn't get latest version of osm file" +
self.logger.log_ap)
return (False, None)
elif nb_iter == 1:
return (True, xml_change)
else:
# TODO: we should return a merge of all xml change files
return (True, None)
except:
shutil.rmtree(dir_country_tmp, ignore_errors=True)
self.logger.err("got error, aborting")
shutil.copyfile(os.path.join(diff_path, "state.txt.old"),
os.path.join(diff_path, "state.txt"))
raise
def run_diff(self, conf):
self.logger.log(self.logger.log_av_r+"run osmosis replication"+self.logger.log_ap)
diff_path = conf.download["diff_path"]
xml_change = os.path.join(diff_path, "change.osc.gz")
tmp_pbf_file = conf.download["dst"] + ".tmp"
shutil.copyfile(os.path.join(diff_path, "state.txt"),
os.path.join(diff_path, "state.txt.old"))
dir_country_tmp = os.path.join(self.conf.dir_tmp, self.db_schema)
shutil.rmtree(dir_country_tmp, ignore_errors=True)
os.makedirs(dir_country_tmp)
try:
is_uptodate = False
nb_iter = 0
osm_state = OsmState(os.path.join(diff_path, "state.txt"))
prev_state_ts = osm_state.timestamp()
cur_ts = datetime.datetime.today()
print("state: ", osm_state.timestamp(), end=' ')
if osm_state.timestamp() < (cur_ts - datetime.timedelta(days=10)):
# Skip updates, and directly download .pbf file if extract is too old
self.logger.log(self.logger.log_av_r + "stop updates, to download full extract" + self.logger.log_ap)
return (False, None)
while not is_uptodate and nb_iter < 10:
nb_iter += 1
self.logger.log("iteration=%d" % nb_iter)
try:
cmd = [conf.bin_osmosis]
cmd += ["--read-replication-interval", "workingDirectory=%s" % diff_path]
cmd += ["--simplify-change", "--write-xml-change", "file=%s" % xml_change]
cmd += ["-quiet"]
self.logger.execute_err(cmd)
except:
self.logger.log("waiting 2 minutes")
time.sleep(2*60)
continue
cmd = [conf.bin_osmosis]
cmd += ["--read-xml-change", "file=%s" % xml_change]
cmd += ["--read-pbf", "file=%s" % conf.download["dst"] ]
cmd += ["--apply-change", "--buffer"]
cmd += ["--write-pbf", "file=%s" % tmp_pbf_file]
cmd += ["-quiet"]
self.logger.execute_err(cmd)
shutil.move(tmp_pbf_file, conf.download["dst"])
# find if state.txt is more recent than one day
osm_state = OsmState(os.path.join(diff_path, "state.txt"))
cur_ts = datetime.datetime.today()
print("state: ", nb_iter, " - ", osm_state.timestamp(), end=' ')
if prev_state_ts != None:
print(" ", prev_state_ts - osm_state.timestamp())
if osm_state.timestamp() > (cur_ts - datetime.timedelta(days=1)):
is_uptodate = True
elif prev_state_ts == osm_state.timestamp():
is_uptodate = True
else:
prev_state_ts = osm_state.timestamp()
if not is_uptodate:
# we didn't get the latest version of the pbf file
self.logger.log(self.logger.log_av_r + "didn't get latest version of osm file" + self.logger.log_ap)
return (False, None)
elif nb_iter == 1:
return (True, xml_change)
else:
# TODO: we should return a merge of all xml change files
return (True, None)
except:
shutil.rmtree(dir_country_tmp, ignore_errors=True)
self.logger.err("got error, aborting")
shutil.copyfile(os.path.join(diff_path, "state.txt.old"),
os.path.join(diff_path, "state.txt"))
raise
shutil.rmtree(dir_country_tmp, ignore_errors=True)
