def init_diff(self, conf):
self.logger.log(self.logger.log_av_r+"init osmosis replication for diff"+self.logger.log_ap)
diff_path = conf.download["diff_path"]
for f_name in ["configuration.txt", "download.lock", "state.txt"]:
f = os.path.join(diff_path, f_name)
if os.path.exists(f):
os.remove(f)
cmd = [conf.bin_osmosis]
cmd += ["--read-replication-interval-init", "workingDirectory=%s" % diff_path]
cmd += ["-quiet"]
self.logger.execute_err(cmd)
for line in fileinput.input(os.path.join(diff_path, "configuration.txt"), inplace=1):
if line.startswith("baseUrl"):
sys.stdout.write("baseUrl=" + conf.download["diff"])
elif line.startswith("maxInterval"):
if "geofabrik" in conf.download["diff"]:
# on daily diffs provided by Geofabrik, we should apply only one diff at a time
sys.stdout.write("maxInterval=" + str(60*60*24/2)) # 1/2 day at most
else:
sys.stdout.write("maxInterval=" + str(7*60*60*24)) # 7 day at most
else:
sys.stdout.write(line)
fileinput.close()
download.dl(conf.download["state.txt"],
os.path.join(diff_path, "state.txt"),
self.logger.sub(),
min_file_size=10)
def init_osmosis_diff(conf):
logger.log(log_av_r + "init osmosis replication for diff" + log_ap)
diff_path = conf.download["diff_path"]
if os.path.exists(diff_path):
for f_name in ["configuration.txt", "download.lock", "state.txt"]:
f = os.path.join(diff_path, f_name)
if os.path.exists(f):
os.remove(f)
else:
os.makedirs(diff_path)
cmd = [conf.osmosis_bin]
cmd += ["--read-replication-interval-init", "workingDirectory=%s" % diff_path]
cmd += ["-quiet"]
logger.execute_err(cmd)
for line in fileinput.input(os.path.join(diff_path, "configuration.txt"), inplace=1):
if line.startswith("baseUrl"):
sys.stdout.write("baseUrl=" + conf.download["diff"])
elif line.startswith("maxInterval"):
if "geofabrik" in conf.download["diff"]:
# on daily diffs provided by Geofabrik, we should apply only one diff at a time
sys.stdout.write("maxInterval=" + str(60 * 60 * 24 / 2)) # 1/2 day at most
else:
sys.stdout.write("maxInterval=" + str(7 * 60 * 60 * 24)) # 7 day at most
else:
sys.stdout.write(line)
fileinput.close()
if conf.download["diff"].endswith("minute/"):
from modules import OsmTs
OsmTs.run(conf.download["dst"], os.path.join(diff_path, "state.txt"), "minute", logger)
else:
download.dl(
conf.download["diff"] + "state.txt", os.path.join(diff_path, "state.txt"), logger.sub(), min_file_size=10
)
elif "url" in conf.download:
newer = False
xml_change = None
if options.diff and check_osmosis_diff(conf, logger) and os.path.exists(conf.download["dst"]):
(status, xml_change) = run_osmosis_diff(conf, logger)
if status:
newer = True
if not newer and options.skip_download:
logger.sub().log("skip download")
newer = True
if not newer:
logger.log(logger.log_av_r+u"downloading"+logger.log_ap)
newer = download.dl(conf.download["url"], conf.download["dst"], logger.sub())
if not newer:
return 0
init_database(conf, logger)
if options.change:
init_osmosis_change(conf, logger)
elif options.diff and not xml_change:
init_osmosis_diff(conf, logger)
if hasattr(conf, "sql_post_scripts"):
logger.log(logger.log_av_r+"import post scripts"+logger.log_ap)
for script in conf.sql_post_scripts:
cmd = ["psql"]
def execc(conf, logger, analysers, options, osmosis_manager):
err_code = 0
## download and create database
xml_change = None
if not options.skip_init:
if options.change and osmosis_manager.check_change(
conf) and not options.change_init:
xml_change = osmosis_manager.run_change(conf)
elif "url" in conf.download:
newer = False
xml_change = None
if not newer and options.skip_download:
logger.sub().log("skip download")
newer = True
if not newer and options.diff and os.path.exists(
conf.download["dst"]):
status = False
if options.pbf_update_tool == 'osmosis':
if osmosis_manager.check_osmosis_diff(conf):
(status,
xml_change) = osmosis_manager.run_osmosis_diff(conf)
else:
if osmosis_manager.check_osmium_diff(conf):
(status,
xml_change) = osmosis_manager.run_osmium_diff(conf)
if status:
newer = True
if not newer:
logger.log(logger.log_av_r + u"downloading" + logger.log_ap)
newer = download.dl(conf.download["url"],
conf.download["dst"],
logger.sub(),
min_file_size=8 * 1024)
if newer and options.diff:
if options.pbf_update_tool == 'osmosis':
osmosis_manager.init_osmosis_diff(conf)
if "/minute/" in conf.download[
"diff"] or "/hour/" in conf.download["diff"]:
# update extract with any more recent available diff
if options.pbf_update_tool == 'osmosis':
osmosis_manager.run_osmosis_diff(conf)
else:
osmosis_manager.run_osmium_diff(conf)
if not newer:
return 0x11
if osmosis_manager:
osmosis_manager.init_database(conf)
if options.change:
osmosis_manager.init_change(conf)
if hasattr(conf, "sql_post_scripts"):
logger.log(logger.log_av_r + "import post scripts" + logger.log_ap)
for script in conf.sql_post_scripts:
osmosis_manager.psql_f(script)
if osmosis_manager:
osmosis_manager.update_metainfo(conf)
if options.resume:
osmosis_manager.run_resume(conf)
##########################################################################
## analyses
version = get_version()
lunched_analyser = []
lunched_analyser_change = []
lunched_analyser_resume = []
for analyser in analysers:
if not options.analyser and analyser not in conf.analyser:
continue
logger.log(logger.log_av_r + conf.country + " : " + analyser +
logger.log_ap)
password = conf.analyser.get(analyser)
if not password or password == "xxx":
logger.sub().err("No password to upload result to %s" %
conf.updt_url)
try:
analyser_conf = analyser_config(conf, options, osmosis_manager,
xml_change)
for name, obj in inspect.getmembers(analysers[analyser]):
if (inspect.isclass(obj)
and obj.__module__ == "analysers.analyser_" + analyser
and (name.startswith("Analyser")
or name.startswith("analyser"))):
analyser_name = name[len("Analyser_"):]
resume = options.resume or (options.resume_analyser
and analyser
in options.resume_analyser)
dst = os.path.join(conf.dir_results,
name + "-" + conf.country)
analyser_conf.error_file = issues_file_from_fromat(
dst,
options.result_format,
bz2=True,
version=version,
polygon_id=analyser_conf.polygon_id)
# analyse
if not options.skip_analyser:
with obj(analyser_conf, logger.sub()) as analyser_obj:
remote_timestamp = None
if not options.skip_frontend_check:
url = modules.config.url_frontend_update + "/../../control/status/%s/%s?%s" % (
conf.country, analyser_name,
'objects=true' if resume else '')
resp = downloader.get(url)
if not resp.ok:
logger.sub().err(
"Fails to get status from frontend: {0}"
.format(resp.status_code))
else:
try:
status = resp.json()
remote_timestamp = dateutil.parser.parse(
status['timestamp']
) if status else None
remote_analyser_version = int(
status['analyser_version'])
except Exception as e:
logger.sub().err(e)
if analyser_obj.timestamp(
) and remote_timestamp and analyser_obj.timestamp(
) <= remote_timestamp and analyser_obj.analyser_version(
) == remote_analyser_version:
logger.sub().warn(
"Skip, frontend is already up to date")
continue
if resume and remote_timestamp and analyser_obj.analyser_version(
) == remote_analyser_version:
already_issued_objects = {
'N': status['nodes'] or [],
'W': status['ways'] or [],
'R': status['relations'] or []
}
analyser_obj.analyser_resume(
remote_timestamp, already_issued_objects)
lunched_analyser_resume.append(
[obj, analyser_conf])
else:
if resume:
if not remote_timestamp:
logger.sub().err(
"No remote timestamp to resume from, start a full run"
)
elif analyser_obj.analyser_version(
) == remote_analyser_version:
logger.sub().err(
"Analyser version changed, start a full run"
)
if not options.change or not xml_change:
analyser_obj.analyser()
lunched_analyser.append(
[obj, analyser_conf])
else:
analyser_obj.analyser_change()
lunched_analyser_change.append(
[obj, analyser_conf])
# update
if not options.skip_upload and password != "xxx":
logger.sub().log("update")
if analyser in conf.analyser_updt_url:
list_urls = conf.analyser_updt_url[analyser]
else:
list_urls = [conf.updt_url]
for url in list_urls:
update_finished = False
nb_iter = 0
was_on_timeout = False
while not update_finished and nb_iter < 3:
time.sleep(nb_iter * 15)
nb_iter += 1
logger.sub().sub().log("iteration=%d" %
nb_iter)
try:
u = url + '?analyser=' + analyser_name + '&country=' + conf.country
r = requests.post(
u,
timeout=1800,
data={
'analyser': analyser_name,
'country': conf.country,
'code': password
},
files={
'content':
open(analyser_conf.error_file.dst,
'rb')
})
r.raise_for_status()
logger.sub().sub().log(r.text.strip())
update_finished = True
except requests.exceptions.HTTPError as e:
if e.response.status_code == 504:
was_on_timeout = True
logger.sub().sub().sub().err(
'got an HTTP timeout status')
else:
dt = r.text.strip()
logger.sub().sub().sub().err(
u"UPDATE ERROR %s/%s : %s\n" %
(conf.country, analyser_name, dt))
if dt == "FAIL: Already up to date":
update_finished = True
if not was_on_timeout:
err_code |= 4
except Exception as e:
if isinstance(
e, requests.exceptions.
ConnectTimeout):
was_on_timeout = True
logger.sub().sub().sub().err(
'got a connection timeout')
else:
tb = traceback.format_exc()
logger.sub().err('error on update...')
for l in tb.splitlines():
logger.sub().sub().log(l)
if not update_finished:
err_code |= 1
except:
tb = traceback.format_exc()
logger.sub().err("error on analyse {0}...".format(analyser))
for l in tb.splitlines():
logger.sub().sub().log(l)
err_code |= 2
continue
if not options.no_clean:
for (obj, analyser_conf) in lunched_analyser:
analyser_conf.error_file = None
with obj(analyser_conf, logger.sub()) as analyser_obj:
analyser_obj.analyser_deferred_clean()
for (obj, analyser_conf) in lunched_analyser_change:
analyser_conf.error_file = None
with obj(analyser_conf, logger.sub()) as analyser_obj:
analyser_obj.analyser_deferred_clean()
for (obj, analyser_conf) in lunched_analyser_resume:
analyser_conf.error_file = None
with obj(analyser_conf, logger.sub()) as analyser_obj:
analyser_obj.analyser_deferred_clean()
return err_code
def run(conf, logger, options):
err_code = 0
country = conf.country
try:
version = get_version()
except:
version = None
if not check_database(conf, logger):
logger.log(logger.log_av_r + u"error in database initialisation" + logger.log_ap)
return 0x10
##########################################################################
## check for working dirs and creates when needed
dirs = [conf.dir_tmp, conf.dir_cache, conf.dir_results, conf.dir_extracts, conf.dir_diffs]
if "diff_path" in conf.download:
dirs.append(conf.download["diff_path"])
for i in dirs:
if not os.path.exists(i):
try:
os.makedirs(i)
except OSError as e:
sys.exit("%s\nCheck 'dir_work' in modules/config.py and its permissions" % str(e))
# variable used by osmosis
if not "JAVACMD_OPTIONS" in os.environ:
os.environ["JAVACMD_OPTIONS"] = ""
os.environ["JAVACMD_OPTIONS"] += " -Djava.io.tmpdir=" + conf.dir_tmp
##########################################################################
## download and create database
if options.skip_init:
pass
elif options.change and check_osmosis_change(conf, logger) and not options.change_init:
xml_change = run_osmosis_change(conf, logger)
elif "url" in conf.download:
newer = False
xml_change = None
if options.diff and check_osmosis_diff(conf, logger) and os.path.exists(conf.download["dst"]):
(status, xml_change) = run_osmosis_diff(conf, logger)
if status:
newer = True
if not newer and options.skip_download:
logger.sub().log("skip download")
newer = True
if not newer:
logger.log(logger.log_av_r + u"downloading" + logger.log_ap)
newer = download.dl(conf.download["url"], conf.download["dst"], logger.sub(), min_file_size=8 * 1024)
if not newer:
return 0
init_database(conf, logger)
if options.change:
init_osmosis_change(conf, logger)
elif options.diff and not xml_change:
init_osmosis_diff(conf, logger)
if hasattr(conf, "sql_post_scripts"):
logger.log(logger.log_av_r + "import post scripts" + logger.log_ap)
for script in conf.sql_post_scripts:
cmd = ["psql"]
cmd += conf.db_psql_args
cmd += ["-f", script]
logger.execute_out(cmd)
##########################################################################
## analyses
for analyser, password in conf.analyser.iteritems():
logger.log(logger.log_av_r + country + " : " + analyser + logger.log_ap)
if not "analyser_" + analyser in analysers:
logger.sub().log("skipped")
continue
if password == "xxx":
logger.sub().log("code is not correct - won't upload to %s" % conf.updt_url)
elif not conf.results_url and not has_poster_lib:
logger.sub().log("results_url is not correct - won't upload to %s" % conf.updt_url)
try:
analyser_conf = analyser_config()
analyser_conf.dst_dir = conf.dir_results
analyser_conf.db_string = conf.db_string
analyser_conf.db_user = conf.db_user
if conf.db_schema:
analyser_conf.db_schema = conf.db_schema
else:
analyser_conf.db_schema = country
analyser_conf.dir_scripts = conf.dir_scripts
analyser_conf.options = conf.analyser_options
analyser_conf.polygon_id = conf.polygon_id
if options.change and xml_change:
analyser_conf.src = xml_change
elif "dst" in conf.download:
analyser_conf.src = conf.download["dst"]
for name, obj in inspect.getmembers(analysers["analyser_" + analyser]):
if (
inspect.isclass(obj)
and obj.__module__ == "analyser_" + analyser
and (name.startswith("Analyser") or name.startswith("analyser"))
):
# analyse
analyser_conf.dst_file = name + "-" + country + ".xml"
analyser_conf.dst_file += ".bz2"
analyser_conf.dst = os.path.join(conf.dir_results, analyser_conf.dst_file)
analyser_conf.version = version
analyser_conf.verbose = options.verbose
with obj(analyser_conf, logger.sub()) as analyser_obj:
if not options.change or not xml_change:
analyser_obj.analyser()
else:
analyser_obj.analyser_change()
# update
if (conf.results_url or has_poster_lib) and password != "xxx":
logger.sub().log("update")
update_finished = False
nb_iter = 0
while not update_finished and nb_iter < 3:
time.sleep(nb_iter * 15)
nb_iter += 1
logger.sub().sub().log("iteration=%d" % nb_iter)
try:
tmp_src = "%s-%s" % (analyser, country)
if has_poster_lib:
(tmp_dat, tmp_headers) = poster.encode.multipart_encode(
{"content": open(analyser_conf.dst, "rb"), "source": tmp_src, "code": password}
)
tmp_req = urllib2.Request(conf.updt_url, tmp_dat, tmp_headers)
fd = urllib2.urlopen(tmp_req, timeout=1800)
else:
tmp_req = urllib2.Request(conf.updt_url)
tmp_url = os.path.join(conf.results_url, analyser_conf.dst_file)
tmp_dat = urllib.urlencode(
[("url", tmp_url), ("source", tmp_src), ("code", password)]
)
fd = urllib2.urlopen(tmp_req, tmp_dat, timeout=1800)
dt = fd.read().decode("utf8").strip()
if dt[-2:] != "OK":
sys.stderr.write(
(u"UPDATE ERROR %s/%s : %s\n" % (country, analyser, dt)).encode("utf8")
)
err_code |= 4
else:
logger.sub().sub().log(dt)
update_finished = True
except socket.timeout:
logger.sub().sub().sub().log("got a timeout")
pass
except:
s = StringIO()
traceback.print_exc(file=s)
logger.sub().log("error on update...")
for l in s.getvalue().decode("utf8").split("\n"):
logger.sub().sub().log(l)
if not update_finished:
err_code |= 1
except:
s = StringIO()
traceback.print_exc(file=s)
logger.sub().log("error on analyse...")
for l in s.getvalue().decode("utf8").split("\n"):
logger.sub().sub().log(l)
err_code |= 2
continue
##########################################################################
## vidange
logger.log(logger.log_av_r + u"cleaning : " + country + logger.log_ap)
if options.change:
pass
else:
clean_database(conf, logger, options.no_clean or not conf.clean_at_end)
if options.diff:
# don't erase any file
return err_code
# remove files
if "url" in conf.download and "dst" in conf.download and not options.no_clean:
f = ".osm".join(conf.download["dst"].split(".osm")[:-1])
for ext in ["osm", "osm.bz2", "osm.pbf"]:
try:
os.remove("%s.%s" % (f, ext))
logger.sub().log("DROP FILE %s.%s" % (f, ext))
except:
pass
return err_code
def execc(conf, logger, options, osmosis_manager):
err_code = 0
version = get_version()
logger.log("osmose backend version: %s" % version)
## download and create database
country = conf.country
if options.skip_init:
pass
elif options.change and osmosis_manager.check_change(
conf) and not options.change_init:
xml_change = osmosis_manager.run_change(conf)
elif "url" in conf.download:
newer = False
xml_change = None
if not newer and options.skip_download:
logger.sub().log("skip download")
newer = True
if not newer and options.diff and osmosis_manager.check_diff(
conf) and os.path.exists(conf.download["dst"]):
(status, xml_change) = osmosis_manager.run_diff(conf)
if status:
newer = True
if not newer:
logger.log(logger.log_av_r + u"downloading" + logger.log_ap)
newer = download.dl(conf.download["url"],
conf.download["dst"],
logger.sub(),
min_file_size=8 * 1024)
if newer and options.diff:
osmosis_manager.init_diff(conf)
if "/minute/" in conf.download[
"diff"] or "/hour/" in conf.download["diff"]:
# update extract with any more recent available diff
osmosis_manager.run_diff(conf)
if not newer:
return 0x11
if osmosis_manager:
osmosis_manager.init_database(conf)
if options.change:
osmosis_manager.init_change(conf)
if hasattr(conf, "sql_post_scripts"):
logger.log(logger.log_av_r + "import post scripts" + logger.log_ap)
for script in conf.sql_post_scripts:
osmosis_manager.psql_f(script)
if not options.skip_init and osmosis_manager:
osmosis_manager.update_metainfo(conf)
if options.resume:
osmosis_manager.run_resume(conf)
##########################################################################
## analyses
lunched_analyser = []
lunched_analyser_change = []
lunched_analyser_resume = []
for analyser, password in conf.analyser.items():
logger.log(logger.log_av_r + country + " : " + analyser +
logger.log_ap)
if not "analyser_" + analyser in analysers:
logger.sub().log("skipped")
continue
if password == "xxx":
logger.sub().log("code is not correct - won't upload to %s" %
conf.updt_url)
try:
analyser_conf = analyser_config()
analyser_conf.dst_dir = conf.dir_results
analyser_conf.osmosis_manager = osmosis_manager
analyser_conf.db_user = conf.db_user
if conf.db_schema:
analyser_conf.db_schema = conf.db_schema
else:
analyser_conf.db_schema = country
analyser_conf.db_schema_path = conf.db_schema_path
analyser_conf.dir_scripts = conf.dir_scripts
analyser_conf.options = conf.analyser_options
analyser_conf.polygon_id = conf.polygon_id
if options.change and xml_change:
analyser_conf.src = xml_change
elif "dst" in conf.download:
analyser_conf.src = conf.download["dst"]
if "diff_path" in conf.download:
analyser_conf.src_state = os.path.join(
conf.download["diff_path"], "state.txt")
for name, obj in inspect.getmembers(analysers["analyser_" +
analyser]):
if (inspect.isclass(obj)
and obj.__module__ == "analysers.analyser_" + analyser
and (name.startswith("Analyser")
or name.startswith("analyser"))):
analyser_name = name[len("Analyser_"):]
analyser_conf.dst_file = name + "-" + country + ".xml"
analyser_conf.dst_file += ".bz2"
analyser_conf.dst = os.path.join(conf.dir_results,
analyser_conf.dst_file)
analyser_conf.version = version
analyser_conf.verbose = options.verbose
# analyse
if not options.skip_analyser:
with obj(analyser_conf, logger.sub()) as analyser_obj:
remote_timestamp = None
if not options.skip_frontend_check:
url = modules.config.url_frontend_update + "/../../control/status/%s/%s?%s" % (
country, analyser_name,
'objects=true' if options.resume else '')
resp = downloader.get(url)
if not resp.ok:
logger.sub().err(
"Fails to get status from frontend: {0}"
.format(resp.status_code))
else:
try:
status = resp.json()
remote_timestamp = dateutil.parser.parse(
status['timestamp']
) if status else None
except e:
logger.sub().err(e)
if options.resume:
if remote_timestamp:
already_issued_objects = {
'N': status['nodes'] or [],
'W': status['ways'] or [],
'R': status['relations'] or []
}
analyser_obj.analyser_resume(
remote_timestamp,
already_issued_objects)
lunched_analyser_resume.append(
[obj, analyser_conf])
continue
else:
logger.sub().err("Not able to resume")
if analyser_obj.timestamp(
) and remote_timestamp and analyser_obj.timestamp(
) <= remote_timestamp:
logger.sub().warn(
"Skip, frontend is already up to date")
continue
if not options.change or not xml_change:
analyser_obj.analyser()
lunched_analyser.append([obj, analyser_conf])
else:
analyser_obj.analyser_change()
lunched_analyser_change.append(
[obj, analyser_conf])
# update
if not options.skip_upload and password != "xxx":
logger.sub().log("update")
if analyser in conf.analyser_updt_url:
list_urls = conf.analyser_updt_url[analyser]
else:
list_urls = [conf.updt_url]
for url in list_urls:
update_finished = False
nb_iter = 0
was_on_timeout = False
while not update_finished and nb_iter < 3:
time.sleep(nb_iter * 15)
nb_iter += 1
logger.sub().sub().log("iteration=%d" %
nb_iter)
try:
u = url + '?name=' + name + '&country=' + (
conf.db_schema or conf.country)
r = requests.post(
u,
timeout=1800,
data={
'analyser': analyser_name,
'country': country,
'code': password
},
files={
'content':
open(analyser_conf.dst, 'rb')
})
r.raise_for_status()
dt = r.text.strip()
if dt == "FAIL: Already up to date" and was_on_timeout:
logger.sub().sub().sub().err(
(u"UPDATE ERROR %s/%s : %s\n" %
(country, analyser_name,
dt)).encode("utf8"))
# Log error, but do not set err_code
elif dt[-2:] != "OK":
logger.sub().sub().sub().err(
(u"UPDATE ERROR %s/%s : %s\n" %
(country, analyser_name,
dt)).encode("utf8"))
err_code |= 4
else:
logger.sub().sub().log(dt)
update_finished = True
except Exception as e:
if isinstance(
e,
requests.exceptions.ConnectTimeout
) or (isinstance(
e, requests.exceptions.HTTPError)
and e.response.status_code == 504):
was_on_timeout = True
logger.sub().sub().sub().err(
'got a timeout')
else:
tb = traceback.format_exc()
logger.sub().err('error on update...')
for l in tb.splitlines():
logger.sub().sub().log(l)
if not update_finished:
err_code |= 1
except:
tb = traceback.format_exc()
logger.sub().err("error on analyse {0}...".format(analyser))
for l in tb.splitlines():
logger.sub().sub().log(l)
err_code |= 2
continue
if not options.no_clean:
for (obj, analyser_conf) in lunched_analyser:
analyser_conf.dst = None
with obj(analyser_conf, logger.sub()) as analyser_obj:
analyser_obj.analyser_deferred_clean()
for (obj, analyser_conf) in lunched_analyser_change:
analyser_conf.dst = None
with obj(analyser_conf, logger.sub()) as analyser_obj:
analyser_obj.analyser_deferred_clean()
for (obj, analyser_conf) in lunched_analyser_resume:
analyser_conf.dst = None
with obj(analyser_conf, logger.sub()) as analyser_obj:
analyser_obj.analyser_deferred_clean()
return err_code
def run(conf, logger, options):
err_code = 0
country = conf.country
try:
version = get_version()
except:
version = None
if not check_database(conf, logger):
logger.log(logger.log_av_r+u"error in database initialisation"+logger.log_ap)
return 0x10
##########################################################################
## check for working dirs and creates when needed
dirs = [conf.dir_tmp, conf.dir_cache, conf.dir_results, conf.dir_extracts, conf.dir_diffs]
if "diff_path" in conf.download:
dirs.append(conf.download["diff_path"])
for i in dirs:
if not os.path.exists(i):
try:
os.makedirs(i)
except OSError as e:
sys.exit("%s\nCheck 'dir_work' in modules/config.py and its permissions" % str(e))
# variable used by osmosis
if not "JAVACMD_OPTIONS" in os.environ:
os.environ["JAVACMD_OPTIONS"] = ""
os.environ["JAVACMD_OPTIONS"] += " -Djava.io.tmpdir="+conf.dir_tmp
##########################################################################
## download and create database
if options.skip_init:
pass
elif options.change and check_osmosis_change(conf, logger) and not options.change_init:
xml_change = run_osmosis_change(conf, logger)
elif "url" in conf.download:
newer = False
xml_change = None
if options.diff and check_osmosis_diff(conf, logger) and os.path.exists(conf.download["dst"]):
(status, xml_change) = run_osmosis_diff(conf, logger)
if status:
newer = True
if not newer and options.skip_download:
logger.sub().log("skip download")
newer = True
if not newer:
logger.log(logger.log_av_r+u"downloading"+logger.log_ap)
newer = download.dl(conf.download["url"], conf.download["dst"], logger.sub(),
min_file_size=8*1024)
if not newer:
return 0
init_database(conf, logger)
if options.change:
init_osmosis_change(conf, logger)
elif options.diff and not xml_change:
init_osmosis_diff(conf, logger)
if hasattr(conf, "sql_post_scripts"):
logger.log(logger.log_av_r+"import post scripts"+logger.log_ap)
for script in conf.sql_post_scripts:
cmd = ["psql"]
cmd += conf.db_psql_args
cmd += ["-f", script]
logger.execute_out(cmd)
##########################################################################
## analyses
for analyser, password in conf.analyser.iteritems():
logger.log(logger.log_av_r + country + " : " + analyser + logger.log_ap)
if not "analyser_" + analyser in analysers:
logger.sub().log("skipped")
continue
if password == "xxx":
logger.sub().log("code is not correct - won't upload to %s" % conf.updt_url)
elif not conf.results_url and not has_poster_lib:
logger.sub().log("results_url is not correct - won't upload to %s" % conf.updt_url)
try:
analyser_conf = analyser_config()
analyser_conf.dst_dir = conf.dir_results
analyser_conf.db_string = conf.db_string
analyser_conf.db_user = conf.db_user
if conf.db_schema:
analyser_conf.db_schema = conf.db_schema
else:
analyser_conf.db_schema = country
analyser_conf.dir_scripts = conf.dir_scripts
analyser_conf.options = conf.analyser_options
analyser_conf.polygon_id = conf.polygon_id
if options.change and xml_change:
analyser_conf.src = xml_change
elif "dst" in conf.download:
analyser_conf.src = conf.download["dst"]
for name, obj in inspect.getmembers(analysers["analyser_" + analyser]):
if (inspect.isclass(obj) and obj.__module__ == "analyser_" + analyser and
(name.startswith("Analyser") or name.startswith("analyser"))):
# analyse
analyser_conf.dst_file = name + "-" + country + ".xml"
analyser_conf.dst_file += ".bz2"
analyser_conf.dst = os.path.join(conf.dir_results, analyser_conf.dst_file)
analyser_conf.version = version
analyser_conf.verbose = options.verbose
with obj(analyser_conf, logger.sub()) as analyser_obj:
if not options.change or not xml_change:
analyser_obj.analyser()
else:
analyser_obj.analyser_change()
# update
if (conf.results_url or has_poster_lib) and password != "xxx":
logger.sub().log("update")
if analyser in conf.analyser_updt_url:
list_urls = conf.analyser_updt_url[analyser]
else:
list_urls = [conf.updt_url]
for url in list_urls:
update_finished = False
nb_iter = 0
while not update_finished and nb_iter < 3:
time.sleep(nb_iter * 15)
nb_iter += 1
logger.sub().sub().log("iteration=%d" % nb_iter)
try:
tmp_src = "%s-%s" % (analyser, country)
if has_poster_lib:
(tmp_dat, tmp_headers) = poster.encode.multipart_encode(
{"content": open(analyser_conf.dst, "rb"),
"source": tmp_src,
"code": password})
tmp_req = urllib2.Request(url, tmp_dat, tmp_headers)
fd = urllib2.urlopen(tmp_req, timeout=1800)
else:
tmp_req = urllib2.Request(url)
tmp_url = os.path.join(conf.results_url, analyser_conf.dst_file)
tmp_dat = urllib.urlencode([('url', tmp_url),
('source', tmp_src),
('code', password)])
fd = urllib2.urlopen(tmp_req, tmp_dat, timeout=1800)
dt = fd.read().decode("utf8").strip()
if dt[-2:] != "OK":
sys.stderr.write((u"UPDATE ERROR %s/%s : %s\n"%(country, analyser, dt)).encode("utf8"))
err_code |= 4
else:
logger.sub().sub().log(dt)
update_finished = True
except socket.timeout:
logger.sub().sub().sub().log("got a timeout")
pass
except:
s = StringIO()
traceback.print_exc(file=s)
logger.sub().log("error on update...")
for l in s.getvalue().decode("utf8").split("\n"):
logger.sub().sub().log(l)
if not update_finished:
err_code |= 1
except:
s = StringIO()
traceback.print_exc(file=s)
logger.sub().log("error on analyse...")
for l in s.getvalue().decode("utf8").split("\n"):
logger.sub().sub().log(l)
err_code |= 2
continue
##########################################################################
## vidange
logger.log(logger.log_av_r + u"cleaning : " + country + logger.log_ap)
if options.change:
pass
else:
clean_database(conf, logger, options.no_clean or not conf.clean_at_end)
if options.diff:
# don't erase any file
return err_code
# remove files
if "url" in conf.download and "dst" in conf.download and not options.no_clean:
f = ".osm".join(conf.download["dst"].split(".osm")[:-1])
for ext in ["osm", "osm.bz2", "osm.pbf"]:
try:
os.remove("%s.%s"%(f, ext))
logger.sub().log("DROP FILE %s.%s"%(f, ext))
except:
pass
return err_code
def execc(conf, logger, options, osmosis_manager):
err_code = 0
version = get_version()
logger.log("osmose backend version: %s" % version)
## download and create database
country = conf.country
if options.skip_init:
pass
elif options.change and osmosis_manager.check_change(conf) and not options.change_init:
xml_change = osmosis_manager.run_change(conf)
elif "url" in conf.download:
newer = False
xml_change = None
if not newer and options.skip_download:
logger.sub().log("skip download")
newer = True
if not newer and options.diff and osmosis_manager.check_diff(conf) and os.path.exists(conf.download["dst"]):
(status, xml_change) = osmosis_manager.run_diff(conf)
if status:
newer = True
if not newer:
logger.log(logger.log_av_r+u"downloading"+logger.log_ap)
newer = download.dl(conf.download["url"], conf.download["dst"], logger.sub(),
min_file_size=8*1024)
if newer and options.diff:
osmosis_manager.init_diff(conf)
if "/minute/" in conf.download["diff"] or "/hour/" in conf.download["diff"]:
# update extract with any more recent available diff
osmosis_manager.run_diff(conf)
if not newer:
return 0x11
if osmosis_manager:
osmosis_manager.init_database(conf)
if options.change:
osmosis_manager.init_change(conf)
if hasattr(conf, "sql_post_scripts"):
logger.log(logger.log_av_r+"import post scripts"+logger.log_ap)
for script in conf.sql_post_scripts:
osmosis_manager.psql_f(script)
if not options.skip_init and osmosis_manager:
osmosis_manager.update_metainfo(conf)
if options.resume:
osmosis_manager.run_resume(conf)
##########################################################################
## analyses
lunched_analyser = []
lunched_analyser_change = []
lunched_analyser_resume = []
for analyser, password in conf.analyser.items():
logger.log(logger.log_av_r + country + " : " + analyser + logger.log_ap)
if not "analyser_" + analyser in analysers:
logger.sub().log("skipped")
continue
if password == "xxx":
logger.sub().log("code is not correct - won't upload to %s" % conf.updt_url)
try:
analyser_conf = analyser_config()
analyser_conf.dst_dir = conf.dir_results
analyser_conf.osmosis_manager = osmosis_manager
analyser_conf.db_user = conf.db_user
if conf.db_schema:
analyser_conf.db_schema = conf.db_schema
else:
analyser_conf.db_schema = country
analyser_conf.db_schema_path = conf.db_schema_path
analyser_conf.dir_scripts = conf.dir_scripts
analyser_conf.options = conf.analyser_options
analyser_conf.polygon_id = conf.polygon_id
if options.change and xml_change:
analyser_conf.src = xml_change
elif "dst" in conf.download:
analyser_conf.src = conf.download["dst"]
if "diff_path" in conf.download:
analyser_conf.src_state = os.path.join(conf.download["diff_path"], "state.txt")
for name, obj in inspect.getmembers(analysers["analyser_" + analyser]):
if (inspect.isclass(obj) and obj.__module__ == "analysers.analyser_" + analyser and
(name.startswith("Analyser") or name.startswith("analyser"))):
analyser_name = name[len("Analyser_"):]
analyser_conf.dst_file = name + "-" + country + ".xml"
analyser_conf.dst_file += ".bz2"
analyser_conf.dst = os.path.join(conf.dir_results, analyser_conf.dst_file)
analyser_conf.version = version
analyser_conf.verbose = options.verbose
# analyse
if not options.skip_analyser:
with obj(analyser_conf, logger.sub()) as analyser_obj:
remote_timestamp = None
if not options.skip_frontend_check:
url = modules.config.url_frontend_update + "/../../control/status/%s/%s?%s" % (country, analyser_name, 'objects=true' if options.resume else '')
resp = downloader.get(url)
if not resp.ok:
logger.sub().err("Fails to get status from frontend: {0}".format(resp.status_code))
else:
try:
status = resp.json()
remote_timestamp = dateutil.parser.parse(status['timestamp']) if status else None
except e:
logger.sub().err(e)
if options.resume:
if remote_timestamp:
already_issued_objects = {'N': status['nodes'] or [], 'W': status['ways'] or [], 'R': status['relations'] or []}
analyser_obj.analyser_resume(remote_timestamp, already_issued_objects)
lunched_analyser_resume.append([obj, analyser_conf])
continue
else:
logger.sub().err("Not able to resume")
if analyser_obj.timestamp() and remote_timestamp and analyser_obj.timestamp() <= remote_timestamp:
logger.sub().warn("Skip, frontend is already up to date")
continue
if not options.change or not xml_change:
analyser_obj.analyser()
lunched_analyser.append([obj, analyser_conf])
else:
analyser_obj.analyser_change()
lunched_analyser_change.append([obj, analyser_conf])
# update
if not options.skip_upload and password != "xxx":
logger.sub().log("update")
if analyser in conf.analyser_updt_url:
list_urls = conf.analyser_updt_url[analyser]
else:
list_urls = [conf.updt_url]
for url in list_urls:
update_finished = False
nb_iter = 0
was_on_timeout = False
while not update_finished and nb_iter < 3:
time.sleep(nb_iter * 15)
nb_iter += 1
logger.sub().sub().log("iteration=%d" % nb_iter)
try:
u = url + '?name=' + name + '&country=' + (conf.db_schema or conf.country)
r = requests.post(u, timeout=1800, data={
'analyser': analyser_name,
'country': country,
'code': password
}, files={
'content': open(analyser_conf.dst, 'rb')
})
r.raise_for_status()
dt = r.text.strip()
if dt == "FAIL: Already up to date" and was_on_timeout:
logger.sub().sub().sub().err((u"UPDATE ERROR %s/%s : %s\n"%(country, analyser_name, dt)).encode("utf8"))
# Log error, but do not set err_code
elif dt[-2:] != "OK":
logger.sub().sub().sub().err((u"UPDATE ERROR %s/%s : %s\n"%(country, analyser_name, dt)).encode("utf8"))
err_code |= 4
else:
logger.sub().sub().log(dt)
update_finished = True
except Exception as e:
if isinstance(e, requests.exceptions.ConnectTimeout) or (isinstance(e, requests.exceptions.HTTPError) and e.response.status_code == 504):
was_on_timeout = True
logger.sub().sub().sub().err('got a timeout')
else:
tb = traceback.format_exc()
logger.sub().err('error on update...')
for l in tb.splitlines():
logger.sub().sub().log(l)
if not update_finished:
err_code |= 1
except:
tb = traceback.format_exc()
logger.sub().err("error on analyse {0}...".format(analyser))
for l in tb.splitlines():
logger.sub().sub().log(l)
err_code |= 2
continue
if not options.no_clean:
for (obj, analyser_conf) in lunched_analyser:
analyser_conf.dst = None
with obj(analyser_conf, logger.sub()) as analyser_obj:
analyser_obj.analyser_deferred_clean()
for (obj, analyser_conf) in lunched_analyser_change:
analyser_conf.dst = None
with obj(analyser_conf, logger.sub()) as analyser_obj:
analyser_obj.analyser_deferred_clean()
for (obj, analyser_conf) in lunched_analyser_resume:
analyser_conf.dst = None
with obj(analyser_conf, logger.sub()) as analyser_obj:
analyser_obj.analyser_deferred_clean()
return err_code
def run(conf, logger, options):
err_code = 0
country = conf.country
if not check_database(conf, logger):
logger.log(logger.log_av_r+u"error in database initialisation"+logger.log_ap)
return 0x10
if not os.path.exists(conf.dir_tmp):
os.makedirs(conf.dir_tmp)
# variable used by osmosis
os.environ["JAVACMD_OPTIONS"] = "-Xms2048M -Xmx2048M -XX:MaxPermSize=2048M -Djava.io.tmpdir="+conf.dir_tmp
if "http_proxy" in os.environ:
(_tmp, host, port) = os.environ["http_proxy"].split(":")
host = host.split("/")[2]
os.environ["JAVACMD_OPTIONS"] += " -Dhttp.proxyHost=%s -Dhttp.proxyPort=%s" % (host, port)
os.environ["JAVACMD_OPTIONS"] += " -Djava.net.preferIPv6Addresses=false"
##########################################################################
## download and create database
if options.skip_init:
pass
elif options.change and check_osmosis_change(conf, logger) and not options.change_init:
xml_change = run_osmosis_change(conf, logger)
elif "url" in conf.download:
newer = False
xml_change = None
if options.diff and check_osmosis_diff(conf, logger) and os.path.exists(conf.download["dst"]):
(status, xml_change) = run_osmosis_diff(conf, logger)
if status:
newer = True
if not newer and options.skip_download:
logger.sub().log("skip download")
newer = True
if not newer:
logger.log(logger.log_av_r+u"downloading"+logger.log_ap)
newer = download.dl(conf.download["url"], conf.download["dst"], logger.sub())
if not newer:
return 0
init_database(conf, logger)
if options.change:
init_osmosis_change(conf, logger)
elif options.diff and not xml_change:
init_osmosis_diff(conf, logger)
if hasattr(conf, "sql_post_scripts"):
logger.log(logger.log_av_r+"import post scripts"+logger.log_ap)
for script in conf.sql_post_scripts:
cmd = ["psql"]
cmd += ["-d", conf.db_base]
cmd += ["-U", conf.db_user]
cmd += ["-h", conf.db_host]
cmd += ["-f", script]
logger.execute_out(cmd)
##########################################################################
## analyses
for analyser, password in conf.analyser.iteritems():
logger.log(logger.log_av_r + country + " : " + analyser + logger.log_ap)
if not "analyser_" + analyser in analysers:
logger.sub().log("skipped")
continue
if password == "xxx":
logger.sub().log("code is not correct - won't upload to %s" % conf.updt_url)
elif not conf.results_url:
logger.sub().log("results_url is not correct - won't upload to %s" % conf.updt_url)
if not os.path.exists(conf.dir_results):
os.makedirs(conf.dir_results)
try:
analyser_conf = analyser_config()
analyser_conf.dst_dir = conf.dir_results
analyser_conf.db_string = conf.db_string
analyser_conf.db_user = conf.db_user
if conf.db_schema:
analyser_conf.db_schema = conf.db_schema
else:
analyser_conf.db_schema = country
analyser_conf.dir_scripts = conf.dir_scripts
analyser_conf.options = conf.analyser_options
analyser_conf.polygon_id = conf.polygon_id
if options.change and xml_change:
analyser_conf.src = xml_change
elif "dst" in conf.download:
analyser_conf.src = conf.download["dst"]
for name, obj in inspect.getmembers(analysers["analyser_" + analyser]):
if (inspect.isclass(obj) and obj.__module__ == "analyser_" + analyser and
(name.startswith("Analyser") or name.startswith("analyser"))):
# analyse
analyser_conf.dst_file = name + "-" + country + ".xml"
analyser_conf.dst_file += ".bz2"
analyser_conf.dst = os.path.join(conf.dir_results, analyser_conf.dst_file)
with obj(analyser_conf, logger.sub()) as analyser_obj:
if not options.change or not xml_change:
analyser_obj.analyser()
else:
analyser_obj.analyser_change()
# update
if conf.results_url and password != "xxx":
logger.sub().log("update")
try:
tmp_req = urllib2.Request(conf.updt_url)
tmp_url = os.path.join(conf.results_url, analyser_conf.dst_file)
tmp_src = "%s-%s" % (analyser, country)
tmp_dat = urllib.urlencode([('url', tmp_url),
('source', tmp_src),
('code', password)])
fd = urllib2.urlopen(tmp_req, tmp_dat, timeout=1800)
dt = fd.read().decode("utf8").strip()
if dt[-2:] != "OK":
sys.stderr.write((u"UPDATE ERROR %s/%s : %s\n"%(country, analyser, dt)).encode("utf8"))
err_code |= 4
else:
logger.sub().sub().log(dt)
except:
s = StringIO()
traceback.print_exc(file=s)
logger.sub().log("error on update...")
for l in s.getvalue().decode("utf8").split("\n"):
logger.sub().sub().log(l)
err_code |= 1
continue
except:
s = StringIO()
traceback.print_exc(file=s)
logger.sub().log("error on analyse...")
for l in s.getvalue().decode("utf8").split("\n"):
logger.sub().sub().log(l)
err_code |= 2
continue
##########################################################################
## vidange
logger.log(logger.log_av_r + u"cleaning : " + country + logger.log_ap)
if options.change:
pass
else:
clean_database(conf, logger, options.no_clean or not conf.clean_at_end)
if options.diff:
# don't erase any file
return err_code
# remove files
if "url" in conf.download and "dst" in conf.download and not options.no_clean:
f = ".osm".join(conf.download["dst"].split(".osm")[:-1])
for ext in ["osm", "osm.bz2", "osm.pbf"]:
try:
os.remove("%s.%s"%(f, ext))
logger.sub().log("DROP FILE %s.%s"%(f, ext))
except:
pass
return err_code