def __init__(self, folder, mode = "r"):
self._mode = mode
self._folder = folder
self._reldir = os.path.join(folder, "relation")
self._fNode_crd = open(os.path.join(folder, "node.crd"), {"w":"rb+", "r":"rb"}[mode])
self._fWay_idx = open(os.path.join(folder, "way.idx"), {"w":"rb+", "r":"rb"}[mode])
self._fWay_data = open(os.path.join(folder, "way.data"), {"w":"rb+", "r":"rb"}[mode])
self._fWay_data_size = os.stat(os.path.join(folder, "way.data")).st_size
if self._mode == "w":
lock_file = os.path.join(folder, "lock")
self._lock = lockfile(lock_file)
self._ReadFree()
self.node_id_size = 5
def lock_database(self):
osmosis_lock = False
for trial in range(60):
# acquire lock
try:
lfil = "/tmp/osmose-osmosis_import"
osmosis_lock = lockfile(lfil)
break
except:
self.logger.err("can't lock %s" % lfil)
self.logger.log("waiting 2 minutes")
time.sleep(2 * 60)
if not osmosis_lock:
self.logger.err("definitively can't lock")
raise
return osmosis_lock
def lock_database(self):
osmosis_lock = False
for trial in range(60):
# acquire lock
try:
lfil = "/tmp/osmose-osmosis_import"
osmosis_lock = lockfile(lfil)
break
except:
self.logger.err("can't lock %s" % lfil)
self.logger.log("waiting 2 minutes")
time.sleep(2*60)
if not osmosis_lock:
self.logger.err("definitively can't lock")
raise
return osmosis_lock
def main(options):
analysers_path = os.path.join(os.path.dirname(__file__), "analysers")
if options.list_analyser:
for fn in sorted(os.listdir(analysers_path)):
if fn.startswith("analyser_") and fn.endswith(".py"):
print(fn[9:-3])
return 0
if options.list_country:
for k in sorted(config.config.keys()):
print(k)
return 0
if options.cron:
output = sys.stdout
logger = OsmoseLog.logger(output, False)
else:
output = sys.stdout
logger = OsmoseLog.logger(output, True)
if options.change_init and not options.change:
logger.log(logger.log_av_b+"--change must be specified "+logger.log_ap)
return 1
#=====================================
# Load of analysers
err_code = 0
logger.log("osmose backend version: %s" % get_version())
old_path = list(sys.path)
sys.path.insert(0, analysers_path)
logger.log(logger.log_av_green+"loading analyses "+logger.log_ap)
analysers = {}
for fn in os.listdir(analysers_path):
if fn.startswith("analyser_") and fn.endswith(".py"):
if options.analyser and fn[9:-3] not in options.analyser:
continue
logger.log(" load "+fn[9:-3])
try:
analysers[fn[9:-3]] = importlib.import_module("analysers." + fn[:-3])
except ImportError as e:
logger.log(e)
logger.log("Fails to load analysers {0}".format(fn[:-3]))
if options.analyser:
count = 0
for k in options.analyser:
if k not in analysers:
logger.log(logger.log_av_b+"not found "+k+logger.log_ap)
count += 1
# user is passing only non-existent analysers
if len(options.analyser) == count:
logger.log(logger.log_av_b+"No valid analysers specified"+logger.log_ap)
return 1
sys.path[:] = old_path # restore previous path
#=====================================
# analyser
for country in options.country:
country_conf = config.config[country]
# acquire lock
try:
base = '|'.join(map(str, [country_conf.db_base, country_conf.db_host]))
lfil = "/tmp/analyse-{0}-{1}".format(country, base)
lock = lockfile(lfil)
except:
logger.err("can't lock {0} ({1})".format(country, lfil))
if options.cron:
sys.stderr.write("can't lock %s\n" % country)
for l in open(lfil).read().rstrip().split("\n"):
logger.log(" "+l)
if options.cron:
sys.stderr.write(" "+l+"\n")
if options.cron:
sys.stderr.flush()
err_code |= 0x80
continue
country_conf.init()
options.diff = not options.change and "diff" in country_conf.download
# analyse
err_code |= run(country_conf, logger, analysers, options)
# free lock
del lock
logger.log(logger.log_av_green+u"end of analyses"+logger.log_ap)
return err_code
#=====================================
# analyse
for country, country_conf in config.config.iteritems():
# filter
if options.country and country not in options.country:
continue
# acquire lock
try:
base = '|'.join(
map(str, [country_conf.db_base, country_conf.db_host]))
lfil = "/tmp/analyse-{0}-{1}".format(country, base)
lock = lockfile(lfil)
except:
logger.log(logger.log_av_r + "can't lock %s" % country +
logger.log_ap)
if options.cron:
sys.stderr.write("can't lock %s\n" % country)
for l in open(lfil).read().rstrip().split("\n"):
logger.log(" " + l)
if options.cron:
sys.stderr.write(" " + l + "\n")
if options.cron:
sys.stderr.flush()
err_code |= 0x80
continue
country_conf.init()
sys.path[:] = old_path # restore previous path
#=====================================
# analyse
for country, country_conf in config.config.items():
# filter
if options.country and country not in options.country:
continue
# acquire lock
try:
base = '|'.join(map(str, [country_conf.db_base, country_conf.db_host]))
lfil = "/tmp/analyse-{0}-{1}".format(country, base)
lock = lockfile(lfil)
except:
logger.err("can't lock %s"%country)
if options.cron:
sys.stderr.write("can't lock %s\n"%country)
for l in open(lfil).read().rstrip().split("\n"):
logger.log(" "+l)
if options.cron:
sys.stderr.write(" "+l+"\n")
if options.cron:
sys.stderr.flush()
err_code |= 0x80
continue
country_conf.init()
options.diff = not options.change and "diff" in country_conf.download
