Exemplo n.º 1
0
  def init_diff(self, conf):
    self.logger.log(self.logger.log_av_r+"init osmosis replication for diff"+self.logger.log_ap)
    diff_path = conf.download["diff_path"]

    for f_name in ["configuration.txt", "download.lock", "state.txt"]:
        f = os.path.join(diff_path, f_name)
        if os.path.exists(f):
            os.remove(f)

    cmd  = [conf.bin_osmosis]
    cmd += ["--read-replication-interval-init", "workingDirectory=%s" % diff_path]
    cmd += ["-quiet"]
    self.logger.execute_err(cmd)

    for line in fileinput.input(os.path.join(diff_path, "configuration.txt"), inplace=1):
        if line.startswith("baseUrl"):
            sys.stdout.write("baseUrl=" + conf.download["diff"])
        elif line.startswith("maxInterval"):
            if "geofabrik" in conf.download["diff"]:
                # on daily diffs provided by Geofabrik, we should apply only one diff at a time
                sys.stdout.write("maxInterval=" + str(60*60*24/2)) # 1/2 day at most
            else:
                sys.stdout.write("maxInterval=" + str(7*60*60*24)) # 7 day at most
        else:
            sys.stdout.write(line)
    fileinput.close()

    download.dl(conf.download["state.txt"],
                os.path.join(diff_path, "state.txt"),
                self.logger.sub(),
                min_file_size=10)
Exemplo n.º 2
0
def init_osmosis_diff(conf):

    logger.log(log_av_r + "init osmosis replication for diff" + log_ap)
    diff_path = conf.download["diff_path"]
    if os.path.exists(diff_path):
        for f_name in ["configuration.txt", "download.lock", "state.txt"]:
            f = os.path.join(diff_path, f_name)
            if os.path.exists(f):
                os.remove(f)
    else:
        os.makedirs(diff_path)
    cmd = [conf.osmosis_bin]
    cmd += ["--read-replication-interval-init", "workingDirectory=%s" % diff_path]
    cmd += ["-quiet"]
    logger.execute_err(cmd)

    for line in fileinput.input(os.path.join(diff_path, "configuration.txt"), inplace=1):
        if line.startswith("baseUrl"):
            sys.stdout.write("baseUrl=" + conf.download["diff"])
        elif line.startswith("maxInterval"):
            if "geofabrik" in conf.download["diff"]:
                # on daily diffs provided by Geofabrik, we should apply only one diff at a time
                sys.stdout.write("maxInterval=" + str(60 * 60 * 24 / 2))  # 1/2 day at most
            else:
                sys.stdout.write("maxInterval=" + str(7 * 60 * 60 * 24))  # 7 day at most
        else:
            sys.stdout.write(line)
    fileinput.close()

    if conf.download["diff"].endswith("minute/"):
        from modules import OsmTs

        OsmTs.run(conf.download["dst"], os.path.join(diff_path, "state.txt"), "minute", logger)

    else:
        download.dl(
            conf.download["diff"] + "state.txt", os.path.join(diff_path, "state.txt"), logger.sub(), min_file_size=10
        )
Exemplo n.º 3
0
    elif "url" in conf.download:
        newer = False
        xml_change = None
        if options.diff and check_osmosis_diff(conf, logger) and os.path.exists(conf.download["dst"]):
            (status, xml_change) = run_osmosis_diff(conf, logger)
            if status:
                newer = True

        if not newer and options.skip_download:
            logger.sub().log("skip download")
            newer = True

        if not newer:
            logger.log(logger.log_av_r+u"downloading"+logger.log_ap)
            newer = download.dl(conf.download["url"], conf.download["dst"], logger.sub())

        if not newer:
            return 0

        init_database(conf, logger)

        if options.change:
            init_osmosis_change(conf, logger)
        elif options.diff and not xml_change:
            init_osmosis_diff(conf, logger)

    if hasattr(conf, "sql_post_scripts"):
        logger.log(logger.log_av_r+"import post scripts"+logger.log_ap)
        for script in conf.sql_post_scripts:
            cmd  = ["psql"]
Exemplo n.º 4
0
def execc(conf, logger, analysers, options, osmosis_manager):
    err_code = 0

    ## download and create database

    xml_change = None
    if not options.skip_init:
        if options.change and osmosis_manager.check_change(
                conf) and not options.change_init:
            xml_change = osmosis_manager.run_change(conf)

        elif "url" in conf.download:
            newer = False
            xml_change = None

            if not newer and options.skip_download:
                logger.sub().log("skip download")
                newer = True

            if not newer and options.diff and os.path.exists(
                    conf.download["dst"]):
                status = False
                if options.pbf_update_tool == 'osmosis':
                    if osmosis_manager.check_osmosis_diff(conf):
                        (status,
                         xml_change) = osmosis_manager.run_osmosis_diff(conf)
                else:
                    if osmosis_manager.check_osmium_diff(conf):
                        (status,
                         xml_change) = osmosis_manager.run_osmium_diff(conf)
                if status:
                    newer = True

            if not newer:
                logger.log(logger.log_av_r + u"downloading" + logger.log_ap)
                newer = download.dl(conf.download["url"],
                                    conf.download["dst"],
                                    logger.sub(),
                                    min_file_size=8 * 1024)

                if newer and options.diff:
                    if options.pbf_update_tool == 'osmosis':
                        osmosis_manager.init_osmosis_diff(conf)
                    if "/minute/" in conf.download[
                            "diff"] or "/hour/" in conf.download["diff"]:
                        # update extract with any more recent available diff
                        if options.pbf_update_tool == 'osmosis':
                            osmosis_manager.run_osmosis_diff(conf)
                        else:
                            osmosis_manager.run_osmium_diff(conf)

            if not newer:
                return 0x11

            if osmosis_manager:
                osmosis_manager.init_database(conf)

            if options.change:
                osmosis_manager.init_change(conf)

        if hasattr(conf, "sql_post_scripts"):
            logger.log(logger.log_av_r + "import post scripts" + logger.log_ap)
            for script in conf.sql_post_scripts:
                osmosis_manager.psql_f(script)

        if osmosis_manager:
            osmosis_manager.update_metainfo(conf)

        if options.resume:
            osmosis_manager.run_resume(conf)

    ##########################################################################
    ## analyses

    version = get_version()

    lunched_analyser = []
    lunched_analyser_change = []
    lunched_analyser_resume = []

    for analyser in analysers:
        if not options.analyser and analyser not in conf.analyser:
            continue

        logger.log(logger.log_av_r + conf.country + " : " + analyser +
                   logger.log_ap)

        password = conf.analyser.get(analyser)
        if not password or password == "xxx":
            logger.sub().err("No password to upload result to %s" %
                             conf.updt_url)

        try:
            analyser_conf = analyser_config(conf, options, osmosis_manager,
                                            xml_change)

            for name, obj in inspect.getmembers(analysers[analyser]):
                if (inspect.isclass(obj)
                        and obj.__module__ == "analysers.analyser_" + analyser
                        and (name.startswith("Analyser")
                             or name.startswith("analyser"))):
                    analyser_name = name[len("Analyser_"):]
                    resume = options.resume or (options.resume_analyser
                                                and analyser
                                                in options.resume_analyser)

                    dst = os.path.join(conf.dir_results,
                                       name + "-" + conf.country)
                    analyser_conf.error_file = issues_file_from_fromat(
                        dst,
                        options.result_format,
                        bz2=True,
                        version=version,
                        polygon_id=analyser_conf.polygon_id)

                    # analyse
                    if not options.skip_analyser:
                        with obj(analyser_conf, logger.sub()) as analyser_obj:
                            remote_timestamp = None
                            if not options.skip_frontend_check:
                                url = modules.config.url_frontend_update + "/../../control/status/%s/%s?%s" % (
                                    conf.country, analyser_name,
                                    'objects=true' if resume else '')
                                resp = downloader.get(url)
                                if not resp.ok:
                                    logger.sub().err(
                                        "Fails to get status from frontend: {0}"
                                        .format(resp.status_code))
                                else:
                                    try:
                                        status = resp.json()
                                        remote_timestamp = dateutil.parser.parse(
                                            status['timestamp']
                                        ) if status else None
                                        remote_analyser_version = int(
                                            status['analyser_version'])
                                    except Exception as e:
                                        logger.sub().err(e)

                            if analyser_obj.timestamp(
                            ) and remote_timestamp and analyser_obj.timestamp(
                            ) <= remote_timestamp and analyser_obj.analyser_version(
                            ) == remote_analyser_version:
                                logger.sub().warn(
                                    "Skip, frontend is already up to date")
                                continue

                            if resume and remote_timestamp and analyser_obj.analyser_version(
                            ) == remote_analyser_version:
                                already_issued_objects = {
                                    'N': status['nodes'] or [],
                                    'W': status['ways'] or [],
                                    'R': status['relations'] or []
                                }
                                analyser_obj.analyser_resume(
                                    remote_timestamp, already_issued_objects)
                                lunched_analyser_resume.append(
                                    [obj, analyser_conf])
                            else:
                                if resume:
                                    if not remote_timestamp:
                                        logger.sub().err(
                                            "No remote timestamp to resume from, start a full run"
                                        )
                                    elif analyser_obj.analyser_version(
                                    ) == remote_analyser_version:
                                        logger.sub().err(
                                            "Analyser version changed, start a full run"
                                        )

                                if not options.change or not xml_change:
                                    analyser_obj.analyser()
                                    lunched_analyser.append(
                                        [obj, analyser_conf])
                                else:
                                    analyser_obj.analyser_change()
                                    lunched_analyser_change.append(
                                        [obj, analyser_conf])

                    # update
                    if not options.skip_upload and password != "xxx":
                        logger.sub().log("update")

                        if analyser in conf.analyser_updt_url:
                            list_urls = conf.analyser_updt_url[analyser]
                        else:
                            list_urls = [conf.updt_url]

                        for url in list_urls:
                            update_finished = False
                            nb_iter = 0
                            was_on_timeout = False
                            while not update_finished and nb_iter < 3:
                                time.sleep(nb_iter * 15)
                                nb_iter += 1
                                logger.sub().sub().log("iteration=%d" %
                                                       nb_iter)
                                try:
                                    u = url + '?analyser=' + analyser_name + '&country=' + conf.country
                                    r = requests.post(
                                        u,
                                        timeout=1800,
                                        data={
                                            'analyser': analyser_name,
                                            'country': conf.country,
                                            'code': password
                                        },
                                        files={
                                            'content':
                                            open(analyser_conf.error_file.dst,
                                                 'rb')
                                        })
                                    r.raise_for_status()
                                    logger.sub().sub().log(r.text.strip())
                                    update_finished = True
                                except requests.exceptions.HTTPError as e:
                                    if e.response.status_code == 504:
                                        was_on_timeout = True
                                        logger.sub().sub().sub().err(
                                            'got an HTTP timeout status')
                                    else:
                                        dt = r.text.strip()
                                        logger.sub().sub().sub().err(
                                            u"UPDATE ERROR %s/%s : %s\n" %
                                            (conf.country, analyser_name, dt))
                                        if dt == "FAIL: Already up to date":
                                            update_finished = True
                                        if not was_on_timeout:
                                            err_code |= 4
                                except Exception as e:
                                    if isinstance(
                                            e, requests.exceptions.
                                            ConnectTimeout):
                                        was_on_timeout = True
                                        logger.sub().sub().sub().err(
                                            'got a connection timeout')
                                    else:
                                        tb = traceback.format_exc()
                                        logger.sub().err('error on update...')
                                        for l in tb.splitlines():
                                            logger.sub().sub().log(l)

                        if not update_finished:
                            err_code |= 1

        except:
            tb = traceback.format_exc()
            logger.sub().err("error on analyse {0}...".format(analyser))
            for l in tb.splitlines():
                logger.sub().sub().log(l)
            err_code |= 2
            continue

    if not options.no_clean:
        for (obj, analyser_conf) in lunched_analyser:
            analyser_conf.error_file = None
            with obj(analyser_conf, logger.sub()) as analyser_obj:
                analyser_obj.analyser_deferred_clean()
        for (obj, analyser_conf) in lunched_analyser_change:
            analyser_conf.error_file = None
            with obj(analyser_conf, logger.sub()) as analyser_obj:
                analyser_obj.analyser_deferred_clean()
        for (obj, analyser_conf) in lunched_analyser_resume:
            analyser_conf.error_file = None
            with obj(analyser_conf, logger.sub()) as analyser_obj:
                analyser_obj.analyser_deferred_clean()

    return err_code
Exemplo n.º 5
0
def run(conf, logger, options):

    err_code = 0
    country = conf.country
    try:
        version = get_version()
    except:
        version = None

    if not check_database(conf, logger):
        logger.log(logger.log_av_r + u"error in database initialisation" + logger.log_ap)
        return 0x10

    ##########################################################################
    ## check for working dirs and creates when needed

    dirs = [conf.dir_tmp, conf.dir_cache, conf.dir_results, conf.dir_extracts, conf.dir_diffs]
    if "diff_path" in conf.download:
        dirs.append(conf.download["diff_path"])

    for i in dirs:
        if not os.path.exists(i):
            try:
                os.makedirs(i)
            except OSError as e:
                sys.exit("%s\nCheck 'dir_work' in modules/config.py and its permissions" % str(e))

    # variable used by osmosis
    if not "JAVACMD_OPTIONS" in os.environ:
        os.environ["JAVACMD_OPTIONS"] = ""
    os.environ["JAVACMD_OPTIONS"] += " -Djava.io.tmpdir=" + conf.dir_tmp

    ##########################################################################
    ## download and create database

    if options.skip_init:
        pass

    elif options.change and check_osmosis_change(conf, logger) and not options.change_init:
        xml_change = run_osmosis_change(conf, logger)

    elif "url" in conf.download:
        newer = False
        xml_change = None
        if options.diff and check_osmosis_diff(conf, logger) and os.path.exists(conf.download["dst"]):
            (status, xml_change) = run_osmosis_diff(conf, logger)
            if status:
                newer = True

        if not newer and options.skip_download:
            logger.sub().log("skip download")
            newer = True

        if not newer:
            logger.log(logger.log_av_r + u"downloading" + logger.log_ap)
            newer = download.dl(conf.download["url"], conf.download["dst"], logger.sub(), min_file_size=8 * 1024)

        if not newer:
            return 0

        init_database(conf, logger)

        if options.change:
            init_osmosis_change(conf, logger)
        elif options.diff and not xml_change:
            init_osmosis_diff(conf, logger)

    if hasattr(conf, "sql_post_scripts"):
        logger.log(logger.log_av_r + "import post scripts" + logger.log_ap)
        for script in conf.sql_post_scripts:
            cmd = ["psql"]
            cmd += conf.db_psql_args
            cmd += ["-f", script]
            logger.execute_out(cmd)

    ##########################################################################
    ## analyses

    for analyser, password in conf.analyser.iteritems():
        logger.log(logger.log_av_r + country + " : " + analyser + logger.log_ap)

        if not "analyser_" + analyser in analysers:
            logger.sub().log("skipped")
            continue

        if password == "xxx":
            logger.sub().log("code is not correct - won't upload to %s" % conf.updt_url)
        elif not conf.results_url and not has_poster_lib:
            logger.sub().log("results_url is not correct - won't upload to %s" % conf.updt_url)

        try:
            analyser_conf = analyser_config()
            analyser_conf.dst_dir = conf.dir_results

            analyser_conf.db_string = conf.db_string
            analyser_conf.db_user = conf.db_user
            if conf.db_schema:
                analyser_conf.db_schema = conf.db_schema
            else:
                analyser_conf.db_schema = country

            analyser_conf.dir_scripts = conf.dir_scripts
            analyser_conf.options = conf.analyser_options

            analyser_conf.polygon_id = conf.polygon_id

            if options.change and xml_change:
                analyser_conf.src = xml_change
            elif "dst" in conf.download:
                analyser_conf.src = conf.download["dst"]

            for name, obj in inspect.getmembers(analysers["analyser_" + analyser]):
                if (
                    inspect.isclass(obj)
                    and obj.__module__ == "analyser_" + analyser
                    and (name.startswith("Analyser") or name.startswith("analyser"))
                ):
                    # analyse
                    analyser_conf.dst_file = name + "-" + country + ".xml"
                    analyser_conf.dst_file += ".bz2"
                    analyser_conf.dst = os.path.join(conf.dir_results, analyser_conf.dst_file)
                    analyser_conf.version = version
                    analyser_conf.verbose = options.verbose
                    with obj(analyser_conf, logger.sub()) as analyser_obj:
                        if not options.change or not xml_change:
                            analyser_obj.analyser()
                        else:
                            analyser_obj.analyser_change()

                    # update
                    if (conf.results_url or has_poster_lib) and password != "xxx":
                        logger.sub().log("update")
                        update_finished = False
                        nb_iter = 0

                        while not update_finished and nb_iter < 3:
                            time.sleep(nb_iter * 15)
                            nb_iter += 1
                            logger.sub().sub().log("iteration=%d" % nb_iter)
                            try:
                                tmp_src = "%s-%s" % (analyser, country)
                                if has_poster_lib:
                                    (tmp_dat, tmp_headers) = poster.encode.multipart_encode(
                                        {"content": open(analyser_conf.dst, "rb"), "source": tmp_src, "code": password}
                                    )
                                    tmp_req = urllib2.Request(conf.updt_url, tmp_dat, tmp_headers)
                                    fd = urllib2.urlopen(tmp_req, timeout=1800)

                                else:
                                    tmp_req = urllib2.Request(conf.updt_url)
                                    tmp_url = os.path.join(conf.results_url, analyser_conf.dst_file)
                                    tmp_dat = urllib.urlencode(
                                        [("url", tmp_url), ("source", tmp_src), ("code", password)]
                                    )
                                    fd = urllib2.urlopen(tmp_req, tmp_dat, timeout=1800)

                                dt = fd.read().decode("utf8").strip()
                                if dt[-2:] != "OK":
                                    sys.stderr.write(
                                        (u"UPDATE ERROR %s/%s : %s\n" % (country, analyser, dt)).encode("utf8")
                                    )
                                    err_code |= 4
                                else:
                                    logger.sub().sub().log(dt)
                                update_finished = True
                            except socket.timeout:
                                logger.sub().sub().sub().log("got a timeout")
                                pass
                            except:
                                s = StringIO()
                                traceback.print_exc(file=s)
                                logger.sub().log("error on update...")
                                for l in s.getvalue().decode("utf8").split("\n"):
                                    logger.sub().sub().log(l)

                        if not update_finished:
                            err_code |= 1

        except:
            s = StringIO()
            traceback.print_exc(file=s)
            logger.sub().log("error on analyse...")
            for l in s.getvalue().decode("utf8").split("\n"):
                logger.sub().sub().log(l)
            err_code |= 2
            continue

    ##########################################################################
    ## vidange

    logger.log(logger.log_av_r + u"cleaning : " + country + logger.log_ap)

    if options.change:
        pass
    else:
        clean_database(conf, logger, options.no_clean or not conf.clean_at_end)

    if options.diff:
        # don't erase any file
        return err_code

    # remove files
    if "url" in conf.download and "dst" in conf.download and not options.no_clean:
        f = ".osm".join(conf.download["dst"].split(".osm")[:-1])
        for ext in ["osm", "osm.bz2", "osm.pbf"]:
            try:
                os.remove("%s.%s" % (f, ext))
                logger.sub().log("DROP FILE %s.%s" % (f, ext))
            except:
                pass

    return err_code
Exemplo n.º 6
0
def execc(conf, logger, options, osmosis_manager):
    err_code = 0

    version = get_version()

    logger.log("osmose backend version: %s" % version)

    ## download and create database

    country = conf.country

    if options.skip_init:
        pass

    elif options.change and osmosis_manager.check_change(
            conf) and not options.change_init:
        xml_change = osmosis_manager.run_change(conf)

    elif "url" in conf.download:
        newer = False
        xml_change = None

        if not newer and options.skip_download:
            logger.sub().log("skip download")
            newer = True

        if not newer and options.diff and osmosis_manager.check_diff(
                conf) and os.path.exists(conf.download["dst"]):
            (status, xml_change) = osmosis_manager.run_diff(conf)
            if status:
                newer = True

        if not newer:
            logger.log(logger.log_av_r + u"downloading" + logger.log_ap)
            newer = download.dl(conf.download["url"],
                                conf.download["dst"],
                                logger.sub(),
                                min_file_size=8 * 1024)

            if newer and options.diff:
                osmosis_manager.init_diff(conf)
                if "/minute/" in conf.download[
                        "diff"] or "/hour/" in conf.download["diff"]:
                    # update extract with any more recent available diff
                    osmosis_manager.run_diff(conf)

        if not newer:
            return 0x11

        if osmosis_manager:
            osmosis_manager.init_database(conf)

        if options.change:
            osmosis_manager.init_change(conf)

    if hasattr(conf, "sql_post_scripts"):
        logger.log(logger.log_av_r + "import post scripts" + logger.log_ap)
        for script in conf.sql_post_scripts:
            osmosis_manager.psql_f(script)

    if not options.skip_init and osmosis_manager:
        osmosis_manager.update_metainfo(conf)

    if options.resume:
        osmosis_manager.run_resume(conf)

    ##########################################################################
    ## analyses

    lunched_analyser = []
    lunched_analyser_change = []
    lunched_analyser_resume = []

    for analyser, password in conf.analyser.items():
        logger.log(logger.log_av_r + country + " : " + analyser +
                   logger.log_ap)

        if not "analyser_" + analyser in analysers:
            logger.sub().log("skipped")
            continue

        if password == "xxx":
            logger.sub().log("code is not correct - won't upload to %s" %
                             conf.updt_url)

        try:
            analyser_conf = analyser_config()
            analyser_conf.dst_dir = conf.dir_results

            analyser_conf.osmosis_manager = osmosis_manager
            analyser_conf.db_user = conf.db_user
            if conf.db_schema:
                analyser_conf.db_schema = conf.db_schema
            else:
                analyser_conf.db_schema = country
            analyser_conf.db_schema_path = conf.db_schema_path

            analyser_conf.dir_scripts = conf.dir_scripts
            analyser_conf.options = conf.analyser_options
            analyser_conf.polygon_id = conf.polygon_id

            if options.change and xml_change:
                analyser_conf.src = xml_change
            elif "dst" in conf.download:
                analyser_conf.src = conf.download["dst"]
                if "diff_path" in conf.download:
                    analyser_conf.src_state = os.path.join(
                        conf.download["diff_path"], "state.txt")

            for name, obj in inspect.getmembers(analysers["analyser_" +
                                                          analyser]):
                if (inspect.isclass(obj)
                        and obj.__module__ == "analysers.analyser_" + analyser
                        and (name.startswith("Analyser")
                             or name.startswith("analyser"))):
                    analyser_name = name[len("Analyser_"):]
                    analyser_conf.dst_file = name + "-" + country + ".xml"
                    analyser_conf.dst_file += ".bz2"
                    analyser_conf.dst = os.path.join(conf.dir_results,
                                                     analyser_conf.dst_file)
                    analyser_conf.version = version
                    analyser_conf.verbose = options.verbose

                    # analyse
                    if not options.skip_analyser:
                        with obj(analyser_conf, logger.sub()) as analyser_obj:
                            remote_timestamp = None
                            if not options.skip_frontend_check:
                                url = modules.config.url_frontend_update + "/../../control/status/%s/%s?%s" % (
                                    country, analyser_name,
                                    'objects=true' if options.resume else '')
                                resp = downloader.get(url)
                                if not resp.ok:
                                    logger.sub().err(
                                        "Fails to get status from frontend: {0}"
                                        .format(resp.status_code))
                                else:
                                    try:
                                        status = resp.json()
                                        remote_timestamp = dateutil.parser.parse(
                                            status['timestamp']
                                        ) if status else None
                                    except e:
                                        logger.sub().err(e)

                            if options.resume:
                                if remote_timestamp:
                                    already_issued_objects = {
                                        'N': status['nodes'] or [],
                                        'W': status['ways'] or [],
                                        'R': status['relations'] or []
                                    }
                                    analyser_obj.analyser_resume(
                                        remote_timestamp,
                                        already_issued_objects)
                                    lunched_analyser_resume.append(
                                        [obj, analyser_conf])
                                    continue
                                else:
                                    logger.sub().err("Not able to resume")

                            if analyser_obj.timestamp(
                            ) and remote_timestamp and analyser_obj.timestamp(
                            ) <= remote_timestamp:
                                logger.sub().warn(
                                    "Skip, frontend is already up to date")
                                continue

                            if not options.change or not xml_change:
                                analyser_obj.analyser()
                                lunched_analyser.append([obj, analyser_conf])
                            else:
                                analyser_obj.analyser_change()
                                lunched_analyser_change.append(
                                    [obj, analyser_conf])

                    # update
                    if not options.skip_upload and password != "xxx":
                        logger.sub().log("update")

                        if analyser in conf.analyser_updt_url:
                            list_urls = conf.analyser_updt_url[analyser]
                        else:
                            list_urls = [conf.updt_url]

                        for url in list_urls:
                            update_finished = False
                            nb_iter = 0
                            was_on_timeout = False
                            while not update_finished and nb_iter < 3:
                                time.sleep(nb_iter * 15)
                                nb_iter += 1
                                logger.sub().sub().log("iteration=%d" %
                                                       nb_iter)
                                try:
                                    u = url + '?name=' + name + '&country=' + (
                                        conf.db_schema or conf.country)
                                    r = requests.post(
                                        u,
                                        timeout=1800,
                                        data={
                                            'analyser': analyser_name,
                                            'country': country,
                                            'code': password
                                        },
                                        files={
                                            'content':
                                            open(analyser_conf.dst, 'rb')
                                        })
                                    r.raise_for_status()

                                    dt = r.text.strip()
                                    if dt == "FAIL: Already up to date" and was_on_timeout:
                                        logger.sub().sub().sub().err(
                                            (u"UPDATE ERROR %s/%s : %s\n" %
                                             (country, analyser_name,
                                              dt)).encode("utf8"))
                                        # Log error, but do not set err_code
                                    elif dt[-2:] != "OK":
                                        logger.sub().sub().sub().err(
                                            (u"UPDATE ERROR %s/%s : %s\n" %
                                             (country, analyser_name,
                                              dt)).encode("utf8"))
                                        err_code |= 4
                                    else:
                                        logger.sub().sub().log(dt)
                                    update_finished = True
                                except Exception as e:
                                    if isinstance(
                                            e,
                                            requests.exceptions.ConnectTimeout
                                    ) or (isinstance(
                                            e, requests.exceptions.HTTPError)
                                          and e.response.status_code == 504):
                                        was_on_timeout = True
                                        logger.sub().sub().sub().err(
                                            'got a timeout')
                                    else:
                                        tb = traceback.format_exc()
                                        logger.sub().err('error on update...')
                                        for l in tb.splitlines():
                                            logger.sub().sub().log(l)

                        if not update_finished:
                            err_code |= 1

        except:
            tb = traceback.format_exc()
            logger.sub().err("error on analyse {0}...".format(analyser))
            for l in tb.splitlines():
                logger.sub().sub().log(l)
            err_code |= 2
            continue

    if not options.no_clean:
        for (obj, analyser_conf) in lunched_analyser:
            analyser_conf.dst = None
            with obj(analyser_conf, logger.sub()) as analyser_obj:
                analyser_obj.analyser_deferred_clean()
        for (obj, analyser_conf) in lunched_analyser_change:
            analyser_conf.dst = None
            with obj(analyser_conf, logger.sub()) as analyser_obj:
                analyser_obj.analyser_deferred_clean()
        for (obj, analyser_conf) in lunched_analyser_resume:
            analyser_conf.dst = None
            with obj(analyser_conf, logger.sub()) as analyser_obj:
                analyser_obj.analyser_deferred_clean()

    return err_code
Exemplo n.º 7
0
def run(conf, logger, options):

    err_code = 0
    country = conf.country
    try:
      version = get_version()
    except:
      version = None

    if not check_database(conf, logger):
        logger.log(logger.log_av_r+u"error in database initialisation"+logger.log_ap)
        return 0x10


    ##########################################################################
    ## check for working dirs and creates when needed

    dirs = [conf.dir_tmp, conf.dir_cache, conf.dir_results, conf.dir_extracts, conf.dir_diffs]
    if "diff_path" in conf.download:
        dirs.append(conf.download["diff_path"])

    for i in dirs:
        if not os.path.exists(i):
            try:
                os.makedirs(i)
            except OSError as e:
                sys.exit("%s\nCheck 'dir_work' in modules/config.py and its permissions" % str(e))

    # variable used by osmosis
    if not "JAVACMD_OPTIONS" in os.environ:
        os.environ["JAVACMD_OPTIONS"] = ""
    os.environ["JAVACMD_OPTIONS"] += " -Djava.io.tmpdir="+conf.dir_tmp

    ##########################################################################
    ## download and create database

    if options.skip_init:
        pass

    elif options.change and check_osmosis_change(conf, logger) and not options.change_init:
        xml_change = run_osmosis_change(conf, logger)

    elif "url" in conf.download:
        newer = False
        xml_change = None
        if options.diff and check_osmosis_diff(conf, logger) and os.path.exists(conf.download["dst"]):
            (status, xml_change) = run_osmosis_diff(conf, logger)
            if status:
                newer = True

        if not newer and options.skip_download:
            logger.sub().log("skip download")
            newer = True

        if not newer:
            logger.log(logger.log_av_r+u"downloading"+logger.log_ap)
            newer = download.dl(conf.download["url"], conf.download["dst"], logger.sub(),
                                min_file_size=8*1024)

        if not newer:
            return 0

        init_database(conf, logger)

        if options.change:
            init_osmosis_change(conf, logger)
        elif options.diff and not xml_change:
            init_osmosis_diff(conf, logger)

    if hasattr(conf, "sql_post_scripts"):
        logger.log(logger.log_av_r+"import post scripts"+logger.log_ap)
        for script in conf.sql_post_scripts:
            cmd  = ["psql"]
            cmd += conf.db_psql_args
            cmd += ["-f", script]
            logger.execute_out(cmd)

    ##########################################################################
    ## analyses

    for analyser, password in conf.analyser.iteritems():
        logger.log(logger.log_av_r + country + " : " + analyser + logger.log_ap)

        if not "analyser_" + analyser in analysers:
            logger.sub().log("skipped")
            continue

        if password == "xxx":
            logger.sub().log("code is not correct - won't upload to %s" % conf.updt_url)
        elif not conf.results_url and not has_poster_lib:
            logger.sub().log("results_url is not correct - won't upload to %s" % conf.updt_url)

        try:
            analyser_conf = analyser_config()
            analyser_conf.dst_dir = conf.dir_results

            analyser_conf.db_string = conf.db_string
            analyser_conf.db_user = conf.db_user
            if conf.db_schema:
                analyser_conf.db_schema = conf.db_schema
            else:
                analyser_conf.db_schema = country

            analyser_conf.dir_scripts = conf.dir_scripts
            analyser_conf.options = conf.analyser_options

            analyser_conf.polygon_id = conf.polygon_id

            if options.change and xml_change:
                analyser_conf.src = xml_change
            elif "dst" in conf.download:
                analyser_conf.src = conf.download["dst"]

            for name, obj in inspect.getmembers(analysers["analyser_" + analyser]):
                if (inspect.isclass(obj) and obj.__module__ == "analyser_" + analyser and
                    (name.startswith("Analyser") or name.startswith("analyser"))):
                    # analyse
                    analyser_conf.dst_file = name + "-" + country + ".xml"
                    analyser_conf.dst_file += ".bz2"
                    analyser_conf.dst = os.path.join(conf.dir_results, analyser_conf.dst_file)
                    analyser_conf.version = version
                    analyser_conf.verbose = options.verbose
                    with obj(analyser_conf, logger.sub()) as analyser_obj:
                        if not options.change or not xml_change:
                            analyser_obj.analyser()
                        else:
                            analyser_obj.analyser_change()

                    # update
                    if (conf.results_url or has_poster_lib) and password != "xxx":
                        logger.sub().log("update")

                        if analyser in conf.analyser_updt_url:
                            list_urls = conf.analyser_updt_url[analyser]
                        else:
                            list_urls = [conf.updt_url]

                        for url in list_urls:
                            update_finished = False
                            nb_iter = 0
                            while not update_finished and nb_iter < 3:
                                time.sleep(nb_iter * 15)
                                nb_iter += 1
                                logger.sub().sub().log("iteration=%d" % nb_iter)
                                try:
                                    tmp_src = "%s-%s" % (analyser, country)
                                    if has_poster_lib:
                                        (tmp_dat, tmp_headers) = poster.encode.multipart_encode(
                                                                    {"content": open(analyser_conf.dst, "rb"),
                                                                     "source": tmp_src,
                                                                     "code": password})
                                        tmp_req = urllib2.Request(url, tmp_dat, tmp_headers)
                                        fd = urllib2.urlopen(tmp_req, timeout=1800)

                                    else:
                                        tmp_req = urllib2.Request(url)
                                        tmp_url = os.path.join(conf.results_url, analyser_conf.dst_file)
                                        tmp_dat = urllib.urlencode([('url', tmp_url),
                                                                    ('source', tmp_src),
                                                                    ('code', password)])
                                        fd = urllib2.urlopen(tmp_req, tmp_dat, timeout=1800)

                                    dt = fd.read().decode("utf8").strip()
                                    if dt[-2:] != "OK":
                                        sys.stderr.write((u"UPDATE ERROR %s/%s : %s\n"%(country, analyser, dt)).encode("utf8"))
                                        err_code |= 4
                                    else:
                                        logger.sub().sub().log(dt)
                                    update_finished = True
                                except socket.timeout:
                                    logger.sub().sub().sub().log("got a timeout")
                                    pass
                                except:
                                    s = StringIO()
                                    traceback.print_exc(file=s)
                                    logger.sub().log("error on update...")
                                    for l in s.getvalue().decode("utf8").split("\n"):
                                        logger.sub().sub().log(l)

                        if not update_finished:
                            err_code |= 1

        except:
            s = StringIO()
            traceback.print_exc(file=s)
            logger.sub().log("error on analyse...")
            for l in s.getvalue().decode("utf8").split("\n"):
                logger.sub().sub().log(l)
            err_code |= 2
            continue

    ##########################################################################
    ## vidange

    logger.log(logger.log_av_r + u"cleaning : " + country + logger.log_ap)

    if options.change:
        pass
    else:
        clean_database(conf, logger, options.no_clean or not conf.clean_at_end)

    if options.diff:
        # don't erase any file
        return err_code

    # remove files
    if "url" in conf.download and "dst" in conf.download and not options.no_clean:
        f = ".osm".join(conf.download["dst"].split(".osm")[:-1])
        for ext in ["osm", "osm.bz2", "osm.pbf"]:
            try:
                os.remove("%s.%s"%(f, ext))
                logger.sub().log("DROP FILE %s.%s"%(f, ext))
            except:
                pass

    return err_code
Exemplo n.º 8
0
def execc(conf, logger, options, osmosis_manager):
    err_code = 0

    version = get_version()

    logger.log("osmose backend version: %s" % version)

    ## download and create database

    country = conf.country

    if options.skip_init:
        pass

    elif options.change and osmosis_manager.check_change(conf) and not options.change_init:
        xml_change = osmosis_manager.run_change(conf)

    elif "url" in conf.download:
        newer = False
        xml_change = None

        if not newer and options.skip_download:
            logger.sub().log("skip download")
            newer = True

        if not newer and options.diff and osmosis_manager.check_diff(conf) and os.path.exists(conf.download["dst"]):
            (status, xml_change) = osmosis_manager.run_diff(conf)
            if status:
                newer = True

        if not newer:
            logger.log(logger.log_av_r+u"downloading"+logger.log_ap)
            newer = download.dl(conf.download["url"], conf.download["dst"], logger.sub(),
                                min_file_size=8*1024)

            if newer and options.diff:
                osmosis_manager.init_diff(conf)
                if "/minute/" in conf.download["diff"] or "/hour/" in conf.download["diff"]:
                    # update extract with any more recent available diff
                    osmosis_manager.run_diff(conf)

        if not newer:
            return 0x11

        if osmosis_manager:
            osmosis_manager.init_database(conf)

        if options.change:
            osmosis_manager.init_change(conf)

    if hasattr(conf, "sql_post_scripts"):
        logger.log(logger.log_av_r+"import post scripts"+logger.log_ap)
        for script in conf.sql_post_scripts:
            osmosis_manager.psql_f(script)

    if not options.skip_init and osmosis_manager:
        osmosis_manager.update_metainfo(conf)

    if options.resume:
        osmosis_manager.run_resume(conf)

    ##########################################################################
    ## analyses

    lunched_analyser = []
    lunched_analyser_change = []
    lunched_analyser_resume = []

    for analyser, password in conf.analyser.items():
        logger.log(logger.log_av_r + country + " : " + analyser + logger.log_ap)

        if not "analyser_" + analyser in analysers:
            logger.sub().log("skipped")
            continue

        if password == "xxx":
            logger.sub().log("code is not correct - won't upload to %s" % conf.updt_url)

        try:
            analyser_conf = analyser_config()
            analyser_conf.dst_dir = conf.dir_results

            analyser_conf.osmosis_manager = osmosis_manager
            analyser_conf.db_user = conf.db_user
            if conf.db_schema:
                analyser_conf.db_schema = conf.db_schema
            else:
                analyser_conf.db_schema = country
            analyser_conf.db_schema_path = conf.db_schema_path

            analyser_conf.dir_scripts = conf.dir_scripts
            analyser_conf.options = conf.analyser_options
            analyser_conf.polygon_id = conf.polygon_id

            if options.change and xml_change:
                analyser_conf.src = xml_change
            elif "dst" in conf.download:
                analyser_conf.src = conf.download["dst"]
                if "diff_path" in conf.download:
                    analyser_conf.src_state = os.path.join(conf.download["diff_path"], "state.txt")

            for name, obj in inspect.getmembers(analysers["analyser_" + analyser]):
                if (inspect.isclass(obj) and obj.__module__ == "analysers.analyser_" + analyser and
                    (name.startswith("Analyser") or name.startswith("analyser"))):
                    analyser_name = name[len("Analyser_"):]
                    analyser_conf.dst_file = name + "-" + country + ".xml"
                    analyser_conf.dst_file += ".bz2"
                    analyser_conf.dst = os.path.join(conf.dir_results, analyser_conf.dst_file)
                    analyser_conf.version = version
                    analyser_conf.verbose = options.verbose

                    # analyse
                    if not options.skip_analyser:
                        with obj(analyser_conf, logger.sub()) as analyser_obj:
                            remote_timestamp = None
                            if not options.skip_frontend_check:
                                url = modules.config.url_frontend_update + "/../../control/status/%s/%s?%s" % (country, analyser_name, 'objects=true' if options.resume else '')
                                resp = downloader.get(url)
                                if not resp.ok:
                                    logger.sub().err("Fails to get status from frontend: {0}".format(resp.status_code))
                                else:
                                    try:
                                        status = resp.json()
                                        remote_timestamp = dateutil.parser.parse(status['timestamp']) if status else None
                                    except e:
                                        logger.sub().err(e)

                            if options.resume:
                                if remote_timestamp:
                                    already_issued_objects = {'N': status['nodes'] or [], 'W': status['ways'] or [], 'R': status['relations'] or []}
                                    analyser_obj.analyser_resume(remote_timestamp, already_issued_objects)
                                    lunched_analyser_resume.append([obj, analyser_conf])
                                    continue
                                else:
                                    logger.sub().err("Not able to resume")

                            if analyser_obj.timestamp() and remote_timestamp and analyser_obj.timestamp() <= remote_timestamp:
                                logger.sub().warn("Skip, frontend is already up to date")
                                continue

                            if not options.change or not xml_change:
                                analyser_obj.analyser()
                                lunched_analyser.append([obj, analyser_conf])
                            else:
                                analyser_obj.analyser_change()
                                lunched_analyser_change.append([obj, analyser_conf])

                    # update
                    if not options.skip_upload and password != "xxx":
                        logger.sub().log("update")

                        if analyser in conf.analyser_updt_url:
                            list_urls = conf.analyser_updt_url[analyser]
                        else:
                            list_urls = [conf.updt_url]

                        for url in list_urls:
                            update_finished = False
                            nb_iter = 0
                            was_on_timeout = False
                            while not update_finished and nb_iter < 3:
                                time.sleep(nb_iter * 15)
                                nb_iter += 1
                                logger.sub().sub().log("iteration=%d" % nb_iter)
                                try:
                                    u = url + '?name=' + name + '&country=' + (conf.db_schema or conf.country)
                                    r = requests.post(u, timeout=1800, data={
                                        'analyser': analyser_name,
                                        'country': country,
                                        'code': password
                                    }, files={
                                        'content': open(analyser_conf.dst, 'rb')
                                    })
                                    r.raise_for_status()

                                    dt = r.text.strip()
                                    if dt == "FAIL: Already up to date" and was_on_timeout:
                                        logger.sub().sub().sub().err((u"UPDATE ERROR %s/%s : %s\n"%(country, analyser_name, dt)).encode("utf8"))
                                        # Log error, but do not set err_code
                                    elif dt[-2:] != "OK":
                                        logger.sub().sub().sub().err((u"UPDATE ERROR %s/%s : %s\n"%(country, analyser_name, dt)).encode("utf8"))
                                        err_code |= 4
                                    else:
                                        logger.sub().sub().log(dt)
                                    update_finished = True
                                except Exception as e:
                                    if isinstance(e, requests.exceptions.ConnectTimeout) or (isinstance(e, requests.exceptions.HTTPError) and e.response.status_code == 504):
                                        was_on_timeout = True
                                        logger.sub().sub().sub().err('got a timeout')
                                    else:
                                        tb = traceback.format_exc()
                                        logger.sub().err('error on update...')
                                        for l in tb.splitlines():
                                            logger.sub().sub().log(l)

                        if not update_finished:
                            err_code |= 1

        except:
            tb = traceback.format_exc()
            logger.sub().err("error on analyse {0}...".format(analyser))
            for l in tb.splitlines():
                logger.sub().sub().log(l)
            err_code |= 2
            continue

    if not options.no_clean:
        for (obj, analyser_conf) in lunched_analyser:
            analyser_conf.dst = None
            with obj(analyser_conf, logger.sub()) as analyser_obj:
                analyser_obj.analyser_deferred_clean()
        for (obj, analyser_conf) in lunched_analyser_change:
            analyser_conf.dst = None
            with obj(analyser_conf, logger.sub()) as analyser_obj:
                analyser_obj.analyser_deferred_clean()
        for (obj, analyser_conf) in lunched_analyser_resume:
            analyser_conf.dst = None
            with obj(analyser_conf, logger.sub()) as analyser_obj:
                analyser_obj.analyser_deferred_clean()

    return err_code
Exemplo n.º 9
0
def run(conf, logger, options):

    err_code = 0
    country = conf.country

    if not check_database(conf, logger):
        logger.log(logger.log_av_r+u"error in database initialisation"+logger.log_ap)
        return 0x10

    if not os.path.exists(conf.dir_tmp):
        os.makedirs(conf.dir_tmp)

    # variable used by osmosis
    os.environ["JAVACMD_OPTIONS"] = "-Xms2048M -Xmx2048M -XX:MaxPermSize=2048M -Djava.io.tmpdir="+conf.dir_tmp
    if "http_proxy" in os.environ:
        (_tmp, host, port) = os.environ["http_proxy"].split(":")
        host = host.split("/")[2]
        os.environ["JAVACMD_OPTIONS"] += " -Dhttp.proxyHost=%s -Dhttp.proxyPort=%s" % (host, port)
        os.environ["JAVACMD_OPTIONS"] += " -Djava.net.preferIPv6Addresses=false"

    ##########################################################################
    ## download and create database

    if options.skip_init:
        pass
   
    elif options.change and check_osmosis_change(conf, logger) and not options.change_init:
        xml_change = run_osmosis_change(conf, logger)

    elif "url" in conf.download:
        newer = False
        xml_change = None
        if options.diff and check_osmosis_diff(conf, logger) and os.path.exists(conf.download["dst"]):
            (status, xml_change) = run_osmosis_diff(conf, logger)
            if status:
                newer = True

        if not newer and options.skip_download:
            logger.sub().log("skip download")
            newer = True

        if not newer:
            logger.log(logger.log_av_r+u"downloading"+logger.log_ap)
            newer = download.dl(conf.download["url"], conf.download["dst"], logger.sub())

        if not newer:
            return 0

        init_database(conf, logger)

        if options.change:
            init_osmosis_change(conf, logger)
        elif options.diff and not xml_change:
            init_osmosis_diff(conf, logger)

    if hasattr(conf, "sql_post_scripts"):
        logger.log(logger.log_av_r+"import post scripts"+logger.log_ap)
        for script in conf.sql_post_scripts:
            cmd  = ["psql"]
            cmd += ["-d", conf.db_base]
            cmd += ["-U", conf.db_user]
            cmd += ["-h", conf.db_host]
            cmd += ["-f", script]
            logger.execute_out(cmd)

    ##########################################################################
    ## analyses
    
    for analyser, password in conf.analyser.iteritems():
        logger.log(logger.log_av_r + country + " : " + analyser + logger.log_ap)

        if not "analyser_" + analyser in analysers:
            logger.sub().log("skipped")
            continue

        if password == "xxx":
            logger.sub().log("code is not correct - won't upload to %s" % conf.updt_url)
        elif not conf.results_url:
            logger.sub().log("results_url is not correct - won't upload to %s" % conf.updt_url)

        if not os.path.exists(conf.dir_results):
            os.makedirs(conf.dir_results)

        try:
            analyser_conf = analyser_config()
            analyser_conf.dst_dir = conf.dir_results

            analyser_conf.db_string = conf.db_string
            analyser_conf.db_user = conf.db_user
            if conf.db_schema:
                analyser_conf.db_schema = conf.db_schema
            else:
                analyser_conf.db_schema = country

            analyser_conf.dir_scripts = conf.dir_scripts
            analyser_conf.options = conf.analyser_options

            analyser_conf.polygon_id = conf.polygon_id

            if options.change and xml_change:
                analyser_conf.src = xml_change
            elif "dst" in conf.download:
                analyser_conf.src = conf.download["dst"]

            for name, obj in inspect.getmembers(analysers["analyser_" + analyser]):
                if (inspect.isclass(obj) and obj.__module__ == "analyser_" + analyser and
                    (name.startswith("Analyser") or name.startswith("analyser"))):
                    # analyse
                    analyser_conf.dst_file = name + "-" + country + ".xml"
                    analyser_conf.dst_file += ".bz2"
                    analyser_conf.dst = os.path.join(conf.dir_results, analyser_conf.dst_file)
                    with obj(analyser_conf, logger.sub()) as analyser_obj:
                        if not options.change or not xml_change:
                            analyser_obj.analyser()
                        else:
                            analyser_obj.analyser_change()

                    # update
                    if conf.results_url and password != "xxx":
                        logger.sub().log("update")
                        try:
                            tmp_req = urllib2.Request(conf.updt_url)
                            tmp_url = os.path.join(conf.results_url, analyser_conf.dst_file)
                            tmp_src = "%s-%s" % (analyser, country)
                            tmp_dat = urllib.urlencode([('url', tmp_url),
                                                        ('source', tmp_src),
                                                        ('code', password)])
                            fd = urllib2.urlopen(tmp_req, tmp_dat, timeout=1800)
                            dt = fd.read().decode("utf8").strip()
                            if dt[-2:] != "OK":
                                sys.stderr.write((u"UPDATE ERROR %s/%s : %s\n"%(country, analyser, dt)).encode("utf8"))
                                err_code |= 4
                            else:
                                logger.sub().sub().log(dt)
                        except:
                            s = StringIO()
                            traceback.print_exc(file=s)
                            logger.sub().log("error on update...")
                            for l in s.getvalue().decode("utf8").split("\n"):
                                logger.sub().sub().log(l)
                            err_code |= 1
                            continue

        except:
            s = StringIO()
            traceback.print_exc(file=s)
            logger.sub().log("error on analyse...")
            for l in s.getvalue().decode("utf8").split("\n"):
                logger.sub().sub().log(l)
            err_code |= 2
            continue

    ##########################################################################
    ## vidange
    
    logger.log(logger.log_av_r + u"cleaning : " + country + logger.log_ap)
    
    if options.change:
        pass
    else:
        clean_database(conf, logger, options.no_clean or not conf.clean_at_end)

    if options.diff:
        # don't erase any file
        return err_code

    # remove files
    if "url" in conf.download and "dst" in conf.download and not options.no_clean:
        f = ".osm".join(conf.download["dst"].split(".osm")[:-1])
        for ext in ["osm", "osm.bz2", "osm.pbf"]:
            try:
                os.remove("%s.%s"%(f, ext))
                logger.sub().log("DROP FILE %s.%s"%(f, ext))
            except:
                pass

    return err_code