def fetch(self): fd, tmp_file = tempfile.mkstemp() pip = PointInPolygon(self.polygon_id, 60) traffic_signs = [] reader = json.loads(open(self.mapping, 'r').read()) try: for row in reader: traffic_signs += row['object'] except: self.logger.err(row) raise with open(tmp_file, 'w') as csvfile: writer = csv.writer(csvfile) writer.writerow(['accuracy', 'direction', 'image_key', 'first_seen_at', 'last_seen_at', 'value', 'X', 'Y']) slice = lambda A, n: [A[i:i+n] for i in range(0, len(A), n)] bboxes = pip.bboxes() start_time = (datetime.today() - timedelta(days=365*2)).isoformat()[0:10] b = 0 for traffic_signs_ in slice(traffic_signs, 10): b = b + 1 self.logger.log('Batch {0}/{1}: {2}'.format(b, round(len(traffic_signs) / 10 + 0.5), ','.join(traffic_signs_))) for bbox in bboxes: url = 'https://a.mapillary.com/v3/map_features?bbox={bbox}&client_id={client_id}&layers={layer}&per_page=1000&start_time={start_time}&values={values}'.format(bbox=','.join(map(str, bbox)), layer=self.layer, client_id='MEpmMTFQclBTUWlacjV6RTUxWWMtZzo5OTc2NjY2MmRiMDUwYmMw', start_time=start_time, values=','.join(traffic_signs_)) print(url) with open(tmp_file, 'a') as csvfile: writer = csv.writer(csvfile) r = None page = 0 while(url): page = page + 1 self.logger.log("Page {0}".format(page)) r = downloader.get(url) url = r.links['next']['url'] if 'next' in r.links else None features = r.json()['features'] filtered = 0 self.logger.log('{0} features fetched'.format(len(features))) for j in features: p = j['properties'] image_key = p['detections'][0]['image_key'] gc = j['geometry']['coordinates'] row = [p['accuracy'], p['direction'] if 'direction' in p else None, image_key, p['first_seen_at'], p['last_seen_at'], p['value']] + gc if row[0] > 0.01 and pip.point_inside_polygon(gc[0], gc[1]): writer.writerow(row) filtered = filtered + 1 self.logger.log('{0} keeped'.format(filtered)) return tmp_file
def fetch(self, url, tmp_file, date_string=None): pip = PointInPolygon(self.polygon_id, 60) traffic_signs = [] reader = json.loads(open(self.mapping, 'r').read()) try: for row in reader: traffic_signs += row['object'] except: self.logger.err(row) raise with open(tmp_file, 'w') as csvfile: writer = csv.writer(csvfile) writer.writerow(['accuracy', 'direction', 'image_key', 'first_seen_at', 'last_seen_at', 'value', 'X', 'Y']) slice = lambda A, n: [A[i:i+n] for i in range(0, len(A), n)] bboxes = pip.bboxes() start_time = (datetime.today() - timedelta(days=365*2)).isoformat()[0:10] b = 0 for traffic_signs_ in slice(traffic_signs, 10): b = b + 1 self.logger.log('Batch {0}/{1}: {2}'.format(b, ceil(len(traffic_signs) / 10.0), ','.join(traffic_signs_))) for bbox in bboxes: url = 'https://a.mapillary.com/v3/map_features?bbox={bbox}&client_id={client_id}&layers={layer}&per_page=1000&start_time={start_time}&values={values}'.format(bbox=','.join(map(str, bbox)), layer=self.layer, client_id='MEpmMTFQclBTUWlacjV6RTUxWWMtZzo5OTc2NjY2MmRiMDUwYmMw', start_time=start_time, values=','.join(traffic_signs_)) self.logger.log(url) with open(tmp_file, 'a') as csvfile: writer = csv.writer(csvfile) r = None page = 0 while(url): page = page + 1 self.logger.log("Page {0}".format(page)) r = downloader.get(url) url = r.links['next']['url'] if 'next' in r.links else None features = r.json()['features'] filtered = 0 self.logger.log('{0} features fetched'.format(len(features))) for j in features: p = j['properties'] image_key = p['detections'][0]['image_key'] gc = j['geometry']['coordinates'] row = [p['accuracy'], p['direction'] if 'direction' in p else None, image_key, p['first_seen_at'], p['last_seen_at'], p['value']] + gc if row[0] > 0.01 and pip.point_inside_polygon(gc[0], gc[1]): writer.writerow(row) filtered = filtered + 1 self.logger.log('{0} keeped'.format(filtered)) return True
def execc(conf, logger, analysers, options, osmosis_manager): err_code = 0 ## download and create database xml_change = None if not options.skip_init: if options.change and osmosis_manager.check_change(conf) and not options.change_init: xml_change = osmosis_manager.run_change(conf) elif "url" in conf.download: newer = False xml_change = None if not newer and options.skip_download: logger.sub().log("skip download") newer = True if not newer and options.diff and os.path.exists(conf.download["dst"]): status = False if options.pbf_update_tool == 'osmosis': if osmosis_manager.check_osmosis_diff(conf): (status, xml_change) = osmosis_manager.run_osmosis_diff(conf) else: if osmosis_manager.check_osmium_diff(conf): (status, xml_change) = osmosis_manager.run_osmium_diff(conf) if status: newer = True if not newer: logger.log(logger.log_av_r+u"downloading"+logger.log_ap) newer = download.dl(conf.download["url"], conf.download["dst"], logger.sub(), min_file_size=8*1024) if newer and options.diff: if options.pbf_update_tool == 'osmosis': osmosis_manager.init_osmosis_diff(conf) if "/minute/" in conf.download["diff"] or "/hour/" in conf.download["diff"]: # update extract with any more recent available diff if options.pbf_update_tool == 'osmosis': osmosis_manager.run_osmosis_diff(conf) else: osmosis_manager.run_osmium_diff(conf) if not newer: return 0x11 if osmosis_manager: osmosis_manager.init_database(conf) if options.change: osmosis_manager.init_change(conf) if hasattr(conf, "sql_post_scripts"): logger.log(logger.log_av_r+"import post scripts"+logger.log_ap) for script in conf.sql_post_scripts: osmosis_manager.psql_f(script) if osmosis_manager: osmosis_manager.update_metainfo(conf) if options.resume: osmosis_manager.run_resume(conf) ########################################################################## ## analyses version = get_version() lunched_analyser = [] lunched_analyser_change = [] lunched_analyser_resume = [] for analyser in analysers: if not options.analyser and analyser not in conf.analyser: continue logger.log(logger.log_av_r + conf.country + " : " + analyser + logger.log_ap) password = conf.analyser.get(analyser) if not options.skip_upload and (not password or password == "xxx"): logger.sub().err("No password to upload result to %s" % conf.updt_url) try: analyser_conf = analyser_config(conf, options, osmosis_manager, xml_change) for name, obj in inspect.getmembers(analysers[analyser]): if (inspect.isclass(obj) and obj.__module__ == "analysers.analyser_" + analyser and (name.startswith("Analyser") or name.startswith("analyser"))): analyser_name = name[len("Analyser_"):] resume = options.resume or (options.resume_analyser and analyser in options.resume_analyser) dst = os.path.join(conf.dir_results, name + "-" + conf.country) analyser_conf.error_file = issues_file_from_fromat(dst, options.result_format, bz2 = True, version = version, polygon_id = analyser_conf.polygon_id) # analyse if not options.skip_analyser: with obj(analyser_conf, logger.sub()) as analyser_obj: remote_timestamp = None if not options.skip_frontend_check: url = modules.config.url_frontend_update + "/../../control/status/%s/%s?%s" % (conf.country, analyser_name, 'objects=true' if resume else '') resp = downloader.get(url) if not resp.ok: logger.sub().err("Fails to get status from frontend: {0}".format(resp.status_code)) else: try: status = resp.json() remote_timestamp = dateutil.parser.parse(status['timestamp']) if status else None remote_analyser_version = int(status['analyser_version']) except Exception as e: logger.sub().err(e) if analyser_obj.timestamp() and remote_timestamp and analyser_obj.timestamp() <= remote_timestamp and analyser_obj.analyser_version() == remote_analyser_version: logger.sub().warn("Skip, frontend is already up to date") continue if resume and remote_timestamp and analyser_obj.analyser_version() == remote_analyser_version: already_issued_objects = {'N': status['nodes'] or [], 'W': status['ways'] or [], 'R': status['relations'] or []} analyser_obj.analyser_resume(remote_timestamp, already_issued_objects) lunched_analyser_resume.append([obj, analyser_conf]) else: if resume: if not remote_timestamp: logger.sub().err("No remote timestamp to resume from, start a full run") elif analyser_obj.analyser_version() == remote_analyser_version: logger.sub().err("Analyser version changed, start a full run") if not options.change or not xml_change: analyser_obj.analyser() lunched_analyser.append([obj, analyser_conf]) else: analyser_obj.analyser_change() lunched_analyser_change.append([obj, analyser_conf]) # update if not options.skip_upload and password != "xxx": logger.sub().log("update") if analyser in conf.analyser_updt_url: list_urls = conf.analyser_updt_url[analyser] else: list_urls = [conf.updt_url] for url in list_urls: update_finished = False nb_iter = 0 was_on_timeout = False while not update_finished and nb_iter < 3: time.sleep(nb_iter * 15) nb_iter += 1 logger.sub().sub().log("iteration=%d" % nb_iter) try: u = url + '?analyser=' + analyser_name + '&country=' + conf.country r = requests.post(u, timeout=1800, data={ 'analyser': analyser_name, 'country': conf.country, 'code': password }, files={ 'content': open(analyser_conf.error_file.dst, 'rb') }) r.raise_for_status() logger.sub().sub().log(r.text.strip()) update_finished = True except requests.exceptions.HTTPError as e: if e.response.status_code == 504: was_on_timeout = True logger.sub().sub().sub().err('got an HTTP timeout status') else: dt = r.text.strip() logger.sub().sub().sub().err(u"UPDATE ERROR %s/%s : %s\n" % (conf.country, analyser_name, dt)) if dt == "FAIL: Already up to date": update_finished = True if not was_on_timeout: err_code |= 4 except Exception as e: if isinstance(e, requests.exceptions.ConnectTimeout): was_on_timeout = True logger.sub().sub().sub().err('got a connection timeout') else: tb = traceback.format_exc() logger.sub().err('error on update...') for l in tb.splitlines(): logger.sub().sub().log(l) if not update_finished: err_code |= 1 except: tb = traceback.format_exc() logger.sub().err("error on analyse {0}...".format(analyser)) for l in tb.splitlines(): logger.sub().sub().log(l) err_code |= 2 continue if not options.no_clean: for (obj, analyser_conf) in lunched_analyser: analyser_conf.error_file = None with obj(analyser_conf, logger.sub()) as analyser_obj: analyser_obj.analyser_deferred_clean() for (obj, analyser_conf) in lunched_analyser_change: analyser_conf.error_file = None with obj(analyser_conf, logger.sub()) as analyser_obj: analyser_obj.analyser_deferred_clean() for (obj, analyser_conf) in lunched_analyser_resume: analyser_conf.error_file = None with obj(analyser_conf, logger.sub()) as analyser_obj: analyser_obj.analyser_deferred_clean() return err_code
def dl(url, local, logger=OsmoseLog.logger(), min_file_size=10 * 1024): unzip = False convert_pbf = False # file names file_ts = local + ".ts" url_ext = os.path.splitext(url)[1] local_ext = os.path.splitext(local)[1] if (url_ext in [".bz2"]) and (local_ext not in [".bz2"]): file_dl = local + url_ext unzip = True elif (url_ext in [".pbf"]) and (local_ext not in [".pbf"]): file_dl = local + url_ext convert_pbf = True else: file_dl = local headers = {} # make the download conditional if os.path.exists(file_dl) and os.path.exists(file_ts): headers["If-Modified-Since"] = open(file_ts).read() # request fails with a 304 error when the file wasn't modified # Retry on 404, workaround Geofabrik update in progress answer = downloader.get( url, headers=headers, session=downloader.requests_retry_session( status_forcelist=downloader.DEFAULT_RETRY_ON + (404, ))) if answer.status_code == 304: logger.log(u"not newer") return False if not answer.ok: logger.log(u"got error %d" % answer.status_code) logger.log(u" URL=%s" % url) answer.raise_for_status() url_ts = answer.headers.get('Last-Modified') file_size = int(answer.headers.get('Content-Length')) if file_size < min_file_size: # file must be bigger than 100 KB logger.log("File is not big enough: %d B" % file_size) raise SystemError # write the file with open(file_dl, "wb") as outfile: for data in answer.iter_content(chunk_size=None): outfile.write(data) if not answer.headers.get( 'Content-Encoding') and file_size != os.path.getsize(file_dl): logger.log( u"error: Download file (%d) not of the expected size (%d) for %s" % (os.path.getsize(file_dl), file_size, url)) os.remove(file_dl) return False # uncompress if unzip: logger.log(u"bunzip2") subprocess.check_output(['bunzip2', '-f', file_dl]) # convert pbf to osm if convert_pbf: logger.log(u"osmconvert") subprocess.check_output("{} {} > {}".format(config.bin_osmconvert, file_dl, local), shell=True) os.remove(file_dl) # set timestamp open(file_ts, "w").write(url_ts) return True
def execc(conf, logger, options, osmosis_manager): err_code = 0 version = get_version() logger.log("osmose backend version: %s" % version) ## download and create database country = conf.country if options.skip_init: pass elif options.change and osmosis_manager.check_change( conf) and not options.change_init: xml_change = osmosis_manager.run_change(conf) elif "url" in conf.download: newer = False xml_change = None if not newer and options.skip_download: logger.sub().log("skip download") newer = True if not newer and options.diff and osmosis_manager.check_diff( conf) and os.path.exists(conf.download["dst"]): (status, xml_change) = osmosis_manager.run_diff(conf) if status: newer = True if not newer: logger.log(logger.log_av_r + u"downloading" + logger.log_ap) newer = download.dl(conf.download["url"], conf.download["dst"], logger.sub(), min_file_size=8 * 1024) if newer and options.diff: osmosis_manager.init_diff(conf) if "/minute/" in conf.download[ "diff"] or "/hour/" in conf.download["diff"]: # update extract with any more recent available diff osmosis_manager.run_diff(conf) if not newer: return 0x11 if osmosis_manager: osmosis_manager.init_database(conf) if options.change: osmosis_manager.init_change(conf) if hasattr(conf, "sql_post_scripts"): logger.log(logger.log_av_r + "import post scripts" + logger.log_ap) for script in conf.sql_post_scripts: osmosis_manager.psql_f(script) if not options.skip_init and osmosis_manager: osmosis_manager.update_metainfo(conf) if options.resume: osmosis_manager.run_resume(conf) ########################################################################## ## analyses lunched_analyser = [] lunched_analyser_change = [] lunched_analyser_resume = [] for analyser, password in conf.analyser.items(): logger.log(logger.log_av_r + country + " : " + analyser + logger.log_ap) if not "analyser_" + analyser in analysers: logger.sub().log("skipped") continue if password == "xxx": logger.sub().log("code is not correct - won't upload to %s" % conf.updt_url) try: analyser_conf = analyser_config() analyser_conf.dst_dir = conf.dir_results analyser_conf.osmosis_manager = osmosis_manager analyser_conf.db_user = conf.db_user if conf.db_schema: analyser_conf.db_schema = conf.db_schema else: analyser_conf.db_schema = country analyser_conf.db_schema_path = conf.db_schema_path analyser_conf.dir_scripts = conf.dir_scripts analyser_conf.options = conf.analyser_options analyser_conf.polygon_id = conf.polygon_id if options.change and xml_change: analyser_conf.src = xml_change elif "dst" in conf.download: analyser_conf.src = conf.download["dst"] if "diff_path" in conf.download: analyser_conf.src_state = os.path.join( conf.download["diff_path"], "state.txt") for name, obj in inspect.getmembers(analysers["analyser_" + analyser]): if (inspect.isclass(obj) and obj.__module__ == "analysers.analyser_" + analyser and (name.startswith("Analyser") or name.startswith("analyser"))): analyser_name = name[len("Analyser_"):] analyser_conf.dst_file = name + "-" + country + ".xml" analyser_conf.dst_file += ".bz2" analyser_conf.dst = os.path.join(conf.dir_results, analyser_conf.dst_file) analyser_conf.version = version analyser_conf.verbose = options.verbose # analyse if not options.skip_analyser: with obj(analyser_conf, logger.sub()) as analyser_obj: remote_timestamp = None if not options.skip_frontend_check: url = modules.config.url_frontend_update + "/../../control/status/%s/%s?%s" % ( country, analyser_name, 'objects=true' if options.resume else '') resp = downloader.get(url) if not resp.ok: logger.sub().err( "Fails to get status from frontend: {0}" .format(resp.status_code)) else: try: status = resp.json() remote_timestamp = dateutil.parser.parse( status['timestamp'] ) if status else None except e: logger.sub().err(e) if options.resume: if remote_timestamp: already_issued_objects = { 'N': status['nodes'] or [], 'W': status['ways'] or [], 'R': status['relations'] or [] } analyser_obj.analyser_resume( remote_timestamp, already_issued_objects) lunched_analyser_resume.append( [obj, analyser_conf]) continue else: logger.sub().err("Not able to resume") if analyser_obj.timestamp( ) and remote_timestamp and analyser_obj.timestamp( ) <= remote_timestamp: logger.sub().warn( "Skip, frontend is already up to date") continue if not options.change or not xml_change: analyser_obj.analyser() lunched_analyser.append([obj, analyser_conf]) else: analyser_obj.analyser_change() lunched_analyser_change.append( [obj, analyser_conf]) # update if not options.skip_upload and password != "xxx": logger.sub().log("update") if analyser in conf.analyser_updt_url: list_urls = conf.analyser_updt_url[analyser] else: list_urls = [conf.updt_url] for url in list_urls: update_finished = False nb_iter = 0 was_on_timeout = False while not update_finished and nb_iter < 3: time.sleep(nb_iter * 15) nb_iter += 1 logger.sub().sub().log("iteration=%d" % nb_iter) try: u = url + '?name=' + name + '&country=' + ( conf.db_schema or conf.country) r = requests.post( u, timeout=1800, data={ 'analyser': analyser_name, 'country': country, 'code': password }, files={ 'content': open(analyser_conf.dst, 'rb') }) r.raise_for_status() dt = r.text.strip() if dt == "FAIL: Already up to date" and was_on_timeout: logger.sub().sub().sub().err( (u"UPDATE ERROR %s/%s : %s\n" % (country, analyser_name, dt)).encode("utf8")) # Log error, but do not set err_code elif dt[-2:] != "OK": logger.sub().sub().sub().err( (u"UPDATE ERROR %s/%s : %s\n" % (country, analyser_name, dt)).encode("utf8")) err_code |= 4 else: logger.sub().sub().log(dt) update_finished = True except Exception as e: if isinstance( e, requests.exceptions.ConnectTimeout ) or (isinstance( e, requests.exceptions.HTTPError) and e.response.status_code == 504): was_on_timeout = True logger.sub().sub().sub().err( 'got a timeout') else: tb = traceback.format_exc() logger.sub().err('error on update...') for l in tb.splitlines(): logger.sub().sub().log(l) if not update_finished: err_code |= 1 except: tb = traceback.format_exc() logger.sub().err("error on analyse {0}...".format(analyser)) for l in tb.splitlines(): logger.sub().sub().log(l) err_code |= 2 continue if not options.no_clean: for (obj, analyser_conf) in lunched_analyser: analyser_conf.dst = None with obj(analyser_conf, logger.sub()) as analyser_obj: analyser_obj.analyser_deferred_clean() for (obj, analyser_conf) in lunched_analyser_change: analyser_conf.dst = None with obj(analyser_conf, logger.sub()) as analyser_obj: analyser_obj.analyser_deferred_clean() for (obj, analyser_conf) in lunched_analyser_resume: analyser_conf.dst = None with obj(analyser_conf, logger.sub()) as analyser_obj: analyser_obj.analyser_deferred_clean() return err_code
def dl(url, local, logger=OsmoseLog.logger(), min_file_size=10*1024): unzip = False convert_pbf = False # file names file_ts = local+".ts" url_ext = os.path.splitext(url)[1] local_ext = os.path.splitext(local)[1] if (url_ext in [".bz2"]) and (local_ext not in [".bz2"]) : file_dl = local + url_ext unzip = True elif (url_ext in [".pbf"]) and (local_ext not in [".pbf"]) : file_dl = local + url_ext convert_pbf = True else: file_dl = local headers = {} # make the download conditional if os.path.exists(file_dl) and os.path.exists(file_ts): headers["If-Modified-Since"] = open(file_ts).read() # request fails with a 304 error when the file wasn't modified answer = downloader.get(url, headers=headers) if answer.status_code == 304: logger.log(u"not newer") return False if not answer.ok: logger.log(u"got error %d" % answer.status_code) logger.log(u" URL=%s" % url) answer.raise_for_status() url_ts = answer.headers.get('Last-Modified') file_size = int(answer.headers.get('Content-Length')) if file_size < min_file_size: # file must be bigger than 100 KB logger.log("File is not big enough: %d B" % file_size) raise SystemError # write the file outfile = open(file_dl, "wb") try: for data in answer.iter_content(chunk_size=None): outfile.write(data) finally: outfile.close() if not answer.headers.get('Content-Encoding') and file_size != os.path.getsize(file_dl): logger.log(u"error: Download file (%d) not of the expected size (%d) for %s" % (os.path.getsize(file_dl), file_size, url)) os.remove(file_dl) return False # uncompress if unzip: logger.log(u"bunzip2") subprocess.check_output(['bunzip2', '-f', file_dl]) # convert pbf to osm if convert_pbf: logger.log(u"osmconvert") subprocess.check_output("{} {} > {}".format(config.bin_osmconvert, file_dl, local), shell=True) os.remove(file_dl) # set timestamp open(file_ts, "w").write(url_ts) return True
def execc(conf, logger, options, osmosis_manager): err_code = 0 version = get_version() logger.log("osmose backend version: %s" % version) ## download and create database country = conf.country if options.skip_init: pass elif options.change and osmosis_manager.check_change(conf) and not options.change_init: xml_change = osmosis_manager.run_change(conf) elif "url" in conf.download: newer = False xml_change = None if not newer and options.skip_download: logger.sub().log("skip download") newer = True if not newer and options.diff and osmosis_manager.check_diff(conf) and os.path.exists(conf.download["dst"]): (status, xml_change) = osmosis_manager.run_diff(conf) if status: newer = True if not newer: logger.log(logger.log_av_r+u"downloading"+logger.log_ap) newer = download.dl(conf.download["url"], conf.download["dst"], logger.sub(), min_file_size=8*1024) if newer and options.diff: osmosis_manager.init_diff(conf) if "/minute/" in conf.download["diff"] or "/hour/" in conf.download["diff"]: # update extract with any more recent available diff osmosis_manager.run_diff(conf) if not newer: return 0x11 if osmosis_manager: osmosis_manager.init_database(conf) if options.change: osmosis_manager.init_change(conf) if hasattr(conf, "sql_post_scripts"): logger.log(logger.log_av_r+"import post scripts"+logger.log_ap) for script in conf.sql_post_scripts: osmosis_manager.psql_f(script) if not options.skip_init and osmosis_manager: osmosis_manager.update_metainfo(conf) if options.resume: osmosis_manager.run_resume(conf) ########################################################################## ## analyses lunched_analyser = [] lunched_analyser_change = [] lunched_analyser_resume = [] for analyser, password in conf.analyser.items(): logger.log(logger.log_av_r + country + " : " + analyser + logger.log_ap) if not "analyser_" + analyser in analysers: logger.sub().log("skipped") continue if password == "xxx": logger.sub().log("code is not correct - won't upload to %s" % conf.updt_url) try: analyser_conf = analyser_config() analyser_conf.dst_dir = conf.dir_results analyser_conf.osmosis_manager = osmosis_manager analyser_conf.db_user = conf.db_user if conf.db_schema: analyser_conf.db_schema = conf.db_schema else: analyser_conf.db_schema = country analyser_conf.db_schema_path = conf.db_schema_path analyser_conf.dir_scripts = conf.dir_scripts analyser_conf.options = conf.analyser_options analyser_conf.polygon_id = conf.polygon_id if options.change and xml_change: analyser_conf.src = xml_change elif "dst" in conf.download: analyser_conf.src = conf.download["dst"] if "diff_path" in conf.download: analyser_conf.src_state = os.path.join(conf.download["diff_path"], "state.txt") for name, obj in inspect.getmembers(analysers["analyser_" + analyser]): if (inspect.isclass(obj) and obj.__module__ == "analysers.analyser_" + analyser and (name.startswith("Analyser") or name.startswith("analyser"))): analyser_name = name[len("Analyser_"):] analyser_conf.dst_file = name + "-" + country + ".xml" analyser_conf.dst_file += ".bz2" analyser_conf.dst = os.path.join(conf.dir_results, analyser_conf.dst_file) analyser_conf.version = version analyser_conf.verbose = options.verbose # analyse if not options.skip_analyser: with obj(analyser_conf, logger.sub()) as analyser_obj: remote_timestamp = None if not options.skip_frontend_check: url = modules.config.url_frontend_update + "/../../control/status/%s/%s?%s" % (country, analyser_name, 'objects=true' if options.resume else '') resp = downloader.get(url) if not resp.ok: logger.sub().err("Fails to get status from frontend: {0}".format(resp.status_code)) else: try: status = resp.json() remote_timestamp = dateutil.parser.parse(status['timestamp']) if status else None except e: logger.sub().err(e) if options.resume: if remote_timestamp: already_issued_objects = {'N': status['nodes'] or [], 'W': status['ways'] or [], 'R': status['relations'] or []} analyser_obj.analyser_resume(remote_timestamp, already_issued_objects) lunched_analyser_resume.append([obj, analyser_conf]) continue else: logger.sub().err("Not able to resume") if analyser_obj.timestamp() and remote_timestamp and analyser_obj.timestamp() <= remote_timestamp: logger.sub().warn("Skip, frontend is already up to date") continue if not options.change or not xml_change: analyser_obj.analyser() lunched_analyser.append([obj, analyser_conf]) else: analyser_obj.analyser_change() lunched_analyser_change.append([obj, analyser_conf]) # update if not options.skip_upload and password != "xxx": logger.sub().log("update") if analyser in conf.analyser_updt_url: list_urls = conf.analyser_updt_url[analyser] else: list_urls = [conf.updt_url] for url in list_urls: update_finished = False nb_iter = 0 was_on_timeout = False while not update_finished and nb_iter < 3: time.sleep(nb_iter * 15) nb_iter += 1 logger.sub().sub().log("iteration=%d" % nb_iter) try: u = url + '?name=' + name + '&country=' + (conf.db_schema or conf.country) r = requests.post(u, timeout=1800, data={ 'analyser': analyser_name, 'country': country, 'code': password }, files={ 'content': open(analyser_conf.dst, 'rb') }) r.raise_for_status() dt = r.text.strip() if dt == "FAIL: Already up to date" and was_on_timeout: logger.sub().sub().sub().err((u"UPDATE ERROR %s/%s : %s\n"%(country, analyser_name, dt)).encode("utf8")) # Log error, but do not set err_code elif dt[-2:] != "OK": logger.sub().sub().sub().err((u"UPDATE ERROR %s/%s : %s\n"%(country, analyser_name, dt)).encode("utf8")) err_code |= 4 else: logger.sub().sub().log(dt) update_finished = True except Exception as e: if isinstance(e, requests.exceptions.ConnectTimeout) or (isinstance(e, requests.exceptions.HTTPError) and e.response.status_code == 504): was_on_timeout = True logger.sub().sub().sub().err('got a timeout') else: tb = traceback.format_exc() logger.sub().err('error on update...') for l in tb.splitlines(): logger.sub().sub().log(l) if not update_finished: err_code |= 1 except: tb = traceback.format_exc() logger.sub().err("error on analyse {0}...".format(analyser)) for l in tb.splitlines(): logger.sub().sub().log(l) err_code |= 2 continue if not options.no_clean: for (obj, analyser_conf) in lunched_analyser: analyser_conf.dst = None with obj(analyser_conf, logger.sub()) as analyser_obj: analyser_obj.analyser_deferred_clean() for (obj, analyser_conf) in lunched_analyser_change: analyser_conf.dst = None with obj(analyser_conf, logger.sub()) as analyser_obj: analyser_obj.analyser_deferred_clean() for (obj, analyser_conf) in lunched_analyser_resume: analyser_conf.dst = None with obj(analyser_conf, logger.sub()) as analyser_obj: analyser_obj.analyser_deferred_clean() return err_code