def __init__(self, templates_dir, images_dir, results_dir = './results'): self.templates_dir = templates_dir self.images_dir = images_dir self.results_dir = results_dir if templates_dir is None or not os.path.exists(templates_dir): logging.error("Invalid path: {0}".format(templates_dir)) sys.exit(1) if images_dir is None or not os.path.exists(images_dir): logging.error("Invalid path: {0}".format(images_dir)) sys.exit(1) FileHelper.create_or_clear_dir(results_dir) # # try: # if os.path.exists(results_dir): # FileHelper.remove_files_in_dir(results_dir) # else: # os.makedirs(results_dir) # # except OSError: # # pass # self.detector = cv2.SIFT(3200) self.detector = cv2.SIFT(900, edgeThreshold=30) # self.detector = cv2.SURF(300) # self.detector = cv2.SURF(30, upright=1) # self.detector = cv2.BRISK() # matcher = cv2.BFMatcher(cv2.NORM_L2) FLANN_INDEX_KDTREE = 0 index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=15) search_params = dict(checks=50) self.matcher = cv2.FlannBasedMatcher(index_params, search_params) self.templatesFeatures = None
def images_to_csv_file( self, dir_positive, dir_negative, filename_positive="positive.csv", filename_negative="negative.csv" ): positive_images = FileHelper.read_images_in_dir(dir_positive) negative_images = FileHelper.read_images_in_dir(dir_negative) if len(negative_images) > len(positive_images) * 3: negative_images = negative_images[: len(positive_images) * 3] # print "cut negative images" with open(filename_positive, "wb") as f: for filename in positive_images: im = imread(os.path.join(dir_positive, filename), True) f.write(self.__image_to_csv_string(im, 1)) f.write("\n") with open(filename_negative, "w+b") as f: for filename in negative_images: im = imread(os.path.join(dir_negative, filename), True) f.write(self.__image_to_csv_string(im, 0)) f.write("\n")
def __getFeaturesFromTemplates(self): features = {} images = FileHelper.read_images_in_dir(self.templates_dir) for filename in images: im = cv2.imread(os.path.join(self.templates_dir, filename), cv2.IMREAD_GRAYSCALE) kp, desc = self.detector.detectAndCompute(im, None) logging.debug('{0} - {1} features'.format(filename, len(kp))) # features.append((kp, desc)) features[filename] = (kp, desc) return features
def resize_images(self, dir_in, dir_out, h_size, v_size, make_grey=True): image_files = FileHelper.read_images_in_dir(dir_in) i = 0 for filename in image_files: print filename.split(".")[0] + "_%04d.jpg" # im = imread(os.path.join(in_dir, filename), as_grey= True) im = resize(imread(os.path.join(dir_in, filename)), (h_size, v_size)) if make_grey: im = rgb2gray(im) imsave(os.path.join(dir_out, "image%05d.jpg" % i), im) i += 1
def cut_frames_from_video(path, videofile): videofile_abspath = os.path.join(path,videofile) if os.path.exists(videofile_abspath) == False: logging.error("File "+ videofile_abspath + " doesn't exists or permission denied") return None parts = videofile.split('.') frames_path = os.path.join(path, parts[0]) if os.path.exists(frames_path): FileHelper.remove_files_in_dir(frames_path) else: os.mkdir(frames_path) try: images_template = frames_path + "/{0}_%05d.jpg".format(parts[0][:10]) #ffmpeg -i input.flv -f image2 -vf fps=fps=1 out%04d.png cmd = "ffmpeg -i {0} -f image2 -vf fps=fps=1 {1}".format(videofile_abspath, images_template) logging.debug(cmd) os.system(cmd) return 1 except: logging.error(sys.exc_info()) return None
def __init__(self, root_url): # 待爬取的数据不限长度 self.url_queue = Queue.Queue(maxsize = -1) # url过滤器 self.url_filter = set() # 已爬取过的url self.old_urls = set() # 读取配置文件中被排除的url self.exclude_urls = CommonUtil.get_dict_value(FileHelper.read_setting_info(), "exclude_url").split() for exclude_url in self.exclude_urls: self.url_filter.add(exclude_url) self.domain = re.match(r"^(http(s)?://)?([\w-]+\.)+[\w-]+/?",root_url,re.M|re.I).group() self.__add_new_url(SpiderUrl(root_url, 0))
def slice_images(self, dir_in, dir_out, h_size, v_size): """ 1. Read images from directory and slice they into small images """ # test # A = np.arange(120).reshape((10, 12)) # print A # slice_image_to_small_images(A, out_dir+'image_%04d.jpg') image_files = FileHelper.read_images_in_dir(dir_in) i = 0 for filename in image_files: print filename.split(".")[0] + "_%04d.jpg" # im = imread(os.path.join(dir_in, filename)) # im = rgb2gray(im) im = Image(os.path.join(dir_in, filename)) im.prepare() self.__slice_image_to_small_images( im.image, os.path.join(dir_out, filename.split(".")[0] + "_%04d.jpg"), h_size, v_size )
def load_tiles(self, zoom_level, layer_filter, load_mask_layer=False, merge_tiles=True, apply_styles=True, max_tiles=None, bounds=None, limit_reacher_handler=None): """ * Loads the vector tiles from either a file or a URL and adds them to QGIS :param zoom_level: The zoom level to load :param layer_filter: A list of layers. If any layers are set, only these will be loaded. If the list is empty, all available layers will be loaded :param load_mask_layer: If True the mask layer will also be loaded :param merge_tiles: If True neighbouring tiles and features will be merged :param apply_styles: If True the default styles will be applied :param max_tiles: The maximum number of tiles to load :param bounds: :param limit_reacher_handler: :return: """ self.cancel_requested = False self.feature_collections_by_layer_path = {} self._qgis_layer_groups_by_name = {} self._update_progress(show_dialog=True, title="Loading '{}'".format(os.path.basename(self.source.name()))) min_zoom = self.source.min_zoom() max_zoom = self.source.max_zoom() if min_zoom is not None and zoom_level < min_zoom: zoom_level = min_zoom if max_zoom is not None and zoom_level > max_zoom: zoom_level = max_zoom all_tiles = get_all_tiles(bounds, lambda: self.cancel_requested) tiles_to_load = set() tiles = [] for t in all_tiles: if self.cancel_requested: break file_name = self._get_tile_cache_name(zoom_level, t[0], t[1]) tile = FileHelper.get_cached_tile(file_name) if tile and tile.decoded_data: tiles.append(tile) else: tiles_to_load.add(t) debug("{} cache hits. {} will be loaded from the source.", len(tiles), len(tiles_to_load)) debug("Loading extent {} for zoom level '{}' of: {}", zoom_level, self.source.name()) tile_data_tuples = [] if len(tiles_to_load) > 0: tile_data_tuples = self.source.load_tiles(zoom_level=zoom_level, tiles_to_load=tiles_to_load, max_tiles=max_tiles, for_each=QApplication.processEvents, limit_reacher_handler=limit_reacher_handler) if len(tiles) == 0 and (not tile_data_tuples or len(tile_data_tuples) == 0): QMessageBox.information(None, "No tiles found", "What a pity, no tiles could be found!") if load_mask_layer: mask_level = self.source.mask_level() if mask_level is not None and mask_level != zoom_level: debug("Mapping {} tiles to mask level", len(all_tiles)) scheme = self.source.scheme() crs = self.source.crs() mask_tiles = map( lambda t: change_zoom(zoom_level, int(mask_level), t, scheme, crs), all_tiles) debug("Mapping done") mask_tiles_to_load = set() for t in mask_tiles: file_name = self._get_tile_cache_name(mask_level, t[0], t[1]) tile = FileHelper.get_cached_tile(file_name) if tile and tile.decoded_data: tiles.append(tile) else: mask_tiles_to_load.add(t) debug("Loading mask layer (zoom_level={})", mask_level) tile_data_tuples = [] if len(mask_tiles_to_load) > 0: mask_layer_data = self.source.load_tiles(zoom_level=mask_level, tiles_to_load=mask_tiles_to_load, max_tiles=max_tiles, for_each=QApplication.processEvents) debug("Mask layer loaded") tile_data_tuples.extend(mask_layer_data) if tile_data_tuples and len(tile_data_tuples) > 0: if not self.cancel_requested: decoded_tiles = self._decode_tiles(tile_data_tuples) tiles.extend(decoded_tiles) if len(tiles) > 0: if not self.cancel_requested: self._process_tiles(tiles, layer_filter) if not self.cancel_requested: self._create_qgis_layers(merge_features=merge_tiles, apply_styles=apply_styles) self._update_progress(show_dialog=False) if self.cancel_requested: info("Import cancelled") else: info("Import complete")
def __init__(self): self.YOUTUBE_VIDEO_ID_LIST = LocalSettingsLoader( ).LOCAL_SETTINGS['YOUTUBE_VIDEO_ID_LIST'] self.file_helper_obj = FileHelper() self.shell_executor = ShellExecutor()
def load_states(): data = FileHelper.load('data/states.json') return {k: np.array(v) for k, v in data.items()}
def copy_button_clicked(self): self.copy_button['text'] = 'Loading...' self.copy_button['state'] = tk.DISABLED parser = Parser() url = self.url.get() directory = self.directory.get() css_directory = directory + self.css_path js_directory = directory + self.js_path if FileHelper.directory_exists(directory): parser.set_url(url) try: content = parser.get_content() self.write_log("Received HTML page from " + url) FileHelper.create_file('index.html', directory, content.decode("UTF-8")) self.write_log("Saved HTML page to " + directory + '/index.html') FileHelper.delete_folder_if_exists(css_directory) FileHelper.create_path_recursively(css_directory) self.write_log("Created directory for saving CSS stylesheets") link_nodes = parser.get_nodes('link') css_regexp = re.compile("\.css$", re.IGNORECASE) for node in link_nodes: link = Parser.form_script_url(url, node.get('href')) link = str(link) if css_regexp.search(link): script_name = re.findall('\w+\.css', link, re.IGNORECASE) if len(script_name) > 0: script_name = script_name[0] else: script_name = uuid.uuid4().hex css_content = Parser.get_content_from_url(link) if css_content: FileHelper.create_file(script_name, directory + self.css_path, css_content.decode("UTF-8")) self.write_log("Saved CSS - " + self.css_path + script_name) else: self.write_log("Can't get content from - " + link) FileHelper.delete_folder_if_exists(js_directory) FileHelper.create_path_recursively(js_directory) self.write_log("Created directory for saving JS scripts") script_nodes = parser.get_nodes('script') js_regexp = re.compile("\.js", re.IGNORECASE) for node in script_nodes: src = node.get('src') link = None if src: link = Parser.form_script_url(url, src) link = str(link) if js_regexp.search(link): script_name = re.findall('\w+\.js', str(link), re.IGNORECASE); if len(script_name) > 0: script_name = script_name[0] else: script_name = uuid.uuid4().hex js_content = Parser.get_content_from_url(link) if js_content: FileHelper.create_file(script_name, directory + self.js_path, js_content.decode("UTF-8")) self.write_log("Saved JS - " + self.js_path + script_name) else: self.write_log("Can't get content from - " + link) except Exception as e: self.write_log("ERROR") self.write_log(e) self.show_error_message(e) else: self.write_log("ERROR") self.write_log("Directory '" + directory + "' not exists") self.show_error_message("Directory '" + directory + "' not exists") self.copy_button['text'] = 'Start' self.copy_button['state'] = tk.NORMAL
class ECMLMachine: def __init__(self, file_name): self.utils_cl = MyUtils() self.this_file_dir = os.path.dirname(os.path.realpath(__file__)) self.fh = FileHelper() self.hm = HmmMachine() self.cm = ClusteringMachine() self.pm = PredictMachine() self.my_metric = "euclidean" self.file_name = file_name self.out_fcasts_f = os.path.join(self.this_file_dir, "res", "fcasts", file_name, "ecml") self.fh.ensure_dirs_exist([self.out_fcasts_f]) logging.info("Instantiated ECML_operator") def get_results_univ(self, df, mean_day, n_pt_one_period, n_serie_concatenated = 1): # Create datasets in the format we need logging.info("Computing ECML results") logging.info("Find best number of cluster") (df_train, df_valid, df_test) = self.utils_cl.app_valid_test(df, n_pt_one_period) (df_train_compar, df_test_compar) = self.utils_cl.app_test(df, n_pt_one_period) last_day = df_train_compar["val_"][-len(df_test_compar):] # run algos using df_valid to find the better num of kmeans tb-used mean_mse = self.do_it( [2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 40, 60, 70, 80, 100, 150, 200], df_train, df_valid, mean_day, n_pt_one_period) best_num_of_kmean = min(mean_mse, key = lambda t: t[1])[0] logging.info("Found best number of cluster: %s", best_num_of_kmean) # run algos using df_test with previously found best kmean res = self.do_it( [best_num_of_kmean], df_train, df_test, mean_day, n_pt_one_period) # Retrieve results (mse_colin, mse_fake, mse_mean) = (res[0][1], res[0][2], res[0][3]) (mae_colin, mae_fake, mae_mean) = (res[0][4], res[0][5], res[0][6]) (mase_colin, mase_fake, mase_mean) = (res[0][7], res[0][8], res[0][9]) (std_mse_colin, std_mse_fake, std_mse_mean) = (res[0][10], res[0][11], res[0][12]) (std_mae_colin, std_mae_fake, std_mae_mean) = (res[0][13], res[0][14], res[0][15]) (std_mase_colin, std_mase_fake, std_mase_mean) = (res[0][16], res[0][17], res[0][18]) logging.info("Retrieve results") # Compute baselines logging.info("Compute baselines") (_, mse_ar_error, mae_ar_error, mase_ar_error) = self.pm.do_forecast_ar_model( last_day, df_train_compar["val_"], df_test_compar["val_"]) (mse_hw, mae_hw, mase_hw) = -1, -1, -1 return ( mse_colin, mse_fake, mse_mean, mae_colin, mae_fake, mae_mean, mase_colin, mase_fake, mase_mean, mse_ar_error, mae_ar_error, mase_ar_error, mse_hw, mae_hw, mase_hw, best_num_of_kmean, std_mse_colin, std_mse_fake, std_mse_mean, std_mae_colin, std_mae_fake, std_mae_mean, std_mase_colin, std_mase_fake, std_mase_mean ) def do_it(self, km_sizes, df_train, df_valid_ou_test, mean_day, n_pt_one_period, n_serie_concatenated = 1): mean_mse = [] for my_km_size in km_sizes: logging.info("ECML with km_size of %s", my_km_size) n_day_found_train = len(df_train["n_day_"].unique()) logging.debug("There are %s days in train", n_day_found_train) ################################# # I. TRAIN ################################# (km, comprehensive_clusters_df_train) = self.cm.do_kmeans_wrapper( df_train, km_size = my_km_size ) #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # MM #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # compute MMs raw_hmm_data_df_train = list( self.hm.compute_raw_hmm(comprehensive_clusters_df_train, order = 1)) # compute transition matrix transition_mat = self.hm.compute_hmm_transition_mat_1d( raw_hmm_data_df_train, my_km_size) ################################# # II. VALID ################################# # 1. apply km on df_valid data y_pred = self.cm.apply_clustering(df_valid_ou_test, km) # create tuple of known/wanted: # a) for ts data itself s_arr=[] for c in range(df_valid_ou_test["n_day_"].min() + 1, df_valid_ou_test["n_day_"].max() - 1): s_arr.append( ( df_valid_ou_test[df_valid_ou_test["n_day_"] == c - 1]["val_"].values, df_valid_ou_test[df_valid_ou_test["n_day_"] == c]["val_"].values, df_valid_ou_test[df_valid_ou_test["n_day_"] <= c]["val_"].values ) ) # b) for ts labels s_l_arr=[] for c in range(1, len(y_pred) - 1): s_l_arr.append((y_pred[c - 1], y_pred[c])) precision_colin = [] precision_fake = [] precision_mean = [] precision_colin_mae = [] precision_fake_mae = [] precision_mean_mae = [] precision_colin_mase = [] precision_fake_mase = [] precision_mean_mase = [] # compute predictions and mse for count in range(0, len(s_arr)): path_count = os.path.join(self.out_fcasts_f, str(count) + "_fcast_" + str(my_km_size) + "_kmsize.csv") (known, guess, before_w) = s_arr[count] known_w = np.pad(known, (0,len(before_w) - len(known)), "constant", constant_values=-42.42) guess_w = np.pad(guess, (0,len(before_w) - len(guess)), "constant", constant_values=-42.42) (known_l, guess_l) = s_l_arr[count] pd.DataFrame({"known": known_w[:], "guess": guess_w[:], "before": before_w[:]}) .to_csv(path_count, sep = ";", index_label = False, index = False) pred = self.pm.predict_median_hmm(known_l, transition_mat, km) pred_fake = self.pm.predict_median_hmm( known_l, transition_mat, km, real_class_of_following_day = guess_l) precision_colin.append(self.utils_cl.compute_mse(guess, pred)) precision_fake .append(self.utils_cl.compute_mse(guess, pred_fake)) precision_mean .append(self.utils_cl.compute_mse(guess, mean_day)) precision_colin_mae.append(self.utils_cl.compute_mae(guess, pred)) precision_fake_mae .append(self.utils_cl.compute_mae(guess, pred_fake)) precision_mean_mae .append(self.utils_cl.compute_mae(guess, mean_day)) precision_colin_mase.append(self.utils_cl.compute_mase(known, guess, pred)) precision_fake_mase .append(self.utils_cl.compute_mase(known, guess, pred_fake)) precision_mean_mase .append(self.utils_cl.compute_mase(known, guess, mean_day)) mean_mse.append( ( my_km_size, np.mean(precision_colin), np.mean(precision_fake), np.mean(precision_mean), np.mean(precision_colin_mae), np.mean(precision_fake_mae), np.mean(precision_mean_mae), np.mean(precision_colin_mase), np.mean(precision_fake_mase),np.mean(precision_mean_mase), np.std(precision_colin), np.std(precision_fake), np.std(precision_mean), np.std(precision_colin_mae), np.std(precision_fake_mae), np.std(precision_mean_mae), np.std(precision_colin_mase), np.std(precision_fake_mase), np.std(precision_mean_mase) ) ) return mean_mse
def _add_path_to_icons(self): icons_directory = FileHelper.get_icons_directory() current_paths = QgsApplication.svgPaths() if icons_directory not in current_paths: current_paths.append(icons_directory) QgsApplication.setDefaultSvgPaths(current_paths)
def _add_path_to_icons(self): icons_directory = FileHelper.get_icons_directory()
class VtReader: geo_types = { 1: GeoTypes.POINT, 2: GeoTypes.LINE_STRING, 3: GeoTypes.POLYGON } layer_sort_ids = [ "poi", "boundary", "transportation", "transportation_name", "housenumber", "building", "place", "aeroway", "park", "water_name", "waterway", "water", "landcover", "landuse" ] _extent = 4096 _layers_to_dissolve = [] # todo: remove hardcoded path after testing _file_path = "{}/sample data/zurich_switzerland.mbtiles".format( FileHelper.get_directory()) def __init__(self, iface): self.iface = iface self._counter = 0 self._bool = True self._mbtile_id = "name" self.features_by_path = {} self.qgis_layer_groups_by_feature_path = {} def reinit(self): """ * Reinitializes the VtReader >> Cleans the temp directory >> Cleans the feature cache >> Cleans the qgis group cache """ FileHelper.clear_temp_dir() self.features_by_path = {} self.qgis_layer_groups_by_feature_path = {} @staticmethod def _get_empty_feature_collection(): """ * Returns an empty GeoJSON FeatureCollection """ crs = { # crs = coordinate reference system "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::3857" } } return {"type": "FeatureCollection", "crs": crs, "features": []} def load_vector_tiles_default(self, zoom_level): self.load_vector_tiles(zoom_level, self._file_path) def load_vector_tiles(self, zoom_level, path): self._file_path = path debug("Loading vector tiles: {}".format(path)) self.reinit() self._connect_to_db(path) mask_level = self._get_mask_layer_id() tile_data_tuples = self._load_tiles_from_db(zoom_level, mask_level) # if mask_level: # mask_layer_data = self._load_tiles_from_db(mask_level) # tile_data_tuples.extend(mask_layer_data) tiles = self._decode_all_tiles(tile_data_tuples) self._process_tiles(tiles) self._create_qgis_layer_hierarchy() print("Import complete!") def _connect_to_db(self, path): """ * Since an mbtile file is a sqlite database, we can connect to it """ try: self.conn = sqlite3.connect(path) self.conn.row_factory = sqlite3.Row print "Successfully connected to db" except: print "Db connection failed:", sys.exc_info() return def _get_mask_layer_id(self): sql_command = "select value as 'masklevel' from metadata where name = 'maskLevel'" mask_level = None rows = self._get_from_db(sql=sql_command) if rows: mask_level = rows[0]["masklevel"] return mask_level def _load_tiles_from_db(self, zoom_level, mask_layer): print("Reading tiles of zoom level {}".format(zoom_level)) if zoom_level != mask_layer: # sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {} and tile_row = 10638 and tile_column=8568;".format(zoom_level) # sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {} and tile_row = 10644 and tile_column=8581;".format(zoom_level) # sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {} and tile_row >= 10640 and tile_row <= 10645 and tile_column>=8580 and tile_column<= 8582;".format(zoom_level) sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {} LIMIT 5;".format( zoom_level) # sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {};".format(zoom_level) else: sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {};".format( zoom_level) # sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {} LIMIT 10;".format(zoom_level) tile_data_tuples = [] rows = self._get_from_db(sql=sql_command) for row in rows: tile_data_tuples.append(self._create_tile(row)) return tile_data_tuples @staticmethod def _create_tile(row): zoom_level = row["zoom_level"] tile_col = row["tile_column"] tile_row = row["tile_row"] binary_data = row["tile_data"] tile = VectorTile(zoom_level, tile_col, tile_row) return tile, binary_data def _get_from_db(self, sql): try: cur = self.conn.cursor() cur.execute(sql) return cur.fetchall() except: print "Getting data from db failed:", sys.exc_info() def _decode_all_tiles(self, tiles_with_encoded_data): tiles = [] total_nr_tiles = len(tiles_with_encoded_data) print "Decoding {} tiles".format(total_nr_tiles) for index, tile_data_tuple in enumerate(tiles_with_encoded_data): tile = tile_data_tuple[0] encoded_data = tile_data_tuple[1] tile.decoded_data = self._decode_binary_tile_data(encoded_data) if tile.decoded_data: tiles.append(tile) print "Progress: {0:.1f}%".format(100.0 / total_nr_tiles * (index + 1)) return tiles def _process_tiles(self, tiles): total_nr_tiles = len(tiles) print "Processing {} tiles".format(total_nr_tiles) for index, tile in enumerate(tiles): self._write_features(tile) print "Progress: {0:.1f}%".format(100.0 / total_nr_tiles * (index + 1)) def _decode_binary_tile_data(self, data): try: # The offset of 32 signals to the zlib header that the gzip header is expected but skipped. file_content = zlib.decompress(data, 32 + zlib.MAX_WBITS) decoded_data = mapbox_vector_tile.decode(file_content) except: print "decoding data with mapbox_vector_tile failed", sys.exc_info( ) return return decoded_data def _create_qgis_layer_hierarchy(self): """ * Creates a hierarchy of groups and layers in qgis """ print("Creating hierarchy in qgis") print("Layers to dissolve: {}".format(self._layers_to_dissolve)) root = QgsProject.instance().layerTreeRoot() group_name = os.path.splitext(os.path.basename(self._file_path))[0] root_group = root.addGroup(group_name) feature_paths = sorted( self.features_by_path.keys(), key=lambda path: VtReader._get_feature_sort_id(path)) for feature_path in feature_paths: target_group, layer_name = self._get_group_for_path( feature_path, root_group) feature_collection = self.features_by_path[feature_path] file_src = FileHelper.get_unique_file_name() with open(file_src, "w") as f: json.dump(feature_collection, f) layer = self._add_vector_layer(file_src, layer_name, target_group, feature_path) VtReader._load_named_style(layer) @staticmethod def _get_feature_sort_id(feature_path): first_node = feature_path.split(".")[0] sort_id = 999 if first_node in VtReader.layer_sort_ids: sort_id = VtReader.layer_sort_ids.index(first_node) return sort_id def _get_group_for_path(self, path, root_group): """ * Returns the group for the specified path >> If the group not already exists, it will be created >> The path has to be delimited by '.' >> The last element in the path will be used as name for the vector layer, the other elements will be used to create the group hierarchy >> Example: The path 'zurich.poi.police' will create two groups 'zurich' and 'poi' (if not already existing) and 'police' will be returned as name for the layer to create """ group_names = path.split(".") current_group = root_group current_path = "" target_layer_name = "" for index, name in enumerate(group_names): target_layer_name = name is_last = index == len(group_names) - 1 if is_last: break current_path += "." + name if current_path not in self.qgis_layer_groups_by_feature_path: self.qgis_layer_groups_by_feature_path[ current_path] = current_group.addGroup(name) current_group = self.qgis_layer_groups_by_feature_path[ current_path] return current_group, target_layer_name @staticmethod def _load_named_style(layer): try: name = layer.name().split("_")[0] style_name = "{}.qml".format(name) # style_name = "{}.qml".format(layer.name()) style_path = os.path.join(FileHelper.get_directory(), "styles/{}".format(style_name)) if os.path.isfile(style_path): res = layer.loadNamedStyle(style_path) if res[1]: # Style loaded layer.setCustomProperty("layerStyle", style_path) print("Style successfully applied: {}".format(style_name)) except: print("Loading style failed: {}".format(sys.exc_info())) def _add_vector_layer(self, json_src, layer_name, layer_target_group, feature_path): """ * Creates a QgsVectorLayer and adds it to the group specified by layer_target_group * Invalid geometries will be removed during the process of merging features over tile boundaries """ layer = QgsVectorLayer(json_src, layer_name, "ogr") if feature_path in self._layers_to_dissolve: layer = FeatureMerger().merge_features(layer) layer.setName(layer_name) QgsMapLayerRegistry.instance().addMapLayer(layer, False) layer_target_group.addLayer(layer) return layer def _write_features(self, tile): """ * Transforms all features of the specified tile into GeoJSON and writes it into the dictionary :param tile: :return: """ # iterate through all the features of the data and build proper gejson conform objects. for layer_name in tile.decoded_data: tile_features = tile.decoded_data[layer_name]["features"] for index, feature in enumerate(tile_features): geojson_feature, geo_type = VtReader._create_geojson_feature( feature, tile) if geojson_feature: feature_path = VtReader._get_feature_path( layer_name, geojson_feature, tile.zoom_level) if feature_path not in self.features_by_path: self.features_by_path[ feature_path] = VtReader._get_empty_feature_collection( ) self.features_by_path[feature_path]["features"].append( geojson_feature) geotypes_to_dissolve = [GeoTypes.POLYGON] if geo_type in geotypes_to_dissolve and feature_path not in self._layers_to_dissolve: self._layers_to_dissolve.append(feature_path) @staticmethod def _get_feature_class_and_subclass(feature): feature_class = None feature_subclass = None properties = feature["properties"] if "class" in properties: feature_class = properties["class"] if "subclass" in properties: feature_subclass = properties["subclass"] if feature_subclass == feature_class: feature_subclass = None if feature_subclass: assert feature_class, "A feature with a subclass should also have a class" return feature_class, feature_subclass @staticmethod def _get_feature_path(layer_name, feature, zoom_level): feature_class, feature_subclass = VtReader._get_feature_class_and_subclass( feature) feature_path = layer_name if feature_class: feature_path += "." + feature_class if feature_subclass: feature_path += "." + feature_subclass feature_path += "_{}".format(zoom_level) return feature_path total_feature_count = 0 @staticmethod def _create_geojson_feature(feature, tile): """ Creates a proper GeoJSON feature for the specified feature """ geo_type = VtReader.geo_types[feature["type"]] coordinates = feature["geometry"] # # todo: remove after testing # if geo_type != GeoTypes.POLYGON: # return None, None coordinates = VtReader._map_coordinates_recursive( coordinates=coordinates, func=lambda coords: VtReader._transform_to_epsg3857(coords, tile)) if geo_type == GeoTypes.POINT: # Due to mercator_geometrys nature, the point will be displayed in a List "[[]]", remove the outer bracket. coordinates = coordinates[0] properties = feature["properties"] properties["zoomLevel"] = tile.zoom_level properties["featureNr"] = VtReader.total_feature_count properties["col"] = tile.column properties["row"] = tile.row VtReader.total_feature_count += 1 feature_json = VtReader._create_geojson_feature_from_coordinates( geo_type, coordinates, properties) return feature_json, geo_type @staticmethod def _get_is_multi(geo_type, coordinates): if geo_type == GeoTypes.POINT: is_single = len(coordinates) == 2 and all( isinstance(c, int) for c in coordinates) return not is_single elif geo_type == GeoTypes.LINE_STRING: is_array_of_tuples = all( len(c) == 2 and all(isinstance(ci, int) for ci in c) for c in coordinates) is_single = is_array_of_tuples return not is_single elif geo_type == GeoTypes.POLYGON: is_multi = VtReader.get_array_depth(coordinates, 0) >= 2 return is_multi return False @staticmethod def get_array_depth(arr, depth): if all(isinstance(c, numbers.Real) for c in arr[0]): return depth else: depth += 1 return VtReader.get_array_depth(arr[0], depth) @staticmethod def _create_geojson_feature_from_coordinates(geo_type, coordinates, properties): is_multi = VtReader._get_is_multi(geo_type, coordinates) type_string = geo_type if is_multi: type_string = "Multi{}".format(geo_type) feature_json = { "type": "Feature", "geometry": { "type": type_string, "coordinates": coordinates }, "properties": properties } return feature_json @staticmethod def _map_coordinates_recursive(coordinates, func): """ Recursively traverses the array of coordinates (depth first) and applies the specified function """ tmp = [] for coord in coordinates: is_coordinate_tuple = len(coord) == 2 and all( isinstance(c, int) for c in coord) if is_coordinate_tuple: newval = func(coord) tmp.append(newval) else: tmp.append(VtReader._map_coordinates_recursive(coord, func)) return tmp @staticmethod def _transform_to_epsg3857(coordinates, tile): """ Does a mercator transformation on the specified coordinate tuple """ # calculate the mercator geometry using external library # geometry:: 0: zoom, 1: easting, 2: northing tmp = GlobalMercator().TileBounds(tile.column, tile.row, tile.zoom_level) delta_x = tmp[2] - tmp[0] delta_y = tmp[3] - tmp[1] merc_easting = int(tmp[0] + delta_x / VtReader._extent * coordinates[0]) merc_northing = int(tmp[1] + delta_y / VtReader._extent * coordinates[1]) return [merc_easting, merc_northing]
def fetch_nc_geojson(): # refresh road status data = FileHelper.load_json_file('data/nc_parkway_data.geojson') return json.dumps({'data': data}), 200, {'ContentType': 'application/json'}
def _dissolve(layer): debug("Dissolving layer") target_file = FileHelper.get_unique_file_name() processing.runalg("qgis:dissolve", layer, False, "dissolveGroup", target_file) dissolved_layer = QgsVectorLayer(target_file, "Dissolved", "ogr") return dissolved_layer
def _load_tiles(self): try: # recreate source to assure the source belongs to the new thread, SQLite3 isn't happy about it otherwise self.source = self._create_source(self.source.source()) zoom_level = self._loading_options["zoom_level"] bounds = self._loading_options["bounds"] load_mask_layer = self._loading_options["load_mask_layer"] merge_tiles = self._loading_options["merge_tiles"] clip_tiles = self._loading_options["clip_tiles"] apply_styles = self._loading_options["apply_styles"] max_tiles = self._loading_options["max_tiles"] layer_filter = self._loading_options["layer_filter"] self.cancel_requested = False self.feature_collections_by_layer_name_and_geotype = {} self._qgis_layer_groups_by_name = {} self._update_progress(show_dialog=True, title="Loading '{}'".format(os.path.basename(self.source.name()))) self._clip_tiles_at_tile_bounds = clip_tiles min_zoom = self.source.min_zoom() max_zoom = self.source.max_zoom() zoom_level = clamp(zoom_level, low=min_zoom, high=max_zoom) all_tiles = get_all_tiles( bounds=bounds, is_cancel_requested_handler=lambda: self.cancel_requested, ) tiles_to_load = set() tiles = [] tiles_to_ignore = set() for t in all_tiles: if self.cancel_requested or (max_tiles and len(tiles) >= max_tiles): break file_name = self._get_tile_cache_name(zoom_level, t[0], t[1]) tile = FileHelper.get_cached_tile(file_name) if tile and tile.decoded_data: tiles.append(tile) tiles_to_ignore.add((tile.column, tile.row)) else: tiles_to_load.add(t) remaining_nr_of_tiles = len(tiles_to_load) if max_tiles: if len(tiles) + len(tiles_to_load) >= max_tiles: remaining_nr_of_tiles = max_tiles - len(tiles) if remaining_nr_of_tiles < 0: remaining_nr_of_tiles = 0 debug("{} cache hits. {} may potentially be loaded.", len(tiles), remaining_nr_of_tiles) debug("Loading data for zoom level '{}' source '{}'", zoom_level, self.source.name()) tile_data_tuples = [] if remaining_nr_of_tiles > 0: tile_data_tuples = self.source.load_tiles(zoom_level=zoom_level, tiles_to_load=tiles_to_load, max_tiles=remaining_nr_of_tiles) if len(tiles) == 0 and (not tile_data_tuples or len(tile_data_tuples) == 0): QMessageBox.information(None, "No tiles found", "What a pity, no tiles could be found!") if load_mask_layer: mask_level = self.source.mask_level() if mask_level is not None and mask_level != zoom_level: debug("Mapping {} tiles to mask level", len(all_tiles)) scheme = self.source.scheme() mask_tiles = map( lambda t: change_zoom(zoom_level, int(mask_level), t, scheme), all_tiles) debug("Mapping done") mask_tiles_to_load = set() for t in mask_tiles: file_name = self._get_tile_cache_name(mask_level, t[0], t[1]) tile = FileHelper.get_cached_tile(file_name) if tile and tile.decoded_data: tiles.append(tile) else: mask_tiles_to_load.add(t) debug("Loading mask layer (zoom_level={})", mask_level) tile_data_tuples = [] if len(mask_tiles_to_load) > 0: mask_layer_data = self.source.load_tiles(zoom_level=mask_level, tiles_to_load=mask_tiles_to_load, max_tiles=max_tiles) debug("Mask layer loaded") tile_data_tuples.extend(mask_layer_data) if tile_data_tuples and len(tile_data_tuples) > 0: if not self.cancel_requested: decoded_tiles = self._decode_tiles(tile_data_tuples) tiles.extend(decoded_tiles) if len(tiles) > 0: if not self.cancel_requested: self._process_tiles(tiles, layer_filter) if not self.cancel_requested: self._create_qgis_layers(merge_features=merge_tiles, apply_styles=apply_styles) self._update_progress(show_dialog=False) if self.cancel_requested: info("Import cancelled") self.cancelled.emit() else: info("Import complete") loaded_tiles_x = map(lambda t: t.coord()[0], tiles) loaded_tiles_y = map(lambda t: t.coord()[1], tiles) if len(loaded_tiles_x) == 0 or len(loaded_tiles_y) == 0: return None loaded_extent = {"x_min": int(min(loaded_tiles_x)), "x_max": int(max(loaded_tiles_x)), "y_min": int(min(loaded_tiles_y)), "y_max": int(max(loaded_tiles_y)), "zoom": int(zoom_level) } loaded_extent["width"] = loaded_extent["x_max"] - loaded_extent["x_min"] + 1 loaded_extent["height"] = loaded_extent["y_max"] - loaded_extent["y_min"] + 1 self.loading_finished.emit(zoom_level, loaded_extent) except Exception as e: critical("An exception occured: {}, {}", e, traceback.format_exc()) self.cancelled.emit()
class KhiopsManager: def __init__(self): logging.info(pk.getKhiopsInfo()) self.this_file_dir = os.path.dirname(os.path.realpath(__file__)) # path mgmt # use timestamp of each exec in paths self.fh = FileHelper() self.dictionary_file = os.path.join(self.this_file_dir, "dic", "series.kdic") self.classif_res = os.path.join(self.this_file_dir, "res", "khiops_res", "classif") self.coclus_res = os.path.join(self.this_file_dir, "res", "khiops_res", "coclus") self.pred_res = os.path.join(self.this_file_dir, "res", "khiops_res", "pred_res") self.fh.ensure_dirs_exist([ self.dictionary_file, self.classif_res, self.coclus_res, self.pred_res ]) self.ccr = CoclusteringResults() self.utils = MyUtils() logging.info("Khiops manager instantiated") logging.info("dictionary_file used: %s", self.dictionary_file) """KHIOPS COCLUSTERING TRAIN AND SIMPLIFICATIONS""" def train_coclustering(self, f): """ Train a coclustering model in the simplest way possible """ file_name = self.fh.get_file_name(f) logging.info("Train of coclustering for file %s", file_name) # Train coclustering model for variables "SampleId" and "Char" pk.trainCoclustering(dictionaryFile=self.dictionary_file, dictionary="train", dataTable=f, coclusteringVariables=["time_", "n_day_", "val_"], resultsDir=self.coclus_res, fieldSeparator=";", samplePercentage=100, resultsPrefix=file_name + "_") def simplify_coclustering(self, file_name, mpi=100, mcn=999999): """ Simplify a coclustering model in the simplest way possible """ base_path = os.path.join(self.coclus_res, file_name) self.fh.ensure_dirs_exist([base_path]) cf = os.path.join(self.coclus_res, file_name + "_Coclustering.khc") logging.info("Simplify coclustering for file %s", file_name) logging.info("MCN=%s, MPI=%s", mcn, mpi) if mcn != 999999: scf = file_name + "_Simplified-" + str(mcn) + ".khc" else: scf = file_name + "_Simplified-" + str(mpi) + ".khc" logging.info("scf=%s", scf) pk.simplifyCoclustering( coclusteringFile=cf, simplifiedCoclusteringFile=scf, resultsDir=base_path, maxCellNumber=mcn, maxPreservedInformation=mpi, ) return str(os.path.join(base_path, scf)).replace(".khc", ".json") def deploy_coclustering(self, file_name, identifier): """ Deploy a coclustering model """ base_path = os.path.join(self.coclus_res, file_name) self.fh.ensure_dirs_exist([base_path]) logging.info("Deploy coclustering for file %s", file_name) scf = os.path.join( self.coclus_res, file_name, file_name + "_Simplified-" + str(identifier) + ".khc") dep_prefix = file_name + "_Deployed-" + str(identifier) + "-" pk.prepareCoclusteringDeployment(dictionaryFile=self.dictionary_file, dictionary="root", coclusteringFile=scf, tableVariable="secondary", deployedVariable="n_day_", buildDistanceVariables=True, resultsPrefix=dep_prefix, resultsDir=base_path) return os.path.join(base_path, dep_prefix + "Coclustering.kdic") def transfer_database(self, d, deployed_path, f, identifier): """ Transfer a coclustering model to use it """ file_name = self.fh.get_file_name(f) out_path = os.path.join( self.coclus_res, file_name, file_name + "_Transf-" + str(identifier) + ".csv") logging.info("Transfering of coclustering for file %s into %s", file_name, out_path) pk.transferDatabase(dictionaryFile=d, dictionary="root", dataTable=deployed_path, additionalDataTables={"root`secondary": f}, fieldSeparator=";", outputFieldSeparator=";", outputDataTable=out_path) return out_path """JSON COCLUSTERING MANIPULATIONS""" def get_clusters(self, path): """ From a given Khiops json file, extract clusters, and detailed info about them. Returns: * zips: more info about each cluster zipped together (e.g all tuples that goes together 'aligned') => tuples (clus, values, value_frequencies, value_typicalities) * cluster_and_values: dic(cluster_num => values_associated) """ cluster_and_values = {} zips = [] with open(path) as f: data = json.load(f)["coclusteringReport"]["dimensionPartitions"] n_days = list(filter(lambda d: d['name'] in "n_day_", data)) for idx, n_day in enumerate(n_days[0]["valueGroups"], start=1): # Ugly turnaround to manipulate floats and int # see https://goo.gl/8tYfhn values = list(map(int, map(float, n_day["values"]))) clus = n_day["cluster"] value_frequencies = n_day["valueFrequencies"] value_typicalities = n_day["valueTypicalities"] zips.append( list( zip(clus, values, value_frequencies, value_typicalities))) cluster_and_values[clus] = values return self.utils.flattify(zips), cluster_and_values @staticmethod def get_clusters_from_dep(path): """ From a given Khiops transferred thing, extract clusters, and detailed info about them. Returns: * cluster_and_values: dic(cluster_num => values_associated) """ with open(path) as f: df = pd.read_csv(f, sep=";") k = df["n_day_PredictedLabel"].unique() cluster_and_values = dict((key, []) for key in k) for index, row in df.iterrows(): n_day_ = row["n_day_"] clus = row["n_day_PredictedLabel"] cluster_and_values[clus].append(n_day_) return cluster_and_values def get_cells_number(self, file_name, mpi=None): """ From a given Khiops json file, extract number of cells for all dimentions return: - cells """ if mpi is not None: p = os.path.join(self.coclus_res, file_name, file_name + "_Simplified-" + str(mpi) + ".json") else: p = os.path.join(self.coclus_res, file_name + "_Coclustering.json") with open(p) as f: cells = json.load(f)["coclusteringReport"]["summary"]["cells"] return int(cells) def get_cluster_number(self, file_name, ref_id=None): """ From a given Khiops json file, extract number of cluster for dim "n_day_" return: - nb_of_cluster_found """ if ref_id is not None: p = os.path.join( self.coclus_res, file_name, file_name + "_Simplified-" + str(ref_id) + ".json") else: p = os.path.join(self.coclus_res, file_name + "_Coclustering.json") with open(p) as f: data = json.load(f)["coclusteringReport"]["dimensionSummaries"] n_days = list(filter(lambda d: d['name'] in "n_day_", data)) if not n_days: return False else: return int(n_days[0]["parts"]) """UTILS""" @staticmethod def _compute_accuracy(len_y_pred, y_pred, y_pred_target): """ Compute the accuracy of a classifier. """ c = 0 for i in range(0, len_y_pred): pred_group_day_ahead = str(y_pred.iloc[i]["Predictedy"]) real_group_day_ahead = str(y_pred_target.values[i]) if pred_group_day_ahead != real_group_day_ahead: c = c + 1 train_acc = 100 - c * 100 / len_y_pred logging.debug("%s errors over %s values", c, len_y_pred) logging.debug("That is %s perc of accuracy", train_acc) return train_acc @staticmethod def __get_typicalitie(n_day, my_zip): """ From a list of tuples my_zip, find the tuple which concern n_day and retrieve its valTyp. PARAMETERS ------------------------ - my_zip: tuples (group, n_day, valFreq, valTyp) - n_day: int """ items = [i for i in my_zip if n_day == i[1]] return items[0][3] @staticmethod def compute_centroids(c_a_v, df, l_ref): """ This method computes mean days and centroids for given values PARAMETERS ------------------------ * c_a_v: dic(cluster_num => values_associated) * df: dataset studied * l_ref: len of one ref day RETURNS ------------------------- * centroids: dic(cluster_id: int => centroid: []) """ centroids = {} e = False for k, v in c_a_v.items(): # If there is at least one day in the cluster considered! if len(v) != 0: logging.info("Computing cluster %s centroid", k) centroids[k] = df[df['n_day_'].isin(map( str, v))].groupby('time_')['val_'].agg('mean').values # Otherwise, mean day = 0 else: logging.info("Cluster %s is empty", k) e = True centroids[k] = [0] * l_ref logging.info("Keys of centroids are %s", list(centroids.keys())) logging.debug("Centroids are %s", centroids) return centroids, e def process_pred_proba(self, y_pred, y_pred_target, clusts_names, centroids, days_to_predict, nb_cluster_found, n_pt_per_day, mean_day, o=False): """ This method process results from Khiops classification method to create predictions and compute mses. If Oracle mode is "True", then use a mock for classifier Y (e.g the classifier knows exactly the Y columns) WHAT IT DOES ------------------------ * First thing (A): Compute given classifier accuracy regarding known y_pred_target. * Second thing (B): Process probalistic classifier results but not using the probabilities and only the most probable class for day n + 1. NPA * Third thing (C): Process probalistic classifier results using probabilities affected to each cluster (e.g. day n + 1 could be 10% in cluster 1, 50% in cluster 2, etc). Use ponderations to sum up each centroids and produce result. PA * Last thing (D): Wrap up, means computations => compute results... PARAMETERS ------------------------ * y_pred: Matrix results from classifier * y_pred_target_test: Known y_pred_target for classifier for test data * clusts_names: names of all clusters * centroids: clusters centroids * days_to_predict: list of days that have been used for test * nb_cluster_found: value exctracted from Khiops coclustering file; used to parameterize loops and computations. * n_pt_per_day: if there is one point per hour of the day, then this value will be 24. * mean_day: pre-computed mean day for all training ensemble. * oracle: are we in oracle mode? ('Y' known) RETURN ------------------------- * Tuple(mean_mse_non_proba, mean_mse_proba, classifier_acc, mean_mse_mean_day) => Meaned MSE ! :param y_pred: :param clusts_names: :param centroids: :param days_to_predict: :param nb_cluster_found: :param n_pt_per_day: :param mean_day: :param y_pred_target: :param o: """ len_y_pred = len(y_pred) # (A): Compute classifier accuracy classifier_acc = self._compute_accuracy(len_y_pred, y_pred, y_pred_target) if classifier_acc == 0.0: logging.info( "This classifier is very not good and failed all the time") # Init empty arrays mses_non_proba = [] mses_proba = [] mses_mean_day = [] mae_non_proba = [] mae_proba = [] mae_mean_day = [] mase_non_proba = [] mase_proba = [] mase_mean_day = [] # Think about removing last day, which has not days after n_days_ = days_to_predict["n_day_"].unique()[:-1] # Also got to remove the first day, which has not day before # Wrangling data to have good types. n_days_int = list(map(int, n_days_)) n_days_int.remove(min(n_days_int)) n_days_ = list(map(str, n_days_int)) for ref_in_classif_ensemble, z in enumerate(n_days_): """ (B): NPA approach """ indice_today = str(int(z) - 1) today_vals = days_to_predict[days_to_predict["n_day_"] == indice_today]["val_"].values tomor_vals = days_to_predict[days_to_predict["n_day_"] == z]["val_"].values tom_class_pred_cluster_y = y_pred.iloc[ref_in_classif_ensemble][ "Predictedy"] tom_pred_y = centroids[str(tom_class_pred_cluster_y)] # Append mses mses_non_proba.append( self.utils.compute_mse(tomor_vals, tom_pred_y)) mses_mean_day.append(self.utils.compute_mse(tomor_vals, mean_day)) mae_non_proba.append(self.utils.compute_mae( tomor_vals, tom_pred_y)) mae_mean_day.append(self.utils.compute_mae(tomor_vals, mean_day)) mase_non_proba.append( self.utils.compute_mase(today_vals, tomor_vals, tom_pred_y)) mase_mean_day.append( self.utils.compute_mase(today_vals, tomor_vals, mean_day)) mean_days_pond = [] p_tot = 0 """ (C): PA approach """ for c in clusts_names: try: # Get probability to be in groupe number "c" p = y_pred.iloc[ref_in_classif_ensemble]["Proby" + str(c)] # p_tot variable for debug purposes p_tot = p_tot + p except: logging.debug("Did not find %s in the prediction matrix", c) p = 0 pass # Retrieve mean day for cluster c day_pond = centroids[c].copy() # Scale according to proba day_pond[:] = [x * p for x in day_pond] mean_days_pond.append(day_pond) logging.debug("p_tot is %s", p_tot) # Here it should be 100. # Sum up everything to create prediction tomo_pred_with_pond = [0] * n_pt_per_day for d_p in mean_days_pond: for idx, e in enumerate(d_p): tomo_pred_with_pond[idx] = tomo_pred_with_pond[idx] + e mses_proba.append( self.utils.compute_mse(tomor_vals, tomo_pred_with_pond)) mae_proba.append( self.utils.compute_mae(tomor_vals, tomo_pred_with_pond)) mase_proba.append( self.utils.compute_mase(today_vals, tomor_vals, tomo_pred_with_pond)) # (D): Wrap up: mean MSEs (so we have, at the end, Meaned Mean Square # Error) # MMSES mean_mse_non_proba = np.mean(mses_non_proba) mean_mse_proba = np.mean(mses_proba) mean_mse_mean_day = np.mean(mses_mean_day) mean_mae_non_proba = np.mean(mae_non_proba) mean_mae_proba = np.mean(mae_proba) mean_mae_mean_day = np.mean(mae_mean_day) mean_mase_non_proba = np.mean(mase_non_proba) mean_mase_proba = np.mean(mase_proba) mean_mase_mean_day = np.mean(mase_mean_day) # MSTD std_mse_non_proba = np.std(mses_non_proba) std_mse_proba = np.std(mses_proba) std_mse_mean_day = np.std(mses_mean_day) std_mae_non_proba = np.std(mae_non_proba) std_mae_proba = np.std(mae_proba) std_mae_mean_day = np.std(mae_mean_day) std_mase_non_proba = np.std(mase_non_proba) std_mase_proba = np.std(mase_proba) std_mase_mean_day = np.std(mase_mean_day) return ((mean_mse_non_proba, mean_mse_proba, mean_mse_mean_day), (mean_mae_non_proba, mean_mae_proba, mean_mae_mean_day), (mean_mase_non_proba, mean_mase_proba, mean_mase_mean_day), classifier_acc, (std_mse_non_proba, std_mse_proba, std_mse_mean_day), (std_mae_non_proba, std_mae_proba, std_mae_mean_day), (std_mase_non_proba, std_mase_proba, std_mase_mean_day))
class VtReader(QObject): progress_changed = pyqtSignal(int, name='progressChanged') max_progress_changed = pyqtSignal(int, name='maxProgressChanged') message_changed = pyqtSignal('QString', name='messageChanged') title_changed = pyqtSignal('QString', name='titleChanged') show_progress_changed = pyqtSignal(bool, name='titleChanged') loading_finished = pyqtSignal(int, object, name='loadingFinished') tile_limit_reached = pyqtSignal(int, name='tile_limit_reached') cancelled = pyqtSignal(name='cancelled') omt_layer_ordering = [ "place", "mountain_peak", "housenumber", "water_name", "transportation_name", "poi", "boundary", "transportation", "building", "aeroway", "park", "water", "waterway", "landcover", "landuse" ] _loading_options = { 'zoom_level': None, 'layer_filter': None, 'load_mask_layer': None, 'merge_tiles': None, 'clip_tiles': None, 'apply_styles': None, 'max_tiles': None, 'bounds': None } _layers_to_dissolve = [] _zoom_level_delimiter = "*" _DEFAULT_EXTENT = 4096 _id = str(uuid.uuid4()) _styles = FileHelper.get_styles() flush_layers_of_other_zoom_level = False def __init__(self, iface, path_or_url): """ * The mbtiles_path can also be an URL in zxy format: z=zoom, x=tile column, y=tile row :param iface: :param path_or_url: """ QObject.__init__(self) if not path_or_url: raise RuntimeError("The datasource is required") self.source = self._create_source(path_or_url) FileHelper.assure_temp_dirs_exist() self.iface = iface self.feature_collections_by_layer_name_and_geotype = {} self._qgis_layer_groups_by_name = {} self.cancel_requested = False self._loaded_pois_by_id = {} self._clip_tiles_at_tile_bounds = None self._always_overwrite_geojson = False self._root_group_name = None self._flush = False def _create_source(self, path_or_url): is_web_source = path_or_url.lower().startswith("http://") or path_or_url.lower().startswith("https://") if is_web_source: source = ServerSource(url=path_or_url) else: if os.path.isfile(path_or_url): source = MBTilesSource(path=path_or_url) else: source = TrexCacheSource(path=path_or_url) source.progress_changed.connect(self._source_progress_changed) source.max_progress_changed.connect(self._source_max_progress_changed) source.message_changed.connect(self._source_message_changed) source.tile_limit_reached.connect(self._source_tile_limit_reached) return source def shutdown(self): info("Shutdown reader") self.source.progress_changed.disconnect() self.source.max_progress_changed.disconnect() self.source.message_changed.disconnect() self.source.close_connection() def id(self): return self._id @pyqtSlot(int) def _source_tile_limit_reached(self): self.tile_limit_reached.emit(self._loading_options["max_tiles"]) @pyqtSlot(int) def _source_progress_changed(self, progress): self._update_progress(progress=progress) @pyqtSlot(int) def _source_max_progress_changed(self, max_progress): self._update_progress(max_progress=max_progress) @pyqtSlot('QString') def _source_message_changed(self, msg): self._update_progress(msg=msg) def set_root_group_name(self, name): self._root_group_name = name def _update_progress(self, title=None, show_dialog=None, progress=None, max_progress=None, msg=None): if progress is not None: self.progress_changed.emit(progress) if max_progress is not None: self.max_progress_changed.emit(max_progress) if title: self.title_changed.emit(title) if msg: self.message_changed.emit(msg) if show_dialog: self.show_progress_changed.emit(show_dialog) def _get_empty_feature_collection(self, layer_name, zoom_level): """ * Returns an empty GeoJSON FeatureCollection with the coordinate reference system (crs) set to EPSG3857 """ # todo: when improving CRS handling: the correct CRS of the source has to be set here source_crs = self.source.crs() if source_crs: epsg_id = get_code_from_epsg(source_crs) else: epsg_id = 3857 crs = { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::{}".format(epsg_id)}} return { "tiles": [], "source": self.source.name(), "scheme": self.source.scheme(), "layer": layer_name, "zoom_level": zoom_level, "type": "FeatureCollection", "crs": crs, "features": []} def always_overwrite_geojson(self, enabled): """ * If activated, the geoJson written to the disk will always be overwritten, with each load * As a result of this, only the latest loaded extent will be visible in qgis :return: """ self._always_overwrite_geojson = enabled def cancel(self): """ * Cancels the loading process. :return: """ self.cancel_requested = True if self.source: self.source.cancel() def _get_tile_cache_name(self, zoom_level, col, row): return FileHelper.get_cached_tile_file_name(self.source.name(), zoom_level, col, row) def _load_tiles(self): try: # recreate source to assure the source belongs to the new thread, SQLite3 isn't happy about it otherwise self.source = self._create_source(self.source.source()) zoom_level = self._loading_options["zoom_level"] bounds = self._loading_options["bounds"] load_mask_layer = self._loading_options["load_mask_layer"] merge_tiles = self._loading_options["merge_tiles"] clip_tiles = self._loading_options["clip_tiles"] apply_styles = self._loading_options["apply_styles"] max_tiles = self._loading_options["max_tiles"] layer_filter = self._loading_options["layer_filter"] self.cancel_requested = False self.feature_collections_by_layer_name_and_geotype = {} self._qgis_layer_groups_by_name = {} self._update_progress(show_dialog=True, title="Loading '{}'".format(os.path.basename(self.source.name()))) self._clip_tiles_at_tile_bounds = clip_tiles min_zoom = self.source.min_zoom() max_zoom = self.source.max_zoom() zoom_level = clamp(zoom_level, low=min_zoom, high=max_zoom) all_tiles = get_all_tiles( bounds=bounds, is_cancel_requested_handler=lambda: self.cancel_requested, ) tiles_to_load = set() tiles = [] tiles_to_ignore = set() for t in all_tiles: if self.cancel_requested or (max_tiles and len(tiles) >= max_tiles): break file_name = self._get_tile_cache_name(zoom_level, t[0], t[1]) tile = FileHelper.get_cached_tile(file_name) if tile and tile.decoded_data: tiles.append(tile) tiles_to_ignore.add((tile.column, tile.row)) else: tiles_to_load.add(t) remaining_nr_of_tiles = len(tiles_to_load) if max_tiles: if len(tiles) + len(tiles_to_load) >= max_tiles: remaining_nr_of_tiles = max_tiles - len(tiles) if remaining_nr_of_tiles < 0: remaining_nr_of_tiles = 0 debug("{} cache hits. {} may potentially be loaded.", len(tiles), remaining_nr_of_tiles) debug("Loading data for zoom level '{}' source '{}'", zoom_level, self.source.name()) tile_data_tuples = [] if remaining_nr_of_tiles > 0: tile_data_tuples = self.source.load_tiles(zoom_level=zoom_level, tiles_to_load=tiles_to_load, max_tiles=remaining_nr_of_tiles) if len(tiles) == 0 and (not tile_data_tuples or len(tile_data_tuples) == 0): QMessageBox.information(None, "No tiles found", "What a pity, no tiles could be found!") if load_mask_layer: mask_level = self.source.mask_level() if mask_level is not None and mask_level != zoom_level: debug("Mapping {} tiles to mask level", len(all_tiles)) scheme = self.source.scheme() mask_tiles = map( lambda t: change_zoom(zoom_level, int(mask_level), t, scheme), all_tiles) debug("Mapping done") mask_tiles_to_load = set() for t in mask_tiles: file_name = self._get_tile_cache_name(mask_level, t[0], t[1]) tile = FileHelper.get_cached_tile(file_name) if tile and tile.decoded_data: tiles.append(tile) else: mask_tiles_to_load.add(t) debug("Loading mask layer (zoom_level={})", mask_level) tile_data_tuples = [] if len(mask_tiles_to_load) > 0: mask_layer_data = self.source.load_tiles(zoom_level=mask_level, tiles_to_load=mask_tiles_to_load, max_tiles=max_tiles) debug("Mask layer loaded") tile_data_tuples.extend(mask_layer_data) if tile_data_tuples and len(tile_data_tuples) > 0: if not self.cancel_requested: decoded_tiles = self._decode_tiles(tile_data_tuples) tiles.extend(decoded_tiles) if len(tiles) > 0: if not self.cancel_requested: self._process_tiles(tiles, layer_filter) if not self.cancel_requested: self._create_qgis_layers(merge_features=merge_tiles, apply_styles=apply_styles) self._update_progress(show_dialog=False) if self.cancel_requested: info("Import cancelled") self.cancelled.emit() else: info("Import complete") loaded_tiles_x = map(lambda t: t.coord()[0], tiles) loaded_tiles_y = map(lambda t: t.coord()[1], tiles) if len(loaded_tiles_x) == 0 or len(loaded_tiles_y) == 0: return None loaded_extent = {"x_min": int(min(loaded_tiles_x)), "x_max": int(max(loaded_tiles_x)), "y_min": int(min(loaded_tiles_y)), "y_max": int(max(loaded_tiles_y)), "zoom": int(zoom_level) } loaded_extent["width"] = loaded_extent["x_max"] - loaded_extent["x_min"] + 1 loaded_extent["height"] = loaded_extent["y_max"] - loaded_extent["y_min"] + 1 self.loading_finished.emit(zoom_level, loaded_extent) except Exception as e: critical("An exception occured: {}, {}", e, traceback.format_exc()) self.cancelled.emit() def set_options(self, load_mask_layer=False, merge_tiles=True, clip_tiles=False, apply_styles=True, max_tiles=None, layer_filter=None): self._loading_options = { 'load_mask_layer': load_mask_layer, 'merge_tiles': merge_tiles, 'clip_tiles': clip_tiles, 'apply_styles': apply_styles, 'max_tiles': max_tiles, 'layer_filter': layer_filter } def load_tiles_async(self, zoom_level, bounds): """ * Loads the vector tiles from either a file or a URL and adds them to QGIS :param clip_tiles: :param zoom_level: The zoom level to load :param layer_filter: A list of layers. If any layers are set, only these will be loaded. If the list is empty, all available layers will be loaded :param load_mask_layer: If True the mask layer will also be loaded :param merge_tiles: If True neighbouring tiles and features will be merged :param apply_styles: If True the default styles will be applied :param max_tiles: The maximum number of tiles to load :param bounds: :return: """ self._loading_options["zoom_level"] = zoom_level self._loading_options["bounds"] = bounds _worker_thread = QThread(self.iface.mainWindow()) self.moveToThread(_worker_thread) _worker_thread.started.connect(self._load_tiles) _worker_thread.start() def _decode_tiles(self, tiles_with_encoded_data): """ * Decodes the PBF data from all the specified tiles and reports the progress * If a tile is loaded from the cache, the decoded_data is already set and doesn't have to be encoded :param tiles_with_encoded_data: :return: """ total_nr_tiles = len(tiles_with_encoded_data) self._update_progress(progress=0, max_progress=100, msg="Decoding {} tiles...".format(total_nr_tiles)) nr_processors = 4 try: nr_processors = mp.cpu_count() except NotImplementedError: info("CPU count cannot be retrieved. Falling back to default = 4") tiles_with_encoded_data = map(lambda t: (t[0], self._unzip(t[1])), tiles_with_encoded_data) pool = mp.Pool(nr_processors) tiles = [] rs = pool.map_async(decode_tile, tiles_with_encoded_data, callback=tiles.extend) pool.close() current_progress = 0 while not rs.ready() and not self.cancel_requested: if self.cancel_requested: pool.terminate() break else: QApplication.processEvents() remaining = rs._number_left index = total_nr_tiles - remaining progress = int(100.0 / total_nr_tiles * (index + 1)) if progress != current_progress: current_progress = progress self._update_progress(progress=progress) tiles = filter(lambda ti: ti.decoded_data is not None, tiles) for t in tiles: if self.cancel_requested: break else: cache_file_name = self._get_tile_cache_name(t.zoom_level, t.column, t.row) if not os.path.isfile(cache_file_name): FileHelper.cache_tile(t, cache_file_name) return tiles def _unzip(self, data): """ * If the passed data is gzipped, it will be unzipped. Otherwise it will be returned untouched :param data: :return: """ is_gzipped = FileHelper.is_gzipped(data) if is_gzipped: file_content = GzipFile('', 'r', 0, StringIO(data)).read() else: file_content = data return file_content def _process_tiles(self, tiles, layer_filter): """ * Creates GeoJSON for all the specified tiles and reports the progress :param tiles: :return: """ total_nr_tiles = len(tiles) info("Processing {} tiles", total_nr_tiles) self._update_progress(progress=0, max_progress=100, msg="Processing features...") current_progress = -1 for index, tile in enumerate(tiles): if self.cancel_requested: break self._create_geojson(tile, layer_filter) progress = int(100.0 / total_nr_tiles * (index + 1)) if progress != current_progress: current_progress = progress self._update_progress(progress=progress) def _get_omt_layer_sort_id(self, layer_name): """ * Returns the cartographic sort id for the specified layer. * This sort id is the position of the layer in the omt_layer_ordering collection. * If the layer isn't present in the collection, the sort id wil be 999 and therefore the layer will be added at the bottom. :param layer_name: :return: """ sort_id = 999 if layer_name in self.omt_layer_ordering: sort_id = self.omt_layer_ordering.index(layer_name) return sort_id def _assure_qgis_groups_exist(self, manual_layer_name=None, sort_layers=False): """ * Createss a group for each layer that is given by the layer source scheme >> mbtiles: value 'JSON' in metadata table, array 'vector_layers' >> TileJSON: value 'vector_layers' :return: """ root = QgsProject.instance().layerTreeRoot() name = self._root_group_name if not name: name = self.source.name() root_group = root.findGroup(name) if not root_group: root_group = root.addGroup(name) if not manual_layer_name: layers = map(lambda l: l["id"], self.source.vector_layers()) else: layers = [manual_layer_name] if sort_layers: layers = sorted(layers, key=lambda n: self._get_omt_layer_sort_id(n)) for index, layer_name in enumerate(layers): group = root_group.findGroup(layer_name) if not group: group = root_group.addGroup(layer_name) self._qgis_layer_groups_by_name[layer_name] = group def _get_geojson_filename(self, layer_name, geo_type): return "{}.{}.{}".format(self.source.name().replace(" ", "_"), layer_name, geo_type) def _create_qgis_layers(self, merge_features, apply_styles): """ * Creates a hierarchy of groups and layers in qgis """ debug("Creating hierarchy in QGIS") self._assure_qgis_groups_exist(sort_layers=apply_styles) qgis_layers = QgsMapLayerRegistry.instance().mapLayers() vt_qgis_name_layer_tuples = filter(lambda (n, l): l.customProperty("vector_tile_source") == self.source.source(), qgis_layers.iteritems()) own_layers = map(lambda (n, l): l, vt_qgis_name_layer_tuples) for l in own_layers: name = l.name() geo_type = l.customProperty("geo_type") if (name, geo_type) not in self.feature_collections_by_layer_name_and_geotype: self._update_layer_source(l.source(), self._get_empty_feature_collection(0, l.name())) self._update_progress(progress=0, max_progress=len(self.feature_collections_by_layer_name_and_geotype), msg="Creating layers...") new_layers = [] count = 0 for layer_name, geo_type in self.feature_collections_by_layer_name_and_geotype: count += 1 if self.cancel_requested: break feature_collection = self.feature_collections_by_layer_name_and_geotype[(layer_name, geo_type)] zoom_level = feature_collection file_name = self._get_geojson_filename(layer_name, geo_type) file_path = FileHelper.get_geojson_file_name(file_name) layer = None if os.path.isfile(file_path): # file exists already. add the features of the collection to the existing collection # get the layer from qgis and update its source layer = self._get_layer_by_source(own_layers, layer_name, file_path) if layer: self._update_layer_source(file_path, feature_collection) if not layer: self._update_layer_source(file_path, feature_collection) layer = self._create_named_layer(file_path, layer_name, zoom_level, merge_features, geo_type) new_layers.append((layer_name, geo_type, layer)) self._update_progress(progress=count+1) QgsMapLayerRegistry.instance().reloadAllLayers() if len(new_layers) > 0: only_layers = list(map(lambda layer_name_tuple: layer_name_tuple[2], new_layers)) QgsMapLayerRegistry.instance().addMapLayers(only_layers, False) for name, geo_type, layer in new_layers: target_group = self._qgis_layer_groups_by_name[name] target_group.addLayer(layer) if apply_styles: count = 0 self._update_progress(progress=0, max_progress=len(new_layers), msg="Styling layers...") for name, geo_type, layer in new_layers: count += 1 if self.cancel_requested: break VtReader._apply_named_style(layer, geo_type) self._update_progress(progress=count) def _update_layer_source(self, layer_source, feature_collection): """ * Updates the layers GeoJSON source file :param layer_source: :param feature_collections_by_tile_coord: :return: """ with open(layer_source, "w") as f: f.write(json.dumps(feature_collection)) @staticmethod def _merge_feature_collections(current_feature_collection, feature_collections_by_tile_coord): """ * Merges the features of multiple tiles into the current_feature_collection if not already present. :param current_feature_collection: :param feature_collections_by_tile_coord: :return: """ for tile_coord in feature_collections_by_tile_coord: if tile_coord not in current_feature_collection["tiles"]: feature_collection = feature_collections_by_tile_coord[tile_coord] current_feature_collection["tiles"].extend(feature_collection["tiles"]) current_feature_collection["features"].extend(feature_collection["features"]) @staticmethod def _get_layer_by_source(all_layers, layer_name, layer_source_file): """ * Returns the layer from QGIS whose name and layer_source matches the specified parameters :param layer_name: :param layer_source_file: :return: """ layers = filter(lambda l: l.name() == layer_name and l.source() == layer_source_file, all_layers) if len(layers) > 0: return layers[0] return None @staticmethod def _apply_named_style(layer, geo_type): """ * Looks for a styles with the same name as the layer and if one is found, it is applied to the layer :param layer: :param layer_path: e.g. 'transportation.service' or 'transportation_name.path' :return: """ try: name = layer.name().lower() styles = [ "{}.{}".format(name, geo_type.lower()), name ] for p in styles: style_name = "{}.qml".format(p).lower() if style_name in VtReader._styles: style_path = os.path.join(FileHelper.get_plugin_directory(), "styles/{}".format(style_name)) res = layer.loadNamedStyle(style_path) if res[1]: # Style loaded layer.setCustomProperty("layerStyle", style_path) if layer.customProperty("layerStyle") == style_path: debug("Style successfully applied: {}", style_name) break except: critical("Loading style failed: {}", sys.exc_info()) def _create_named_layer(self, json_src, layer_name, zoom_level, merge_features, geo_type): """ * Creates a QgsVectorLayer and adds it to the group specified by layer_target_group * Invalid geometries will be removed during the process of merging features over tile boundaries """ # layer_with_zoom = "{}{}{}".format(layer_name, VtReader._zoom_level_delimiter, zoom_level) layer = QgsVectorLayer(json_src, layer_name, "ogr") layer.setCustomProperty("vector_tile_source", self.source.source()) layer.setCustomProperty("zoom_level", zoom_level) layer.setShortName(layer_name) layer.setDataUrl(self.source.source()) if self.source.name() and "openmaptiles" in self.source.name().lower(): layer.setDataUrl(remove_key(self.source.source())) layer.setAttribution(u"Vector Tiles © Klokan Technologies GmbH (CC-BY), Data © OpenStreetMap contributors (ODbL)") layer.setAttributionUrl("https://openmaptiles.com/hosting/") if merge_features and geo_type in [GeoTypes.LINE_STRING, GeoTypes.POLYGON]: layer = FeatureMerger().merge_features(layer) layer.setName(layer_name) return layer def _create_geojson(self, tile, layer_filter): """ * Transforms all features of the specified tile into GeoJSON * The resulting GeoJSON feature will be applied to the features of the corresponding GeoJSON FeatureCollection :param tile: :return: """ for layer_name in tile.decoded_data: if layer_filter and len(layer_filter) > 0: if layer_name not in layer_filter: continue layer = tile.decoded_data[layer_name] if "extent" in layer: extent = layer["extent"] else: extent = self._DEFAULT_EXTENT tile_features = layer["features"] tile_id = tile.id() for feature in tile_features: if self._is_duplicate_feature(feature, tile) or self.cancel_requested: continue geojson_features, geo_type = self._create_geojson_feature(feature, tile, extent) if geojson_features and len(geojson_features) > 0: name_and_geotype = (layer_name, geo_type) if name_and_geotype not in self.feature_collections_by_layer_name_and_geotype: self.feature_collections_by_layer_name_and_geotype[name_and_geotype] = self._get_empty_feature_collection(tile.zoom_level, layer_name) feature_collection = self.feature_collections_by_layer_name_and_geotype[name_and_geotype] feature_collection["features"].extend(geojson_features) if tile_id not in feature_collection["tiles"]: feature_collection["tiles"].append(tile_id) @staticmethod def _get_feature_class_and_subclass(feature): feature_class = None feature_subclass = None properties = feature["properties"] if "class" in properties: feature_class = properties["class"] if "subclass" in properties: feature_subclass = properties["subclass"] if feature_subclass == feature_class: feature_subclass = None if feature_subclass: assert feature_class, "A feature with a subclass should also have a class" return feature_class, feature_subclass def _create_geojson_feature(self, feature, tile, current_layer_tile_extent): """ Creates a GeoJSON feature for the specified feature """ geo_type = geo_types[feature["type"]] coordinates = feature["geometry"] properties = feature["properties"] properties["_col"] = tile.column properties["_row"] = tile.row if "id" in properties and properties["id"] < 0: properties["id"] = 0 if geo_type == GeoTypes.POINT: coordinates = coordinates[0] properties["_symbol"] = self._get_poi_icon(feature) if self._clip_tiles_at_tile_bounds and not all(0 <= c <= current_layer_tile_extent for c in coordinates): return None, None all_out_of_bounds = [] coordinates = map_coordinates_recursive(coordinates=coordinates, tile_extent=current_layer_tile_extent, mapper_func=lambda coords: VtReader._get_absolute_coordinates(coords, tile, current_layer_tile_extent), all_out_of_bounds_func=lambda out_of_bounds: all_out_of_bounds.append( out_of_bounds)) if self._clip_tiles_at_tile_bounds and all(c is True for c in all_out_of_bounds): return None, None split_geometries = self._loading_options["merge_tiles"] geojson_features = VtReader._create_geojson_feature_from_coordinates(geo_type, coordinates, properties, split_geometries) return geojson_features, geo_type def _get_poi_icon(self, feature): """ * Returns the name of the svg icon that will be applied in QGIS. * The resulting icon is determined based on class and subclass of the specified feature. :param feature: :return: """ feature_class, feature_subclass = self._get_feature_class_and_subclass(feature) root_path = FileHelper.get_icons_directory() class_icon = "{}.svg".format(feature_class) class_subclass_icon = "{}.{}.svg".format(feature_class, feature_subclass) icon_name = "poi.svg" if os.path.isfile(os.path.join(root_path, class_subclass_icon)): icon_name = class_subclass_icon elif os.path.isfile(os.path.join(root_path, class_icon)): icon_name = class_icon return icon_name def _is_duplicate_feature(self, feature, tile): """ * Returns true if the same feature has already been loaded * If the feature has not been loaded, it is marked as loaded by calling this function * A feature is identified by the tuple: (feature_name, feature_class, feature_subclass) * A feature is only loaded if the same feature identifier doesn't occur on the same or a neighbouring tile :param feature: :param tile: :return: """ geo_type = geo_types[feature["type"]] is_poi = geo_type == GeoTypes.POINT is_loaded = False if is_poi and VtReader._feature_name(feature): feature_id = VtReader._feature_id(feature) if feature_id in self._loaded_pois_by_id: locations = self._loaded_pois_by_id[feature_id] for loc in locations: distance_x = math.fabs(loc["col"] - tile.column) distance_y = math.fabs(loc["row"] - tile.row) distance_threshold = 2 is_loaded = distance_x <= distance_threshold and distance_y <= distance_threshold if is_loaded: break if not is_loaded: if feature_id not in self._loaded_pois_by_id: self._loaded_pois_by_id[feature_id] = [] self._loaded_pois_by_id[feature_id].append({'col': tile.column, 'row': tile.row}) return is_loaded @staticmethod def _feature_id(feature): name = VtReader._feature_name(feature) feature_class, feature_subclass = VtReader._get_feature_class_and_subclass(feature) feature_id = (name, feature_class, feature_subclass) return feature_id @staticmethod def _feature_name(feature): """ * Returns the 'name' property of the feature :param feature: :return: """ name = None properties = feature["properties"] if "name" in properties: name = properties["name"] return name @staticmethod def _create_geojson_feature_from_coordinates(geo_type, coordinates, properties, split_multi_geometries): """ * Returns a JSON object that represents a GeoJSON feature :param geo_type: :param coordinates: :param properties: :return: """ assert coordinates is not None all_features = [] coordinate_sets = [coordinates] type_string = geo_type is_multi_geometry = is_multi(geo_type, coordinates) if is_multi_geometry and not split_multi_geometries: type_string = "Multi{}".format(geo_type) elif is_multi_geometry and split_multi_geometries: coordinate_sets = [] for coord_array in coordinates: coordinate_sets.append(coord_array) for c in coordinate_sets: feature_json = { "type": "Feature", "geometry": { "type": type_string, "coordinates": c }, "properties": properties } all_features.append(feature_json) return all_features @staticmethod def _get_absolute_coordinates(coordinates, tile, extent): """ * The coordinates of a geometry, are relative to the tile the feature is located on. * Due to this, we've to get the absolute coordinates of the geometry. """ delta_x = tile.extent[2] - tile.extent[0] delta_y = tile.extent[3] - tile.extent[1] merc_easting = int(tile.extent[0] + delta_x / extent * coordinates[0]) merc_northing = int(tile.extent[1] + delta_y / extent * coordinates[1]) return [merc_easting, merc_northing]
out_path_res = os.path.join(this_file_dir, "res") in_dir = os.path.join(this_file_dir, "data", "csvs") out_dir_res = os.path.join(this_file_dir, "res") out_dir_khiops = os.path.join(this_file_dir, "res", "khiops_res") out_dir_fcasts = os.path.join(this_file_dir, "res", "fcasts") glob_csv_res = os.path.join(out_path_res, "all.csv") vali_csv_res = os.path.join(out_path_res, "valid.csv") import pandas as pd from sklearn.tree import DecisionTreeClassifier from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB # Bunch of cleanings for empty files from file_helper import FileHelper fh = FileHelper(tstmp) fh.clean_zips_folder() fh.clean_res_folder(out_dir_res) fh.ensure_dirs_exist([out_path_res, in_dir, out_dir_res, out_dir_khiops, out_dir_fcasts]) # After cleaning, zip the code which is executed now fh.zip_code() # Init objects from khiops import KhiopsManager km = KhiopsManager() from my_utils import MyUtils utils = MyUtils() from ecml_machine import ECMLMachine
def _create_qgis_layers(self, merge_features, apply_styles): """ * Creates a hierarchy of groups and layers in qgis """ debug("Creating hierarchy in QGIS") self._assure_qgis_groups_exist(sort_layers=apply_styles) qgis_layers = QgsMapLayerRegistry.instance().mapLayers() vt_qgis_name_layer_tuples = filter(lambda (n, l): l.customProperty("vector_tile_source") == self.source.source(), qgis_layers.iteritems()) own_layers = map(lambda (n, l): l, vt_qgis_name_layer_tuples) for l in own_layers: name = l.name() geo_type = l.customProperty("geo_type") if (name, geo_type) not in self.feature_collections_by_layer_name_and_geotype: self._update_layer_source(l.source(), self._get_empty_feature_collection(0, l.name())) self._update_progress(progress=0, max_progress=len(self.feature_collections_by_layer_name_and_geotype), msg="Creating layers...") new_layers = [] count = 0 for layer_name, geo_type in self.feature_collections_by_layer_name_and_geotype: count += 1 if self.cancel_requested: break feature_collection = self.feature_collections_by_layer_name_and_geotype[(layer_name, geo_type)] zoom_level = feature_collection file_name = self._get_geojson_filename(layer_name, geo_type) file_path = FileHelper.get_geojson_file_name(file_name) layer = None if os.path.isfile(file_path): # file exists already. add the features of the collection to the existing collection # get the layer from qgis and update its source layer = self._get_layer_by_source(own_layers, layer_name, file_path) if layer: self._update_layer_source(file_path, feature_collection) if not layer: self._update_layer_source(file_path, feature_collection) layer = self._create_named_layer(file_path, layer_name, zoom_level, merge_features, geo_type) new_layers.append((layer_name, geo_type, layer)) self._update_progress(progress=count+1) QgsMapLayerRegistry.instance().reloadAllLayers() if len(new_layers) > 0: only_layers = list(map(lambda layer_name_tuple: layer_name_tuple[2], new_layers)) QgsMapLayerRegistry.instance().addMapLayers(only_layers, False) for name, geo_type, layer in new_layers: target_group = self._qgis_layer_groups_by_name[name] target_group.addLayer(layer) if apply_styles: count = 0 self._update_progress(progress=0, max_progress=len(new_layers), msg="Styling layers...") for name, geo_type, layer in new_layers: count += 1 if self.cancel_requested: break VtReader._apply_named_style(layer, geo_type) self._update_progress(progress=count)
def _save_recently_used(self): recently_used = FileHelper.get_recently_used_file() with open(recently_used, 'w') as f: for path in self.recently_used: f.write("{}\n".format(path))
def load_data_from_file(self): contents = FileHelper.load_json_file(self.path()) self.road_data = contents['road_data'] self.last_update = parse(contents['last_update'])
def save_states(agent): data = {k: v.tolist() for k, v in agent.states.items()} FileHelper.save(data, 'data/states.json')
def save_data_to_file(self): data = { 'last_update': self.last_update, 'road_data': self.road_data } FileHelper.save_json_file(self.path(), data)
from file_helper import FileHelper #创建程序存储用户输入的个人信息 u_name = input('input your name\n') u_gender = input('input your gender\n') u_age = input('input your age\n') f_h = FileHelper(r'study_11\user_info.txt') f_h.write_append(u_name + '\n') f_h.write_append(u_gender + '\n') f_h.write_append(u_age + '\n') print('==========读取个人信息=========') content = f_h.read() print(content) print('==========序列化存储=========') f_h.remove_content() user = {'name': u_name, 'gender': u_gender, 'age': u_age} f_h.write_to_json(user) obj = f_h.json_to_object() print(obj['name'] + '_' + obj['gender'] + '_' + obj['age'])
def move_old_file(self): date = self.last_update.strftime('%Y%m%d') new_path = 'old_data/' + date + '_' + self.file_name FileHelper.move_file(self.path(), new_path)
def _get_tile_cache_name(self, zoom_level, col, row): return FileHelper.get_cached_tile_file_name(self.source.name(), zoom_level, col, row)
#!/usr/bin/python # -*- coding: utf-8 -*- ''' THIS is USED FOR GENERATE BINGACCOUNT DIMENSSION ''' import maxminddb from file_helper import FileHelper reader = maxminddb.open_database('GeoLite2-Country.mmdb') geo = reader.get('47.88.18.160') reader.close() print(geo) country = geo['country'] countryCode = country['iso_code'] print(countryCode) ipList = FileHelper.loadFileList('d:/tmp/stem/ip_uniq.txt') countryList = [] reader = maxminddb.open_database('GeoLite2-Country.mmdb') for ip in ipList: geo = reader.get(ip) if geo != None and 'country' in geo: country = geo['country'] countryName = geo['country']['names']['en'] countryCode = country['iso_code'] else: countryCode = 'na' countryList.append(countryName) reader.close() FileHelper.saveFileList('d:/tmp/stem/country.txt', countryList, 'w')
class VtReader: cartographic_layer_ordering = [ "place", "mountain_peak", "housenumber", "water_name", "transportation_name", "poi", "boundary", "transportation", "building", "aeroway", "park", "water", "waterway", "landcover", "landuse" ] _layers_to_dissolve = [] _zoom_level_delimiter = "*" _styles = FileHelper.get_styles() def __init__(self, iface, path_or_url, progress_handler): """ * The mbtiles_path can also be an URL in zxy format: z=zoom, x=tile column, y=tile row :param iface: :param path_or_url: """ if not path_or_url: raise RuntimeError("The datasource is required") is_web_source = path_or_url.lower().startswith("http://") or path_or_url.lower().startswith("https://") if is_web_source: self.source = ServerSource(url=path_or_url) else: self.source = MBTilesSource(path=path_or_url) self.source.set_progress_handler(self._update_progress) FileHelper.assure_temp_dirs_exist() self.iface = iface self.cartographic_ordering_enabled = True self.progress_handler = progress_handler self.feature_collections_by_layer_path = {} self._qgis_layer_groups_by_name = {} self.cancel_requested = False self._loaded_pois_by_id = {} def _update_progress(self, title=None, show_dialog=None, progress=None, max_progress=None, msg=None): if self.progress_handler: self.progress_handler(title, progress, max_progress, msg, show_dialog) def _get_empty_feature_collection(self, zoom_level, layer_name): """ * Returns an empty GeoJSON FeatureCollection with the coordinate reference system (crs) set to EPSG3857 """ # todo: when improving CRS handling: the correct CRS of the source has to be set here source_crs = self.source.crs() if source_crs: epsg_id = get_code_from_epsg(source_crs) else: epsg_id = 3857 crs = { "type": "name", "properties": { "name": "urn:ogc:def:crs:EPSG::{}".format(epsg_id)}} return { "tiles": [], "source": self.source.name(), "scheme": self.source.scheme(), "layer": layer_name, "zoom_level": zoom_level, "type": "FeatureCollection", "crs": crs, "features": []} def cancel(self): """ * Cancels the loading process. :return: """ self.cancel_requested = True if self.source: self.source.cancel() def _get_tile_cache_name(self, zoom_level, col, row): return FileHelper.get_cached_tile_file_name(self.source.name(), zoom_level, col, row) def load_tiles(self, zoom_level, layer_filter, load_mask_layer=False, merge_tiles=True, apply_styles=True, max_tiles=None, bounds=None, limit_reacher_handler=None): """ * Loads the vector tiles from either a file or a URL and adds them to QGIS :param zoom_level: The zoom level to load :param layer_filter: A list of layers. If any layers are set, only these will be loaded. If the list is empty, all available layers will be loaded :param load_mask_layer: If True the mask layer will also be loaded :param merge_tiles: If True neighbouring tiles and features will be merged :param apply_styles: If True the default styles will be applied :param max_tiles: The maximum number of tiles to load :param bounds: :param limit_reacher_handler: :return: """ self.cancel_requested = False self.feature_collections_by_layer_path = {} self._qgis_layer_groups_by_name = {} self._update_progress(show_dialog=True, title="Loading '{}'".format(os.path.basename(self.source.name()))) min_zoom = self.source.min_zoom() max_zoom = self.source.max_zoom() if min_zoom is not None and zoom_level < min_zoom: zoom_level = min_zoom if max_zoom is not None and zoom_level > max_zoom: zoom_level = max_zoom all_tiles = get_all_tiles(bounds, lambda: self.cancel_requested) tiles_to_load = set() tiles = [] for t in all_tiles: if self.cancel_requested: break file_name = self._get_tile_cache_name(zoom_level, t[0], t[1]) tile = FileHelper.get_cached_tile(file_name) if tile and tile.decoded_data: tiles.append(tile) else: tiles_to_load.add(t) debug("{} cache hits. {} will be loaded from the source.", len(tiles), len(tiles_to_load)) debug("Loading extent {} for zoom level '{}' of: {}", zoom_level, self.source.name()) tile_data_tuples = [] if len(tiles_to_load) > 0: tile_data_tuples = self.source.load_tiles(zoom_level=zoom_level, tiles_to_load=tiles_to_load, max_tiles=max_tiles, for_each=QApplication.processEvents, limit_reacher_handler=limit_reacher_handler) if len(tiles) == 0 and (not tile_data_tuples or len(tile_data_tuples) == 0): QMessageBox.information(None, "No tiles found", "What a pity, no tiles could be found!") if load_mask_layer: mask_level = self.source.mask_level() if mask_level is not None and mask_level != zoom_level: debug("Mapping {} tiles to mask level", len(all_tiles)) scheme = self.source.scheme() crs = self.source.crs() mask_tiles = map( lambda t: change_zoom(zoom_level, int(mask_level), t, scheme, crs), all_tiles) debug("Mapping done") mask_tiles_to_load = set() for t in mask_tiles: file_name = self._get_tile_cache_name(mask_level, t[0], t[1]) tile = FileHelper.get_cached_tile(file_name) if tile and tile.decoded_data: tiles.append(tile) else: mask_tiles_to_load.add(t) debug("Loading mask layer (zoom_level={})", mask_level) tile_data_tuples = [] if len(mask_tiles_to_load) > 0: mask_layer_data = self.source.load_tiles(zoom_level=mask_level, tiles_to_load=mask_tiles_to_load, max_tiles=max_tiles, for_each=QApplication.processEvents) debug("Mask layer loaded") tile_data_tuples.extend(mask_layer_data) if tile_data_tuples and len(tile_data_tuples) > 0: if not self.cancel_requested: decoded_tiles = self._decode_tiles(tile_data_tuples) tiles.extend(decoded_tiles) if len(tiles) > 0: if not self.cancel_requested: self._process_tiles(tiles, layer_filter) if not self.cancel_requested: self._create_qgis_layers(merge_features=merge_tiles, apply_styles=apply_styles) self._update_progress(show_dialog=False) if self.cancel_requested: info("Import cancelled") else: info("Import complete") def _decode_tiles(self, tiles_with_encoded_data): """ * Decodes the PBF data from all the specified tiles and reports the progress * If a tile is loaded from the cache, the decoded_data is already set and doesn't have to be encoded :param tiles_with_encoded_data: :return: """ total_nr_tiles = len(tiles_with_encoded_data) info("Decoding {} tiles", total_nr_tiles) self._update_progress(progress=0, max_progress=100, msg="Decoding tiles...") nr_processors = 4 try: nr_processors = mp.cpu_count() except NotImplementedError: info("CPU count cannot be retrieved. Falling back to default = 4") tiles_with_encoded_data = map(lambda t: (t[0], self._unzip(t[1])), tiles_with_encoded_data) pool = mp.Pool(nr_processors) tiles = [] rs = pool.map_async(decode_tile, tiles_with_encoded_data, callback=tiles.extend) pool.close() current_progress = 0 while not rs.ready() and not self.cancel_requested: QApplication.processEvents() remaining = rs._number_left index = total_nr_tiles - remaining progress = int(100.0 / total_nr_tiles * (index + 1)) if progress != current_progress: current_progress = progress self._update_progress(progress=progress) for t in tiles: cache_file_name = self._get_tile_cache_name(t.zoom_level, t.column, t.row) if not os.path.isfile(cache_file_name): FileHelper.cache_tile(t, cache_file_name) return tiles def _unzip(self, data): """ * If the passed data is gzipped, it will be unzipped. Otherwise it will be returned untouched :param data: :return: """ is_gzipped = FileHelper.is_gzipped(data) if is_gzipped: file_content = GzipFile('', 'r', 0, StringIO(data)).read() else: file_content = data return file_content def _process_tiles(self, tiles, layer_filter): """ * Creates GeoJSON for all the specified tiles and reports the progress :param tiles: :return: """ total_nr_tiles = len(tiles) info("Processing {} tiles", total_nr_tiles) self._update_progress(progress=0, max_progress=100, msg="Processing features...") current_progress = -1 for index, tile in enumerate(tiles): QApplication.processEvents() if self.cancel_requested: break self._create_geojson(tile, layer_filter) progress = int(100.0 / total_nr_tiles * (index + 1)) if progress != current_progress: current_progress = progress self._update_progress(progress=progress) def _get_cartographic_layer_sort_id(self, layer_name): """ * Returns the cartographic sort id for the specified layer. * This sort id is the position of the layer in the cartographic_layer_ordering collection. * If the layer isn't present in the collection, the sort id wil be 999 and therefore the layer will be added at the bottom. :param layer_name: :return: """ sort_id = 999 if layer_name in self.cartographic_layer_ordering: sort_id = self.cartographic_layer_ordering.index(layer_name) return sort_id def _assure_qgis_groups_exist(self): """ * Createss a group for each layer that is given by the layer source scheme >> mbtiles: value 'JSON' in metadata table, array 'vector_layers' >> TileJSON: value 'vector_layers' :return: """ root = QgsProject.instance().layerTreeRoot() root_group = root.findGroup(self.source.name()) if not root_group: root_group = root.addGroup(self.source.name()) layers = map(lambda l: l["id"], self.source.vector_layers()) layers = sorted(layers, key=lambda x: self._get_cartographic_layer_sort_id(x)) for index, layer_name in enumerate(layers): group = root_group.findGroup(layer_name) if not group: group = root_group.addGroup(layer_name) self._qgis_layer_groups_by_name[layer_name] = group def _get_geojson_filename(self, layer_name, geo_type, zoom_level): return "{}.{}.{}.{}".format(self.source.name(), layer_name, geo_type, zoom_level) def _create_qgis_layers(self, merge_features, apply_styles): """ * Creates a hierarchy of groups and layers in qgis """ debug("Creating hierarchy in QGIS") self._assure_qgis_groups_exist() self._update_progress(progress=0, max_progress=len(self.feature_collections_by_layer_path), msg="Creating layers...") layers = [] for index, layer_name_and_type in enumerate(self.feature_collections_by_layer_path): layer_name_and_zoom = layer_name_and_type[0] geo_type = layer_name_and_type[1] layer_name = layer_name_and_zoom.split(VtReader._zoom_level_delimiter)[0] zoom_level = layer_name_and_zoom.split(VtReader._zoom_level_delimiter)[1] QApplication.processEvents() if self.cancel_requested: break target_group = self._qgis_layer_groups_by_name[layer_name] feature_collections_by_tile_coord = self.feature_collections_by_layer_path[layer_name_and_type] file_name = self._get_geojson_filename(layer_name, geo_type, zoom_level) file_path = FileHelper.get_geojson_file_name(file_name) layer = None if os.path.isfile(file_path): # file exists already. add the features of the collection to the existing collection # get the layer from qgis and update its source layer = self._get_layer_by_source(layer_name_and_zoom, file_path) if layer: self._update_layer_source(file_path, feature_collections_by_tile_coord) layer.reload() if not layer: complete_collection = self._get_empty_feature_collection(zoom_level, layer_name) self._merge_feature_collections(current_feature_collection=complete_collection, feature_collections_by_tile_coord=feature_collections_by_tile_coord) with open(file_path, "w") as f: f.write(json.dumps(complete_collection)) layer = self._add_vector_layer_to_qgis(file_path, layer_name, zoom_level, target_group, merge_features, geo_type) if apply_styles: layers.append((layer_name_and_type, layer)) self._update_progress(progress=index+1) if apply_styles: self._update_progress(progress=0, max_progress=len(layers), msg="Styling layers...") for index, layer_path_tuple in enumerate(layers): QApplication.processEvents() if self.cancel_requested: break path_and_type = layer_path_tuple[0] geo_type = path_and_type[1] layer = layer_path_tuple[1] VtReader._apply_named_style(layer, geo_type) self._update_progress(progress=index+1) @staticmethod def _update_layer_source(layer_source, feature_collections_by_tile_coord): """ * Updates the layers GeoJSON source file :param layer_source: :param feature_collections_by_tile_coord: :return: """ with open(layer_source, "r") as f: current_feature_collection = json.load(f) VtReader._merge_feature_collections(current_feature_collection, feature_collections_by_tile_coord) if current_feature_collection: with open(layer_source, "w") as f: json.dump(current_feature_collection, f) @staticmethod def _merge_feature_collections(current_feature_collection, feature_collections_by_tile_coord): """ * Merges the features of multiple tiles into the current_feature_collection if not already present. :param current_feature_collection: :param feature_collections_by_tile_coord: :return: """ for tile_coord in feature_collections_by_tile_coord: if tile_coord not in current_feature_collection["tiles"]: feature_collection = feature_collections_by_tile_coord[tile_coord] current_feature_collection["tiles"].extend(feature_collection["tiles"]) current_feature_collection["features"].extend(feature_collection["features"]) @staticmethod def _get_layer_by_source(layer_name, layer_source_file): """ * Returns the layer from QGIS whose name and layer_source matches the specified parameters :param layer_name: :param layer_source_file: :return: """ matching_layer = None layers = QgsMapLayerRegistry.instance().mapLayersByName(layer_name) for l in layers: if l.source() == layer_source_file: matching_layer = l break return matching_layer @staticmethod def _apply_named_style(layer, geo_type): """ * Looks for a styles with the same name as the layer and if one is found, it is applied to the layer :param layer: :param layer_path: e.g. 'transportation.service' or 'transportation_name.path' :return: """ try: name = layer.name().split(VtReader._zoom_level_delimiter)[0].lower() styles = [ "{}.{}".format(name, geo_type.lower()), name ] for p in styles: style_name = "{}.qml".format(p).lower() if style_name in VtReader._styles: style_path = os.path.join(FileHelper.get_plugin_directory(), "styles/{}".format(style_name)) res = layer.loadNamedStyle(style_path) if res[1]: # Style loaded layer.setCustomProperty("layerStyle", style_path) if layer.customProperty("layerStyle") == style_path: debug("Style successfully applied: {}", style_name) break except: critical("Loading style failed: {}", sys.exc_info()) def _add_vector_layer_to_qgis(self, json_src, layer_name, zoom_level, layer_target_group, merge_features, geo_type): """ * Creates a QgsVectorLayer and adds it to the group specified by layer_target_group * Invalid geometries will be removed during the process of merging features over tile boundaries """ layer_with_zoom = "{}{}{}".format(layer_name, VtReader._zoom_level_delimiter, zoom_level) layer = QgsVectorLayer(json_src, layer_with_zoom, "ogr") if merge_features and geo_type in [GeoTypes.LINE_STRING, GeoTypes.POLYGON]: layer = FeatureMerger().merge_features(layer) layer.setName(layer_name) QgsMapLayerRegistry.instance().addMapLayer(layer, False) layer_target_group.addLayer(layer) layer.setCustomProperty("vector_tile_source", self.source.source()) layer.setShortName(layer_name) layer.setDataUrl(self.source.source()) if self.source.name() and "openmaptiles" in self.source.name().lower(): layer.setDataUrl(remove_key(self.source.source())) layer.setAttribution(u"Vector Tiles © Klokan Technologies GmbH (CC-BY), Data © OpenStreetMap contributors (ODbL)") layer.setAttributionUrl("https://openmaptiles.com/hosting/") return layer def _create_geojson(self, tile, layer_filter): """ * Transforms all features of the specified tile into GeoJSON * The resulting GeoJSON feature will be applied to the features of the corresponding GeoJSON FeatureCollection :param tile: :return: """ for layer_name in tile.decoded_data: if layer_filter and len(layer_filter) > 0: if layer_name not in layer_filter: continue tile_features = tile.decoded_data[layer_name]["features"] tile_id = tile.id() feature_path = "{}{}{}".format(layer_name, VtReader._zoom_level_delimiter, tile.zoom_level) for index, feature in enumerate(tile_features): if self._is_duplicate_feature(feature, tile): continue geojson_feature, geo_type = self._create_geojson_feature(feature, tile) if geojson_feature: path_and_type = (feature_path, geo_type) if path_and_type not in self.feature_collections_by_layer_path: self.feature_collections_by_layer_path[path_and_type] = {} collection_dict = self.feature_collections_by_layer_path[path_and_type] if tile_id not in collection_dict: collection_dict[tile_id] = self._get_empty_feature_collection(tile.zoom_level, layer_name) collection = collection_dict[tile_id] collection["features"].append(geojson_feature) if tile_id not in collection["tiles"]: collection["tiles"].append(tile_id) geotypes_to_dissolve = [GeoTypes.POLYGON, GeoTypes.LINE_STRING] if geo_type in geotypes_to_dissolve and feature_path not in self._layers_to_dissolve: self._layers_to_dissolve.append(feature_path) @staticmethod def _get_feature_class_and_subclass(feature): feature_class = None feature_subclass = None properties = feature["properties"] if "class" in properties: feature_class = properties["class"] if "subclass" in properties: feature_subclass = properties["subclass"] if feature_subclass == feature_class: feature_subclass = None if feature_subclass: assert feature_class, "A feature with a subclass should also have a class" return feature_class, feature_subclass def _create_geojson_feature(self, feature, tile): """ Creates a GeoJSON feature for the specified feature """ geo_type = geo_types[feature["type"]] coordinates = feature["geometry"] properties = feature["properties"] if geo_type == GeoTypes.POINT: coordinates = coordinates[0] properties["_symbol"] = self._get_poi_icon(feature) if not all(0 <= c <= tile_extent for c in coordinates): return None, None all_out_of_bounds = [] coordinates = map_coordinates_recursive(coordinates=coordinates, mapper_func=lambda coords: VtReader._get_absolute_coordinates(coords, tile), all_out_of_bounds_func=lambda out_of_bounds: all_out_of_bounds.append( out_of_bounds)) if all(c is True for c in all_out_of_bounds): return None, None feature_json = VtReader._create_geojson_feature_from_coordinates(geo_type, coordinates, properties) return feature_json, geo_type def _get_poi_icon(self, feature): """ * Returns the name of the svg icon that will be applied in QGIS. * The resulting icon is determined based on class and subclass of the specified feature. :param feature: :return: """ feature_class, feature_subclass = self._get_feature_class_and_subclass(feature) root_path = FileHelper.get_icons_directory() class_icon = "{}.svg".format(feature_class) class_subclass_icon = "{}.{}.svg".format(feature_class, feature_subclass) icon_name = "poi.svg" if os.path.isfile(os.path.join(root_path, class_subclass_icon)): icon_name = class_subclass_icon elif os.path.isfile(os.path.join(root_path, class_icon)): icon_name = class_icon return icon_name def _is_duplicate_feature(self, feature, tile): """ * Returns true if the same feature has already been loaded * If the feature has not been loaded, it is marked as loaded by calling this function * A feature is identified by the tuple: (feature_name, feature_class, feature_subclass) * A feature is only loaded if the same feature identifier doesn't occur on the same or a neighbouring tile :param feature: :param tile: :return: """ geo_type = geo_types[feature["type"]] is_poi = geo_type == GeoTypes.POINT is_loaded = False if is_poi and VtReader._feature_name(feature): feature_id = VtReader._feature_id(feature) if feature_id in self._loaded_pois_by_id: locations = self._loaded_pois_by_id[feature_id] for loc in locations: distance_x = math.fabs(loc["col"] - tile.column) distance_y = math.fabs(loc["row"] - tile.row) distance_threshold = 2 is_loaded = distance_x <= distance_threshold and distance_y <= distance_threshold if is_loaded: break if not is_loaded: if feature_id not in self._loaded_pois_by_id: self._loaded_pois_by_id[feature_id] = [] self._loaded_pois_by_id[feature_id].append({'col': tile.column, 'row': tile.row}) return is_loaded @staticmethod def _feature_id(feature): name = VtReader._feature_name(feature) feature_class, feature_subclass = VtReader._get_feature_class_and_subclass(feature) feature_id = (name, feature_class, feature_subclass) return feature_id @staticmethod def _feature_name(feature): """ * Returns the 'name' property of the feature :param feature: :return: """ name = None properties = feature["properties"] if "name" in properties: name = properties["name"] return name @staticmethod def _create_geojson_feature_from_coordinates(geo_type, coordinates, properties): """ * Returns a JSON object that represents a GeoJSON feature :param geo_type: :param coordinates: :param properties: :return: """ type_string = geo_type if is_multi(geo_type, coordinates): type_string = "Multi{}".format(geo_type) feature_json = { "type": "Feature", "geometry": { "type": type_string, "coordinates": coordinates }, "properties": properties } return feature_json @staticmethod def _get_absolute_coordinates(coordinates, tile): """ * The coordinates of a geometry, are relative to the tile the feature is located on. * Due to this, we've to get the absolute coordinates of the geometry. """ delta_x = tile.extent[2] - tile.extent[0] delta_y = tile.extent[3] - tile.extent[1] merc_easting = int(tile.extent[0] + delta_x / tile_extent * coordinates[0]) merc_northing = int(tile.extent[1] + delta_y / tile_extent * coordinates[1]) return [merc_easting, merc_northing] def enable_cartographic_ordering(self, enabled): self.cartographic_ordering_enabled = enabled
def OnSaveSettingInfoButtonClick(self, event): setting_info_dict = self.get_setting_info() FileHelper.write_setting_info(setting_info_dict) self.confirm_dialog(u"保存设置成功!") event.Skip()