def __init__(self, templates_dir, images_dir, results_dir = './results'):
self.templates_dir = templates_dir
self.images_dir = images_dir
self.results_dir = results_dir
if templates_dir is None or not os.path.exists(templates_dir):
logging.error("Invalid path: {0}".format(templates_dir))
sys.exit(1)
if images_dir is None or not os.path.exists(images_dir):
logging.error("Invalid path: {0}".format(images_dir))
sys.exit(1)
FileHelper.create_or_clear_dir(results_dir)
# # try:
# if os.path.exists(results_dir):
# FileHelper.remove_files_in_dir(results_dir)
# else:
# os.makedirs(results_dir)
# # except OSError:
# # pass
# self.detector = cv2.SIFT(3200)
self.detector = cv2.SIFT(900, edgeThreshold=30)
# self.detector = cv2.SURF(300)
# self.detector = cv2.SURF(30, upright=1)
# self.detector = cv2.BRISK()
# matcher = cv2.BFMatcher(cv2.NORM_L2)
FLANN_INDEX_KDTREE = 0
index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=15)
search_params = dict(checks=50)
self.matcher = cv2.FlannBasedMatcher(index_params, search_params)
self.templatesFeatures = None
def images_to_csv_file(
self, dir_positive, dir_negative, filename_positive="positive.csv", filename_negative="negative.csv"
):
positive_images = FileHelper.read_images_in_dir(dir_positive)
negative_images = FileHelper.read_images_in_dir(dir_negative)
if len(negative_images) > len(positive_images) * 3:
negative_images = negative_images[: len(positive_images) * 3]
# print "cut negative images"
with open(filename_positive, "wb") as f:
for filename in positive_images:
im = imread(os.path.join(dir_positive, filename), True)
f.write(self.__image_to_csv_string(im, 1))
f.write("\n")
with open(filename_negative, "w+b") as f:
for filename in negative_images:
im = imread(os.path.join(dir_negative, filename), True)
f.write(self.__image_to_csv_string(im, 0))
f.write("\n")
def __getFeaturesFromTemplates(self):
features = {}
images = FileHelper.read_images_in_dir(self.templates_dir)
for filename in images:
im = cv2.imread(os.path.join(self.templates_dir, filename), cv2.IMREAD_GRAYSCALE)
kp, desc = self.detector.detectAndCompute(im, None)
logging.debug('{0} - {1} features'.format(filename, len(kp)))
# features.append((kp, desc))
features[filename] = (kp, desc)
return features
def resize_images(self, dir_in, dir_out, h_size, v_size, make_grey=True):
image_files = FileHelper.read_images_in_dir(dir_in)
i = 0
for filename in image_files:
print filename.split(".")[0] + "_%04d.jpg"
# im = imread(os.path.join(in_dir, filename), as_grey= True)
im = resize(imread(os.path.join(dir_in, filename)), (h_size, v_size))
if make_grey:
im = rgb2gray(im)
imsave(os.path.join(dir_out, "image%05d.jpg" % i), im)
i += 1
def cut_frames_from_video(path, videofile):
videofile_abspath = os.path.join(path,videofile)
if os.path.exists(videofile_abspath) == False:
logging.error("File "+ videofile_abspath + " doesn't exists or permission denied")
return None
parts = videofile.split('.')
frames_path = os.path.join(path, parts[0])
if os.path.exists(frames_path):
FileHelper.remove_files_in_dir(frames_path)
else:
os.mkdir(frames_path)
try:
images_template = frames_path + "/{0}_%05d.jpg".format(parts[0][:10])
#ffmpeg -i input.flv -f image2 -vf fps=fps=1 out%04d.png
cmd = "ffmpeg -i {0} -f image2 -vf fps=fps=1 {1}".format(videofile_abspath, images_template)
logging.debug(cmd)
os.system(cmd)
return 1
except:
logging.error(sys.exc_info())
return None
def __init__(self, root_url): # 待爬取的数据不限长度 self.url_queue = Queue.Queue(maxsize = -1) # url过滤器 self.url_filter = set() # 已爬取过的url self.old_urls = set() # 读取配置文件中被排除的url self.exclude_urls = CommonUtil.get_dict_value(FileHelper.read_setting_info(), "exclude_url").split() for exclude_url in self.exclude_urls: self.url_filter.add(exclude_url) self.domain = re.match(r"^(http(s)?://)?([\w-]+\.)+[\w-]+/?",root_url,re.M|re.I).group() self.__add_new_url(SpiderUrl(root_url, 0))
def slice_images(self, dir_in, dir_out, h_size, v_size):
"""
1. Read images from directory and slice they into small images
"""
# test
# A = np.arange(120).reshape((10, 12))
# print A
# slice_image_to_small_images(A, out_dir+'image_%04d.jpg')
image_files = FileHelper.read_images_in_dir(dir_in)
i = 0
for filename in image_files:
print filename.split(".")[0] + "_%04d.jpg"
# im = imread(os.path.join(dir_in, filename))
# im = rgb2gray(im)
im = Image(os.path.join(dir_in, filename))
im.prepare()
self.__slice_image_to_small_images(
im.image, os.path.join(dir_out, filename.split(".")[0] + "_%04d.jpg"), h_size, v_size
)
def load_tiles(self, zoom_level, layer_filter, load_mask_layer=False, merge_tiles=True, apply_styles=True, max_tiles=None,
bounds=None, limit_reacher_handler=None):
"""
* Loads the vector tiles from either a file or a URL and adds them to QGIS
:param zoom_level: The zoom level to load
:param layer_filter: A list of layers. If any layers are set, only these will be loaded. If the list is empty,
all available layers will be loaded
:param load_mask_layer: If True the mask layer will also be loaded
:param merge_tiles: If True neighbouring tiles and features will be merged
:param apply_styles: If True the default styles will be applied
:param max_tiles: The maximum number of tiles to load
:param bounds:
:param limit_reacher_handler:
:return:
"""
self.cancel_requested = False
self.feature_collections_by_layer_path = {}
self._qgis_layer_groups_by_name = {}
self._update_progress(show_dialog=True, title="Loading '{}'".format(os.path.basename(self.source.name())))
min_zoom = self.source.min_zoom()
max_zoom = self.source.max_zoom()
if min_zoom is not None and zoom_level < min_zoom:
zoom_level = min_zoom
if max_zoom is not None and zoom_level > max_zoom:
zoom_level = max_zoom
all_tiles = get_all_tiles(bounds, lambda: self.cancel_requested)
tiles_to_load = set()
tiles = []
for t in all_tiles:
if self.cancel_requested:
break
file_name = self._get_tile_cache_name(zoom_level, t[0], t[1])
tile = FileHelper.get_cached_tile(file_name)
if tile and tile.decoded_data:
tiles.append(tile)
else:
tiles_to_load.add(t)
debug("{} cache hits. {} will be loaded from the source.", len(tiles), len(tiles_to_load))
debug("Loading extent {} for zoom level '{}' of: {}", zoom_level, self.source.name())
tile_data_tuples = []
if len(tiles_to_load) > 0:
tile_data_tuples = self.source.load_tiles(zoom_level=zoom_level,
tiles_to_load=tiles_to_load,
max_tiles=max_tiles,
for_each=QApplication.processEvents,
limit_reacher_handler=limit_reacher_handler)
if len(tiles) == 0 and (not tile_data_tuples or len(tile_data_tuples) == 0):
QMessageBox.information(None, "No tiles found", "What a pity, no tiles could be found!")
if load_mask_layer:
mask_level = self.source.mask_level()
if mask_level is not None and mask_level != zoom_level:
debug("Mapping {} tiles to mask level", len(all_tiles))
scheme = self.source.scheme()
crs = self.source.crs()
mask_tiles = map(
lambda t: change_zoom(zoom_level, int(mask_level), t, scheme, crs),
all_tiles)
debug("Mapping done")
mask_tiles_to_load = set()
for t in mask_tiles:
file_name = self._get_tile_cache_name(mask_level, t[0], t[1])
tile = FileHelper.get_cached_tile(file_name)
if tile and tile.decoded_data:
tiles.append(tile)
else:
mask_tiles_to_load.add(t)
debug("Loading mask layer (zoom_level={})", mask_level)
tile_data_tuples = []
if len(mask_tiles_to_load) > 0:
mask_layer_data = self.source.load_tiles(zoom_level=mask_level,
tiles_to_load=mask_tiles_to_load,
max_tiles=max_tiles,
for_each=QApplication.processEvents)
debug("Mask layer loaded")
tile_data_tuples.extend(mask_layer_data)
if tile_data_tuples and len(tile_data_tuples) > 0:
if not self.cancel_requested:
decoded_tiles = self._decode_tiles(tile_data_tuples)
tiles.extend(decoded_tiles)
if len(tiles) > 0:
if not self.cancel_requested:
self._process_tiles(tiles, layer_filter)
if not self.cancel_requested:
self._create_qgis_layers(merge_features=merge_tiles,
apply_styles=apply_styles)
self._update_progress(show_dialog=False)
if self.cancel_requested:
info("Import cancelled")
else:
info("Import complete")
def __init__(self):
self.YOUTUBE_VIDEO_ID_LIST = LocalSettingsLoader(
).LOCAL_SETTINGS['YOUTUBE_VIDEO_ID_LIST']
self.file_helper_obj = FileHelper()
self.shell_executor = ShellExecutor()
def load_states():
data = FileHelper.load('data/states.json')
return {k: np.array(v) for k, v in data.items()}
def copy_button_clicked(self):
self.copy_button['text'] = 'Loading...'
self.copy_button['state'] = tk.DISABLED
parser = Parser()
url = self.url.get()
directory = self.directory.get()
css_directory = directory + self.css_path
js_directory = directory + self.js_path
if FileHelper.directory_exists(directory):
parser.set_url(url)
try:
content = parser.get_content()
self.write_log("Received HTML page from " + url)
FileHelper.create_file('index.html', directory, content.decode("UTF-8"))
self.write_log("Saved HTML page to " + directory + '/index.html')
FileHelper.delete_folder_if_exists(css_directory)
FileHelper.create_path_recursively(css_directory)
self.write_log("Created directory for saving CSS stylesheets")
link_nodes = parser.get_nodes('link')
css_regexp = re.compile("\.css$", re.IGNORECASE)
for node in link_nodes:
link = Parser.form_script_url(url, node.get('href'))
link = str(link)
if css_regexp.search(link):
script_name = re.findall('\w+\.css', link, re.IGNORECASE)
if len(script_name) > 0:
script_name = script_name[0]
else:
script_name = uuid.uuid4().hex
css_content = Parser.get_content_from_url(link)
if css_content:
FileHelper.create_file(script_name, directory + self.css_path, css_content.decode("UTF-8"))
self.write_log("Saved CSS - " + self.css_path + script_name)
else:
self.write_log("Can't get content from - " + link)
FileHelper.delete_folder_if_exists(js_directory)
FileHelper.create_path_recursively(js_directory)
self.write_log("Created directory for saving JS scripts")
script_nodes = parser.get_nodes('script')
js_regexp = re.compile("\.js", re.IGNORECASE)
for node in script_nodes:
src = node.get('src')
link = None
if src:
link = Parser.form_script_url(url, src)
link = str(link)
if js_regexp.search(link):
script_name = re.findall('\w+\.js', str(link), re.IGNORECASE);
if len(script_name) > 0:
script_name = script_name[0]
else:
script_name = uuid.uuid4().hex
js_content = Parser.get_content_from_url(link)
if js_content:
FileHelper.create_file(script_name, directory + self.js_path, js_content.decode("UTF-8"))
self.write_log("Saved JS - " + self.js_path + script_name)
else:
self.write_log("Can't get content from - " + link)
except Exception as e:
self.write_log("ERROR")
self.write_log(e)
self.show_error_message(e)
else:
self.write_log("ERROR")
self.write_log("Directory '" + directory + "' not exists")
self.show_error_message("Directory '" + directory + "' not exists")
self.copy_button['text'] = 'Start'
self.copy_button['state'] = tk.NORMAL
class ECMLMachine:
def __init__(self, file_name):
self.utils_cl = MyUtils()
self.this_file_dir = os.path.dirname(os.path.realpath(__file__))
self.fh = FileHelper()
self.hm = HmmMachine()
self.cm = ClusteringMachine()
self.pm = PredictMachine()
self.my_metric = "euclidean"
self.file_name = file_name
self.out_fcasts_f = os.path.join(self.this_file_dir, "res", "fcasts", file_name, "ecml")
self.fh.ensure_dirs_exist([self.out_fcasts_f])
logging.info("Instantiated ECML_operator")
def get_results_univ(self, df, mean_day, n_pt_one_period, n_serie_concatenated = 1):
# Create datasets in the format we need
logging.info("Computing ECML results")
logging.info("Find best number of cluster")
(df_train, df_valid, df_test) = self.utils_cl.app_valid_test(df, n_pt_one_period)
(df_train_compar, df_test_compar) = self.utils_cl.app_test(df, n_pt_one_period)
last_day = df_train_compar["val_"][-len(df_test_compar):]
# run algos using df_valid to find the better num of kmeans tb-used
mean_mse = self.do_it(
[2, 3, 4, 5, 6, 7, 8, 9, 10, 20, 40, 60, 70, 80, 100, 150, 200],
df_train, df_valid, mean_day,
n_pt_one_period)
best_num_of_kmean = min(mean_mse, key = lambda t: t[1])[0]
logging.info("Found best number of cluster: %s", best_num_of_kmean)
# run algos using df_test with previously found best kmean
res = self.do_it(
[best_num_of_kmean],
df_train, df_test, mean_day,
n_pt_one_period)
# Retrieve results
(mse_colin, mse_fake, mse_mean) = (res[0][1], res[0][2], res[0][3])
(mae_colin, mae_fake, mae_mean) = (res[0][4], res[0][5], res[0][6])
(mase_colin, mase_fake, mase_mean) = (res[0][7], res[0][8], res[0][9])
(std_mse_colin, std_mse_fake, std_mse_mean) = (res[0][10], res[0][11], res[0][12])
(std_mae_colin, std_mae_fake, std_mae_mean) = (res[0][13], res[0][14], res[0][15])
(std_mase_colin, std_mase_fake, std_mase_mean) = (res[0][16], res[0][17], res[0][18])
logging.info("Retrieve results")
# Compute baselines
logging.info("Compute baselines")
(_, mse_ar_error, mae_ar_error, mase_ar_error) = self.pm.do_forecast_ar_model(
last_day, df_train_compar["val_"], df_test_compar["val_"])
(mse_hw, mae_hw, mase_hw) = -1, -1, -1
return (
mse_colin, mse_fake, mse_mean,
mae_colin, mae_fake, mae_mean,
mase_colin, mase_fake, mase_mean,
mse_ar_error, mae_ar_error, mase_ar_error,
mse_hw, mae_hw, mase_hw,
best_num_of_kmean,
std_mse_colin, std_mse_fake, std_mse_mean,
std_mae_colin, std_mae_fake, std_mae_mean,
std_mase_colin, std_mase_fake, std_mase_mean
)
def do_it(self,
km_sizes,
df_train, df_valid_ou_test, mean_day,
n_pt_one_period, n_serie_concatenated = 1):
mean_mse = []
for my_km_size in km_sizes:
logging.info("ECML with km_size of %s", my_km_size)
n_day_found_train = len(df_train["n_day_"].unique())
logging.debug("There are %s days in train", n_day_found_train)
#################################
# I. TRAIN
#################################
(km, comprehensive_clusters_df_train) = self.cm.do_kmeans_wrapper(
df_train,
km_size = my_km_size
)
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# MM
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# compute MMs
raw_hmm_data_df_train = list(
self.hm.compute_raw_hmm(comprehensive_clusters_df_train, order = 1))
# compute transition matrix
transition_mat = self.hm.compute_hmm_transition_mat_1d(
raw_hmm_data_df_train, my_km_size)
#################################
# II. VALID
#################################
# 1. apply km on df_valid data
y_pred = self.cm.apply_clustering(df_valid_ou_test, km)
# create tuple of known/wanted:
# a) for ts data itself
s_arr=[]
for c in range(df_valid_ou_test["n_day_"].min() + 1, df_valid_ou_test["n_day_"].max() - 1):
s_arr.append(
(
df_valid_ou_test[df_valid_ou_test["n_day_"] == c - 1]["val_"].values,
df_valid_ou_test[df_valid_ou_test["n_day_"] == c]["val_"].values,
df_valid_ou_test[df_valid_ou_test["n_day_"] <= c]["val_"].values
)
)
# b) for ts labels
s_l_arr=[]
for c in range(1, len(y_pred) - 1):
s_l_arr.append((y_pred[c - 1], y_pred[c]))
precision_colin = []
precision_fake = []
precision_mean = []
precision_colin_mae = []
precision_fake_mae = []
precision_mean_mae = []
precision_colin_mase = []
precision_fake_mase = []
precision_mean_mase = []
# compute predictions and mse
for count in range(0, len(s_arr)):
path_count = os.path.join(self.out_fcasts_f, str(count) + "_fcast_" + str(my_km_size) + "_kmsize.csv")
(known, guess, before_w) = s_arr[count]
known_w = np.pad(known, (0,len(before_w) - len(known)), "constant", constant_values=-42.42)
guess_w = np.pad(guess, (0,len(before_w) - len(guess)), "constant", constant_values=-42.42)
(known_l, guess_l) = s_l_arr[count]
pd.DataFrame({"known": known_w[:], "guess": guess_w[:], "before": before_w[:]}) .to_csv(path_count, sep = ";", index_label = False, index = False)
pred = self.pm.predict_median_hmm(known_l, transition_mat, km)
pred_fake = self.pm.predict_median_hmm(
known_l, transition_mat, km,
real_class_of_following_day = guess_l)
precision_colin.append(self.utils_cl.compute_mse(guess, pred))
precision_fake .append(self.utils_cl.compute_mse(guess, pred_fake))
precision_mean .append(self.utils_cl.compute_mse(guess, mean_day))
precision_colin_mae.append(self.utils_cl.compute_mae(guess, pred))
precision_fake_mae .append(self.utils_cl.compute_mae(guess, pred_fake))
precision_mean_mae .append(self.utils_cl.compute_mae(guess, mean_day))
precision_colin_mase.append(self.utils_cl.compute_mase(known, guess, pred))
precision_fake_mase .append(self.utils_cl.compute_mase(known, guess, pred_fake))
precision_mean_mase .append(self.utils_cl.compute_mase(known, guess, mean_day))
mean_mse.append(
(
my_km_size, np.mean(precision_colin), np.mean(precision_fake), np.mean(precision_mean),
np.mean(precision_colin_mae), np.mean(precision_fake_mae), np.mean(precision_mean_mae),
np.mean(precision_colin_mase), np.mean(precision_fake_mase),np.mean(precision_mean_mase),
np.std(precision_colin), np.std(precision_fake), np.std(precision_mean),
np.std(precision_colin_mae), np.std(precision_fake_mae), np.std(precision_mean_mae),
np.std(precision_colin_mase), np.std(precision_fake_mase), np.std(precision_mean_mase)
)
)
return mean_mse
def _add_path_to_icons(self):
icons_directory = FileHelper.get_icons_directory()
current_paths = QgsApplication.svgPaths()
if icons_directory not in current_paths:
current_paths.append(icons_directory)
QgsApplication.setDefaultSvgPaths(current_paths)
def _add_path_to_icons(self):
icons_directory = FileHelper.get_icons_directory()
class VtReader:
geo_types = {
1: GeoTypes.POINT,
2: GeoTypes.LINE_STRING,
3: GeoTypes.POLYGON
}
layer_sort_ids = [
"poi", "boundary", "transportation", "transportation_name",
"housenumber", "building", "place", "aeroway", "park", "water_name",
"waterway", "water", "landcover", "landuse"
]
_extent = 4096
_layers_to_dissolve = []
# todo: remove hardcoded path after testing
_file_path = "{}/sample data/zurich_switzerland.mbtiles".format(
FileHelper.get_directory())
def __init__(self, iface):
self.iface = iface
self._counter = 0
self._bool = True
self._mbtile_id = "name"
self.features_by_path = {}
self.qgis_layer_groups_by_feature_path = {}
def reinit(self):
"""
* Reinitializes the VtReader
>> Cleans the temp directory
>> Cleans the feature cache
>> Cleans the qgis group cache
"""
FileHelper.clear_temp_dir()
self.features_by_path = {}
self.qgis_layer_groups_by_feature_path = {}
@staticmethod
def _get_empty_feature_collection():
"""
* Returns an empty GeoJSON FeatureCollection
"""
crs = { # crs = coordinate reference system
"type": "name",
"properties": {
"name": "urn:ogc:def:crs:EPSG::3857"
}
}
return {"type": "FeatureCollection", "crs": crs, "features": []}
def load_vector_tiles_default(self, zoom_level):
self.load_vector_tiles(zoom_level, self._file_path)
def load_vector_tiles(self, zoom_level, path):
self._file_path = path
debug("Loading vector tiles: {}".format(path))
self.reinit()
self._connect_to_db(path)
mask_level = self._get_mask_layer_id()
tile_data_tuples = self._load_tiles_from_db(zoom_level, mask_level)
# if mask_level:
# mask_layer_data = self._load_tiles_from_db(mask_level)
# tile_data_tuples.extend(mask_layer_data)
tiles = self._decode_all_tiles(tile_data_tuples)
self._process_tiles(tiles)
self._create_qgis_layer_hierarchy()
print("Import complete!")
def _connect_to_db(self, path):
"""
* Since an mbtile file is a sqlite database, we can connect to it
"""
try:
self.conn = sqlite3.connect(path)
self.conn.row_factory = sqlite3.Row
print "Successfully connected to db"
except:
print "Db connection failed:", sys.exc_info()
return
def _get_mask_layer_id(self):
sql_command = "select value as 'masklevel' from metadata where name = 'maskLevel'"
mask_level = None
rows = self._get_from_db(sql=sql_command)
if rows:
mask_level = rows[0]["masklevel"]
return mask_level
def _load_tiles_from_db(self, zoom_level, mask_layer):
print("Reading tiles of zoom level {}".format(zoom_level))
if zoom_level != mask_layer:
# sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {} and tile_row = 10638 and tile_column=8568;".format(zoom_level)
# sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {} and tile_row = 10644 and tile_column=8581;".format(zoom_level)
# sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {} and tile_row >= 10640 and tile_row <= 10645 and tile_column>=8580 and tile_column<= 8582;".format(zoom_level)
sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {} LIMIT 5;".format(
zoom_level)
# sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {};".format(zoom_level)
else:
sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {};".format(
zoom_level)
# sql_command = "SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level = {} LIMIT 10;".format(zoom_level)
tile_data_tuples = []
rows = self._get_from_db(sql=sql_command)
for row in rows:
tile_data_tuples.append(self._create_tile(row))
return tile_data_tuples
@staticmethod
def _create_tile(row):
zoom_level = row["zoom_level"]
tile_col = row["tile_column"]
tile_row = row["tile_row"]
binary_data = row["tile_data"]
tile = VectorTile(zoom_level, tile_col, tile_row)
return tile, binary_data
def _get_from_db(self, sql):
try:
cur = self.conn.cursor()
cur.execute(sql)
return cur.fetchall()
except:
print "Getting data from db failed:", sys.exc_info()
def _decode_all_tiles(self, tiles_with_encoded_data):
tiles = []
total_nr_tiles = len(tiles_with_encoded_data)
print "Decoding {} tiles".format(total_nr_tiles)
for index, tile_data_tuple in enumerate(tiles_with_encoded_data):
tile = tile_data_tuple[0]
encoded_data = tile_data_tuple[1]
tile.decoded_data = self._decode_binary_tile_data(encoded_data)
if tile.decoded_data:
tiles.append(tile)
print "Progress: {0:.1f}%".format(100.0 / total_nr_tiles *
(index + 1))
return tiles
def _process_tiles(self, tiles):
total_nr_tiles = len(tiles)
print "Processing {} tiles".format(total_nr_tiles)
for index, tile in enumerate(tiles):
self._write_features(tile)
print "Progress: {0:.1f}%".format(100.0 / total_nr_tiles *
(index + 1))
def _decode_binary_tile_data(self, data):
try:
# The offset of 32 signals to the zlib header that the gzip header is expected but skipped.
file_content = zlib.decompress(data, 32 + zlib.MAX_WBITS)
decoded_data = mapbox_vector_tile.decode(file_content)
except:
print "decoding data with mapbox_vector_tile failed", sys.exc_info(
)
return
return decoded_data
def _create_qgis_layer_hierarchy(self):
"""
* Creates a hierarchy of groups and layers in qgis
"""
print("Creating hierarchy in qgis")
print("Layers to dissolve: {}".format(self._layers_to_dissolve))
root = QgsProject.instance().layerTreeRoot()
group_name = os.path.splitext(os.path.basename(self._file_path))[0]
root_group = root.addGroup(group_name)
feature_paths = sorted(
self.features_by_path.keys(),
key=lambda path: VtReader._get_feature_sort_id(path))
for feature_path in feature_paths:
target_group, layer_name = self._get_group_for_path(
feature_path, root_group)
feature_collection = self.features_by_path[feature_path]
file_src = FileHelper.get_unique_file_name()
with open(file_src, "w") as f:
json.dump(feature_collection, f)
layer = self._add_vector_layer(file_src, layer_name, target_group,
feature_path)
VtReader._load_named_style(layer)
@staticmethod
def _get_feature_sort_id(feature_path):
first_node = feature_path.split(".")[0]
sort_id = 999
if first_node in VtReader.layer_sort_ids:
sort_id = VtReader.layer_sort_ids.index(first_node)
return sort_id
def _get_group_for_path(self, path, root_group):
"""
* Returns the group for the specified path
>> If the group not already exists, it will be created
>> The path has to be delimited by '.'
>> The last element in the path will be used as name for the vector layer, the other elements will be used to create the group hierarchy
>> Example: The path 'zurich.poi.police' will create two groups 'zurich' and 'poi' (if not already existing) and 'police' will be returned as name for the layer to create
"""
group_names = path.split(".")
current_group = root_group
current_path = ""
target_layer_name = ""
for index, name in enumerate(group_names):
target_layer_name = name
is_last = index == len(group_names) - 1
if is_last:
break
current_path += "." + name
if current_path not in self.qgis_layer_groups_by_feature_path:
self.qgis_layer_groups_by_feature_path[
current_path] = current_group.addGroup(name)
current_group = self.qgis_layer_groups_by_feature_path[
current_path]
return current_group, target_layer_name
@staticmethod
def _load_named_style(layer):
try:
name = layer.name().split("_")[0]
style_name = "{}.qml".format(name)
# style_name = "{}.qml".format(layer.name())
style_path = os.path.join(FileHelper.get_directory(),
"styles/{}".format(style_name))
if os.path.isfile(style_path):
res = layer.loadNamedStyle(style_path)
if res[1]: # Style loaded
layer.setCustomProperty("layerStyle", style_path)
print("Style successfully applied: {}".format(style_name))
except:
print("Loading style failed: {}".format(sys.exc_info()))
def _add_vector_layer(self, json_src, layer_name, layer_target_group,
feature_path):
"""
* Creates a QgsVectorLayer and adds it to the group specified by layer_target_group
* Invalid geometries will be removed during the process of merging features over tile boundaries
"""
layer = QgsVectorLayer(json_src, layer_name, "ogr")
if feature_path in self._layers_to_dissolve:
layer = FeatureMerger().merge_features(layer)
layer.setName(layer_name)
QgsMapLayerRegistry.instance().addMapLayer(layer, False)
layer_target_group.addLayer(layer)
return layer
def _write_features(self, tile):
"""
* Transforms all features of the specified tile into GeoJSON and writes it into the dictionary
:param tile:
:return:
"""
# iterate through all the features of the data and build proper gejson conform objects.
for layer_name in tile.decoded_data:
tile_features = tile.decoded_data[layer_name]["features"]
for index, feature in enumerate(tile_features):
geojson_feature, geo_type = VtReader._create_geojson_feature(
feature, tile)
if geojson_feature:
feature_path = VtReader._get_feature_path(
layer_name, geojson_feature, tile.zoom_level)
if feature_path not in self.features_by_path:
self.features_by_path[
feature_path] = VtReader._get_empty_feature_collection(
)
self.features_by_path[feature_path]["features"].append(
geojson_feature)
geotypes_to_dissolve = [GeoTypes.POLYGON]
if geo_type in geotypes_to_dissolve and feature_path not in self._layers_to_dissolve:
self._layers_to_dissolve.append(feature_path)
@staticmethod
def _get_feature_class_and_subclass(feature):
feature_class = None
feature_subclass = None
properties = feature["properties"]
if "class" in properties:
feature_class = properties["class"]
if "subclass" in properties:
feature_subclass = properties["subclass"]
if feature_subclass == feature_class:
feature_subclass = None
if feature_subclass:
assert feature_class, "A feature with a subclass should also have a class"
return feature_class, feature_subclass
@staticmethod
def _get_feature_path(layer_name, feature, zoom_level):
feature_class, feature_subclass = VtReader._get_feature_class_and_subclass(
feature)
feature_path = layer_name
if feature_class:
feature_path += "." + feature_class
if feature_subclass:
feature_path += "." + feature_subclass
feature_path += "_{}".format(zoom_level)
return feature_path
total_feature_count = 0
@staticmethod
def _create_geojson_feature(feature, tile):
"""
Creates a proper GeoJSON feature for the specified feature
"""
geo_type = VtReader.geo_types[feature["type"]]
coordinates = feature["geometry"]
# # todo: remove after testing
# if geo_type != GeoTypes.POLYGON:
# return None, None
coordinates = VtReader._map_coordinates_recursive(
coordinates=coordinates,
func=lambda coords: VtReader._transform_to_epsg3857(coords, tile))
if geo_type == GeoTypes.POINT:
# Due to mercator_geometrys nature, the point will be displayed in a List "[[]]", remove the outer bracket.
coordinates = coordinates[0]
properties = feature["properties"]
properties["zoomLevel"] = tile.zoom_level
properties["featureNr"] = VtReader.total_feature_count
properties["col"] = tile.column
properties["row"] = tile.row
VtReader.total_feature_count += 1
feature_json = VtReader._create_geojson_feature_from_coordinates(
geo_type, coordinates, properties)
return feature_json, geo_type
@staticmethod
def _get_is_multi(geo_type, coordinates):
if geo_type == GeoTypes.POINT:
is_single = len(coordinates) == 2 and all(
isinstance(c, int) for c in coordinates)
return not is_single
elif geo_type == GeoTypes.LINE_STRING:
is_array_of_tuples = all(
len(c) == 2 and all(isinstance(ci, int) for ci in c)
for c in coordinates)
is_single = is_array_of_tuples
return not is_single
elif geo_type == GeoTypes.POLYGON:
is_multi = VtReader.get_array_depth(coordinates, 0) >= 2
return is_multi
return False
@staticmethod
def get_array_depth(arr, depth):
if all(isinstance(c, numbers.Real) for c in arr[0]):
return depth
else:
depth += 1
return VtReader.get_array_depth(arr[0], depth)
@staticmethod
def _create_geojson_feature_from_coordinates(geo_type, coordinates,
properties):
is_multi = VtReader._get_is_multi(geo_type, coordinates)
type_string = geo_type
if is_multi:
type_string = "Multi{}".format(geo_type)
feature_json = {
"type": "Feature",
"geometry": {
"type": type_string,
"coordinates": coordinates
},
"properties": properties
}
return feature_json
@staticmethod
def _map_coordinates_recursive(coordinates, func):
"""
Recursively traverses the array of coordinates (depth first) and applies the specified function
"""
tmp = []
for coord in coordinates:
is_coordinate_tuple = len(coord) == 2 and all(
isinstance(c, int) for c in coord)
if is_coordinate_tuple:
newval = func(coord)
tmp.append(newval)
else:
tmp.append(VtReader._map_coordinates_recursive(coord, func))
return tmp
@staticmethod
def _transform_to_epsg3857(coordinates, tile):
"""
Does a mercator transformation on the specified coordinate tuple
"""
# calculate the mercator geometry using external library
# geometry:: 0: zoom, 1: easting, 2: northing
tmp = GlobalMercator().TileBounds(tile.column, tile.row,
tile.zoom_level)
delta_x = tmp[2] - tmp[0]
delta_y = tmp[3] - tmp[1]
merc_easting = int(tmp[0] +
delta_x / VtReader._extent * coordinates[0])
merc_northing = int(tmp[1] +
delta_y / VtReader._extent * coordinates[1])
return [merc_easting, merc_northing]
def fetch_nc_geojson():
# refresh road status
data = FileHelper.load_json_file('data/nc_parkway_data.geojson')
return json.dumps({'data': data}), 200, {'ContentType': 'application/json'}
def _dissolve(layer):
debug("Dissolving layer")
target_file = FileHelper.get_unique_file_name()
processing.runalg("qgis:dissolve", layer, False, "dissolveGroup", target_file)
dissolved_layer = QgsVectorLayer(target_file, "Dissolved", "ogr")
return dissolved_layer
def _load_tiles(self):
try:
# recreate source to assure the source belongs to the new thread, SQLite3 isn't happy about it otherwise
self.source = self._create_source(self.source.source())
zoom_level = self._loading_options["zoom_level"]
bounds = self._loading_options["bounds"]
load_mask_layer = self._loading_options["load_mask_layer"]
merge_tiles = self._loading_options["merge_tiles"]
clip_tiles = self._loading_options["clip_tiles"]
apply_styles = self._loading_options["apply_styles"]
max_tiles = self._loading_options["max_tiles"]
layer_filter = self._loading_options["layer_filter"]
self.cancel_requested = False
self.feature_collections_by_layer_name_and_geotype = {}
self._qgis_layer_groups_by_name = {}
self._update_progress(show_dialog=True, title="Loading '{}'".format(os.path.basename(self.source.name())))
self._clip_tiles_at_tile_bounds = clip_tiles
min_zoom = self.source.min_zoom()
max_zoom = self.source.max_zoom()
zoom_level = clamp(zoom_level, low=min_zoom, high=max_zoom)
all_tiles = get_all_tiles(
bounds=bounds,
is_cancel_requested_handler=lambda: self.cancel_requested,
)
tiles_to_load = set()
tiles = []
tiles_to_ignore = set()
for t in all_tiles:
if self.cancel_requested or (max_tiles and len(tiles) >= max_tiles):
break
file_name = self._get_tile_cache_name(zoom_level, t[0], t[1])
tile = FileHelper.get_cached_tile(file_name)
if tile and tile.decoded_data:
tiles.append(tile)
tiles_to_ignore.add((tile.column, tile.row))
else:
tiles_to_load.add(t)
remaining_nr_of_tiles = len(tiles_to_load)
if max_tiles:
if len(tiles) + len(tiles_to_load) >= max_tiles:
remaining_nr_of_tiles = max_tiles - len(tiles)
if remaining_nr_of_tiles < 0:
remaining_nr_of_tiles = 0
debug("{} cache hits. {} may potentially be loaded.", len(tiles), remaining_nr_of_tiles)
debug("Loading data for zoom level '{}' source '{}'", zoom_level, self.source.name())
tile_data_tuples = []
if remaining_nr_of_tiles > 0:
tile_data_tuples = self.source.load_tiles(zoom_level=zoom_level,
tiles_to_load=tiles_to_load,
max_tiles=remaining_nr_of_tiles)
if len(tiles) == 0 and (not tile_data_tuples or len(tile_data_tuples) == 0):
QMessageBox.information(None, "No tiles found", "What a pity, no tiles could be found!")
if load_mask_layer:
mask_level = self.source.mask_level()
if mask_level is not None and mask_level != zoom_level:
debug("Mapping {} tiles to mask level", len(all_tiles))
scheme = self.source.scheme()
mask_tiles = map(
lambda t: change_zoom(zoom_level, int(mask_level), t, scheme),
all_tiles)
debug("Mapping done")
mask_tiles_to_load = set()
for t in mask_tiles:
file_name = self._get_tile_cache_name(mask_level, t[0], t[1])
tile = FileHelper.get_cached_tile(file_name)
if tile and tile.decoded_data:
tiles.append(tile)
else:
mask_tiles_to_load.add(t)
debug("Loading mask layer (zoom_level={})", mask_level)
tile_data_tuples = []
if len(mask_tiles_to_load) > 0:
mask_layer_data = self.source.load_tiles(zoom_level=mask_level,
tiles_to_load=mask_tiles_to_load,
max_tiles=max_tiles)
debug("Mask layer loaded")
tile_data_tuples.extend(mask_layer_data)
if tile_data_tuples and len(tile_data_tuples) > 0:
if not self.cancel_requested:
decoded_tiles = self._decode_tiles(tile_data_tuples)
tiles.extend(decoded_tiles)
if len(tiles) > 0:
if not self.cancel_requested:
self._process_tiles(tiles, layer_filter)
if not self.cancel_requested:
self._create_qgis_layers(merge_features=merge_tiles,
apply_styles=apply_styles)
self._update_progress(show_dialog=False)
if self.cancel_requested:
info("Import cancelled")
self.cancelled.emit()
else:
info("Import complete")
loaded_tiles_x = map(lambda t: t.coord()[0], tiles)
loaded_tiles_y = map(lambda t: t.coord()[1], tiles)
if len(loaded_tiles_x) == 0 or len(loaded_tiles_y) == 0:
return None
loaded_extent = {"x_min": int(min(loaded_tiles_x)),
"x_max": int(max(loaded_tiles_x)),
"y_min": int(min(loaded_tiles_y)),
"y_max": int(max(loaded_tiles_y)),
"zoom": int(zoom_level)
}
loaded_extent["width"] = loaded_extent["x_max"] - loaded_extent["x_min"] + 1
loaded_extent["height"] = loaded_extent["y_max"] - loaded_extent["y_min"] + 1
self.loading_finished.emit(zoom_level, loaded_extent)
except Exception as e:
critical("An exception occured: {}, {}", e, traceback.format_exc())
self.cancelled.emit()
class KhiopsManager:
def __init__(self):
logging.info(pk.getKhiopsInfo())
self.this_file_dir = os.path.dirname(os.path.realpath(__file__))
# path mgmt
# use timestamp of each exec in paths
self.fh = FileHelper()
self.dictionary_file = os.path.join(self.this_file_dir, "dic",
"series.kdic")
self.classif_res = os.path.join(self.this_file_dir, "res",
"khiops_res", "classif")
self.coclus_res = os.path.join(self.this_file_dir, "res", "khiops_res",
"coclus")
self.pred_res = os.path.join(self.this_file_dir, "res", "khiops_res",
"pred_res")
self.fh.ensure_dirs_exist([
self.dictionary_file, self.classif_res, self.coclus_res,
self.pred_res
])
self.ccr = CoclusteringResults()
self.utils = MyUtils()
logging.info("Khiops manager instantiated")
logging.info("dictionary_file used: %s", self.dictionary_file)
"""KHIOPS COCLUSTERING TRAIN AND SIMPLIFICATIONS"""
def train_coclustering(self, f):
"""
Train a coclustering model in the simplest way possible
"""
file_name = self.fh.get_file_name(f)
logging.info("Train of coclustering for file %s", file_name)
# Train coclustering model for variables "SampleId" and "Char"
pk.trainCoclustering(dictionaryFile=self.dictionary_file,
dictionary="train",
dataTable=f,
coclusteringVariables=["time_", "n_day_", "val_"],
resultsDir=self.coclus_res,
fieldSeparator=";",
samplePercentage=100,
resultsPrefix=file_name + "_")
def simplify_coclustering(self, file_name, mpi=100, mcn=999999):
"""
Simplify a coclustering model in the simplest way possible
"""
base_path = os.path.join(self.coclus_res, file_name)
self.fh.ensure_dirs_exist([base_path])
cf = os.path.join(self.coclus_res, file_name + "_Coclustering.khc")
logging.info("Simplify coclustering for file %s", file_name)
logging.info("MCN=%s, MPI=%s", mcn, mpi)
if mcn != 999999:
scf = file_name + "_Simplified-" + str(mcn) + ".khc"
else:
scf = file_name + "_Simplified-" + str(mpi) + ".khc"
logging.info("scf=%s", scf)
pk.simplifyCoclustering(
coclusteringFile=cf,
simplifiedCoclusteringFile=scf,
resultsDir=base_path,
maxCellNumber=mcn,
maxPreservedInformation=mpi,
)
return str(os.path.join(base_path, scf)).replace(".khc", ".json")
def deploy_coclustering(self, file_name, identifier):
"""
Deploy a coclustering model
"""
base_path = os.path.join(self.coclus_res, file_name)
self.fh.ensure_dirs_exist([base_path])
logging.info("Deploy coclustering for file %s", file_name)
scf = os.path.join(
self.coclus_res, file_name,
file_name + "_Simplified-" + str(identifier) + ".khc")
dep_prefix = file_name + "_Deployed-" + str(identifier) + "-"
pk.prepareCoclusteringDeployment(dictionaryFile=self.dictionary_file,
dictionary="root",
coclusteringFile=scf,
tableVariable="secondary",
deployedVariable="n_day_",
buildDistanceVariables=True,
resultsPrefix=dep_prefix,
resultsDir=base_path)
return os.path.join(base_path, dep_prefix + "Coclustering.kdic")
def transfer_database(self, d, deployed_path, f, identifier):
"""
Transfer a coclustering model to use it
"""
file_name = self.fh.get_file_name(f)
out_path = os.path.join(
self.coclus_res, file_name,
file_name + "_Transf-" + str(identifier) + ".csv")
logging.info("Transfering of coclustering for file %s into %s",
file_name, out_path)
pk.transferDatabase(dictionaryFile=d,
dictionary="root",
dataTable=deployed_path,
additionalDataTables={"root`secondary": f},
fieldSeparator=";",
outputFieldSeparator=";",
outputDataTable=out_path)
return out_path
"""JSON COCLUSTERING MANIPULATIONS"""
def get_clusters(self, path):
"""
From a given Khiops json file, extract clusters, and detailed info about
them.
Returns:
* zips: more info about each cluster zipped together (e.g all tuples that goes together 'aligned') => tuples (clus, values, value_frequencies, value_typicalities)
* cluster_and_values: dic(cluster_num => values_associated)
"""
cluster_and_values = {}
zips = []
with open(path) as f:
data = json.load(f)["coclusteringReport"]["dimensionPartitions"]
n_days = list(filter(lambda d: d['name'] in "n_day_", data))
for idx, n_day in enumerate(n_days[0]["valueGroups"], start=1):
# Ugly turnaround to manipulate floats and int
# see https://goo.gl/8tYfhn
values = list(map(int, map(float, n_day["values"])))
clus = n_day["cluster"]
value_frequencies = n_day["valueFrequencies"]
value_typicalities = n_day["valueTypicalities"]
zips.append(
list(
zip(clus, values, value_frequencies,
value_typicalities)))
cluster_and_values[clus] = values
return self.utils.flattify(zips), cluster_and_values
@staticmethod
def get_clusters_from_dep(path):
"""
From a given Khiops transferred thing, extract clusters, and detailed
info about them.
Returns:
* cluster_and_values: dic(cluster_num => values_associated)
"""
with open(path) as f:
df = pd.read_csv(f, sep=";")
k = df["n_day_PredictedLabel"].unique()
cluster_and_values = dict((key, []) for key in k)
for index, row in df.iterrows():
n_day_ = row["n_day_"]
clus = row["n_day_PredictedLabel"]
cluster_and_values[clus].append(n_day_)
return cluster_and_values
def get_cells_number(self, file_name, mpi=None):
"""
From a given Khiops json file, extract number of cells for all
dimentions
return:
- cells
"""
if mpi is not None:
p = os.path.join(self.coclus_res, file_name,
file_name + "_Simplified-" + str(mpi) + ".json")
else:
p = os.path.join(self.coclus_res, file_name + "_Coclustering.json")
with open(p) as f:
cells = json.load(f)["coclusteringReport"]["summary"]["cells"]
return int(cells)
def get_cluster_number(self, file_name, ref_id=None):
"""
From a given Khiops json file, extract number of cluster for dim
"n_day_"
return:
- nb_of_cluster_found
"""
if ref_id is not None:
p = os.path.join(
self.coclus_res, file_name,
file_name + "_Simplified-" + str(ref_id) + ".json")
else:
p = os.path.join(self.coclus_res, file_name + "_Coclustering.json")
with open(p) as f:
data = json.load(f)["coclusteringReport"]["dimensionSummaries"]
n_days = list(filter(lambda d: d['name'] in "n_day_", data))
if not n_days:
return False
else:
return int(n_days[0]["parts"])
"""UTILS"""
@staticmethod
def _compute_accuracy(len_y_pred, y_pred, y_pred_target):
"""
Compute the accuracy of a classifier.
"""
c = 0
for i in range(0, len_y_pred):
pred_group_day_ahead = str(y_pred.iloc[i]["Predictedy"])
real_group_day_ahead = str(y_pred_target.values[i])
if pred_group_day_ahead != real_group_day_ahead:
c = c + 1
train_acc = 100 - c * 100 / len_y_pred
logging.debug("%s errors over %s values", c, len_y_pred)
logging.debug("That is %s perc of accuracy", train_acc)
return train_acc
@staticmethod
def __get_typicalitie(n_day, my_zip):
"""
From a list of tuples my_zip, find the
tuple which concern n_day and retrieve its valTyp.
PARAMETERS
------------------------
- my_zip: tuples (group, n_day, valFreq, valTyp)
- n_day: int
"""
items = [i for i in my_zip if n_day == i[1]]
return items[0][3]
@staticmethod
def compute_centroids(c_a_v, df, l_ref):
"""
This method computes mean days and centroids for given values
PARAMETERS
------------------------
* c_a_v: dic(cluster_num => values_associated)
* df: dataset studied
* l_ref: len of one ref day
RETURNS
-------------------------
* centroids: dic(cluster_id: int => centroid: [])
"""
centroids = {}
e = False
for k, v in c_a_v.items():
# If there is at least one day in the cluster considered!
if len(v) != 0:
logging.info("Computing cluster %s centroid", k)
centroids[k] = df[df['n_day_'].isin(map(
str, v))].groupby('time_')['val_'].agg('mean').values
# Otherwise, mean day = 0
else:
logging.info("Cluster %s is empty", k)
e = True
centroids[k] = [0] * l_ref
logging.info("Keys of centroids are %s", list(centroids.keys()))
logging.debug("Centroids are %s", centroids)
return centroids, e
def process_pred_proba(self,
y_pred,
y_pred_target,
clusts_names,
centroids,
days_to_predict,
nb_cluster_found,
n_pt_per_day,
mean_day,
o=False):
"""
This method process results from Khiops classification method to create
predictions and compute mses. If Oracle mode is "True", then use a mock
for classifier Y (e.g the classifier knows exactly the Y columns)
WHAT IT DOES
------------------------
* First thing (A): Compute given classifier accuracy regarding known
y_pred_target.
* Second thing (B): Process probalistic classifier results but not
using the probabilities and only the most probable class for day n +
1. NPA
* Third thing (C): Process probalistic classifier results using
probabilities affected to each cluster (e.g. day n + 1 could be 10%
in cluster 1, 50% in cluster 2, etc). Use ponderations to sum up
each centroids and produce result. PA
* Last thing (D): Wrap up, means computations => compute results...
PARAMETERS
------------------------
* y_pred: Matrix results from classifier
* y_pred_target_test: Known y_pred_target for classifier for test data
* clusts_names: names of all clusters
* centroids: clusters centroids
* days_to_predict: list of days that have been used for test
* nb_cluster_found: value exctracted from Khiops coclustering file;
used to parameterize loops and computations.
* n_pt_per_day: if there is one point per hour of the day, then this
value will be 24.
* mean_day: pre-computed mean day for all training ensemble.
* oracle: are we in oracle mode? ('Y' known)
RETURN
-------------------------
* Tuple(mean_mse_non_proba, mean_mse_proba, classifier_acc,
mean_mse_mean_day) => Meaned MSE !
:param y_pred:
:param clusts_names:
:param centroids:
:param days_to_predict:
:param nb_cluster_found:
:param n_pt_per_day:
:param mean_day:
:param y_pred_target:
:param o:
"""
len_y_pred = len(y_pred)
# (A): Compute classifier accuracy
classifier_acc = self._compute_accuracy(len_y_pred, y_pred,
y_pred_target)
if classifier_acc == 0.0:
logging.info(
"This classifier is very not good and failed all the time")
# Init empty arrays
mses_non_proba = []
mses_proba = []
mses_mean_day = []
mae_non_proba = []
mae_proba = []
mae_mean_day = []
mase_non_proba = []
mase_proba = []
mase_mean_day = []
# Think about removing last day, which has not days after
n_days_ = days_to_predict["n_day_"].unique()[:-1]
# Also got to remove the first day, which has not day before
# Wrangling data to have good types.
n_days_int = list(map(int, n_days_))
n_days_int.remove(min(n_days_int))
n_days_ = list(map(str, n_days_int))
for ref_in_classif_ensemble, z in enumerate(n_days_):
"""
(B): NPA approach
"""
indice_today = str(int(z) - 1)
today_vals = days_to_predict[days_to_predict["n_day_"] ==
indice_today]["val_"].values
tomor_vals = days_to_predict[days_to_predict["n_day_"] ==
z]["val_"].values
tom_class_pred_cluster_y = y_pred.iloc[ref_in_classif_ensemble][
"Predictedy"]
tom_pred_y = centroids[str(tom_class_pred_cluster_y)]
# Append mses
mses_non_proba.append(
self.utils.compute_mse(tomor_vals, tom_pred_y))
mses_mean_day.append(self.utils.compute_mse(tomor_vals, mean_day))
mae_non_proba.append(self.utils.compute_mae(
tomor_vals, tom_pred_y))
mae_mean_day.append(self.utils.compute_mae(tomor_vals, mean_day))
mase_non_proba.append(
self.utils.compute_mase(today_vals, tomor_vals, tom_pred_y))
mase_mean_day.append(
self.utils.compute_mase(today_vals, tomor_vals, mean_day))
mean_days_pond = []
p_tot = 0
"""
(C): PA approach
"""
for c in clusts_names:
try:
# Get probability to be in groupe number "c"
p = y_pred.iloc[ref_in_classif_ensemble]["Proby" + str(c)]
# p_tot variable for debug purposes
p_tot = p_tot + p
except:
logging.debug("Did not find %s in the prediction matrix",
c)
p = 0
pass
# Retrieve mean day for cluster c
day_pond = centroids[c].copy()
# Scale according to proba
day_pond[:] = [x * p for x in day_pond]
mean_days_pond.append(day_pond)
logging.debug("p_tot is %s", p_tot) # Here it should be 100.
# Sum up everything to create prediction
tomo_pred_with_pond = [0] * n_pt_per_day
for d_p in mean_days_pond:
for idx, e in enumerate(d_p):
tomo_pred_with_pond[idx] = tomo_pred_with_pond[idx] + e
mses_proba.append(
self.utils.compute_mse(tomor_vals, tomo_pred_with_pond))
mae_proba.append(
self.utils.compute_mae(tomor_vals, tomo_pred_with_pond))
mase_proba.append(
self.utils.compute_mase(today_vals, tomor_vals,
tomo_pred_with_pond))
# (D): Wrap up: mean MSEs (so we have, at the end, Meaned Mean Square
# Error)
# MMSES
mean_mse_non_proba = np.mean(mses_non_proba)
mean_mse_proba = np.mean(mses_proba)
mean_mse_mean_day = np.mean(mses_mean_day)
mean_mae_non_proba = np.mean(mae_non_proba)
mean_mae_proba = np.mean(mae_proba)
mean_mae_mean_day = np.mean(mae_mean_day)
mean_mase_non_proba = np.mean(mase_non_proba)
mean_mase_proba = np.mean(mase_proba)
mean_mase_mean_day = np.mean(mase_mean_day)
# MSTD
std_mse_non_proba = np.std(mses_non_proba)
std_mse_proba = np.std(mses_proba)
std_mse_mean_day = np.std(mses_mean_day)
std_mae_non_proba = np.std(mae_non_proba)
std_mae_proba = np.std(mae_proba)
std_mae_mean_day = np.std(mae_mean_day)
std_mase_non_proba = np.std(mase_non_proba)
std_mase_proba = np.std(mase_proba)
std_mase_mean_day = np.std(mase_mean_day)
return ((mean_mse_non_proba, mean_mse_proba, mean_mse_mean_day),
(mean_mae_non_proba, mean_mae_proba,
mean_mae_mean_day), (mean_mase_non_proba, mean_mase_proba,
mean_mase_mean_day), classifier_acc,
(std_mse_non_proba, std_mse_proba, std_mse_mean_day),
(std_mae_non_proba, std_mae_proba, std_mae_mean_day),
(std_mase_non_proba, std_mase_proba, std_mase_mean_day))
class VtReader(QObject):
progress_changed = pyqtSignal(int, name='progressChanged')
max_progress_changed = pyqtSignal(int, name='maxProgressChanged')
message_changed = pyqtSignal('QString', name='messageChanged')
title_changed = pyqtSignal('QString', name='titleChanged')
show_progress_changed = pyqtSignal(bool, name='titleChanged')
loading_finished = pyqtSignal(int, object, name='loadingFinished')
tile_limit_reached = pyqtSignal(int, name='tile_limit_reached')
cancelled = pyqtSignal(name='cancelled')
omt_layer_ordering = [
"place",
"mountain_peak",
"housenumber",
"water_name",
"transportation_name",
"poi",
"boundary",
"transportation",
"building",
"aeroway",
"park",
"water",
"waterway",
"landcover",
"landuse"
]
_loading_options = {
'zoom_level': None,
'layer_filter': None,
'load_mask_layer': None,
'merge_tiles': None,
'clip_tiles': None,
'apply_styles': None,
'max_tiles': None,
'bounds': None
}
_layers_to_dissolve = []
_zoom_level_delimiter = "*"
_DEFAULT_EXTENT = 4096
_id = str(uuid.uuid4())
_styles = FileHelper.get_styles()
flush_layers_of_other_zoom_level = False
def __init__(self, iface, path_or_url):
"""
* The mbtiles_path can also be an URL in zxy format: z=zoom, x=tile column, y=tile row
:param iface:
:param path_or_url:
"""
QObject.__init__(self)
if not path_or_url:
raise RuntimeError("The datasource is required")
self.source = self._create_source(path_or_url)
FileHelper.assure_temp_dirs_exist()
self.iface = iface
self.feature_collections_by_layer_name_and_geotype = {}
self._qgis_layer_groups_by_name = {}
self.cancel_requested = False
self._loaded_pois_by_id = {}
self._clip_tiles_at_tile_bounds = None
self._always_overwrite_geojson = False
self._root_group_name = None
self._flush = False
def _create_source(self, path_or_url):
is_web_source = path_or_url.lower().startswith("http://") or path_or_url.lower().startswith("https://")
if is_web_source:
source = ServerSource(url=path_or_url)
else:
if os.path.isfile(path_or_url):
source = MBTilesSource(path=path_or_url)
else:
source = TrexCacheSource(path=path_or_url)
source.progress_changed.connect(self._source_progress_changed)
source.max_progress_changed.connect(self._source_max_progress_changed)
source.message_changed.connect(self._source_message_changed)
source.tile_limit_reached.connect(self._source_tile_limit_reached)
return source
def shutdown(self):
info("Shutdown reader")
self.source.progress_changed.disconnect()
self.source.max_progress_changed.disconnect()
self.source.message_changed.disconnect()
self.source.close_connection()
def id(self):
return self._id
@pyqtSlot(int)
def _source_tile_limit_reached(self):
self.tile_limit_reached.emit(self._loading_options["max_tiles"])
@pyqtSlot(int)
def _source_progress_changed(self, progress):
self._update_progress(progress=progress)
@pyqtSlot(int)
def _source_max_progress_changed(self, max_progress):
self._update_progress(max_progress=max_progress)
@pyqtSlot('QString')
def _source_message_changed(self, msg):
self._update_progress(msg=msg)
def set_root_group_name(self, name):
self._root_group_name = name
def _update_progress(self, title=None, show_dialog=None, progress=None, max_progress=None, msg=None):
if progress is not None:
self.progress_changed.emit(progress)
if max_progress is not None:
self.max_progress_changed.emit(max_progress)
if title:
self.title_changed.emit(title)
if msg:
self.message_changed.emit(msg)
if show_dialog:
self.show_progress_changed.emit(show_dialog)
def _get_empty_feature_collection(self, layer_name, zoom_level):
"""
* Returns an empty GeoJSON FeatureCollection with the coordinate reference system (crs) set to EPSG3857
"""
# todo: when improving CRS handling: the correct CRS of the source has to be set here
source_crs = self.source.crs()
if source_crs:
epsg_id = get_code_from_epsg(source_crs)
else:
epsg_id = 3857
crs = {
"type": "name",
"properties": {
"name": "urn:ogc:def:crs:EPSG::{}".format(epsg_id)}}
return {
"tiles": [],
"source": self.source.name(),
"scheme": self.source.scheme(),
"layer": layer_name,
"zoom_level": zoom_level,
"type": "FeatureCollection",
"crs": crs,
"features": []}
def always_overwrite_geojson(self, enabled):
"""
* If activated, the geoJson written to the disk will always be overwritten, with each load
* As a result of this, only the latest loaded extent will be visible in qgis
:return:
"""
self._always_overwrite_geojson = enabled
def cancel(self):
"""
* Cancels the loading process.
:return:
"""
self.cancel_requested = True
if self.source:
self.source.cancel()
def _get_tile_cache_name(self, zoom_level, col, row):
return FileHelper.get_cached_tile_file_name(self.source.name(), zoom_level, col, row)
def _load_tiles(self):
try:
# recreate source to assure the source belongs to the new thread, SQLite3 isn't happy about it otherwise
self.source = self._create_source(self.source.source())
zoom_level = self._loading_options["zoom_level"]
bounds = self._loading_options["bounds"]
load_mask_layer = self._loading_options["load_mask_layer"]
merge_tiles = self._loading_options["merge_tiles"]
clip_tiles = self._loading_options["clip_tiles"]
apply_styles = self._loading_options["apply_styles"]
max_tiles = self._loading_options["max_tiles"]
layer_filter = self._loading_options["layer_filter"]
self.cancel_requested = False
self.feature_collections_by_layer_name_and_geotype = {}
self._qgis_layer_groups_by_name = {}
self._update_progress(show_dialog=True, title="Loading '{}'".format(os.path.basename(self.source.name())))
self._clip_tiles_at_tile_bounds = clip_tiles
min_zoom = self.source.min_zoom()
max_zoom = self.source.max_zoom()
zoom_level = clamp(zoom_level, low=min_zoom, high=max_zoom)
all_tiles = get_all_tiles(
bounds=bounds,
is_cancel_requested_handler=lambda: self.cancel_requested,
)
tiles_to_load = set()
tiles = []
tiles_to_ignore = set()
for t in all_tiles:
if self.cancel_requested or (max_tiles and len(tiles) >= max_tiles):
break
file_name = self._get_tile_cache_name(zoom_level, t[0], t[1])
tile = FileHelper.get_cached_tile(file_name)
if tile and tile.decoded_data:
tiles.append(tile)
tiles_to_ignore.add((tile.column, tile.row))
else:
tiles_to_load.add(t)
remaining_nr_of_tiles = len(tiles_to_load)
if max_tiles:
if len(tiles) + len(tiles_to_load) >= max_tiles:
remaining_nr_of_tiles = max_tiles - len(tiles)
if remaining_nr_of_tiles < 0:
remaining_nr_of_tiles = 0
debug("{} cache hits. {} may potentially be loaded.", len(tiles), remaining_nr_of_tiles)
debug("Loading data for zoom level '{}' source '{}'", zoom_level, self.source.name())
tile_data_tuples = []
if remaining_nr_of_tiles > 0:
tile_data_tuples = self.source.load_tiles(zoom_level=zoom_level,
tiles_to_load=tiles_to_load,
max_tiles=remaining_nr_of_tiles)
if len(tiles) == 0 and (not tile_data_tuples or len(tile_data_tuples) == 0):
QMessageBox.information(None, "No tiles found", "What a pity, no tiles could be found!")
if load_mask_layer:
mask_level = self.source.mask_level()
if mask_level is not None and mask_level != zoom_level:
debug("Mapping {} tiles to mask level", len(all_tiles))
scheme = self.source.scheme()
mask_tiles = map(
lambda t: change_zoom(zoom_level, int(mask_level), t, scheme),
all_tiles)
debug("Mapping done")
mask_tiles_to_load = set()
for t in mask_tiles:
file_name = self._get_tile_cache_name(mask_level, t[0], t[1])
tile = FileHelper.get_cached_tile(file_name)
if tile and tile.decoded_data:
tiles.append(tile)
else:
mask_tiles_to_load.add(t)
debug("Loading mask layer (zoom_level={})", mask_level)
tile_data_tuples = []
if len(mask_tiles_to_load) > 0:
mask_layer_data = self.source.load_tiles(zoom_level=mask_level,
tiles_to_load=mask_tiles_to_load,
max_tiles=max_tiles)
debug("Mask layer loaded")
tile_data_tuples.extend(mask_layer_data)
if tile_data_tuples and len(tile_data_tuples) > 0:
if not self.cancel_requested:
decoded_tiles = self._decode_tiles(tile_data_tuples)
tiles.extend(decoded_tiles)
if len(tiles) > 0:
if not self.cancel_requested:
self._process_tiles(tiles, layer_filter)
if not self.cancel_requested:
self._create_qgis_layers(merge_features=merge_tiles,
apply_styles=apply_styles)
self._update_progress(show_dialog=False)
if self.cancel_requested:
info("Import cancelled")
self.cancelled.emit()
else:
info("Import complete")
loaded_tiles_x = map(lambda t: t.coord()[0], tiles)
loaded_tiles_y = map(lambda t: t.coord()[1], tiles)
if len(loaded_tiles_x) == 0 or len(loaded_tiles_y) == 0:
return None
loaded_extent = {"x_min": int(min(loaded_tiles_x)),
"x_max": int(max(loaded_tiles_x)),
"y_min": int(min(loaded_tiles_y)),
"y_max": int(max(loaded_tiles_y)),
"zoom": int(zoom_level)
}
loaded_extent["width"] = loaded_extent["x_max"] - loaded_extent["x_min"] + 1
loaded_extent["height"] = loaded_extent["y_max"] - loaded_extent["y_min"] + 1
self.loading_finished.emit(zoom_level, loaded_extent)
except Exception as e:
critical("An exception occured: {}, {}", e, traceback.format_exc())
self.cancelled.emit()
def set_options(self, load_mask_layer=False, merge_tiles=True, clip_tiles=False,
apply_styles=True, max_tiles=None, layer_filter=None):
self._loading_options = {
'load_mask_layer': load_mask_layer,
'merge_tiles': merge_tiles,
'clip_tiles': clip_tiles,
'apply_styles': apply_styles,
'max_tiles': max_tiles,
'layer_filter': layer_filter
}
def load_tiles_async(self, zoom_level, bounds):
"""
* Loads the vector tiles from either a file or a URL and adds them to QGIS
:param clip_tiles:
:param zoom_level: The zoom level to load
:param layer_filter: A list of layers. If any layers are set, only these will be loaded. If the list is empty,
all available layers will be loaded
:param load_mask_layer: If True the mask layer will also be loaded
:param merge_tiles: If True neighbouring tiles and features will be merged
:param apply_styles: If True the default styles will be applied
:param max_tiles: The maximum number of tiles to load
:param bounds:
:return:
"""
self._loading_options["zoom_level"] = zoom_level
self._loading_options["bounds"] = bounds
_worker_thread = QThread(self.iface.mainWindow())
self.moveToThread(_worker_thread)
_worker_thread.started.connect(self._load_tiles)
_worker_thread.start()
def _decode_tiles(self, tiles_with_encoded_data):
"""
* Decodes the PBF data from all the specified tiles and reports the progress
* If a tile is loaded from the cache, the decoded_data is already set and doesn't have to be encoded
:param tiles_with_encoded_data:
:return:
"""
total_nr_tiles = len(tiles_with_encoded_data)
self._update_progress(progress=0, max_progress=100, msg="Decoding {} tiles...".format(total_nr_tiles))
nr_processors = 4
try:
nr_processors = mp.cpu_count()
except NotImplementedError:
info("CPU count cannot be retrieved. Falling back to default = 4")
tiles_with_encoded_data = map(lambda t: (t[0], self._unzip(t[1])), tiles_with_encoded_data)
pool = mp.Pool(nr_processors)
tiles = []
rs = pool.map_async(decode_tile, tiles_with_encoded_data, callback=tiles.extend)
pool.close()
current_progress = 0
while not rs.ready() and not self.cancel_requested:
if self.cancel_requested:
pool.terminate()
break
else:
QApplication.processEvents()
remaining = rs._number_left
index = total_nr_tiles - remaining
progress = int(100.0 / total_nr_tiles * (index + 1))
if progress != current_progress:
current_progress = progress
self._update_progress(progress=progress)
tiles = filter(lambda ti: ti.decoded_data is not None, tiles)
for t in tiles:
if self.cancel_requested:
break
else:
cache_file_name = self._get_tile_cache_name(t.zoom_level, t.column, t.row)
if not os.path.isfile(cache_file_name):
FileHelper.cache_tile(t, cache_file_name)
return tiles
def _unzip(self, data):
"""
* If the passed data is gzipped, it will be unzipped. Otherwise it will be returned untouched
:param data:
:return:
"""
is_gzipped = FileHelper.is_gzipped(data)
if is_gzipped:
file_content = GzipFile('', 'r', 0, StringIO(data)).read()
else:
file_content = data
return file_content
def _process_tiles(self, tiles, layer_filter):
"""
* Creates GeoJSON for all the specified tiles and reports the progress
:param tiles:
:return:
"""
total_nr_tiles = len(tiles)
info("Processing {} tiles", total_nr_tiles)
self._update_progress(progress=0, max_progress=100, msg="Processing features...")
current_progress = -1
for index, tile in enumerate(tiles):
if self.cancel_requested:
break
self._create_geojson(tile, layer_filter)
progress = int(100.0 / total_nr_tiles * (index + 1))
if progress != current_progress:
current_progress = progress
self._update_progress(progress=progress)
def _get_omt_layer_sort_id(self, layer_name):
"""
* Returns the cartographic sort id for the specified layer.
* This sort id is the position of the layer in the omt_layer_ordering collection.
* If the layer isn't present in the collection, the sort id wil be 999 and therefore the layer will be added at the bottom.
:param layer_name:
:return:
"""
sort_id = 999
if layer_name in self.omt_layer_ordering:
sort_id = self.omt_layer_ordering.index(layer_name)
return sort_id
def _assure_qgis_groups_exist(self, manual_layer_name=None, sort_layers=False):
"""
* Createss a group for each layer that is given by the layer source scheme
>> mbtiles: value 'JSON' in metadata table, array 'vector_layers'
>> TileJSON: value 'vector_layers'
:return:
"""
root = QgsProject.instance().layerTreeRoot()
name = self._root_group_name
if not name:
name = self.source.name()
root_group = root.findGroup(name)
if not root_group:
root_group = root.addGroup(name)
if not manual_layer_name:
layers = map(lambda l: l["id"], self.source.vector_layers())
else:
layers = [manual_layer_name]
if sort_layers:
layers = sorted(layers, key=lambda n: self._get_omt_layer_sort_id(n))
for index, layer_name in enumerate(layers):
group = root_group.findGroup(layer_name)
if not group:
group = root_group.addGroup(layer_name)
self._qgis_layer_groups_by_name[layer_name] = group
def _get_geojson_filename(self, layer_name, geo_type):
return "{}.{}.{}".format(self.source.name().replace(" ", "_"), layer_name, geo_type)
def _create_qgis_layers(self, merge_features, apply_styles):
"""
* Creates a hierarchy of groups and layers in qgis
"""
debug("Creating hierarchy in QGIS")
self._assure_qgis_groups_exist(sort_layers=apply_styles)
qgis_layers = QgsMapLayerRegistry.instance().mapLayers()
vt_qgis_name_layer_tuples = filter(lambda (n, l): l.customProperty("vector_tile_source") == self.source.source(), qgis_layers.iteritems())
own_layers = map(lambda (n, l): l, vt_qgis_name_layer_tuples)
for l in own_layers:
name = l.name()
geo_type = l.customProperty("geo_type")
if (name, geo_type) not in self.feature_collections_by_layer_name_and_geotype:
self._update_layer_source(l.source(), self._get_empty_feature_collection(0, l.name()))
self._update_progress(progress=0, max_progress=len(self.feature_collections_by_layer_name_and_geotype), msg="Creating layers...")
new_layers = []
count = 0
for layer_name, geo_type in self.feature_collections_by_layer_name_and_geotype:
count += 1
if self.cancel_requested:
break
feature_collection = self.feature_collections_by_layer_name_and_geotype[(layer_name, geo_type)]
zoom_level = feature_collection
file_name = self._get_geojson_filename(layer_name, geo_type)
file_path = FileHelper.get_geojson_file_name(file_name)
layer = None
if os.path.isfile(file_path):
# file exists already. add the features of the collection to the existing collection
# get the layer from qgis and update its source
layer = self._get_layer_by_source(own_layers, layer_name, file_path)
if layer:
self._update_layer_source(file_path, feature_collection)
if not layer:
self._update_layer_source(file_path, feature_collection)
layer = self._create_named_layer(file_path, layer_name, zoom_level, merge_features, geo_type)
new_layers.append((layer_name, geo_type, layer))
self._update_progress(progress=count+1)
QgsMapLayerRegistry.instance().reloadAllLayers()
if len(new_layers) > 0:
only_layers = list(map(lambda layer_name_tuple: layer_name_tuple[2], new_layers))
QgsMapLayerRegistry.instance().addMapLayers(only_layers, False)
for name, geo_type, layer in new_layers:
target_group = self._qgis_layer_groups_by_name[name]
target_group.addLayer(layer)
if apply_styles:
count = 0
self._update_progress(progress=0, max_progress=len(new_layers), msg="Styling layers...")
for name, geo_type, layer in new_layers:
count += 1
if self.cancel_requested:
break
VtReader._apply_named_style(layer, geo_type)
self._update_progress(progress=count)
def _update_layer_source(self, layer_source, feature_collection):
"""
* Updates the layers GeoJSON source file
:param layer_source:
:param feature_collections_by_tile_coord:
:return:
"""
with open(layer_source, "w") as f:
f.write(json.dumps(feature_collection))
@staticmethod
def _merge_feature_collections(current_feature_collection, feature_collections_by_tile_coord):
"""
* Merges the features of multiple tiles into the current_feature_collection if not already present.
:param current_feature_collection:
:param feature_collections_by_tile_coord:
:return:
"""
for tile_coord in feature_collections_by_tile_coord:
if tile_coord not in current_feature_collection["tiles"]:
feature_collection = feature_collections_by_tile_coord[tile_coord]
current_feature_collection["tiles"].extend(feature_collection["tiles"])
current_feature_collection["features"].extend(feature_collection["features"])
@staticmethod
def _get_layer_by_source(all_layers, layer_name, layer_source_file):
"""
* Returns the layer from QGIS whose name and layer_source matches the specified parameters
:param layer_name:
:param layer_source_file:
:return:
"""
layers = filter(lambda l: l.name() == layer_name and l.source() == layer_source_file, all_layers)
if len(layers) > 0:
return layers[0]
return None
@staticmethod
def _apply_named_style(layer, geo_type):
"""
* Looks for a styles with the same name as the layer and if one is found, it is applied to the layer
:param layer:
:param layer_path: e.g. 'transportation.service' or 'transportation_name.path'
:return:
"""
try:
name = layer.name().lower()
styles = [
"{}.{}".format(name, geo_type.lower()),
name
]
for p in styles:
style_name = "{}.qml".format(p).lower()
if style_name in VtReader._styles:
style_path = os.path.join(FileHelper.get_plugin_directory(), "styles/{}".format(style_name))
res = layer.loadNamedStyle(style_path)
if res[1]: # Style loaded
layer.setCustomProperty("layerStyle", style_path)
if layer.customProperty("layerStyle") == style_path:
debug("Style successfully applied: {}", style_name)
break
except:
critical("Loading style failed: {}", sys.exc_info())
def _create_named_layer(self, json_src, layer_name, zoom_level, merge_features, geo_type):
"""
* Creates a QgsVectorLayer and adds it to the group specified by layer_target_group
* Invalid geometries will be removed during the process of merging features over tile boundaries
"""
# layer_with_zoom = "{}{}{}".format(layer_name, VtReader._zoom_level_delimiter, zoom_level)
layer = QgsVectorLayer(json_src, layer_name, "ogr")
layer.setCustomProperty("vector_tile_source", self.source.source())
layer.setCustomProperty("zoom_level", zoom_level)
layer.setShortName(layer_name)
layer.setDataUrl(self.source.source())
if self.source.name() and "openmaptiles" in self.source.name().lower():
layer.setDataUrl(remove_key(self.source.source()))
layer.setAttribution(u"Vector Tiles © Klokan Technologies GmbH (CC-BY), Data © OpenStreetMap contributors (ODbL)")
layer.setAttributionUrl("https://openmaptiles.com/hosting/")
if merge_features and geo_type in [GeoTypes.LINE_STRING, GeoTypes.POLYGON]:
layer = FeatureMerger().merge_features(layer)
layer.setName(layer_name)
return layer
def _create_geojson(self, tile, layer_filter):
"""
* Transforms all features of the specified tile into GeoJSON
* The resulting GeoJSON feature will be applied to the features of the corresponding GeoJSON FeatureCollection
:param tile:
:return:
"""
for layer_name in tile.decoded_data:
if layer_filter and len(layer_filter) > 0:
if layer_name not in layer_filter:
continue
layer = tile.decoded_data[layer_name]
if "extent" in layer:
extent = layer["extent"]
else:
extent = self._DEFAULT_EXTENT
tile_features = layer["features"]
tile_id = tile.id()
for feature in tile_features:
if self._is_duplicate_feature(feature, tile) or self.cancel_requested:
continue
geojson_features, geo_type = self._create_geojson_feature(feature, tile, extent)
if geojson_features and len(geojson_features) > 0:
name_and_geotype = (layer_name, geo_type)
if name_and_geotype not in self.feature_collections_by_layer_name_and_geotype:
self.feature_collections_by_layer_name_and_geotype[name_and_geotype] = self._get_empty_feature_collection(tile.zoom_level, layer_name)
feature_collection = self.feature_collections_by_layer_name_and_geotype[name_and_geotype]
feature_collection["features"].extend(geojson_features)
if tile_id not in feature_collection["tiles"]:
feature_collection["tiles"].append(tile_id)
@staticmethod
def _get_feature_class_and_subclass(feature):
feature_class = None
feature_subclass = None
properties = feature["properties"]
if "class" in properties:
feature_class = properties["class"]
if "subclass" in properties:
feature_subclass = properties["subclass"]
if feature_subclass == feature_class:
feature_subclass = None
if feature_subclass:
assert feature_class, "A feature with a subclass should also have a class"
return feature_class, feature_subclass
def _create_geojson_feature(self, feature, tile, current_layer_tile_extent):
"""
Creates a GeoJSON feature for the specified feature
"""
geo_type = geo_types[feature["type"]]
coordinates = feature["geometry"]
properties = feature["properties"]
properties["_col"] = tile.column
properties["_row"] = tile.row
if "id" in properties and properties["id"] < 0:
properties["id"] = 0
if geo_type == GeoTypes.POINT:
coordinates = coordinates[0]
properties["_symbol"] = self._get_poi_icon(feature)
if self._clip_tiles_at_tile_bounds and not all(0 <= c <= current_layer_tile_extent for c in coordinates):
return None, None
all_out_of_bounds = []
coordinates = map_coordinates_recursive(coordinates=coordinates,
tile_extent=current_layer_tile_extent,
mapper_func=lambda coords: VtReader._get_absolute_coordinates(coords,
tile,
current_layer_tile_extent),
all_out_of_bounds_func=lambda out_of_bounds: all_out_of_bounds.append(
out_of_bounds))
if self._clip_tiles_at_tile_bounds and all(c is True for c in all_out_of_bounds):
return None, None
split_geometries = self._loading_options["merge_tiles"]
geojson_features = VtReader._create_geojson_feature_from_coordinates(geo_type, coordinates, properties, split_geometries)
return geojson_features, geo_type
def _get_poi_icon(self, feature):
"""
* Returns the name of the svg icon that will be applied in QGIS.
* The resulting icon is determined based on class and subclass of the specified feature.
:param feature:
:return:
"""
feature_class, feature_subclass = self._get_feature_class_and_subclass(feature)
root_path = FileHelper.get_icons_directory()
class_icon = "{}.svg".format(feature_class)
class_subclass_icon = "{}.{}.svg".format(feature_class, feature_subclass)
icon_name = "poi.svg"
if os.path.isfile(os.path.join(root_path, class_subclass_icon)):
icon_name = class_subclass_icon
elif os.path.isfile(os.path.join(root_path, class_icon)):
icon_name = class_icon
return icon_name
def _is_duplicate_feature(self, feature, tile):
"""
* Returns true if the same feature has already been loaded
* If the feature has not been loaded, it is marked as loaded by calling this function
* A feature is identified by the tuple: (feature_name, feature_class, feature_subclass)
* A feature is only loaded if the same feature identifier doesn't occur on the same or a neighbouring tile
:param feature:
:param tile:
:return:
"""
geo_type = geo_types[feature["type"]]
is_poi = geo_type == GeoTypes.POINT
is_loaded = False
if is_poi and VtReader._feature_name(feature):
feature_id = VtReader._feature_id(feature)
if feature_id in self._loaded_pois_by_id:
locations = self._loaded_pois_by_id[feature_id]
for loc in locations:
distance_x = math.fabs(loc["col"] - tile.column)
distance_y = math.fabs(loc["row"] - tile.row)
distance_threshold = 2
is_loaded = distance_x <= distance_threshold and distance_y <= distance_threshold
if is_loaded:
break
if not is_loaded:
if feature_id not in self._loaded_pois_by_id:
self._loaded_pois_by_id[feature_id] = []
self._loaded_pois_by_id[feature_id].append({'col': tile.column, 'row': tile.row})
return is_loaded
@staticmethod
def _feature_id(feature):
name = VtReader._feature_name(feature)
feature_class, feature_subclass = VtReader._get_feature_class_and_subclass(feature)
feature_id = (name, feature_class, feature_subclass)
return feature_id
@staticmethod
def _feature_name(feature):
"""
* Returns the 'name' property of the feature
:param feature:
:return:
"""
name = None
properties = feature["properties"]
if "name" in properties:
name = properties["name"]
return name
@staticmethod
def _create_geojson_feature_from_coordinates(geo_type, coordinates, properties, split_multi_geometries):
"""
* Returns a JSON object that represents a GeoJSON feature
:param geo_type:
:param coordinates:
:param properties:
:return:
"""
assert coordinates is not None
all_features = []
coordinate_sets = [coordinates]
type_string = geo_type
is_multi_geometry = is_multi(geo_type, coordinates)
if is_multi_geometry and not split_multi_geometries:
type_string = "Multi{}".format(geo_type)
elif is_multi_geometry and split_multi_geometries:
coordinate_sets = []
for coord_array in coordinates:
coordinate_sets.append(coord_array)
for c in coordinate_sets:
feature_json = {
"type": "Feature",
"geometry": {
"type": type_string,
"coordinates": c
},
"properties": properties
}
all_features.append(feature_json)
return all_features
@staticmethod
def _get_absolute_coordinates(coordinates, tile, extent):
"""
* The coordinates of a geometry, are relative to the tile the feature is located on.
* Due to this, we've to get the absolute coordinates of the geometry.
"""
delta_x = tile.extent[2] - tile.extent[0]
delta_y = tile.extent[3] - tile.extent[1]
merc_easting = int(tile.extent[0] + delta_x / extent * coordinates[0])
merc_northing = int(tile.extent[1] + delta_y / extent * coordinates[1])
return [merc_easting, merc_northing]
out_path_res = os.path.join(this_file_dir, "res") in_dir = os.path.join(this_file_dir, "data", "csvs") out_dir_res = os.path.join(this_file_dir, "res") out_dir_khiops = os.path.join(this_file_dir, "res", "khiops_res") out_dir_fcasts = os.path.join(this_file_dir, "res", "fcasts") glob_csv_res = os.path.join(out_path_res, "all.csv") vali_csv_res = os.path.join(out_path_res, "valid.csv") import pandas as pd from sklearn.tree import DecisionTreeClassifier from sklearn.linear_model import LogisticRegression from sklearn.naive_bayes import GaussianNB # Bunch of cleanings for empty files from file_helper import FileHelper fh = FileHelper(tstmp) fh.clean_zips_folder() fh.clean_res_folder(out_dir_res) fh.ensure_dirs_exist([out_path_res, in_dir, out_dir_res, out_dir_khiops, out_dir_fcasts]) # After cleaning, zip the code which is executed now fh.zip_code() # Init objects from khiops import KhiopsManager km = KhiopsManager() from my_utils import MyUtils utils = MyUtils() from ecml_machine import ECMLMachine
def _create_qgis_layers(self, merge_features, apply_styles):
"""
* Creates a hierarchy of groups and layers in qgis
"""
debug("Creating hierarchy in QGIS")
self._assure_qgis_groups_exist(sort_layers=apply_styles)
qgis_layers = QgsMapLayerRegistry.instance().mapLayers()
vt_qgis_name_layer_tuples = filter(lambda (n, l): l.customProperty("vector_tile_source") == self.source.source(), qgis_layers.iteritems())
own_layers = map(lambda (n, l): l, vt_qgis_name_layer_tuples)
for l in own_layers:
name = l.name()
geo_type = l.customProperty("geo_type")
if (name, geo_type) not in self.feature_collections_by_layer_name_and_geotype:
self._update_layer_source(l.source(), self._get_empty_feature_collection(0, l.name()))
self._update_progress(progress=0, max_progress=len(self.feature_collections_by_layer_name_and_geotype), msg="Creating layers...")
new_layers = []
count = 0
for layer_name, geo_type in self.feature_collections_by_layer_name_and_geotype:
count += 1
if self.cancel_requested:
break
feature_collection = self.feature_collections_by_layer_name_and_geotype[(layer_name, geo_type)]
zoom_level = feature_collection
file_name = self._get_geojson_filename(layer_name, geo_type)
file_path = FileHelper.get_geojson_file_name(file_name)
layer = None
if os.path.isfile(file_path):
# file exists already. add the features of the collection to the existing collection
# get the layer from qgis and update its source
layer = self._get_layer_by_source(own_layers, layer_name, file_path)
if layer:
self._update_layer_source(file_path, feature_collection)
if not layer:
self._update_layer_source(file_path, feature_collection)
layer = self._create_named_layer(file_path, layer_name, zoom_level, merge_features, geo_type)
new_layers.append((layer_name, geo_type, layer))
self._update_progress(progress=count+1)
QgsMapLayerRegistry.instance().reloadAllLayers()
if len(new_layers) > 0:
only_layers = list(map(lambda layer_name_tuple: layer_name_tuple[2], new_layers))
QgsMapLayerRegistry.instance().addMapLayers(only_layers, False)
for name, geo_type, layer in new_layers:
target_group = self._qgis_layer_groups_by_name[name]
target_group.addLayer(layer)
if apply_styles:
count = 0
self._update_progress(progress=0, max_progress=len(new_layers), msg="Styling layers...")
for name, geo_type, layer in new_layers:
count += 1
if self.cancel_requested:
break
VtReader._apply_named_style(layer, geo_type)
self._update_progress(progress=count)
def _save_recently_used(self):
recently_used = FileHelper.get_recently_used_file()
with open(recently_used, 'w') as f:
for path in self.recently_used:
f.write("{}\n".format(path))
def load_data_from_file(self):
contents = FileHelper.load_json_file(self.path())
self.road_data = contents['road_data']
self.last_update = parse(contents['last_update'])
def save_states(agent):
data = {k: v.tolist() for k, v in agent.states.items()}
FileHelper.save(data, 'data/states.json')
def save_data_to_file(self):
data = {
'last_update': self.last_update,
'road_data': self.road_data
}
FileHelper.save_json_file(self.path(), data)
from file_helper import FileHelper
#创建程序存储用户输入的个人信息
u_name = input('input your name\n')
u_gender = input('input your gender\n')
u_age = input('input your age\n')
f_h = FileHelper(r'study_11\user_info.txt')
f_h.write_append(u_name + '\n')
f_h.write_append(u_gender + '\n')
f_h.write_append(u_age + '\n')
print('==========读取个人信息=========')
content = f_h.read()
print(content)
print('==========序列化存储=========')
f_h.remove_content()
user = {'name': u_name, 'gender': u_gender, 'age': u_age}
f_h.write_to_json(user)
obj = f_h.json_to_object()
print(obj['name'] + '_' + obj['gender'] + '_' + obj['age'])
def move_old_file(self):
date = self.last_update.strftime('%Y%m%d')
new_path = 'old_data/' + date + '_' + self.file_name
FileHelper.move_file(self.path(), new_path)
def _get_tile_cache_name(self, zoom_level, col, row):
return FileHelper.get_cached_tile_file_name(self.source.name(), zoom_level, col, row)
#!/usr/bin/python
# -*- coding: utf-8 -*-
'''
THIS is USED FOR GENERATE BINGACCOUNT DIMENSSION
'''
import maxminddb
from file_helper import FileHelper
reader = maxminddb.open_database('GeoLite2-Country.mmdb')
geo = reader.get('47.88.18.160')
reader.close()
print(geo)
country = geo['country']
countryCode = country['iso_code']
print(countryCode)
ipList = FileHelper.loadFileList('d:/tmp/stem/ip_uniq.txt')
countryList = []
reader = maxminddb.open_database('GeoLite2-Country.mmdb')
for ip in ipList:
geo = reader.get(ip)
if geo != None and 'country' in geo:
country = geo['country']
countryName = geo['country']['names']['en']
countryCode = country['iso_code']
else:
countryCode = 'na'
countryList.append(countryName)
reader.close()
FileHelper.saveFileList('d:/tmp/stem/country.txt', countryList, 'w')
class VtReader:
cartographic_layer_ordering = [
"place",
"mountain_peak",
"housenumber",
"water_name",
"transportation_name",
"poi",
"boundary",
"transportation",
"building",
"aeroway",
"park",
"water",
"waterway",
"landcover",
"landuse"
]
_layers_to_dissolve = []
_zoom_level_delimiter = "*"
_styles = FileHelper.get_styles()
def __init__(self, iface, path_or_url, progress_handler):
"""
* The mbtiles_path can also be an URL in zxy format: z=zoom, x=tile column, y=tile row
:param iface:
:param path_or_url:
"""
if not path_or_url:
raise RuntimeError("The datasource is required")
is_web_source = path_or_url.lower().startswith("http://") or path_or_url.lower().startswith("https://")
if is_web_source:
self.source = ServerSource(url=path_or_url)
else:
self.source = MBTilesSource(path=path_or_url)
self.source.set_progress_handler(self._update_progress)
FileHelper.assure_temp_dirs_exist()
self.iface = iface
self.cartographic_ordering_enabled = True
self.progress_handler = progress_handler
self.feature_collections_by_layer_path = {}
self._qgis_layer_groups_by_name = {}
self.cancel_requested = False
self._loaded_pois_by_id = {}
def _update_progress(self, title=None, show_dialog=None, progress=None, max_progress=None, msg=None):
if self.progress_handler:
self.progress_handler(title, progress, max_progress, msg, show_dialog)
def _get_empty_feature_collection(self, zoom_level, layer_name):
"""
* Returns an empty GeoJSON FeatureCollection with the coordinate reference system (crs) set to EPSG3857
"""
# todo: when improving CRS handling: the correct CRS of the source has to be set here
source_crs = self.source.crs()
if source_crs:
epsg_id = get_code_from_epsg(source_crs)
else:
epsg_id = 3857
crs = {
"type": "name",
"properties": {
"name": "urn:ogc:def:crs:EPSG::{}".format(epsg_id)}}
return {
"tiles": [],
"source": self.source.name(),
"scheme": self.source.scheme(),
"layer": layer_name,
"zoom_level": zoom_level,
"type": "FeatureCollection",
"crs": crs,
"features": []}
def cancel(self):
"""
* Cancels the loading process.
:return:
"""
self.cancel_requested = True
if self.source:
self.source.cancel()
def _get_tile_cache_name(self, zoom_level, col, row):
return FileHelper.get_cached_tile_file_name(self.source.name(), zoom_level, col, row)
def load_tiles(self, zoom_level, layer_filter, load_mask_layer=False, merge_tiles=True, apply_styles=True, max_tiles=None,
bounds=None, limit_reacher_handler=None):
"""
* Loads the vector tiles from either a file or a URL and adds them to QGIS
:param zoom_level: The zoom level to load
:param layer_filter: A list of layers. If any layers are set, only these will be loaded. If the list is empty,
all available layers will be loaded
:param load_mask_layer: If True the mask layer will also be loaded
:param merge_tiles: If True neighbouring tiles and features will be merged
:param apply_styles: If True the default styles will be applied
:param max_tiles: The maximum number of tiles to load
:param bounds:
:param limit_reacher_handler:
:return:
"""
self.cancel_requested = False
self.feature_collections_by_layer_path = {}
self._qgis_layer_groups_by_name = {}
self._update_progress(show_dialog=True, title="Loading '{}'".format(os.path.basename(self.source.name())))
min_zoom = self.source.min_zoom()
max_zoom = self.source.max_zoom()
if min_zoom is not None and zoom_level < min_zoom:
zoom_level = min_zoom
if max_zoom is not None and zoom_level > max_zoom:
zoom_level = max_zoom
all_tiles = get_all_tiles(bounds, lambda: self.cancel_requested)
tiles_to_load = set()
tiles = []
for t in all_tiles:
if self.cancel_requested:
break
file_name = self._get_tile_cache_name(zoom_level, t[0], t[1])
tile = FileHelper.get_cached_tile(file_name)
if tile and tile.decoded_data:
tiles.append(tile)
else:
tiles_to_load.add(t)
debug("{} cache hits. {} will be loaded from the source.", len(tiles), len(tiles_to_load))
debug("Loading extent {} for zoom level '{}' of: {}", zoom_level, self.source.name())
tile_data_tuples = []
if len(tiles_to_load) > 0:
tile_data_tuples = self.source.load_tiles(zoom_level=zoom_level,
tiles_to_load=tiles_to_load,
max_tiles=max_tiles,
for_each=QApplication.processEvents,
limit_reacher_handler=limit_reacher_handler)
if len(tiles) == 0 and (not tile_data_tuples or len(tile_data_tuples) == 0):
QMessageBox.information(None, "No tiles found", "What a pity, no tiles could be found!")
if load_mask_layer:
mask_level = self.source.mask_level()
if mask_level is not None and mask_level != zoom_level:
debug("Mapping {} tiles to mask level", len(all_tiles))
scheme = self.source.scheme()
crs = self.source.crs()
mask_tiles = map(
lambda t: change_zoom(zoom_level, int(mask_level), t, scheme, crs),
all_tiles)
debug("Mapping done")
mask_tiles_to_load = set()
for t in mask_tiles:
file_name = self._get_tile_cache_name(mask_level, t[0], t[1])
tile = FileHelper.get_cached_tile(file_name)
if tile and tile.decoded_data:
tiles.append(tile)
else:
mask_tiles_to_load.add(t)
debug("Loading mask layer (zoom_level={})", mask_level)
tile_data_tuples = []
if len(mask_tiles_to_load) > 0:
mask_layer_data = self.source.load_tiles(zoom_level=mask_level,
tiles_to_load=mask_tiles_to_load,
max_tiles=max_tiles,
for_each=QApplication.processEvents)
debug("Mask layer loaded")
tile_data_tuples.extend(mask_layer_data)
if tile_data_tuples and len(tile_data_tuples) > 0:
if not self.cancel_requested:
decoded_tiles = self._decode_tiles(tile_data_tuples)
tiles.extend(decoded_tiles)
if len(tiles) > 0:
if not self.cancel_requested:
self._process_tiles(tiles, layer_filter)
if not self.cancel_requested:
self._create_qgis_layers(merge_features=merge_tiles,
apply_styles=apply_styles)
self._update_progress(show_dialog=False)
if self.cancel_requested:
info("Import cancelled")
else:
info("Import complete")
def _decode_tiles(self, tiles_with_encoded_data):
"""
* Decodes the PBF data from all the specified tiles and reports the progress
* If a tile is loaded from the cache, the decoded_data is already set and doesn't have to be encoded
:param tiles_with_encoded_data:
:return:
"""
total_nr_tiles = len(tiles_with_encoded_data)
info("Decoding {} tiles", total_nr_tiles)
self._update_progress(progress=0, max_progress=100, msg="Decoding tiles...")
nr_processors = 4
try:
nr_processors = mp.cpu_count()
except NotImplementedError:
info("CPU count cannot be retrieved. Falling back to default = 4")
tiles_with_encoded_data = map(lambda t: (t[0], self._unzip(t[1])), tiles_with_encoded_data)
pool = mp.Pool(nr_processors)
tiles = []
rs = pool.map_async(decode_tile, tiles_with_encoded_data, callback=tiles.extend)
pool.close()
current_progress = 0
while not rs.ready() and not self.cancel_requested:
QApplication.processEvents()
remaining = rs._number_left
index = total_nr_tiles - remaining
progress = int(100.0 / total_nr_tiles * (index + 1))
if progress != current_progress:
current_progress = progress
self._update_progress(progress=progress)
for t in tiles:
cache_file_name = self._get_tile_cache_name(t.zoom_level, t.column, t.row)
if not os.path.isfile(cache_file_name):
FileHelper.cache_tile(t, cache_file_name)
return tiles
def _unzip(self, data):
"""
* If the passed data is gzipped, it will be unzipped. Otherwise it will be returned untouched
:param data:
:return:
"""
is_gzipped = FileHelper.is_gzipped(data)
if is_gzipped:
file_content = GzipFile('', 'r', 0, StringIO(data)).read()
else:
file_content = data
return file_content
def _process_tiles(self, tiles, layer_filter):
"""
* Creates GeoJSON for all the specified tiles and reports the progress
:param tiles:
:return:
"""
total_nr_tiles = len(tiles)
info("Processing {} tiles", total_nr_tiles)
self._update_progress(progress=0, max_progress=100, msg="Processing features...")
current_progress = -1
for index, tile in enumerate(tiles):
QApplication.processEvents()
if self.cancel_requested:
break
self._create_geojson(tile, layer_filter)
progress = int(100.0 / total_nr_tiles * (index + 1))
if progress != current_progress:
current_progress = progress
self._update_progress(progress=progress)
def _get_cartographic_layer_sort_id(self, layer_name):
"""
* Returns the cartographic sort id for the specified layer.
* This sort id is the position of the layer in the cartographic_layer_ordering collection.
* If the layer isn't present in the collection, the sort id wil be 999 and therefore the layer will be added at the bottom.
:param layer_name:
:return:
"""
sort_id = 999
if layer_name in self.cartographic_layer_ordering:
sort_id = self.cartographic_layer_ordering.index(layer_name)
return sort_id
def _assure_qgis_groups_exist(self):
"""
* Createss a group for each layer that is given by the layer source scheme
>> mbtiles: value 'JSON' in metadata table, array 'vector_layers'
>> TileJSON: value 'vector_layers'
:return:
"""
root = QgsProject.instance().layerTreeRoot()
root_group = root.findGroup(self.source.name())
if not root_group:
root_group = root.addGroup(self.source.name())
layers = map(lambda l: l["id"], self.source.vector_layers())
layers = sorted(layers, key=lambda x: self._get_cartographic_layer_sort_id(x))
for index, layer_name in enumerate(layers):
group = root_group.findGroup(layer_name)
if not group:
group = root_group.addGroup(layer_name)
self._qgis_layer_groups_by_name[layer_name] = group
def _get_geojson_filename(self, layer_name, geo_type, zoom_level):
return "{}.{}.{}.{}".format(self.source.name(), layer_name, geo_type, zoom_level)
def _create_qgis_layers(self, merge_features, apply_styles):
"""
* Creates a hierarchy of groups and layers in qgis
"""
debug("Creating hierarchy in QGIS")
self._assure_qgis_groups_exist()
self._update_progress(progress=0, max_progress=len(self.feature_collections_by_layer_path), msg="Creating layers...")
layers = []
for index, layer_name_and_type in enumerate(self.feature_collections_by_layer_path):
layer_name_and_zoom = layer_name_and_type[0]
geo_type = layer_name_and_type[1]
layer_name = layer_name_and_zoom.split(VtReader._zoom_level_delimiter)[0]
zoom_level = layer_name_and_zoom.split(VtReader._zoom_level_delimiter)[1]
QApplication.processEvents()
if self.cancel_requested:
break
target_group = self._qgis_layer_groups_by_name[layer_name]
feature_collections_by_tile_coord = self.feature_collections_by_layer_path[layer_name_and_type]
file_name = self._get_geojson_filename(layer_name, geo_type, zoom_level)
file_path = FileHelper.get_geojson_file_name(file_name)
layer = None
if os.path.isfile(file_path):
# file exists already. add the features of the collection to the existing collection
# get the layer from qgis and update its source
layer = self._get_layer_by_source(layer_name_and_zoom, file_path)
if layer:
self._update_layer_source(file_path, feature_collections_by_tile_coord)
layer.reload()
if not layer:
complete_collection = self._get_empty_feature_collection(zoom_level, layer_name)
self._merge_feature_collections(current_feature_collection=complete_collection,
feature_collections_by_tile_coord=feature_collections_by_tile_coord)
with open(file_path, "w") as f:
f.write(json.dumps(complete_collection))
layer = self._add_vector_layer_to_qgis(file_path, layer_name, zoom_level, target_group, merge_features, geo_type)
if apply_styles:
layers.append((layer_name_and_type, layer))
self._update_progress(progress=index+1)
if apply_styles:
self._update_progress(progress=0, max_progress=len(layers), msg="Styling layers...")
for index, layer_path_tuple in enumerate(layers):
QApplication.processEvents()
if self.cancel_requested:
break
path_and_type = layer_path_tuple[0]
geo_type = path_and_type[1]
layer = layer_path_tuple[1]
VtReader._apply_named_style(layer, geo_type)
self._update_progress(progress=index+1)
@staticmethod
def _update_layer_source(layer_source, feature_collections_by_tile_coord):
"""
* Updates the layers GeoJSON source file
:param layer_source:
:param feature_collections_by_tile_coord:
:return:
"""
with open(layer_source, "r") as f:
current_feature_collection = json.load(f)
VtReader._merge_feature_collections(current_feature_collection, feature_collections_by_tile_coord)
if current_feature_collection:
with open(layer_source, "w") as f:
json.dump(current_feature_collection, f)
@staticmethod
def _merge_feature_collections(current_feature_collection, feature_collections_by_tile_coord):
"""
* Merges the features of multiple tiles into the current_feature_collection if not already present.
:param current_feature_collection:
:param feature_collections_by_tile_coord:
:return:
"""
for tile_coord in feature_collections_by_tile_coord:
if tile_coord not in current_feature_collection["tiles"]:
feature_collection = feature_collections_by_tile_coord[tile_coord]
current_feature_collection["tiles"].extend(feature_collection["tiles"])
current_feature_collection["features"].extend(feature_collection["features"])
@staticmethod
def _get_layer_by_source(layer_name, layer_source_file):
"""
* Returns the layer from QGIS whose name and layer_source matches the specified parameters
:param layer_name:
:param layer_source_file:
:return:
"""
matching_layer = None
layers = QgsMapLayerRegistry.instance().mapLayersByName(layer_name)
for l in layers:
if l.source() == layer_source_file:
matching_layer = l
break
return matching_layer
@staticmethod
def _apply_named_style(layer, geo_type):
"""
* Looks for a styles with the same name as the layer and if one is found, it is applied to the layer
:param layer:
:param layer_path: e.g. 'transportation.service' or 'transportation_name.path'
:return:
"""
try:
name = layer.name().split(VtReader._zoom_level_delimiter)[0].lower()
styles = [
"{}.{}".format(name, geo_type.lower()),
name
]
for p in styles:
style_name = "{}.qml".format(p).lower()
if style_name in VtReader._styles:
style_path = os.path.join(FileHelper.get_plugin_directory(), "styles/{}".format(style_name))
res = layer.loadNamedStyle(style_path)
if res[1]: # Style loaded
layer.setCustomProperty("layerStyle", style_path)
if layer.customProperty("layerStyle") == style_path:
debug("Style successfully applied: {}", style_name)
break
except:
critical("Loading style failed: {}", sys.exc_info())
def _add_vector_layer_to_qgis(self, json_src, layer_name, zoom_level, layer_target_group, merge_features, geo_type):
"""
* Creates a QgsVectorLayer and adds it to the group specified by layer_target_group
* Invalid geometries will be removed during the process of merging features over tile boundaries
"""
layer_with_zoom = "{}{}{}".format(layer_name, VtReader._zoom_level_delimiter, zoom_level)
layer = QgsVectorLayer(json_src, layer_with_zoom, "ogr")
if merge_features and geo_type in [GeoTypes.LINE_STRING, GeoTypes.POLYGON]:
layer = FeatureMerger().merge_features(layer)
layer.setName(layer_name)
QgsMapLayerRegistry.instance().addMapLayer(layer, False)
layer_target_group.addLayer(layer)
layer.setCustomProperty("vector_tile_source", self.source.source())
layer.setShortName(layer_name)
layer.setDataUrl(self.source.source())
if self.source.name() and "openmaptiles" in self.source.name().lower():
layer.setDataUrl(remove_key(self.source.source()))
layer.setAttribution(u"Vector Tiles © Klokan Technologies GmbH (CC-BY), Data © OpenStreetMap contributors (ODbL)")
layer.setAttributionUrl("https://openmaptiles.com/hosting/")
return layer
def _create_geojson(self, tile, layer_filter):
"""
* Transforms all features of the specified tile into GeoJSON
* The resulting GeoJSON feature will be applied to the features of the corresponding GeoJSON FeatureCollection
:param tile:
:return:
"""
for layer_name in tile.decoded_data:
if layer_filter and len(layer_filter) > 0:
if layer_name not in layer_filter:
continue
tile_features = tile.decoded_data[layer_name]["features"]
tile_id = tile.id()
feature_path = "{}{}{}".format(layer_name, VtReader._zoom_level_delimiter, tile.zoom_level)
for index, feature in enumerate(tile_features):
if self._is_duplicate_feature(feature, tile):
continue
geojson_feature, geo_type = self._create_geojson_feature(feature, tile)
if geojson_feature:
path_and_type = (feature_path, geo_type)
if path_and_type not in self.feature_collections_by_layer_path:
self.feature_collections_by_layer_path[path_and_type] = {}
collection_dict = self.feature_collections_by_layer_path[path_and_type]
if tile_id not in collection_dict:
collection_dict[tile_id] = self._get_empty_feature_collection(tile.zoom_level, layer_name)
collection = collection_dict[tile_id]
collection["features"].append(geojson_feature)
if tile_id not in collection["tiles"]:
collection["tiles"].append(tile_id)
geotypes_to_dissolve = [GeoTypes.POLYGON, GeoTypes.LINE_STRING]
if geo_type in geotypes_to_dissolve and feature_path not in self._layers_to_dissolve:
self._layers_to_dissolve.append(feature_path)
@staticmethod
def _get_feature_class_and_subclass(feature):
feature_class = None
feature_subclass = None
properties = feature["properties"]
if "class" in properties:
feature_class = properties["class"]
if "subclass" in properties:
feature_subclass = properties["subclass"]
if feature_subclass == feature_class:
feature_subclass = None
if feature_subclass:
assert feature_class, "A feature with a subclass should also have a class"
return feature_class, feature_subclass
def _create_geojson_feature(self, feature, tile):
"""
Creates a GeoJSON feature for the specified feature
"""
geo_type = geo_types[feature["type"]]
coordinates = feature["geometry"]
properties = feature["properties"]
if geo_type == GeoTypes.POINT:
coordinates = coordinates[0]
properties["_symbol"] = self._get_poi_icon(feature)
if not all(0 <= c <= tile_extent for c in coordinates):
return None, None
all_out_of_bounds = []
coordinates = map_coordinates_recursive(coordinates=coordinates,
mapper_func=lambda coords: VtReader._get_absolute_coordinates(coords,
tile),
all_out_of_bounds_func=lambda out_of_bounds: all_out_of_bounds.append(
out_of_bounds))
if all(c is True for c in all_out_of_bounds):
return None, None
feature_json = VtReader._create_geojson_feature_from_coordinates(geo_type, coordinates, properties)
return feature_json, geo_type
def _get_poi_icon(self, feature):
"""
* Returns the name of the svg icon that will be applied in QGIS.
* The resulting icon is determined based on class and subclass of the specified feature.
:param feature:
:return:
"""
feature_class, feature_subclass = self._get_feature_class_and_subclass(feature)
root_path = FileHelper.get_icons_directory()
class_icon = "{}.svg".format(feature_class)
class_subclass_icon = "{}.{}.svg".format(feature_class, feature_subclass)
icon_name = "poi.svg"
if os.path.isfile(os.path.join(root_path, class_subclass_icon)):
icon_name = class_subclass_icon
elif os.path.isfile(os.path.join(root_path, class_icon)):
icon_name = class_icon
return icon_name
def _is_duplicate_feature(self, feature, tile):
"""
* Returns true if the same feature has already been loaded
* If the feature has not been loaded, it is marked as loaded by calling this function
* A feature is identified by the tuple: (feature_name, feature_class, feature_subclass)
* A feature is only loaded if the same feature identifier doesn't occur on the same or a neighbouring tile
:param feature:
:param tile:
:return:
"""
geo_type = geo_types[feature["type"]]
is_poi = geo_type == GeoTypes.POINT
is_loaded = False
if is_poi and VtReader._feature_name(feature):
feature_id = VtReader._feature_id(feature)
if feature_id in self._loaded_pois_by_id:
locations = self._loaded_pois_by_id[feature_id]
for loc in locations:
distance_x = math.fabs(loc["col"] - tile.column)
distance_y = math.fabs(loc["row"] - tile.row)
distance_threshold = 2
is_loaded = distance_x <= distance_threshold and distance_y <= distance_threshold
if is_loaded:
break
if not is_loaded:
if feature_id not in self._loaded_pois_by_id:
self._loaded_pois_by_id[feature_id] = []
self._loaded_pois_by_id[feature_id].append({'col': tile.column, 'row': tile.row})
return is_loaded
@staticmethod
def _feature_id(feature):
name = VtReader._feature_name(feature)
feature_class, feature_subclass = VtReader._get_feature_class_and_subclass(feature)
feature_id = (name, feature_class, feature_subclass)
return feature_id
@staticmethod
def _feature_name(feature):
"""
* Returns the 'name' property of the feature
:param feature:
:return:
"""
name = None
properties = feature["properties"]
if "name" in properties:
name = properties["name"]
return name
@staticmethod
def _create_geojson_feature_from_coordinates(geo_type, coordinates, properties):
"""
* Returns a JSON object that represents a GeoJSON feature
:param geo_type:
:param coordinates:
:param properties:
:return:
"""
type_string = geo_type
if is_multi(geo_type, coordinates):
type_string = "Multi{}".format(geo_type)
feature_json = {
"type": "Feature",
"geometry": {
"type": type_string,
"coordinates": coordinates
},
"properties": properties
}
return feature_json
@staticmethod
def _get_absolute_coordinates(coordinates, tile):
"""
* The coordinates of a geometry, are relative to the tile the feature is located on.
* Due to this, we've to get the absolute coordinates of the geometry.
"""
delta_x = tile.extent[2] - tile.extent[0]
delta_y = tile.extent[3] - tile.extent[1]
merc_easting = int(tile.extent[0] + delta_x / tile_extent * coordinates[0])
merc_northing = int(tile.extent[1] + delta_y / tile_extent * coordinates[1])
return [merc_easting, merc_northing]
def enable_cartographic_ordering(self, enabled):
self.cartographic_ordering_enabled = enabled
def OnSaveSettingInfoButtonClick(self, event):
setting_info_dict = self.get_setting_info()
FileHelper.write_setting_info(setting_info_dict)
self.confirm_dialog(u"保存设置成功！")
event.Skip()
