def f_jumping_modulo(n, m=10): idxs = list(range(1, n*m+1)) # vals_mod = np.arange(0, n) % m vals = IndexedOrderedDict([(i, -1) for i in range(1, n*m+1)]) # {i: -1 for i in range(1, n*m+1)} vals_mod = [i for j in range(1, 30+1) for i in range(0, j)] smallest_idx = 0 largest_idx = 0 idx_now = 0 # for i, v in enumerate(vals_mod[:50]): j = 1 max_reached = False while not max_reached: for v in range(j-1, -1, -1): # for v in range(0, j): if v > idx_now: idx_now += v else: idx_now -= v if idx_now >= len(idxs): max_reached = True break idx = idxs.pop(idx_now) vals[idx] = v print("idx_now: {}, idx: {}, v: {}".format(idx_now, idx, v)) if smallest_idx < idxs[0]: smallest_idx = idxs[0] # if largest_idx < idx: # largest_idx = idx # lst_temp = list(vals.values())[:largest_idx] # print("i: {}, lst_temp:\n{}".format(i, lst_temp)) j += 1 lst = list(vals.values())[:smallest_idx-1] # print("lst: {}".format(lst)) # print("idxs: {}".format(idxs)) # arr = np.array(lst[:smallest_idx-1]) # % m # arr = np.array(lst[:smallest_idx-1]) # % m return lst
NIB_URL = 'https://kartverket.maplytic.no/tile/_nib/{zoom}/{x}/{y}.jpeg' TOPO4_URL = 'https://opencache.statkart.no/gatekeeper/gk/gk.open_gmaps?layers=topo4&zoom={zoom}&x={x}&y={y}' PROJECTS = IndexedOrderedDict([ ('DTM50', projects.DigitalHeightModel(key='DTM50', name='Terrain model (50 m)')), ('DTM10', projects.DigitalHeightModel(key='DTM10', name='Terrain model (10 m)')), ('DTM1', projects.DigitalHeightModel(key='DTM1', name='Terrain model (1 m)')), ('DOM50', projects.DigitalHeightModel(key='DOM50', name='Object model (50 m)')), ('DOM10', projects.DigitalHeightModel(key='DOM10', name='Object model (10 m)')), ('DOM1', projects.DigitalHeightModel(key='DOM1', name='Object model (1 m)')), ('NIB', projects.TiledImageModel(key='NIB', name='Norge i bilder', url=NIB_URL)), ('TOPO4', projects.TiledImageModel(key='TOPO4', name='Karverket Topo4', url=TOPO4_URL)), ]) filesystem.create_directories(project.key for project in PROJECTS.values()) class Polygon(DeclarativeBase): """ORM representation of a polygon.""" __tablename__ = 'polygon' id = sql.Column(sql.Integer, primary_key=True) name = sql.Column(sql.String, nullable=False) # Sequence of points (counterclockwise order) making up this polygon # The last point must equal the first one. points = orm.relationship( 'Point', order_by='Point.id', back_populates='polygon', lazy='immediate', cascade='save-update, merge, delete, delete-orphan',
pre_cols[headers[i_c].replace(settings.pre_prefix, "")] = i_c if headers[i_c].startswith(settings.post_prefix): post_cols[headers[i_c].replace(settings.post_prefix, "")] = i_c if settings.debug: #print "......To be deleted: " + str(len(cols_to_be_deleted)) + " cols" print "......Index Cols " + str(len(index_cols)) + " cols" print "......Pre Cols " + str(len(pre_cols)) + " cols" print "......Post Cols " + str(len(post_cols)) + " cols" #### #### STEP 3: add count columns, scale age by 5 #### #raw_input("Press Enter to continue...") print utils.time() + "add count cols" IDX_COUNT = matrix[:, index_ATC_cols.values()].sum(axis=1) matrix = hstack([matrix, IDX_COUNT], format="csr") headers.append("IDX_COUNT_ATC") index_cols["COUNT_ATC"] = len(headers) - 1 IDX_COUNT = matrix[:, index_ICD_cols.values()].sum(axis=1) matrix = hstack([matrix, IDX_COUNT], format="csr") headers.append("IDX_COUNT_ICD") index_cols["COUNT_ICD"] = len(headers) - 1 #### #### STEP 4: pre-calc prevalences and related mean ages #### print utils.time() + "Start calculation Prevalences / Mean Ages" prevalences = matrix[:, index_cols.values()].mean( axis=0).A1 #requires index to be 2010 (1 year)
class GenerateComplexPictures(Exception): def __init__( self, modulo=10., # n1_x=500, # scale=16, # scale_y=1., # x_offset=8, # y_offset=8, nx=None, ny=None, max_length=None, x_center=None, y_center=None, delta=0.0001, func_str=None, main_folder="images/", root_folder=None, number=None): self.modulo = modulo self.delta = delta self.all_symbols_16 = np.array(list("0123456789ABCDEF")) self.all_symbols_64 = np.array( list(string.ascii_letters + string.digits + "-_")) self.file_extension_name = "_{}_{}".format( self.get_date_time_str(), self.get_random_string_base_16(16)) # TODO: need a big fix! if nx != None and ny != None and max_length != None and x_center != None and y_center != None: self.nx = nx self.ny = ny self.max_length = max_length scale_x = max_length scale_y = max_length if nx > ny: scale_y = max_length * ny / nx else: scale_x = max_length * nx / ny self.scale_x = scale_x self.scale_y = scale_y self.x_center = x_center self.y_center = y_center self.xs1 = np.arange( 0, self.nx ) / self.nx * self.scale_x - self.scale_x / 2 + self.x_center + self.delta self.ys1 = np.arange( 0, self.ny ) / self.ny * self.scale_y - self.scale_y / 2 + self.y_center + self.delta self.x_min = self.x_center - self.scale_x / 2 self.x_max = self.x_center + self.scale_x / 2 self.y_min = self.y_center - self.scale_y / 2 self.y_max = self.y_center + self.scale_y / 2 # globals()["xs1"] = self.xs1 # globals()["ys1"] = self.ys1 # print("\nself.nx: {}".format(self.nx)) # print("self.ny: {}".format(self.ny)) # print("\nself.max_length: {}".format(self.max_length)) # print("self.scale_x: {}".format(self.scale_x)) # print("self.scale_y: {}".format(self.scale_y)) # print("self.x_center: {}".format(self.x_center)) # print("self.y_center: {}".format(self.y_center)) # print("TEST!") # sys.exit(-10) else: raise Exception # self.n1_x = n1_x # self.scale = scale # self.scale_y = scale_y # self.x_offset = x_offset # self.y_offset = y_offset # self.n1_y = int(self.n1_x*self.scale_y) # self.xs1 = np.arange(0, self.n1_x)/self.n1_x*self.scale-self.x_offset+self.delta # self.ys1 = np.arange(0, self.n1_y)/self.n1_y*self.scale*self.scale_y-self.y_offset+self.delta self.ys1 = self.ys1[::-1] self.ys1_2d = np.zeros((self.ys1.shape[0], self.xs1.shape[0])) self.xs1_2d = self.ys1_2d.copy() self.xs1_2d[:] = self.xs1 self.ys1_2d[:] = self.ys1.reshape((-1, 1)) self.arr_xy_real_imag = np.vectorize(complex)(self.xs1_2d, self.ys1_2d) self.message = self._construct_message() super(GenerateComplexPictures, self).__init__(self.message) self.main_folder = main_folder if main_folder != "images/": main_folder += ("" if "/" == main_folder[-1] else "/") self.main_folder if root_folder == None: self.root_folder = "" else: self.root_folder = root_folder + ("/" if root_folder[-1] != "/" else "") if number != None: self.number = number self.num_str = "_{:03}".format(number) self.z_func_file_name = "z_func{}.txt".format(self.num_str) self.path_folder_images = self.main_folder + self.root_folder + "z_funcs/" self.path_dir_arrs_data = self.main_folder + self.root_folder + "dm_objs/" else: self.number = None self.num_str = "" self.z_func_file_name = "z_func_{}.txt".format( self.file_extension_name) self.path_folder_images = self.main_folder + "{}{}/".format( self.root_folder, self.file_extension_name) self.path_dir_arrs_data = self.path_folder_images if not os.path.exists(self.path_folder_images): os.makedirs(self.path_folder_images) if func_str != None: self.func_str = func_str with open(self.path_folder_images + self.z_func_file_name, "w") as fout: fout.write(self.func_str) else: generate_generic_z_function.main( path_folder=self.path_folder_images, number=self.number, file_extension_name=self.file_extension_name) with open(self.path_folder_images + self.z_func_file_name, "r") as fin: self.func_str = fin.read() print("func_str: {}".format(self.func_str)) self.func_str_complete = "lambda z: " + self.func_str self.f = self.get_f() print("self.path_folder_images: {}".format(self.path_folder_images)) print("self.func_str_complete: {}".format(self.func_str_complete)) # self.num_str = "" # if self.number != None: # self.num_str = "_{:03}".format(self.number) def get_random_string_base_16(self, n): l = np.random.randint(0, 16, (n, )) return "".join(self.all_symbols_16[l]) def delete_image_folder(self): path_folder_images = self.path_folder_images print("Remove the whole folder '{}'!".format(path_folder_images)) shutil.rmtree(path_folder_images) def get_random_string_base_64(self, n): l = np.random.randint(0, 64, (n, )) return "".join(self.all_symbols_64[l]) def get_date_time_str(self): dt = datetime.datetime.now() return "Y{:04}_m{:02}_d{:02}_H{:02}_M{:02}_S{:02}_f{:06}".format( dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.microsecond) # return "Y{}_m{}_d{}_H{}_M{}_S{}_f{}".format(dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, dt.microsecond) # return "{:%Y_%m_%d_%H_%M_%S_%f}".format(datetime.datetime.now()) def get_f(self): modulo = self.modulo f_str = self.func_str_complete f = eval(f_str) def f_temp(z): n_z_orig = f(z) length = np.abs(n_z_orig) # Is needed for a continous modulo calculation! length_mod = length % modulo scale = (length_mod if int(length / modulo) % 2 == 0 else modulo - length_mod) / length return n_z_orig, scale return f_temp def calculate_arrs(self): print("Calculate self.arr_f and self.arr_scales") self.arr_f, self.arr_scales = np.vectorize(self.f)( self.arr_xy_real_imag) print("Calculate arr") arr_angle = np.angle(self.arr_f) idx = arr_angle < 0 arr_angle[idx] = arr_angle[idx] + np.pi * 2 print("np.min(arr_angle): {}".format(np.min(arr_angle))) print("np.max(arr_angle): {}".format(np.max(arr_angle))) arr_angle_norm = arr_angle / (np.pi * 2) print("np.min(arr_angle_norm): {}".format(np.min(arr_angle_norm))) print("np.max(arr_angle_norm): {}".format(np.max(arr_angle_norm))) print("Calculate abs") arr_abs = np.abs(self.arr_f) arr_abs_mod = arr_abs * self.arr_scales arr_abs_norm = arr_abs / np.max(arr_abs) arr_abs_mod_norm = arr_abs_mod / np.max(arr_abs_mod) print("Calculate x and y") f_norm_axis = lambda v: (lambda v2: v / np.max(v))(v - np.min(v)) arr_x = self.arr_f.real arr_y = self.arr_f.imag arr_x_mod = arr_x * self.arr_scales arr_y_mod = arr_y * self.arr_scales arr_x_norm = f_norm_axis(arr_x) arr_y_norm = f_norm_axis(arr_y) arr_x_mod_norm = f_norm_axis(arr_x_mod) arr_y_mod_norm = f_norm_axis(arr_y_mod) self.arrs = IndexedOrderedDict([ ("angle", arr_angle_norm), ("abs", arr_abs_norm), ("abs_mod", arr_abs_mod_norm), ("x", arr_x_norm), ("x_mod", arr_x_mod_norm), ("y", arr_y_norm), ("y_mod", arr_y_mod_norm), ]) def calculate_rgb_pixs(self): print("Getting all data in pix_float array") shape = self.arr_xy_real_imag.shape + (3, ) self.pixs1_dict = IndexedOrderedDict([ ("f", np.ones(shape)), ("f_mod", np.ones(shape)), ("angle", np.ones(shape)), ("abs", np.ones(shape)), ("abs_mod", np.ones(shape)), ("x", np.ones(shape)), ("x_mod", np.ones(shape)), ("y", np.ones(shape)), ("y_mod", np.ones(shape)), ]) self.pixs1_dict["f"][:, :, 0] = self.arrs["angle"] self.pixs1_dict["f"][:, :, 2] = self.arrs["abs"] * 1 / 3 + 2 / 3 self.pixs1_dict["f_mod"][:, :, 0] = self.arrs["angle"] self.pixs1_dict["f_mod"][:, :, 2] = self.arrs["abs_mod"] keys = ["angle", "abs", "abs_mod", "x", "x_mod", "y", "y_mod"] for key in keys: self.pixs1_dict[key][:, :, 0] = self.arrs[key] hsv_to_rgb_vectorized = np.vectorize(colorsys.hsv_to_rgb) print("Convert hsv to rgb and convert to uint8") self.pixs1_dict_rgb = IndexedOrderedDict() for key, value in self.pixs1_dict.items(): pix_float_rgb = np.dstack( hsv_to_rgb_vectorized(value[..., 0], value[..., 1], value[..., 2])) self.pixs1_dict_rgb[key] = (pix_float_rgb * 255.9).astype(np.uint8) self.pixs1 = [v for v in self.pixs1_dict_rgb.values()] self.imgs_orig = [Image.fromarray(pix.copy()) for pix in self.pixs1] def add_coordinate_lines(self): print("Adding coordinates x and y if possible") find_x = 0. find_y = 0. rows, cols = self.pixs1[0].shape[:2] line_col = np.argmin((self.xs1 - find_x)**2) line_row = np.argmin((self.ys1 - find_y)**2) if not (line_row < 0 or line_row >= rows): for pix in self.pixs1: pix[:, line_col] = pix[:, line_col] ^ (0xFF, ) * 3 if not (line_col < 0 or line_col >= cols): for pix in self.pixs1: pix[line_row, :] = pix[line_row, :] ^ (0xFF, ) * 3 def add_side_info(self): print("Creating the side infos") # this is needed for the info on the left side of the graph! pix2 = np.zeros((self.pixs1_dict_rgb["f"].shape[0], 300, 3), dtype=np.uint8) img2 = Image.fromarray(pix2) # get a font fnt = ImageFont.truetype('monofonto.ttf', 16) d = ImageDraw.Draw(img2) d.text((8, 8), "Used function:", font=fnt, fill=(255, 255, 255)) func_str_split = [ self.func_str_complete[30 * i:30 * (i + 1)] for i in range(0, len(self.func_str_complete) // 30 + 1) ] for i, func_str_part in enumerate(func_str_split, 1): d.text((8, 8 + 24 * i), func_str_part, font=fnt, fill=(255, 255, 255)) font_y_next = 8 + 24 * (i + 2) d.text( (8, font_y_next), "x_min: {:3.02f}, x_max: {:3.02f}".format(self.x_min, self.x_max), font=fnt, fill=(255, 255, 255)) # d.text((8, font_y_next), "x_min: {:3.02f}, x_max: {:3.02f}".format(-self.x_offset, self.scale-self.x_offset), font=fnt, fill=(255, 255, 255)) font_y_next += 24 d.text( (8, font_y_next), "y_min: {:3.02f}, y_max: {:3.02f}".format(self.y_min, self.y_max), font=fnt, fill=(255, 255, 255)) # d.text((8, font_y_next), "y_min: {:3.02f}, y_max: {:3.02f}".format(-self.y_offset, self.scale*self.scale_y-self.y_offset), font=fnt, fill=(255, 255, 255)) pix2 = np.array(img2).copy() font_y_next += 24 modulo_str = "modulo: {:3.02f}".format(self.modulo) text_to_write_lst = [("only with f(z)", ), ( "only with f(z)", modulo_str, ), ("only with angle", ), ("only with abs", ), ("only with abs", modulo_str), ("only with x", ), ("only with x", modulo_str), ("only with y", ), ("only with y", modulo_str)] self.pixs2 = [] for text_to_write in text_to_write_lst: img2_temp = img2.copy() d = ImageDraw.Draw(img2_temp) font_y_next_temp = font_y_next for text in text_to_write: d.text((8, font_y_next_temp), text, font=fnt, fill=(255, 255, 255)) font_y_next_temp += 24 self.pixs2.append(np.array(img2_temp)) self.imgs = [] for pix2, pix1 in zip(self.pixs2, self.pixs1): pix3 = np.hstack((pix2, pix1)) self.imgs.append(Image.fromarray(pix3)) def save_all_images(self): self.suffixes = [ "f", "f_mod", "angle", "abs", "abs_mod", "x", "x_mod", "y", "y_mod" ] if self.root_folder != "": for suffix, img in zip(self.suffixes, self.imgs_orig): folder_path_f_orig = self.main_folder + "" + self.root_folder + "orig_" + suffix + "/" if not os.path.exists(folder_path_f_orig): os.makedirs(folder_path_f_orig) img.save(folder_path_f_orig + "{}_{}.png".format(suffix, self.num_str)) # img.save(folder_path_f_orig+"{}{}_{}_{}.png".format(suffix, self.num_str, # self.get_date_time_str(), # self.get_random_string_base_16(16)) # ) for suffix, img in zip(self.suffixes, self.imgs): folder_path = self.main_folder + "" + self.root_folder + "plot_" + suffix + "/" if not os.path.exists(folder_path): os.makedirs(folder_path) img.save(folder_path + "{}_{}.png".format(suffix, self.num_str)) # img.save(folder_path+suffix+"{}_{}_{}.png".format( # self.num_str, # self.get_date_time_str(), # self.get_random_string_base_16(16) # ) # ) else: # print("NO root_folder!!!!") for suffix, img in zip(self.suffixes, self.imgs_orig): folder_path_f_orig = self.main_folder + "plots_originals/orig_" + suffix + "/" if not os.path.exists(folder_path_f_orig): os.makedirs(folder_path_f_orig) img.save(folder_path_f_orig + "{}{}{}.png".format( suffix, self.num_str, self.file_extension_name)) for suffix, img in zip(self.suffixes, self.imgs): folder_path_f_orig = self.main_folder + "plots_with_side_info/orig_" + suffix + "/" if not os.path.exists(folder_path_f_orig): os.makedirs(folder_path_f_orig) img.save(folder_path_f_orig + "{}{}{}.png".format( suffix, self.num_str, self.file_extension_name)) for suffix, img in zip(self.suffixes, self.imgs): if self.number != None: suffix += "_{}".format(self.number) img.save(self.path_folder_images + "{}{}.png".format(suffix, self.file_extension_name)) def save_arrs_data(self): # if self.root_folder == "": if not os.path.exists(self.path_dir_arrs_data): os.makedirs(self.path_dir_arrs_data) print("self.path_dir_arrs_data: {}".format(self.path_dir_arrs_data)) dm_obj = DotMap() dm_obj.func_str = self.func_str dm_obj.modulo = self.modulo dm_obj.delta = self.delta dm_obj.arr_xy_real_imag = self.arr_xy_real_imag dm_obj.arr_f = self.arr_f dm_obj.arr_scales = self.arr_scales dm_obj.nx = self.nx dm_obj.ny = self.ny dm_obj.max_length = self.max_length dm_obj.x_center = self.x_center dm_obj.y_center = self.y_center if self.number != None: self.path_file_arrs = self.path_dir_arrs_data + "dm_obj{}.pkl.gz".format( self.num_str) else: self.path_file_arrs = self.path_dir_arrs_data + "dm_obj{}.pkl.gz".format( self.file_extension_name) print("self.path_file_arrs: {}".format(self.path_file_arrs)) with gzip.open(self.path_file_arrs, "wb") as fout: dill.dump(dm_obj, fout) # TODO: make it so that you can call this functions from outside too # with the self object! So that an extern z func array can be ploted too! def do_calculations(self): self.calculate_arrs() self.calculate_rgb_pixs() self.add_coordinate_lines() self.add_side_info() self.save_all_images() self.save_arrs_data() def save_new_z_function(self): func_str_complete = self.func_str_complete path_folder_data = "data/" if not os.path.exists(path_folder_data): os.makedirs(path_folder_data) path_file_data = path_folder_data + "working_z_functions.pkl.gz" path_file_data_txt = path_folder_data + "working_z_functions.txt" if not os.path.exists(path_file_data): data = DotMap() data.func_str_lst = [func_str_complete] with gzip.open(path_file_data, "wb") as fout: dill.dump(data, fout) else: with gzip.open(path_file_data, "rb") as fin: data = dill.load(fin) data.func_str_lst.append(func_str_complete) with open(path_file_data_txt, "w") as fout: for line in data.func_str_lst: fout.write(line + "\n") with gzip.open(path_file_data, "wb") as fout: dill.dump(data, fout) print("newest founded function:\n{}".format(func_str_complete)) # print("data.func_str_lst:\n{}".format(data.func_str_lst)) # lst = data.func_str_lst # print("Amount of found functions: {}".format(len(lst))) # for i, func_str in enumerate(lst, 1): # print("\ni: {}, func_str: {}".format(i, func_str)) def _construct_message(self): return "IT WORKS!"