def _store_(doc_name, files, a, type): """ 将更改后的文件oldname和newname以section的方式存到directory.ini或factory.ini中 (具体哪个文件夹则根据所给的文件类型type决定,用户调用相应的方法后type自动赋值) :param doc_name:用户传入的文件夹的路径 :param files:文件夹下面的文件或文件夹(具体类型根据type决定)的名字 :param a:文件重命名后新的编码(名字) :return: """ try: config_write = configparser.ConfigParser() if type == 'D': config_write.read('conf' + resource_manager.getSeparator() + 'directory.ini') ftest = open( 'conf' + resource_manager.getSeparator() + 'directory.ini', 'w+') elif type == 'F': config_write.read('conf' + resource_manager.getSeparator() + 'factory.ini') ftest = open( 'conf' + resource_manager.getSeparator() + 'factory.ini', 'w+') check = config_write.sections() n = False if doc_name in check: n = True config_write.set(doc_name, files, str(a)) if n == False: config_write.add_section(doc_name) config_write.set(doc_name, files, str(a)) config_write.write(ftest) ftest.close() except: pass
def _reverse_(doc_name, type): """ 根据reverse进行反向目录生成 :param doc_name: 用户给定文件夹的路径 :param type: 根据用户调用方法的不同对文件和文件夹分开进行重命名;type=‘F(file)’对文件操作,type=‘D(directory)’对文件夹操作 :return: """ try: conf = configparser.ConfigParser() if type == 'D': conf.read('conf' + resource_manager.getSeparator() + 'directory.ini') elif type == 'F': conf.read('conf' + resource_manager.getSeparator() + 'document.ini') options = conf.options(doc_name) if not os.path.exists(doc_name): doc_name = os.path.abspath(doc_name) for option in options: try: str_val = conf.get(doc_name, option) New = os.path.join(doc_name, option) old = os.path.join(doc_name, str_val) os.rename(old, New) except: print(option + " don't exist") except: print("no document has been renamed")
def __init__(self,type): self.configParse=config_parser_extender.CapitalCaseConfigParser() self.type=type if type == 'D': self.configParse.read(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini') # self.f = open(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini', 'w+') elif type == 'F': self.configParse.read(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'document.ini')
def plot_scatter_diagram(which_fig, x, y, x_label='x', y_label='y', title='title', label=None): ''' Plot scatter diagram Args: which_fig : which sub plot x : x array y : y array x_label : label of x pixel y_label : label of y pixel title : title of the plot ''' styles = ['k.', 'g.', 'r.', 'c.', 'm.', 'y.', 'b.'] linestyles = ['-.', '--', 'None', '-', ':'] stylesMarker = pandas.read_csv( resource_manager.Properties.getDefaultDataFold() + "view" + resource_manager.getSeparator() + "style.csv").ix[:, 3] stylesColors = pandas.read_csv( resource_manager.Properties.getDefaultDataFold() + "view" + resource_manager.getSeparator() + "style.csv").ix[:, 2] assert len(x) == len(y) if label != None: assert len(x) == len(label) # and len(stylesMarker) >= len(set(label)) plt.figure(which_fig) plt.clf() if label == None: plt.plot(x, y, styles[0]) else: l = len(label) labelSet = set(label) k = 0 for i in labelSet: xs = [] ys = [] for j in range(l): if i == label[j]: xs.append(x[j]) ys.append(y[j]) k = k + 1 try: plt.scatter(xs, ys, c=stylesColors[k].strip(), marker=r"$ {} $".format(str(i)),label=i) except: log.fatal(stylesMarker) log.fatal(stylesColors) log.fatal(stylesMarker[k]) log.fatal(stylesColors[k]) plt.scatter(xs, ys, c=stylesColors[k].strip(), marker=r"$ {} $".format(str(i)),label=i) exit() plt.title(title) plt.xlabel(x_label) plt.ylabel(y_label) plt.ylim(bottom=0) # plt.legend(loc='upper left') plt.show()
def start_imagexml_builder( source=resource_manager.Properties.getDefaultOperationFold(), path=resource_manager.Properties.getImageXmlResource()): rmfile() log.info("starting to init dataset from " + str(source) + " to running create image-b.xml module.") # path="D:\Projects\\Python\\deeplay\\src\\train\\run\\*"; Data = dir_to_dataset(source) Data = np.array(Data) # Data and labels are read # set a filename afileName = str( os.getcwd()) + resource_manager.getSeparator() + "kmeans.data" # create the file file = open(afileName, 'w') # put the data into a file. # print(len(Data)) xmlFile = str(os.getcwd()) + resource_manager.getSeparator() + "image1.xml" log.info("starinng to build image xml file") if (os.path.exists(xmlFile) == False): document = minidom.Document() document.appendChild( document.createComment("this is used for save a image file")) imagelist = document.createElement("Images") document.appendChild(imagelist) f = open(path, "w") document.writexml(f, addindent=' ' * 4, newl='\n', encoding='utf-8') f.close() root = xml.dom.minidom.parse(path) imagesRoot = root.documentElement c = 0 # root=xml.etree.ElementTree.parse("image-b.xml"); import sys for x in range(0, len(Data)): imageRoot = document.createElement("Image") id = document.createElement("id") data = document.createElement("data") c = c + 1 aRow = Data[x] value = [] for pix in range(1, len(aRow)): file.write(str(aRow[pix])) value.append(int(str(aRow[pix]))) if pix != len(aRow) - 1: file.write(",") # print(len(aRow)) id.appendChild(document.createTextNode(aRow[0])) data.appendChild(document.createTextNode(str(value))) imagesRoot.appendChild(imageRoot) imageRoot.appendChild(id) imageRoot.appendChild(data) file.write("\n") f = open(path, "w") root.writexml(f, addindent=' ' * 4, newl='\n', encoding='utf-8') f.close() log.info("build about " + str(c) + " image file.")
def save(self): if self.type == 'D': f=open(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini', 'w+') self.configParse.write(f) f.close() # self.f = open(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini', 'w+') elif self.type == 'F': f=open(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'document.ini', 'w+') self.configParse.write(f) f.close()
def build_distance_txt(distance, sep=' ', path=resource_manager.Properties.getDefaultDataFold() + "txt" + resource_manager.getSeparator() + "build.txt"): """ 写入文件,具体格式如下: 行下标 分隔符 列下标 分隔符 distance中行下标和列下标对应的值 :param distance: :param path: :param sep:分隔符 :return: """ num_row = 0 num_rank = 0 fr = open(path, 'w') for row in distance: num_row += 1 num_rank = 0 for element in row: if np.isnan(element): continue num_rank += 1 fr.write(str(num_row)) fr.write(sep) fr.write(str(num_rank)) fr.write(sep) fr.write(str(element)) fr.write("\n") fr.close() """
def save(name='default'): """ 保存id和data数据 :return: """ from context.resource_manager import Properties from pandas import DataFrame, Series path = os.path.join(Properties.getDefaultDataFold()+"xml"+resource_manager.getSeparator()+name+".xml") from xml.dom.minidom import parse, parseString images = parse(path) id = [] data = [] for node in images.getElementsByTagName("Image"): idNode = node.getElementsByTagName("id")[0].childNodes[0].data id.append(idNode) dataNode = node.getElementsByTagName("data")[0].childNodes[0].data dataNode = dataNode[1:-1].split(',') data.append(dataNode) id = np.asarray(id) id = Series(id) data = np.asarray(list(map(conv, data)), dtype=np.float) if not os.path.exists(Properties.getDefaultDataFold()+"/cache/"+name): #f=open(Properties.getDefaultDataFold()+"/csv/threshold.csv","w") #f.close() os.makedirs(Properties.getDefaultDataFold()+"/cache/"+name) np.save(Properties.getRootPath() + "/data/cache/"+name+"/id.npy", id) np.save(Properties.getRootPath() + "/data/cache/"+name+"/data.npy", data)
def binaryzation_proxy(cls,path): s = path.split(resource_manager.getSeparator()) s = s[(len(s) - 1)] c=0 if s[0:4]=="data": log.info(" do binaryzation about files in "+str(path)) size=s[4:] size=int(size) for i in os.walk(path, False): for f in i[2]: BinAndCrop.preOp(os.path.join(path, f)); BinAndCrop.single_crop(path+resource_manager.getSeparator()+f,size) c=c+1 return c return 0
def rename_file(url, reverse=True): """ 用户给定文件夹的路径,如果给定路径存在,更改指定文件夹下的文件的名字 :param url: 用户给定的文件夹的路径 :param reverse: 如果reverse=True就进行反向命名,reverse=False就对所给文件夹所包含的文件进行重命名; :return: """ if _exist_(url) and reverse == False: if not os.path.exists(url): url = os.path.abspath(url) a = 1 list_sto = _random_name(url, 'F') conf_sto = configparser.ConfigParser() conf_sto.read('conf' + resource_manager.getSeparator() + 'factory.ini') for ob in list_sto: try: files = conf_sto.get(url, ob) old = os.path.join(url, files) filetype = os.path.splitext(files)[1] d = a if os.path.isfile(old): New = os.path.join(url, str(a) + filetype) a = str(a) + filetype os.rename(old, New) _store_(url, ob, a, 'F') a = d a = a + 1 except: pass elif reverse == True and _exist_(url): _reverse_(url, 'F')
def run(path=resource_manager.Properties.getDefaultDataFold()+"txt"+resource_manager.getSeparator()+"build.txt",sep=' '): ''' return cluster id i,j distance ''' (dist,xxdist,ND,N) = readfile(path, dimensions = 2, sep=sep) XY, eigs = mds(dist) (rho,delta,ordrho,dc,nneigh) = rhodelta(dist, xxdist, ND, N, percent = 2.0) DCplot(dist, XY, ND, rho, delta,ordrho,dc,nneigh,17,0.1)
def rename_dir(url, reverse=False): """" 根据static的值进行文件夹自动重命名,命名规则 directory.ini 网->0 于->1 :param url:,文件夹地址, :param static:给出的是关于地址是否是相对地址 :param reverse:确定是需要进行方向目录生成,还是正向目录生成 :return: Tip:需要判断url是否存在,是否为文件夹,对于conf目录需要注意是否已经存在,没有存在需要进行创建。另外对于directory.ini文件也需要判断是否存在。建议对这里的工作进行定义多个子函数。定义的子函数请以_开头。 在进行一些具体的操作,需要输出相关日志操作 """ "" """ 用户给定文件夹的路径,如果给定路径存在,更改指定文件夹下的文件夹的名字 :param url: 用户给定的文件夹的路径 :param reverse: 如果reverse=True就进行反向命名,reverse=False就对所给文件夹所包含的文件夹进行重命名; :return: """ if _exist_(url) and reverse == False: if not os.path.exists(url): url = os.path.abspath(url) a = 1 list_sto = random_name(url, 'D') con = config_parser.ConfigParser("D") """ while (True): if str(a) in list_sto: a = a + 1 else: break """ conf_sto = configparser.ConfigParser() conf_sto.read('conf' + resource_manager.getSeparator() + 'directory.ini') for ob in list_sto: try: files = conf_sto.get(url, ob) old = os.path.join(url, files) filetype = os.path.splitext(files)[1] d = a if os.path.isdir(old): New = os.path.join(url, str(a) + filetype) a = str(a) + filetype os.rename(old, New) _store_(con, url, ob, a) a = d a = a + 1 except: pass con.save() elif reverse == True and _exist_(url): _reverse_(url, 'D')
def start_image_rename(src=resource_manager.Properties.getDefaultOperationFold(), path=resource_manager.Properties.getDefaultWorkFold(),state=True): if state: log.info("starting to rename file in dir, do not modify the resource path file. Operation Directory:"+str(path)) dirs = file_manage.subdirs(src) c=0 if os.path.exists(path): shutil.rmtree(path) if len(dirs)>0: for d in dirs: files = file_manage.subfilesName(d) for f in files: name = file_manage.random_str(8) + "_" + f try: shutil.copy(os.path.join(d, f), os.path.join(path+resource_manager.getSeparator()+d, name)) except: file_manage.fix_path(path+resource_manager.getSeparator()+d) shutil.copy(os.path.join(d, f), os.path.join(path+resource_manager.getSeparator()+d, name)) c=c+1 log.info("finished copy file to the path"+str(path)+". about "+str(c)+" files.") file_manage.rename_files(path) else: log.info("starting to rename file in dir, modify the resource path file. Operation Directory:"+str(src)) file_manage.rename_files(path)
def random_name(url, type): """ 对文件或文件夹进行随机重命名(防止产生因同名而无法重命名的问题)(具体类型则根据所给的文件类型type决定,用户调用相应的方法后type自动赋值) :param url: 用户传入的文件夹的地址 :return: 返回文件夹中所有文件或文件夹重命名之前的名字的列表 """ if not os.path.exists(url): url = resource_manager.Properties.getRootPath( ) + resource_manager.getSeparator() + url doc = os.listdir(url) if type == 'D': con = config_parser.ConfigParser('D') else: con = config_parser.ConfigParser('F') for files in doc: filetype = os.path.splitext(files)[1] if os.path.exists(url): old = url + resource_manager.getSeparator() + files else: old = resource_manager.Properties.getRootPath( ) + resource_manager.getSeparator( ) + url + resource_manager.getSeparator() + files if os.path.isdir(old) and type == 'D': random = random_string() New = url + resource_manager.getSeparator() + random + filetype os.rename(old, New) _store_(con, url, files, random + filetype) elif os.path.isfile(old) and type == 'F': random = random_string() if os.path.exists(url): New = url + resource_manager.getSeparator() + random + filetype else: New = url + resource_manager.getSeparator() + random os.rename(old, New) _store_(con, url, files, random + filetype) con.save() list = doc return list
def plot_dataframe_scatter_diagram(which_fig, data, x_label='x', y_label='y', title='title', label=None): styles = ['k.', 'g.', 'r.', 'c.', 'm.', 'y.', 'b.'] linestyles = ['-.', '--', 'None', '-', ':'] stylesMarker = markers = ['.', # point ',', # pixel 'o', # circle 'v', # triangle down '^', # triangle up '<', # triangle_left '>', # triangle_right '1', # tri_down '2', # tri_up '3', # tri_left '4', # tri_right '8', # octagon 's', # square 'p', # pentagon '*', # star 'h', # hexagon1 'H', # hexagon2 '+', # plus 'x', # x 'D', # diamond 'd', # thin_diamond '|', # vline ] # styles = [] stylesColors = pandas.read_csv( resource_manager.Properties.getDefaultDataFold() + "view" + resource_manager.getSeparator() + "color.csv").ix[:, 2] plt.figure(which_fig) plt.clf() plt.title(title) plt.xlabel(x_label) plt.ylabel(y_label) plt.ylim(bottom=0) plt.legend(loc='upper left') plt.show()
def renamefiles(src=resource_manager.Properties.getDefaultOperationFold()): log.info("starting rename file job.") directorys = os.listdir(src) cf = config_parser.ConfigParser("F") c = 0 for directory in directorys: if os.path.isdir(src + resource_manager.getSeparator() + directory): files = subfilesName(src + resource_manager.getSeparator() + directory) c += len(files) log.info( str("####" + src + resource_manager.getSeparator() + directory + " directory is doing #### " + str(len(files)))) path = src + resource_manager.getSeparator() + directory for f in files: # log.info(os.path.join(f)+"....") if os.path.isfile(os.path.join(path, f)) == True: c = c + 1 # name = '{0}_{1:0{2}d}'.format(directory, c, 6) name = '_{0}_{1}{2:0{3}d}'.format(directory, _time_name(), c, 4) name = random_string() + name rename( cf, src + resource_manager.getSeparator() + directory + resource_manager.getSeparator(), f, name) else: c = c + 1 s = src.split(resource_manager.getSeparator()) s = s[(len(s) - 1)] # name = '_{0}_{1:0{2}d}'.format(s, _time_name(), 6) name = '_{0}_{1}{2:0{3}d}'.format(s, _time_name(), c, 4) name = random_string() + name rename(cf, src + resource_manager.getSeparator(), directory, name) log.info("finish rename " + str(c) + " job.") cf.save()
def _random_name(url, type): """ 对文件或文件夹进行随机重命名(防止产生因同名而无法重命名的问题)(具体类型则根据所给的文件类型type决定,用户调用相应的方法后type自动赋值) :param url: 用户传入的文件夹的地址 :return: 返回文件夹中所有文件或文件夹重命名之前的名字的列表 """ doc = os.listdir(url) for files in doc: try: filetype = os.path.splitext(files)[1] if not os.path.exists(files): old = url + resource_manager.getSeparator() + files else: old = resource_manager.Properties.getRootPath( ) + resource_manager.getSeparator( ) + url + resource_manager.getSeparator() + files if os.path.isdir(old) and type == 'D': random = file_manage.random_string() New = resource_manager.Properties.getRootPath( ) + resource_manager.getSeparator( ) + url + resource_manager.getSeparator() + random os.rename(old, New) _store_(url, files, random + filetype, 'D') elif os.path.isfile(old) and type == 'F': random = file_manage.random_string() New = resource_manager.Properties.getRootPath( ) + resource_manager.getSeparator( ) + url + resource_manager.getSeparator() + random os.rename(old, New) _store_(url, files, random + filetype, 'F') except: pass list = doc return list
def cluster_view_json_builder(clusters=[]): """ 将用户传入的多个json格式的对象存储在json文件中 newlist【】作为list先读入文件中的数据以防止丢失,然后追加新加入的json对象(以list格式存入newlist) :param clusters: :return: """ for c in clusters: if not isinstance(c, ClusterViwer): raise Exception("错误的数据类型,不是ClusterViwer") else: newlist = [] try: with open( resource_manager.Properties.getRootPath() + resource_manager.getSeparator() + 'data' + resource_manager.getSeparator() + 'json' + resource_manager.getSeparator() + 'ssdfdssdf.json', 'r') as R: readed = json.load(R) for r in readed: newlist.append(r) except: print('The file is empty!') newlist.append(c.tolist()) with open( resource_manager.Properties.getRootPath() + resource_manager.getSeparator() + 'data' + resource_manager.getSeparator() + 'json' + resource_manager.getSeparator() + 'ssdfdssdf.json', 'w+') as f: f.write( json.dumps(newlist, skipkeys=True, sort_keys=True, indent=2))
#f=open(Properties.getDefaultDataFold()+"/csv/threshold.csv","w") #f.close() os.makedirs(Properties.getDefaultDataFold() + "/cache/" + name) np.save(Properties.getRootPath() + "/data/cache/" + name + "/id.npy", id) np.save(Properties.getRootPath() + "/data/cache/" + name + "/data.npy", data) if __name__ == "__main__": from context.resource_manager import Properties from pandas import Series from view import shape_view from view import plot_utils from cluster import density_cluster distance_c = 0.69 name = 'aggregation' save(name) id = np.load(Properties.getRootPath() + "/data/cache/" + name + "/id.npy") data = np.load(Properties.getRootPath() + "/data/cache/" + name + "/data.npy") path = resource_manager.Properties.getDefaultDataFold( ) + "txt" + resource_manager.getSeparator() + "build.txt" id_index = Series(id.tolist()) from cluster import density_cluster N = id_index.count() distance = compute_distance(data) build_distance_txt(distance) # id.values -> 对应的key run(path) log.warn("the init distance_c is: " + str(distance_c))
def load_data(): id = np.load(Properties.getRootPath() + "/data/cache/flame/id.npy") data = np.load(Properties.getRootPath() + "/data/cache/flame/data.npy") return id, data if __name__ == '__main__': id, data = load_data() label = [int(x[0:1]) for x in id] from context.resource_manager import Properties from view import shape_view from view import plot_utils from cluster import density_cluster x = data[:, :1] y = data[:, 1:2] path = resource_manager.Properties.getDefaultDataFold( ) + resource_manager.getSeparator( ) + "result" + resource_manager.getSeparator() + "srv.png" plot_utils.save_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title='scatter figure', label=label, path=path) #plot_utils.plot_scatter_diagram(None,x=x,y=y,x_label='x',y_label='y',title='scatter figure',label=label)
def save_show_cluster(index_id, data, distance_c, pile_id, dataset="/", level="INFO", level_info='scatter figure'): from view import plot_utils from context import resource_manager path = resource_manager.Properties.getDefaultDataFold() + "result" + resource_manager.getSeparator() + "temp/" + dataset + "/" + resource_manager.Properties.name_str_static() + "/" level_path = resource_manager.Properties.getDefaultDataFold() + "result" + resource_manager.getSeparator() + "temp/" + level + "/" + resource_manager.Properties.name_str_static() + "/" + str( distance_c) + "/" if not os.path.exists(path[:path.rfind('/')]): os.makedirs(path[:path.rfind('/')]) if not os.path.exists(level_path[:level_path.rfind('/')]): os.makedirs(level_path[:level_path.rfind('/')]) pile_id = pile_id.sort_values('size', ascending=False) x = [] y = [] label = [] i = 1 for m in range(len(pile_id)): # l=pile_id.irow(m)['pile'] l = pile_id.iloc[m]['pile'] # size=pile_id.irow(m)['size'] size = pile_id.iloc[m]['size'] if pile_id.loc[m]['outlier'] is False: for node in l: index = index_id[node] x.append(data[index][0]) y.append(data[index][1]) label.append(i) i = i + 1 else: for node in l: index = index_id[node] x.append(data[index][0]) y.append(data[index][1]) label.append(0) if level is "SEE": plot_utils.plot_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title=level_info, label=label) if level is "DEBUG": # plot_utils.save_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title='scatter figure', label=label,path=level_path+resource_manager.Properties.name_str_FULL()+".png") plot_utils.save_all_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title=level_info, label=label, path=level_path + resource_manager.Properties.name_str_FULL() + ".png") else: plot_utils.save_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title='scatter figure', label=label, path=path + str( distance_c) + ".png") plot_utils.save_all_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title='scatter figure', label=label, path=path + str( distance_c) + ".png") # plot_utils.plot_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title='scatter figure', label=label) log.debug(("\n") + str(pile_id)) try: p = Properties.getDefaultDataFold() + "/csv/" + dataset + "/" + resource_manager.Properties.name_str_static() + "/" + str( distance_c) + ".csv" pile_id.to_csv(p) except: if not os.path.exists(p[:p.rfind('/')]): pp = p.rfind('/') os.makedirs(p[:pp]) os.mknod(p) pile_id.to_csv(p)
from cluster import density_cluster_dpc if os.path.exists(resource_manager.Properties.getDefaultDataFold() + "result/test"): shutil.rmtree(resource_manager.Properties.getDefaultDataFold() + "result/test") id = np.load(Properties.getRootPath() + "/data/cache/" + name + "/id.npy") data = np.load(Properties.getRootPath() + "/data/cache/" + name + "/data.npy") from pandas import Series, DataFrame id_index = Series(id.tolist()) from cluster import density_cluster index_id = Series(id_index.index, index=id_index.values) distance = density_cluster.compute_distance(data) #np.save(Properties.getRootPath() + "/data/cache/distance/data.npy", distance) path = resource_manager.Properties.getDefaultDataFold( ) + resource_manager.getSeparator( ) + "result" + resource_manager.getSeparator( ) + "name" + resource_manager.getSeparator() + str(distance_c) + ".png" temp = distance.copy() temp[np.isnan(temp)] = 0 stand = np.std(temp) temp = distance.copy() temp[np.isnan(temp)] = stand temp = temp.min(axis=0) next_distance_c = np.std(temp) pile_id = density_cluster.debug_cluster(id_index, index_id, data, distance, distance_c, next_distance_c, dataset=name,
def save_show_cluster(index_id, data, distance_c, pile_id): from view import plot_utils from context import resource_manager path = resource_manager.Properties.getDefaultDataFold( ) + "result" + resource_manager.getSeparator( ) + "temp" + resource_manager.getSeparator( ) + resource_manager.Properties.name_str_static( ) + resource_manager.getSeparator() + str(distance_c) + ".png" if not os.path.exists(resource_manager.Properties.getDefaultDataFold() + "result" + resource_manager.getSeparator() + "temp" + resource_manager.getSeparator() + resource_manager.Properties.name_str_static() + resource_manager.getSeparator()): os.makedirs(resource_manager.Properties.getDefaultDataFold() + "result" + resource_manager.getSeparator() + "temp" + resource_manager.getSeparator() + resource_manager.Properties.name_str_static() + resource_manager.getSeparator()) pile_id = pile_id.sort_values('size', ascending=False) x = [] y = [] label = [] i = 1 for m in range(len(pile_id)): # l=pile_id.irow(m)['pile'] l = pile_id.iloc[m]['pile'] # size=pile_id.irow(m)['size'] size = pile_id.iloc[m]['size'] if size >= 1 and i < 15: for node in l: index = index_id[node] x.append(data[index][0]) y.append(data[index][1]) label.append(i) i = i + 1 else: for node in l: index = index_id[node] x.append(data[index][0]) y.append(data[index][1]) label.append(0) plot_utils.save_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title='scatter figure', label=label, path=path) # plot_utils.plot_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title='scatter figure', label=label) log.debug(pile_id) try: p = Properties.getDefaultDataFold( ) + "/csv/" + resource_manager.Properties.name_str_static( ) + "/" + str(distance_c) + ".csv" pile_id.to_csv(p) except: if not os.path.exists(p): pp = p.rfind('/') os.makedirs(p[:pp]) os.mknod(p) pile_id.to_csv(p)
data = np.asarray(list(map(conv, data)), dtype=np.float) if not os.path.exists(Properties.getDefaultDataFold()+"/cache/"+name): #f=open(Properties.getDefaultDataFold()+"/csv/threshold.csv","w") #f.close() os.makedirs(Properties.getDefaultDataFold()+"/cache/"+name) np.save(Properties.getRootPath() + "/data/cache/"+name+"/id.npy", id) np.save(Properties.getRootPath() + "/data/cache/"+name+"/data.npy", data) if __name__=="__main__": from context.resource_manager import Properties from pandas import Series from view import shape_view from view import plot_utils from cluster import density_cluster distance_c=0.69 name='aggregation' save(name) id = np.load(Properties.getRootPath() + "/data/cache/" + name + "/id.npy") data = np.load(Properties.getRootPath() + "/data/cache/" + name + "/data.npy") path =resource_manager.Properties.getDefaultDataFold()+"txt"+resource_manager.getSeparator()+"build.txt" id_index = Series(id.tolist()) from cluster import density_cluster N = id_index.count() distance = compute_distance(data) build_distance_txt(distance) # id.values -> 对应的key run(path) log.warn("the init distance_c is: " + str(distance_c))
def start_image_rebuild(src=resource_manager.Properties.getDefaultOperationFold(), path=resource_manager.Properties.getDefaultWorkFold(),target=['data28','data100'],train=70,verify=30,test=30,state=False): log.info("starting to separtion work. rebuild the image directory. At " + src) # path=raw_input("src path:"); # #n=raw_input("input train mount decimals:"); # m=raw_input("input verify mount decimals:"); # l=raw_input("input ssdfdssdf mout decimals:"); if state: log.info("please input the train,verify,test factor:") train = int(input()) verify = int(input()) test = int(input()) train = train * 0.01 verify = verify * 0.01 test = test * 0.01 log.info("this is the factor:"+str(train) + ","+str(verify)+ ","+str(test)) i = 0; dirs = file_manage.subdirs(src) dpath = path + "train" # print dpath; initDirs(dpath) dpath = path + "verify" initDirs(dpath) dpath = path + "test" initDirs(dpath) c=0 if os.path.exists(path): shutil.rmtree(path) if len(dirs)>0: for d in dirs: # files = os.listdir(src) files = file_manage.subfilesName(d) length = len(files) # print "###########",d,"###########" i = 1; for f in files: name = file_manage.random_str(8) + "_" + f if i <= length * train: p = path + "train" elif i <= (length * train + length * verify): p = path + "verify" else: p = path + "test" try: for t in target: shutil.copy(os.path.join(d, f), os.path.join(p +resource_manager.getSeparator()+t, name)) except: for t in target: file_manage.fix_path(p + resource_manager.getSeparator()+ t) shutil.copy(os.path.join(d, f), os.path.join(p +resource_manager.getSeparator()+ t, name)) # os.rename(os.path(join(d,f),os.path.join(p+"/data100",name); i = i + 1 c=c+1 # print i,":",d," file:",f,"--->",p," file:",name else: d=src files = file_manage.subfilesName(d) length=len(files) for f in files: name = file_manage.random_str(8) + "_" + f if i <= length * train: p = path + "train" elif i <= (length * train + length * verify): p = path + "verify" else: p = path + "test" try: for t in target: shutil.copy(os.path.join(d, f), os.path.join(p +resource_manager.getSeparator()+t, name)) except: for t in target: file_manage.fix_path(p +resource_manager.getSeparator()+ t) shutil.copy(os.path.join(d, f), os.path.join(p +resource_manager.getSeparator()+ t, name)) # os.rename(os.path(join(d,f),os.path.join(p+"/data100",name); i = i + 1 c=c+1 log.info("finished copy file to the path"+str(path)+". about "+str(c)+" files.") file_manage.renamefiles(path)
def removeConfigFiles(cls): if os.path.isfile(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini'): os.remove(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini') if os.path.isfile(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'document.ini'): os.remove(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'document.ini')
def reload(self): if type == 'D': self.configParse.read(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini') # self.f = open(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini', 'w+') elif type == 'F': self.configParse.read(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'document.ini')