def _store_(doc_name, files, a, type):
"""
将更改后的文件oldname和newname以section的方式存到directory.ini或factory.ini中
（具体哪个文件夹则根据所给的文件类型type决定，用户调用相应的方法后type自动赋值）
:param doc_name:用户传入的文件夹的路径
:param files:文件夹下面的文件或文件夹（具体类型根据type决定）的名字
:param a:文件重命名后新的编码（名字）
:return:
"""
try:
config_write = configparser.ConfigParser()
if type == 'D':
config_write.read('conf' + resource_manager.getSeparator() +
'directory.ini')
ftest = open(
'conf' + resource_manager.getSeparator() + 'directory.ini',
'w+')
elif type == 'F':
config_write.read('conf' + resource_manager.getSeparator() +
'factory.ini')
ftest = open(
'conf' + resource_manager.getSeparator() + 'factory.ini', 'w+')
check = config_write.sections()
n = False
if doc_name in check:
n = True
config_write.set(doc_name, files, str(a))
if n == False:
config_write.add_section(doc_name)
config_write.set(doc_name, files, str(a))
config_write.write(ftest)
ftest.close()
except:
pass
def _reverse_(doc_name, type):
"""
根据reverse进行反向目录生成
:param doc_name: 用户给定文件夹的路径
:param type: 根据用户调用方法的不同对文件和文件夹分开进行重命名；type=‘F（file）’对文件操作，type=‘D（directory）’对文件夹操作
:return:
"""
try:
conf = configparser.ConfigParser()
if type == 'D':
conf.read('conf' + resource_manager.getSeparator() +
'directory.ini')
elif type == 'F':
conf.read('conf' + resource_manager.getSeparator() +
'document.ini')
options = conf.options(doc_name)
if not os.path.exists(doc_name):
doc_name = os.path.abspath(doc_name)
for option in options:
try:
str_val = conf.get(doc_name, option)
New = os.path.join(doc_name, option)
old = os.path.join(doc_name, str_val)
os.rename(old, New)
except:
print(option + " don't exist")
except:
print("no document has been renamed")
def __init__(self,type):
self.configParse=config_parser_extender.CapitalCaseConfigParser()
self.type=type
if type == 'D':
self.configParse.read(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini')
# self.f = open(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini', 'w+')
elif type == 'F':
self.configParse.read(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'document.ini')
def plot_scatter_diagram(which_fig, x, y, x_label='x', y_label='y', title='title', label=None):
'''
Plot scatter diagram
Args:
which_fig : which sub plot
x : x array
y : y array
x_label : label of x pixel
y_label : label of y pixel
title : title of the plot
'''
styles = ['k.', 'g.', 'r.', 'c.', 'm.', 'y.', 'b.']
linestyles = ['-.', '--', 'None', '-', ':']
stylesMarker = pandas.read_csv(
resource_manager.Properties.getDefaultDataFold() + "view" + resource_manager.getSeparator() + "style.csv").ix[:,
3]
stylesColors = pandas.read_csv(
resource_manager.Properties.getDefaultDataFold() + "view" + resource_manager.getSeparator() + "style.csv").ix[:,
2]
assert len(x) == len(y)
if label != None:
assert len(x) == len(label) # and len(stylesMarker) >= len(set(label))
plt.figure(which_fig)
plt.clf()
if label == None:
plt.plot(x, y, styles[0])
else:
l = len(label)
labelSet = set(label)
k = 0
for i in labelSet:
xs = []
ys = []
for j in range(l):
if i == label[j]:
xs.append(x[j])
ys.append(y[j])
k = k + 1
try:
plt.scatter(xs, ys, c=stylesColors[k].strip(), marker=r"$ {} $".format(str(i)),label=i)
except:
log.fatal(stylesMarker)
log.fatal(stylesColors)
log.fatal(stylesMarker[k])
log.fatal(stylesColors[k])
plt.scatter(xs, ys, c=stylesColors[k].strip(), marker=r"$ {} $".format(str(i)),label=i)
exit()
plt.title(title)
plt.xlabel(x_label)
plt.ylabel(y_label)
plt.ylim(bottom=0)
# plt.legend(loc='upper left')
plt.show()
def start_imagexml_builder(
source=resource_manager.Properties.getDefaultOperationFold(),
path=resource_manager.Properties.getImageXmlResource()):
rmfile()
log.info("starting to init dataset from " + str(source) +
" to running create image-b.xml module.")
# path="D:\Projects\\Python\\deeplay\\src\\train\\run\\*";
Data = dir_to_dataset(source)
Data = np.array(Data)
# Data and labels are read
# set a filename
afileName = str(
os.getcwd()) + resource_manager.getSeparator() + "kmeans.data"
# create the file
file = open(afileName, 'w')
# put the data into a file.
# print(len(Data))
xmlFile = str(os.getcwd()) + resource_manager.getSeparator() + "image1.xml"
log.info("starinng to build image xml file")
if (os.path.exists(xmlFile) == False):
document = minidom.Document()
document.appendChild(
document.createComment("this is used for save a image file"))
imagelist = document.createElement("Images")
document.appendChild(imagelist)
f = open(path, "w")
document.writexml(f, addindent=' ' * 4, newl='\n', encoding='utf-8')
f.close()
root = xml.dom.minidom.parse(path)
imagesRoot = root.documentElement
c = 0
# root=xml.etree.ElementTree.parse("image-b.xml");
import sys
for x in range(0, len(Data)):
imageRoot = document.createElement("Image")
id = document.createElement("id")
data = document.createElement("data")
c = c + 1
aRow = Data[x]
value = []
for pix in range(1, len(aRow)):
file.write(str(aRow[pix]))
value.append(int(str(aRow[pix])))
if pix != len(aRow) - 1:
file.write(",")
# print(len(aRow))
id.appendChild(document.createTextNode(aRow[0]))
data.appendChild(document.createTextNode(str(value)))
imagesRoot.appendChild(imageRoot)
imageRoot.appendChild(id)
imageRoot.appendChild(data)
file.write("\n")
f = open(path, "w")
root.writexml(f, addindent=' ' * 4, newl='\n', encoding='utf-8')
f.close()
log.info("build about " + str(c) + " image file.")
def save(self):
if self.type == 'D':
f=open(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini', 'w+')
self.configParse.write(f)
f.close()
# self.f = open(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini', 'w+')
elif self.type == 'F':
f=open(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'document.ini', 'w+')
self.configParse.write(f)
f.close()
def build_distance_txt(distance,
sep=' ',
path=resource_manager.Properties.getDefaultDataFold() +
"txt" + resource_manager.getSeparator() + "build.txt"):
"""
写入文件，具体格式如下：
行下标 分隔符 列下标 分隔符 distance中行下标和列下标对应的值
:param distance:
:param path:
:param sep:分隔符
:return:
"""
num_row = 0
num_rank = 0
fr = open(path, 'w')
for row in distance:
num_row += 1
num_rank = 0
for element in row:
if np.isnan(element):
continue
num_rank += 1
fr.write(str(num_row))
fr.write(sep)
fr.write(str(num_rank))
fr.write(sep)
fr.write(str(element))
fr.write("\n")
fr.close()
"""
def save(name='default'):
"""
保存id和data数据
:return:
"""
from context.resource_manager import Properties
from pandas import DataFrame, Series
path = os.path.join(Properties.getDefaultDataFold()+"xml"+resource_manager.getSeparator()+name+".xml")
from xml.dom.minidom import parse, parseString
images = parse(path)
id = []
data = []
for node in images.getElementsByTagName("Image"):
idNode = node.getElementsByTagName("id")[0].childNodes[0].data
id.append(idNode)
dataNode = node.getElementsByTagName("data")[0].childNodes[0].data
dataNode = dataNode[1:-1].split(',')
data.append(dataNode)
id = np.asarray(id)
id = Series(id)
data = np.asarray(list(map(conv, data)), dtype=np.float)
if not os.path.exists(Properties.getDefaultDataFold()+"/cache/"+name):
#f=open(Properties.getDefaultDataFold()+"/csv/threshold.csv","w")
#f.close()
os.makedirs(Properties.getDefaultDataFold()+"/cache/"+name)
np.save(Properties.getRootPath() + "/data/cache/"+name+"/id.npy", id)
np.save(Properties.getRootPath() + "/data/cache/"+name+"/data.npy", data)
def binaryzation_proxy(cls,path):
s = path.split(resource_manager.getSeparator())
s = s[(len(s) - 1)]
c=0
if s[0:4]=="data":
log.info(" do binaryzation about files in "+str(path))
size=s[4:]
size=int(size)
for i in os.walk(path, False):
for f in i[2]:
BinAndCrop.preOp(os.path.join(path, f));
BinAndCrop.single_crop(path+resource_manager.getSeparator()+f,size)
c=c+1
return c
return 0
def rename_file(url, reverse=True):
"""
用户给定文件夹的路径，如果给定路径存在，更改指定文件夹下的文件的名字
:param url: 用户给定的文件夹的路径
:param reverse: 如果reverse=True就进行反向命名，reverse=False就对所给文件夹所包含的文件进行重命名；
:return:
"""
if _exist_(url) and reverse == False:
if not os.path.exists(url):
url = os.path.abspath(url)
a = 1
list_sto = _random_name(url, 'F')
conf_sto = configparser.ConfigParser()
conf_sto.read('conf' + resource_manager.getSeparator() + 'factory.ini')
for ob in list_sto:
try:
files = conf_sto.get(url, ob)
old = os.path.join(url, files)
filetype = os.path.splitext(files)[1]
d = a
if os.path.isfile(old):
New = os.path.join(url,
str(a) + filetype)
a = str(a) + filetype
os.rename(old, New)
_store_(url, ob, a, 'F')
a = d
a = a + 1
except:
pass
elif reverse == True and _exist_(url):
_reverse_(url, 'F')
def run(path=resource_manager.Properties.getDefaultDataFold()+"txt"+resource_manager.getSeparator()+"build.txt",sep=' '):
'''
return cluster id
i,j distance
'''
(dist,xxdist,ND,N) = readfile(path, dimensions = 2, sep=sep)
XY, eigs = mds(dist)
(rho,delta,ordrho,dc,nneigh) = rhodelta(dist, xxdist, ND, N, percent = 2.0)
DCplot(dist, XY, ND, rho, delta,ordrho,dc,nneigh,17,0.1)
def rename_dir(url, reverse=False):
""""
根据static的值进行文件夹自动重命名，命名规则
directory.ini
网->0
于->1
:param url:,文件夹地址，
:param static:给出的是关于地址是否是相对地址
:param reverse:确定是需要进行方向目录生成，还是正向目录生成
:return:
Tip：需要判断url是否存在，是否为文件夹,对于conf目录需要注意是否已经存在，没有存在需要进行创建。另外对于directory.ini文件也需要判断是否存在。建议对这里的工作进行定义多个子函数。定义的子函数请以_开头。
在进行一些具体的操作，需要输出相关日志操作
""" ""
"""
用户给定文件夹的路径，如果给定路径存在，更改指定文件夹下的文件夹的名字
:param url: 用户给定的文件夹的路径
:param reverse: 如果reverse=True就进行反向命名，reverse=False就对所给文件夹所包含的文件夹进行重命名；
:return:
"""
if _exist_(url) and reverse == False:
if not os.path.exists(url):
url = os.path.abspath(url)
a = 1
list_sto = random_name(url, 'D')
con = config_parser.ConfigParser("D")
"""
while (True):
if str(a) in list_sto:
a = a + 1
else:
break
"""
conf_sto = configparser.ConfigParser()
conf_sto.read('conf' + resource_manager.getSeparator() +
'directory.ini')
for ob in list_sto:
try:
files = conf_sto.get(url, ob)
old = os.path.join(url, files)
filetype = os.path.splitext(files)[1]
d = a
if os.path.isdir(old):
New = os.path.join(url,
str(a) + filetype)
a = str(a) + filetype
os.rename(old, New)
_store_(con, url, ob, a)
a = d
a = a + 1
except:
pass
con.save()
elif reverse == True and _exist_(url):
_reverse_(url, 'D')
def start_image_rename(src=resource_manager.Properties.getDefaultOperationFold(),
path=resource_manager.Properties.getDefaultWorkFold(),state=True):
if state:
log.info("starting to rename file in dir, do not modify the resource path file. Operation Directory:"+str(path))
dirs = file_manage.subdirs(src)
c=0
if os.path.exists(path):
shutil.rmtree(path)
if len(dirs)>0:
for d in dirs:
files = file_manage.subfilesName(d)
for f in files:
name = file_manage.random_str(8) + "_" + f
try:
shutil.copy(os.path.join(d, f), os.path.join(path+resource_manager.getSeparator()+d, name))
except:
file_manage.fix_path(path+resource_manager.getSeparator()+d)
shutil.copy(os.path.join(d, f), os.path.join(path+resource_manager.getSeparator()+d, name))
c=c+1
log.info("finished copy file to the path"+str(path)+". about "+str(c)+" files.")
file_manage.rename_files(path)
else:
log.info("starting to rename file in dir, modify the resource path file. Operation Directory:"+str(src))
file_manage.rename_files(path)
def random_name(url, type):
"""
对文件或文件夹进行随机重命名（防止产生因同名而无法重命名的问题）（具体类型则根据所给的文件类型type决定，用户调用相应的方法后type自动赋值）
:param url: 用户传入的文件夹的地址
:return: 返回文件夹中所有文件或文件夹重命名之前的名字的列表
"""
if not os.path.exists(url):
url = resource_manager.Properties.getRootPath(
) + resource_manager.getSeparator() + url
doc = os.listdir(url)
if type == 'D':
con = config_parser.ConfigParser('D')
else:
con = config_parser.ConfigParser('F')
for files in doc:
filetype = os.path.splitext(files)[1]
if os.path.exists(url):
old = url + resource_manager.getSeparator() + files
else:
old = resource_manager.Properties.getRootPath(
) + resource_manager.getSeparator(
) + url + resource_manager.getSeparator() + files
if os.path.isdir(old) and type == 'D':
random = random_string()
New = url + resource_manager.getSeparator() + random + filetype
os.rename(old, New)
_store_(con, url, files, random + filetype)
elif os.path.isfile(old) and type == 'F':
random = random_string()
if os.path.exists(url):
New = url + resource_manager.getSeparator() + random + filetype
else:
New = url + resource_manager.getSeparator() + random
os.rename(old, New)
_store_(con, url, files, random + filetype)
con.save()
list = doc
return list
def plot_dataframe_scatter_diagram(which_fig, data, x_label='x', y_label='y', title='title', label=None):
styles = ['k.', 'g.', 'r.', 'c.', 'm.', 'y.', 'b.']
linestyles = ['-.', '--', 'None', '-', ':']
stylesMarker = markers = ['.', # point
',', # pixel
'o', # circle
'v', # triangle down
'^', # triangle up
'<', # triangle_left
'>', # triangle_right
'1', # tri_down
'2', # tri_up
'3', # tri_left
'4', # tri_right
'8', # octagon
's', # square
'p', # pentagon
'*', # star
'h', # hexagon1
'H', # hexagon2
'+', # plus
'x', # x
'D', # diamond
'd', # thin_diamond
'|', # vline
]
# styles = []
stylesColors = pandas.read_csv(
resource_manager.Properties.getDefaultDataFold() + "view" + resource_manager.getSeparator() + "color.csv").ix[:,
2]
plt.figure(which_fig)
plt.clf()
plt.title(title)
plt.xlabel(x_label)
plt.ylabel(y_label)
plt.ylim(bottom=0)
plt.legend(loc='upper left')
plt.show()
def renamefiles(src=resource_manager.Properties.getDefaultOperationFold()):
log.info("starting rename file job.")
directorys = os.listdir(src)
cf = config_parser.ConfigParser("F")
c = 0
for directory in directorys:
if os.path.isdir(src + resource_manager.getSeparator() + directory):
files = subfilesName(src + resource_manager.getSeparator() +
directory)
c += len(files)
log.info(
str("####" + src + resource_manager.getSeparator() +
directory + " directory is doing #### " + str(len(files))))
path = src + resource_manager.getSeparator() + directory
for f in files:
# log.info(os.path.join(f)+"....")
if os.path.isfile(os.path.join(path, f)) == True:
c = c + 1
# name = '{0}_{1:0{2}d}'.format(directory, c, 6)
name = '_{0}_{1}{2:0{3}d}'.format(directory, _time_name(),
c, 4)
name = random_string() + name
rename(
cf, src + resource_manager.getSeparator() + directory +
resource_manager.getSeparator(), f, name)
else:
c = c + 1
s = src.split(resource_manager.getSeparator())
s = s[(len(s) - 1)]
# name = '_{0}_{1:0{2}d}'.format(s, _time_name(), 6)
name = '_{0}_{1}{2:0{3}d}'.format(s, _time_name(), c, 4)
name = random_string() + name
rename(cf, src + resource_manager.getSeparator(), directory, name)
log.info("finish rename " + str(c) + " job.")
cf.save()
def _random_name(url, type):
"""
对文件或文件夹进行随机重命名（防止产生因同名而无法重命名的问题）（具体类型则根据所给的文件类型type决定，用户调用相应的方法后type自动赋值）
:param url: 用户传入的文件夹的地址
:return: 返回文件夹中所有文件或文件夹重命名之前的名字的列表
"""
doc = os.listdir(url)
for files in doc:
try:
filetype = os.path.splitext(files)[1]
if not os.path.exists(files):
old = url + resource_manager.getSeparator() + files
else:
old = resource_manager.Properties.getRootPath(
) + resource_manager.getSeparator(
) + url + resource_manager.getSeparator() + files
if os.path.isdir(old) and type == 'D':
random = file_manage.random_string()
New = resource_manager.Properties.getRootPath(
) + resource_manager.getSeparator(
) + url + resource_manager.getSeparator() + random
os.rename(old, New)
_store_(url, files, random + filetype, 'D')
elif os.path.isfile(old) and type == 'F':
random = file_manage.random_string()
New = resource_manager.Properties.getRootPath(
) + resource_manager.getSeparator(
) + url + resource_manager.getSeparator() + random
os.rename(old, New)
_store_(url, files, random + filetype, 'F')
except:
pass
list = doc
return list
def cluster_view_json_builder(clusters=[]):
"""
将用户传入的多个json格式的对象存储在json文件中
newlist【】作为list先读入文件中的数据以防止丢失，然后追加新加入的json对象（以list格式存入newlist）
:param clusters:
:return:
"""
for c in clusters:
if not isinstance(c, ClusterViwer):
raise Exception("错误的数据类型，不是ClusterViwer")
else:
newlist = []
try:
with open(
resource_manager.Properties.getRootPath() +
resource_manager.getSeparator() + 'data' +
resource_manager.getSeparator() + 'json' +
resource_manager.getSeparator() + 'ssdfdssdf.json',
'r') as R:
readed = json.load(R)
for r in readed:
newlist.append(r)
except:
print('The file is empty!')
newlist.append(c.tolist())
with open(
resource_manager.Properties.getRootPath() +
resource_manager.getSeparator() + 'data' +
resource_manager.getSeparator() + 'json' +
resource_manager.getSeparator() + 'ssdfdssdf.json',
'w+') as f:
f.write(
json.dumps(newlist,
skipkeys=True,
sort_keys=True,
indent=2))
#f=open(Properties.getDefaultDataFold()+"/csv/threshold.csv","w")
#f.close()
os.makedirs(Properties.getDefaultDataFold() + "/cache/" + name)
np.save(Properties.getRootPath() + "/data/cache/" + name + "/id.npy", id)
np.save(Properties.getRootPath() + "/data/cache/" + name + "/data.npy",
data)
if __name__ == "__main__":
from context.resource_manager import Properties
from pandas import Series
from view import shape_view
from view import plot_utils
from cluster import density_cluster
distance_c = 0.69
name = 'aggregation'
save(name)
id = np.load(Properties.getRootPath() + "/data/cache/" + name + "/id.npy")
data = np.load(Properties.getRootPath() + "/data/cache/" + name +
"/data.npy")
path = resource_manager.Properties.getDefaultDataFold(
) + "txt" + resource_manager.getSeparator() + "build.txt"
id_index = Series(id.tolist())
from cluster import density_cluster
N = id_index.count()
distance = compute_distance(data)
build_distance_txt(distance)
# id.values -> 对应的key
run(path)
log.warn("the init distance_c is: " + str(distance_c))
def load_data():
id = np.load(Properties.getRootPath() + "/data/cache/flame/id.npy")
data = np.load(Properties.getRootPath() + "/data/cache/flame/data.npy")
return id, data
if __name__ == '__main__':
id, data = load_data()
label = [int(x[0:1]) for x in id]
from context.resource_manager import Properties
from view import shape_view
from view import plot_utils
from cluster import density_cluster
x = data[:, :1]
y = data[:, 1:2]
path = resource_manager.Properties.getDefaultDataFold(
) + resource_manager.getSeparator(
) + "result" + resource_manager.getSeparator() + "srv.png"
plot_utils.save_scatter_diagram(None,
x=x,
y=y,
x_label='x',
y_label='y',
title='scatter figure',
label=label,
path=path)
#plot_utils.plot_scatter_diagram(None,x=x,y=y,x_label='x',y_label='y',title='scatter figure',label=label)
def save_show_cluster(index_id, data, distance_c, pile_id, dataset="/", level="INFO", level_info='scatter figure'):
from view import plot_utils
from context import resource_manager
path = resource_manager.Properties.getDefaultDataFold() + "result" + resource_manager.getSeparator() + "temp/" + dataset + "/" + resource_manager.Properties.name_str_static() + "/"
level_path = resource_manager.Properties.getDefaultDataFold() + "result" + resource_manager.getSeparator() + "temp/" + level + "/" + resource_manager.Properties.name_str_static() + "/" + str(
distance_c) + "/"
if not os.path.exists(path[:path.rfind('/')]):
os.makedirs(path[:path.rfind('/')])
if not os.path.exists(level_path[:level_path.rfind('/')]):
os.makedirs(level_path[:level_path.rfind('/')])
pile_id = pile_id.sort_values('size', ascending=False)
x = []
y = []
label = []
i = 1
for m in range(len(pile_id)):
# l=pile_id.irow(m)['pile']
l = pile_id.iloc[m]['pile']
# size=pile_id.irow(m)['size']
size = pile_id.iloc[m]['size']
if pile_id.loc[m]['outlier'] is False:
for node in l:
index = index_id[node]
x.append(data[index][0])
y.append(data[index][1])
label.append(i)
i = i + 1
else:
for node in l:
index = index_id[node]
x.append(data[index][0])
y.append(data[index][1])
label.append(0)
if level is "SEE":
plot_utils.plot_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title=level_info, label=label)
if level is "DEBUG":
# plot_utils.save_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title='scatter figure', label=label,path=level_path+resource_manager.Properties.name_str_FULL()+".png")
plot_utils.save_all_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title=level_info, label=label,
path=level_path + resource_manager.Properties.name_str_FULL() + ".png")
else:
plot_utils.save_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title='scatter figure', label=label,
path=path + str(
distance_c) + ".png")
plot_utils.save_all_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title='scatter figure',
label=label,
path=path + str(
distance_c) + ".png")
# plot_utils.plot_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title='scatter figure', label=label)
log.debug(("\n") + str(pile_id))
try:
p = Properties.getDefaultDataFold() + "/csv/" + dataset + "/" + resource_manager.Properties.name_str_static() + "/" + str(
distance_c) + ".csv"
pile_id.to_csv(p)
except:
if not os.path.exists(p[:p.rfind('/')]):
pp = p.rfind('/')
os.makedirs(p[:pp])
os.mknod(p)
pile_id.to_csv(p)
from cluster import density_cluster_dpc
if os.path.exists(resource_manager.Properties.getDefaultDataFold() +
"result/test"):
shutil.rmtree(resource_manager.Properties.getDefaultDataFold() +
"result/test")
id = np.load(Properties.getRootPath() + "/data/cache/" + name + "/id.npy")
data = np.load(Properties.getRootPath() + "/data/cache/" + name +
"/data.npy")
from pandas import Series, DataFrame
id_index = Series(id.tolist())
from cluster import density_cluster
index_id = Series(id_index.index, index=id_index.values)
distance = density_cluster.compute_distance(data)
#np.save(Properties.getRootPath() + "/data/cache/distance/data.npy", distance)
path = resource_manager.Properties.getDefaultDataFold(
) + resource_manager.getSeparator(
) + "result" + resource_manager.getSeparator(
) + "name" + resource_manager.getSeparator() + str(distance_c) + ".png"
temp = distance.copy()
temp[np.isnan(temp)] = 0
stand = np.std(temp)
temp = distance.copy()
temp[np.isnan(temp)] = stand
temp = temp.min(axis=0)
next_distance_c = np.std(temp)
pile_id = density_cluster.debug_cluster(id_index,
index_id,
data,
distance,
distance_c,
next_distance_c,
dataset=name,
def save_show_cluster(index_id, data, distance_c, pile_id):
from view import plot_utils
from context import resource_manager
path = resource_manager.Properties.getDefaultDataFold(
) + "result" + resource_manager.getSeparator(
) + "temp" + resource_manager.getSeparator(
) + resource_manager.Properties.name_str_static(
) + resource_manager.getSeparator() + str(distance_c) + ".png"
if not os.path.exists(resource_manager.Properties.getDefaultDataFold() +
"result" + resource_manager.getSeparator() + "temp" +
resource_manager.getSeparator() +
resource_manager.Properties.name_str_static() +
resource_manager.getSeparator()):
os.makedirs(resource_manager.Properties.getDefaultDataFold() +
"result" + resource_manager.getSeparator() + "temp" +
resource_manager.getSeparator() +
resource_manager.Properties.name_str_static() +
resource_manager.getSeparator())
pile_id = pile_id.sort_values('size', ascending=False)
x = []
y = []
label = []
i = 1
for m in range(len(pile_id)):
# l=pile_id.irow(m)['pile']
l = pile_id.iloc[m]['pile']
# size=pile_id.irow(m)['size']
size = pile_id.iloc[m]['size']
if size >= 1 and i < 15:
for node in l:
index = index_id[node]
x.append(data[index][0])
y.append(data[index][1])
label.append(i)
i = i + 1
else:
for node in l:
index = index_id[node]
x.append(data[index][0])
y.append(data[index][1])
label.append(0)
plot_utils.save_scatter_diagram(None,
x=x,
y=y,
x_label='x',
y_label='y',
title='scatter figure',
label=label,
path=path)
# plot_utils.plot_scatter_diagram(None, x=x, y=y, x_label='x', y_label='y', title='scatter figure', label=label)
log.debug(pile_id)
try:
p = Properties.getDefaultDataFold(
) + "/csv/" + resource_manager.Properties.name_str_static(
) + "/" + str(distance_c) + ".csv"
pile_id.to_csv(p)
except:
if not os.path.exists(p):
pp = p.rfind('/')
os.makedirs(p[:pp])
os.mknod(p)
pile_id.to_csv(p)
data = np.asarray(list(map(conv, data)), dtype=np.float)
if not os.path.exists(Properties.getDefaultDataFold()+"/cache/"+name):
#f=open(Properties.getDefaultDataFold()+"/csv/threshold.csv","w")
#f.close()
os.makedirs(Properties.getDefaultDataFold()+"/cache/"+name)
np.save(Properties.getRootPath() + "/data/cache/"+name+"/id.npy", id)
np.save(Properties.getRootPath() + "/data/cache/"+name+"/data.npy", data)
if __name__=="__main__":
from context.resource_manager import Properties
from pandas import Series
from view import shape_view
from view import plot_utils
from cluster import density_cluster
distance_c=0.69
name='aggregation'
save(name)
id = np.load(Properties.getRootPath() + "/data/cache/" + name + "/id.npy")
data = np.load(Properties.getRootPath() + "/data/cache/" + name + "/data.npy")
path =resource_manager.Properties.getDefaultDataFold()+"txt"+resource_manager.getSeparator()+"build.txt"
id_index = Series(id.tolist())
from cluster import density_cluster
N = id_index.count()
distance = compute_distance(data)
build_distance_txt(distance)
# id.values -> 对应的key
run(path)
log.warn("the init distance_c is: " + str(distance_c))
def start_image_rebuild(src=resource_manager.Properties.getDefaultOperationFold(),
path=resource_manager.Properties.getDefaultWorkFold(),target=['data28','data100'],train=70,verify=30,test=30,state=False):
log.info("starting to separtion work. rebuild the image directory. At " + src)
# path=raw_input("src path:");
# #n=raw_input("input train mount decimals:");
# m=raw_input("input verify mount decimals:");
# l=raw_input("input ssdfdssdf mout decimals:");
if state:
log.info("please input the train,verify,test factor:")
train = int(input())
verify = int(input())
test = int(input())
train = train * 0.01
verify = verify * 0.01
test = test * 0.01
log.info("this is the factor:"+str(train) + ","+str(verify)+ ","+str(test))
i = 0;
dirs = file_manage.subdirs(src)
dpath = path + "train"
# print dpath;
initDirs(dpath)
dpath = path + "verify"
initDirs(dpath)
dpath = path + "test"
initDirs(dpath)
c=0
if os.path.exists(path):
shutil.rmtree(path)
if len(dirs)>0:
for d in dirs:
# files = os.listdir(src)
files = file_manage.subfilesName(d)
length = len(files)
# print "###########",d,"###########"
i = 1;
for f in files:
name = file_manage.random_str(8) + "_" + f
if i <= length * train:
p = path + "train"
elif i <= (length * train + length * verify):
p = path + "verify"
else:
p = path + "test"
try:
for t in target:
shutil.copy(os.path.join(d, f), os.path.join(p +resource_manager.getSeparator()+t, name))
except:
for t in target:
file_manage.fix_path(p + resource_manager.getSeparator()+ t)
shutil.copy(os.path.join(d, f), os.path.join(p +resource_manager.getSeparator()+ t, name))
# os.rename(os.path(join(d,f),os.path.join(p+"/data100",name);
i = i + 1
c=c+1
# print i,":",d," file:",f,"--->",p," file:",name
else:
d=src
files = file_manage.subfilesName(d)
length=len(files)
for f in files:
name = file_manage.random_str(8) + "_" + f
if i <= length * train:
p = path + "train"
elif i <= (length * train + length * verify):
p = path + "verify"
else:
p = path + "test"
try:
for t in target:
shutil.copy(os.path.join(d, f), os.path.join(p +resource_manager.getSeparator()+t, name))
except:
for t in target:
file_manage.fix_path(p +resource_manager.getSeparator()+ t)
shutil.copy(os.path.join(d, f), os.path.join(p +resource_manager.getSeparator()+ t, name))
# os.rename(os.path(join(d,f),os.path.join(p+"/data100",name);
i = i + 1
c=c+1
log.info("finished copy file to the path"+str(path)+". about "+str(c)+" files.")
file_manage.renamefiles(path)
def removeConfigFiles(cls):
if os.path.isfile(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini'):
os.remove(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini')
if os.path.isfile(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'document.ini'):
os.remove(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'document.ini')
def reload(self):
if type == 'D':
self.configParse.read(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini')
# self.f = open(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'directory.ini', 'w+')
elif type == 'F':
self.configParse.read(Properties.getRootPath()+'conf' + resource_manager.getSeparator() + 'document.ini')
