def getMinimalFeatureSet():
"""
Returns a Processing Instance with minimal feature set
"""
proc = Processing()
# channel features
proc.addChannelFeature(MeanChannelValueFeature())
#proc.addChannelFeature(ChannelHistogramFeature())
# channel generators
scale = 1.0
proc.addChannelGenerator(LaplaceChannelGenerator(scale))
proc.addChannelGenerator(GaussianGradientMagnitudeChannelGenerator(scale))
proc.addChannelGenerator(EVofGaussianHessianChannelGenerator(scale))
# supervoxel features
proc.addSupervoxelFeature(SizeFeature())
return proc
def test_typeIsReadOnly(self):
p = Processing("proc1", self.fixture1())
try:
p.type = "AnotherType"
self.fail("Exception expected")
except AttributeError, e:
self.assertEqual("Attribute 'type' is read only", e.args[0])
def Classification(Data, ID):
Answers = []
Message = Data['message']
Answers.append(Data['message'])
Result = bagOfWords(Message)
if Result == 1:
PreProcessing = Pre_Processing(Answers)
Answers = PreProcessing.MainFunction()
del Answers[0]
Features = FeatureExtraction(Answers)
Answers = Features.Test_TFIDF()
Data1 = pd.DataFrame(Answers)
del Answers
Data1 = Data1.fillna(0)
SentimentList = Features.Sentiment()
Data1[315477] = SentimentList
Processing_Object = Processing(Data1)
Results = Processing_Object.Testing()
print(Results[0])
else:
Results = []
Results.append(0)
Message = Data['message']
From = Data['from']
Time = Data['time']
ChatID = Data['ChatID']
Classified = db.collection(u"Chat").document(ChatID).collection(u"chating")
Classified.add({
u"message": Message,
u"from": From,
u"time": Time,
u"Class": str(Results[0])
})
def main():
lines = read_in()
np_lines = np.array(lines)
lines = list(np_lines)
Message = lines[0]
From = lines[1]
Time = lines[2]
ChatID = lines[3]
Answers = []
Answers.append(Message)
PreProcessing = Pre_Processing(Answers)
Answers = PreProcessing.MainFunction()
del Answers[0]
Features = FeatureExtraction(Answers)
Answers = Features.Test_TFIDF()
Data1 = pd.DataFrame(Answers)
del Answers
Data1 = Data1.fillna(0)
SentimentList = Features.Sentiment()
Data1[315477] = SentimentList
Processing_Object = Processing(Data1)
Results = Processing_Object.Testing()
Classified = db.collection(u"Chat").document(ChatID).collection(u"chating")
Classified.add({
u"message": Message,
u"from": From,
u"time": Time,
u"Class": str(Results[0])
})
print("Message Has Been Classified")
def test_connect_from_connectors_wrongDirecction(self):
engine = self.connectivityFixture()
multi1 = Processing("multi1", engine)
multi2 = Processing("multi2", engine)
try:
multi1 < multi2._inports
except BadConnectorDirectionOrder, e:
self.assertEqual(e.message,
"Wrong connectors order: Input < Output")
def test_connect_to_connector_wrongDirection(self):
engine = self.connectivityFixture()
multi1 = Processing("multi1", engine)
csource = Processing("csource", engine)
try:
multi1 > csource.OutControl1
except BadConnectorDirectionOrder, e:
self.assertEqual(e.message,
"Wrong connectors order: Output > Input")
def test_connect_controls_from_slice(self):
engine = self.connectivityFixture()
multi1 = Processing("multi1", engine)
multi2 = Processing("multi2", engine)
self.assertEqual(2, multi1._incontrols < multi2[::2])
self.assertEqual([
("multi2", "OutControl1", "multi1", "InControl1"),
("multi2", "OutControl3", "multi1", "InControl2"),
], engine.controlConnections())
self.assertEqual([], engine.portConnections())
def test_connect_slice_to_slice(self):
engine = self.connectivityFixture()
multi1 = Processing("multi1", engine)
multi2 = Processing("multi2", engine)
self.assertEqual(2, multi2[::2] > multi1[1:])
self.assertEqual([], engine.controlConnections())
self.assertEqual([
("multi2", "OutPort1", "multi1", "InPort2"),
("multi2", "OutPort3", "multi1", "InPort3"),
], engine.portConnections())
def test_connect_from_processing_to_port(self):
engine = self.empty()
engine.addProcessing("ProcessingWithPortsAndControls", "proc1")
engine.addProcessing("DummyPortSink", "proc2")
proc1 = Processing("proc1", engine)
proc2 = Processing("proc2", engine)
self.assertEqual(1, proc1 > proc2.InPort1)
self.assertEqual([], engine.controlConnections())
self.assertEqual([
("proc1", "OutPort1", "proc2", "InPort1"),
], engine.portConnections())
def test_connect_connectors_from_slice(self):
engine = self.connectivityFixture()
multi1 = Processing("multi1", engine)
multi2 = Processing("multi2", engine)
self.assertEqual(3, multi1._inports < multi2[1:])
self.assertEqual([], engine.controlConnections())
self.assertEqual([
("multi2", "OutPort2", "multi1", "InPort1"),
("multi2", "OutPort3", "multi1", "InPort2"),
("multi2", "OutPort4", "multi1", "InPort3"),
], engine.portConnections())
def test_connect_to_connector(self):
engine = self.connectivityFixture()
multi1 = Processing("multi1", engine)
csink = Processing("csink", engine)
self.assertEqual(3, multi1 > csink.InControl1)
self.assertEqual([
("multi1", "OutControl1", "csink", "InControl1"),
("multi1", "OutControl2", "csink", "InControl1"),
("multi1", "OutControl3", "csink", "InControl1"),
], engine.controlConnections())
self.assertEqual([], engine.portConnections())
def test_connect_from_processing(self):
engine = self.connectivityFixture()
p1 = Processing("multi1", engine)
p2 = Processing("multi2", engine)
p2 < p1
self.assertEqual([
("multi1", "OutControl1", "multi2", "InControl1"),
("multi1", "OutControl2", "multi2", "InControl2"),
("multi1", "OutControl3", "multi2", "InControl3"),
], engine.controlConnections())
self.assertEqual([
("multi1", "OutPort1", "multi2", "InPort1"),
("multi1", "OutPort2", "multi2", "InPort2"),
("multi1", "OutPort3", "multi2", "InPort3"),
], engine.portConnections())
def test_dirFunction(self):
p = Processing("proc1", self.fixture1())
self.assertEqual(
sorted([
'name',
'type',
'connect',
'_config',
'_inports',
'_outports',
'_incontrols',
'_outcontrols',
'ConfigParam1',
'ConfigParam2',
'InPort1',
'InPort2',
'InPort3',
'InPort4',
'OutPort1',
'OutPort2',
'InControl1',
'InControl2',
'OutControl1',
'OutControl2',
]), dir(p))
def test_connect_from_connectors(self):
engine = self.connectivityFixture()
multi1 = Processing("multi1", engine)
multi2 = Processing("multi2", engine)
self.assertEqual(3, multi2 < multi1._outcontrols)
self.assertEqual(3, multi2 < multi1._outports)
self.assertEqual([
("multi1", "OutControl1", "multi2", "InControl1"),
("multi1", "OutControl2", "multi2", "InControl2"),
("multi1", "OutControl3", "multi2", "InControl3"),
], engine.controlConnections())
self.assertEqual([
("multi1", "OutPort1", "multi2", "InPort1"),
("multi1", "OutPort2", "multi2", "InPort2"),
("multi1", "OutPort3", "multi2", "InPort3"),
], engine.portConnections())
def test_connect_to_alien(self):
engine = self.connectivityFixture()
proc1 = Processing("multi1", engine)
try:
proc1 > 34
except AssertionError, e:
self.assertEqual(e.message, "Unexpected connection peer: 34")
def test_connect_from_alien(self):
engine = self.connectivityFixture()
proc1 = Processing("multi1", engine)
try:
proc1 < 34
except AttributeError, e:
self.assertEqual(e.message,
"'int' object has no attribute 'connect'")
def bootstrap():
#dataset = Processing().import_data()
for dataset, filename in Processing().import_single_data():
training_data_X, training_data_y, testing_data_X, testing_data_y = Processing(
).separate_data(dataset)
rs = RankSVM().fit(training_data_X, training_data_y)
rs_pred_y = np.around(rs.predict2(testing_data_X))
rs_fpa = PerformanceMeasure(testing_data_y, rs_pred_y).FPA()
rs_aae_result = PerformanceMeasure(testing_data_y, rs_pred_y).AAE()
print('rs_fpa:', rs_fpa, 'rs_aae_result', rs_aae_result)
lr = linear_model.LinearRegression().fit(training_data_X, training_data_y)
lr_pred_y = np.around(lr.predict(testing_data_X))
lr_fpa = PerformanceMeasure(testing_data_y, lr_pred_y).FPA()
lr_aae_result = PerformanceMeasure(testing_data_y, lr_pred_y).AAE()
print('lr_fpa:', lr_fpa, 'lr_aae_result', lr_aae_result)
def ListenThread(Connection, Address):
try:
while True:
Message = Connection.recv(1024).decode("utf-8")
Answers = []
Answers.append(Message)
Answers2 = Answers
PreProcessing = Pre_Processing(Answers)
Answers = PreProcessing.MainFunction()
del Answers[0]
Answers2 = Answers2[0]
Features = FeatureExtraction(Answers)
Answers = Features.Test_TFIDF()
Data = pd.DataFrame(Answers)
Data = Data.fillna(0)
SentimentList = Features.Sentiment()
Data[44247] = SentimentList
Processing_Object = Processing(Data)
Results = Processing_Object.Testing()
Results = str(Results[0])
SendingConnection = None
print(Message)
for i in range(len(SendingIPS)):
if Address != SendingIPS[i]:
SendingConnection = SendingConnections[i]
break
SendThread(SendingConnection, Message, Results)
except:
# print("Client has Disconnected or un expected error ocurred please DEBUG!!!")
# for i in range(len(SendingIPS)):
# if Address == SendingIPS[i]:
# Connection.close()
# del SendingIPS[i]
# print(len(SendingConnections))
# print(len(SendingIPS))
# break
python = sys.executable
os.execl(python, python, *sys.argv)
def main():
for dataset, filename in Processing().dataload():
print(filename)
X, y = convert2numpy(dataset)
print('X:', X.shape)
ratio = 4 # 小于4%的模块合并
k = 5
r = 0.1
ssmote = Ssmote(X=X, y=y, ratio=ratio, k=k, r=r)
synX, synY = ssmote.synthesis()
print(synX)
print(synY)
def bootstrap():
#dataset = Processing().import_data()
for dataset, filename in Processing().import_single_data():
print(filename)
training_data_X, training_data_y, testing_data_X, testing_data_y = Processing(
).separate_data(dataset)
# print('train shape', training_data_X.shape)
training_data_X = training_data_X.tolist()
training_data_y = training_data_y.tolist()
from PyOptimize.General_Opt import Test_function
def LTR(a, **kwargs):
return Test_function().LTR(a, **kwargs)
ga = pyGaft(objfunc=LTR,
var_bounds=[(-2, 2)] * 20,
individual_size=50,
max_iter=10,
max_or_min='max',
X=training_data_X,
y=training_data_y).run()
importlib.reload(best_fit)
a, fitness_value = best_fit.best_fit[-1][-2], best_fit.best_fit[-1][-1]
print('a = {0}'.format(a))
pred_y = []
for test_x in testing_data_X:
pred_y.append(np.dot(test_x, a))
fpa = PerformanceMeasure(testing_data_y, pred_y).FPA()
print('fpa = {0}'.format(fpa))
def proportion():
# 先加入header
with open('proportion2.csv', 'w') as f:
columns = [
'Release', 'Count 0', 'Count 1', 'Count 2', 'Count 3', 'Count 4',
'Count 5', 'Count 6', 'Count 7', 'Count 8', 'Count gt 8'
]
writer = csv.DictWriter(f, fieldnames=columns)
writer.writeheader()
# 添加数据
for dataset, filename in Processing().dataload():
proportion_(dataset, filename, mode=1)
pass
def main(is_print=False):
params = {'root_path': root_path}
processing_ = Processing(params)
stats = Analyze(processing_.dict_, columns=['title',
'paper_id', 'abstract', 'body_text'])
if is_print is True:
print(stats.df_covid.describe(include='all'))
count = 0
for x in processing_.dict_['title']:
if x is "":
count += 1
print("\n\n\n\nQuantity of title is empty: {}".format(count))
return stats
def fiveElements():
""" 统计五个元素
"""
all_filenames = []
all_instances = []
all_defects = []
all_percent_defects = []
all_maxs = []
all_avgs = []
for dataset, filename in Processing().dataload():
data = convert2numpy(dataset)
statics = Statistics(data[1])
# #Instance
# print(filename)
number_instance = statics.numberInstance()
number_defect = statics.numberDefect()
percent_defect = statics.percentDefect()
max_bug = statics.max_()
avg_bug = statics.avg_()
# print('\n')
# print('#Instance :{0}'.format(number_instance))
# print('#Defects :{0}'.format(number_defect))
# print('%Defects :{:.1f}'.format(percent_defect))
# print('max_bug :{0}'.format(max_bug))
# print('avg_bug :{:.2f}'.format(avg_bug))
all_filenames.append(filename)
all_instances.append(number_instance)
all_defects.append(number_defect)
all_percent_defects.append(percent_defect)
all_maxs.append(max_bug)
all_avgs.append(avg_bug)
print(all_filenames)
data_ = {'Release': all_filenames,
'number_instance': all_instances,
'number_defect': all_defects,
'percent_defect': all_percent_defects,
'max_bug': all_maxs,
'avg_bug': all_avgs}
save_(r'fiveElements.csv', filename, **data_)
pass
def bootstrap(dataset):
training_data_X, training_data_y, testing_data_X, testing_data_y = Processing(
).separate_data(dataset)
brr = BayesianRidge()
de = DE(NP=100,
F=0.6,
CR=0.7,
generation=2000,
len_x=3,
ratioRange=[0.5, 1.0, 2.0, 4.0],
kRange=list(range(1, 21)),
value_up_range=5.0,
value_down_range=0.1,
X=training_data_X,
y=training_data_y,
classifier=brr)
de.process()
def on_previewBtn_clicked(self):
print('on_previewBtn_clicked')
# find out what kind of document to create (crb = checked radio btn)
crbName, crbWidget = self.get_checked_radio_button()
doctype = self._radio_choices.get(crbName, None)
filledFields = self.get_form_brief()
# open a popup for progressbar, in which we call a DocBuilder
dialog = QtWidgets.QDialog()
process_bar_dialog = Processing(dialog)
process_bar_dialog.configurate_doc_worker(doctype, filledFields)
process_bar_dialog.run_thread()
dialog.exec_()
# call whatever the system use to open pdf
path_to_file = ('file:///' + os.getcwd().replace('\\', '/') +
'/result.pdf')
QtGui.QDesktopServices.openUrl(QtCore.QUrl(path_to_file))
def run_scraper(self):
self.p.linker(self.p.scraper_dict, 'scraper')
scraper = Scraper(self.p.scraper)
processing = Processing(self.p.scraper)
scraper.run()
self.ProgressBar.setValue(25)
processing.pre_processor(scraper.scraper_df)
self.ProgressBar.setValue(50)
processing.spacy_processor(scraper.scraper_df)
self.ProgressBar.setValue(75)
self.connection = Connect(settings=None, mongo_cfg=self.p.scraper)
self.connection.update_from_df(scraper.scraper_df)
self.mongo_update()
self.ProgressBar.setValue(100)
from Time import Time as t
from Lists import Lists as l
from Processing import Processing as p
### PROCESSING
# get all files from a directory (default is current directory)
print(p.get_files())
int_list = [i for i in range(20)]
str_list = [s for s in "abcdefg"]
### LISTS
# create batches
batched = [i for i in l.batch(int_list)]
print("Batched list: {}".format(batched))
### TIME
# Timeout with printed countdown
t.sleep_countdown(1, print_step=1)
# current timestamp
print("Current timestamp: {}".format(t.timestamp_now()))
# add years to date
from datetime import datetime
d = datetime.datetime(1970, 1, 1)
d_plus = t.date_add_year(d, 2)
print(d)
print(d_plus)
def bootstrap():
#dataset = Processing().import_data()
count = 0
for dataset, filename in Processing().import_single_data():
print(filename)
count += 1
training_data_X, training_data_y, testing_data_X, testing_data_y = Processing(
).separate_data(dataset)
# print('train shape', training_data_X.shape)
# 1.降序排列训练集（Processing中已完成）
# 2.利用transfrom_pairwise() 得到Pi，ri
# P是一个矩阵，每个向量是两个x相减的结果
# r是一个向量 因为排序过，所以结果r = [1,1,1,1,1,1...]
rs = RankSVM()
P, r = rs.transform_pairwise(training_data_X, training_data_y)
#print('p shape ', P.shape, 'r len ', len(r))
P = P.tolist()
r = r.tolist()
print('type of P ', type(P[0][0]), 'type of r ', type(r[0]))
# P = [[1, 1, 2], [1, -1, 3], [3, 2, 1], [1, -5, 1], [2, 1, -2]]
# r = [1, 1, 1, 1, 1]
# 3.用training_data_y计算u,n
u, n = PerformanceMeasure(training_data_y).calc_UN(type='cs')
# print(len(u), len(n))
print(type(u[0]), type(n[0]))
# 4. 将Pi,ri,u,n导入genetic algorithm 计算w
from PyOptimize.General_Opt import Test_function
def Loss(x, **kwargs):
return Test_function().Loss(x, **kwargs)
ga = pyGaft(objfunc=Loss,
var_bounds=[(-2, 2)] * 20,
individual_size=50,
max_iter=200,
max_or_min='min',
P=P,
r=r,
u=u,
n=n).run()
# 5.编写predict3
# w 从best_fit中获得
if count == 1:
import best_fit
else:
importlib.reload(best_fit)
w, fitness = best_fit.best_fit[-1][-2], best_fit.best_fit[-1][-1]
print('w = ', w)
rs_pred_y = RankSVM(w=w).predict3(testing_data_X)
rs_pred_y = np.around(rs_pred_y)
rs_fpa = PerformanceMeasure(testing_data_y, rs_pred_y).FPA()
print('rs_fpa:', rs_fpa)
# f1.append(rs_fpa)
# RankSVM 效果
from sklearn.utils import shuffle
X_shuf, y_shuf = shuffle(training_data_X, training_data_y)
rs2 = RankSVM().fit(X_shuf, y_shuf)
rs_pred_y2 = np.around(rs2.predict2(testing_data_X))
rs_fpa2 = PerformanceMeasure(testing_data_y, rs_pred_y2).FPA()
rs_aae_result = PerformanceMeasure(testing_data_y, rs_pred_y2).AAE()
print('rs_fpa2:', rs_fpa2)
def test_processing():
image = cv2.imread("../images/buffer/output.png", cv2.IMREAD_GRAYSCALE)
assert not np.array_equal(
Processing.process_image('../images/buffer/new.png'),
image
)
def test_IncontrolsGettingByNameAsDictionary(self):
p = Processing("proc1", self.fixture1())
self.assertEqual(p["InControl1"].name, "InControl1")
self.assertEqual(p["InControl1"].kind, "Control")
self.assertEqual(p["InControl1"].direction, "In")
def test_IncontrolsGettingByNameAsAttribute(self):
p = Processing("proc1", self.fixture1())
self.assertEqual(p.InControl2.name, "InControl2")
self.assertEqual(p.InControl2.kind, "Control")
self.assertEqual(p.InControl2.direction, "In")
