def __init__(self, dataset, feature_vector, conf):
self.log = logging.getLogger('SafeDroid.Tuning')
self.sd = dataset
self.fv = feature_vector
self.maxi, self.maxj = self.getMaxDataSetSize(self.fv)
self.models = []
self.config = Config(conf)
def test_pubmed():
from torch_geometric.datasets import Planetoid
dataset = Planetoid('../data/', 'PubMed')
config = Config('../data/PubMed/', True, multilabel=False)
data = prepare_dataset(dataset, config, dataset.data.num_features)
print("train/val/test split: %i/%i/%i" %
(data.train_mask.sum(), data.val_mask.sum(), data.test_mask.sum()))
# data = Data(x=torch.ones(6, 1, dtype=torch.float32),
# edge_index=torch.LongTensor(
# [[2, 2, 3, 4, 0, 1, 4, 2, 3, 5, 2, 4],
# [0, 1, 2, 2, 2, 2, 3, 3, 4, 4, 4, 5]]),
# y=torch.LongTensor([0, 0, 0, 1, 0, 0]))
# data.num_classes = 2
# data.train_mask = torch.ByteTensor([1, 1, 0, 1, 0, 1])
# data.test_mask = torch.ByteTensor([0, 0, 1, 0, 1, 0])
deg = degree(data.edge_index[1].to(torch.long),
data.num_nodes,
dtype=torch.int)
loader = NeighborSampler(data,
size=[10, 10],
deg=deg,
batch_size=50,
shuffle=True)
transductive_sage(loader, config, steps=200, test_every=20)
def AdaBoost(featureVector, sd, tune,ts):
results = []
_target_counter = 0
_ratio_counter = 0
config = Config('model_training.config')
i=0
for matrix in featureVector:
for vector in matrix:
X = vector['vector']
y = sd.target[_target_counter]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=ts, random_state=4)
try:
knn = AdaBoostClassifier(n_estimators=tune['n_estimators'], algorithm = tune['algorithm'])
knn.fit(X_train,y_train)
y_pred = knn.predict(X_test)
accuracy = metrics.accuracy_score(y_test,y_pred)
confusion = metrics.confusion_matrix(y_test,y_pred)
fbk = calculateValues(metrics.confusion_matrix(y_test,y_pred), len(y_test))
f1 = f1Score(y_test, y_pred)
results.append(dict(accuracy=accuracy, supl_info = vector['info'], f1 = f1 , confusion_matrix = confusion , tune = tune , fbk = fbk,fv = X.shape[1]))
except ValueError:
continue
_target_counter +=1
_target_counter = 0
_ratio_counter += 1
return results
def main():
config = Config.load()
exercises = config.exercises.all()
if len(sys.argv) >= 2:
# test specific exercises
exercises = [e for e in exercises if e.slug in sys.argv[1:]]
failures = []
for exercise in exercises:
print('# ', exercise.slug)
if not exercise.test_file:
print('FAIL: File with test cases not found')
failures.append('{} (FileNotFound)'.format(exercise.slug))
else:
if check_assignment(exercise):
failures.append('{} (TestFailed)'.format(exercise))
print('')
print('TestEnvironment:', sys.executable.capitalize(), '\n\n')
if failures:
print('FAILURES: ', ', '.join(failures))
raise SystemExit(1)
else:
print('SUCCESS!')
def test_karate():
from torch_geometric.datasets import KarateClub
dataset = KarateClub('../data/KarateClub')
config = Config('../data/KarateClub/', True, multilabel=False)
data = prepare_dataset(dataset, config, 2)
index = np.arange(data.y.shape[0])
train, test = train_test_split(index,
stratify=data.y,
test_size=0.6,
random_state=0)
data.train_mask = torch.zeros(index.shape[0], dtype=torch.uint8)
data.train_mask[train] = 1
data.test_mask = torch.zeros(index.shape[0], dtype=torch.uint8)
data.test_mask[test] = 1
# ss = sparse.coo_matrix(data.x.numpy())
# i = torch.LongTensor([ss.row, ss.col])
# data.x = torch.sparse.FloatTensor(
# i, torch.from_numpy(ss.data), torch.Size(ss.shape))
import networkx as nx
g = nx.from_edgelist(data.edge_index.numpy().T)
train_set = set(train)
x = []
for nid in sorted(g.nodes):
inter = list(train_set.intersection(g.neighbors(nid)))
ixs, vals = torch.unique(data.y[inter], return_counts=True)
counts = torch.zeros(data.num_classes, dtype=torch.int64)
counts[ixs] = vals
x.append(counts)
data.x = torch.stack(x).to(torch.float)
# sub_edge_index, _ = subgraph(torch.from_numpy(train), data.edge_index)
deg = degree(data.edge_index[1].to(torch.long),
data.num_nodes,
dtype=torch.int)
loader = NeighborSampler(data,
size=[5, 5],
deg=deg,
batch_size=5,
shuffle=True)
transductive_sage(loader, config, steps=200, test_every=20)
def start():
config: Config = Config("./data/config.yaml")
last_entry: LastEntry = LastEntry("./data/last_entry.yaml")
discord: Discord = Discord(token=config.token)
hentai_haven = HentaiHaven(discord, config.hentai_haven, last_entry)
hanime = HAnime(discord, config.hanime, last_entry)
reddit = [Reddit(discord, c) for c in config.reddit]
hentai_haven.start()
hanime.start()
for r in reddit:
r.start()
hentai_haven.join()
hanime.join()
last_entry.write()
for r in reddit:
r.join()
def main():
opts = get_cli().parse_args()
config = Config.load()
status_filter = {ExerciseStatus.Active, ExerciseStatus.Beta}
if opts.include_deprecated:
status_filter.add(ExerciseStatus.Deprecated)
if opts.include_wip:
status_filter.add(ExerciseStatus.WIP)
exercises = config.exercises.all(status_filter)
if opts.exercises:
# test specific exercises
exercises = [
e for e in exercises if e.slug in opts.exercises
]
not_found = [
slug for slug in opts.exercises
if not any(e.slug == slug for e in exercises)
]
if not_found:
for slug in not_found:
if slug not in exercises:
print(f"unknown or disabled exercise '{slug}'")
raise SystemExit(1)
print(f'TestEnvironment: {sys.executable.capitalize()}')
print(f'Runner: {opts.runner}\n\n')
failures = []
for exercise in exercises:
print('# ', exercise.slug)
if not exercise.test_file:
print('FAIL: File with test cases not found')
failures.append('{} (FileNotFound)'.format(exercise.slug))
else:
if check_assignment(exercise, runner=opts.runner, quiet=opts.quiet):
failures.append('{} (TestFailed)'.format(exercise.slug))
print('')
if failures:
print('FAILURES: ', ', '.join(failures))
raise SystemExit(1)
else:
print('SUCCESS!')
class Tune:
def __init__(self, dataset, feature_vector, conf):
self.log = logging.getLogger('SafeDroid.Tuning')
self.sd = dataset
self.fv = feature_vector
self.maxi, self.maxj = self.getMaxDataSetSize(self.fv)
self.models = []
self.config = Config(conf)
def getModels(self):
return self.models
def fineTuneClassifiers(self):
for model in self.models:
self.Learning_Curve_Plot(self.fv[self.maxi][self.maxj]['vector'],
self.sd.target[self.maxj],
model,
display=self.config.display_plots(),
filename=self.config.plot_to_file())
self.Validation_Curve(self.fv[self.maxi][self.maxj]['vector'],
self.sd.target[self.maxj],
model,
display=self.config.display_plots(),
filename=self.config.plot_to_file())
if self.config.display_plots() or self.config.plot_to_file():
self.ROC(self.fv[self.maxi][self.maxj]['vector'],
self.sd.target[self.maxj], model)
return
def tuneClassifiers(self):
results = self.parameter_estimation(
self.fv[self.maxi][self.maxj]['vector'], self.sd.target[self.maxj],
None)
opt_clsf, tuning, acc = self.getBestClassifier(results)
best_tune = self.formTuneDictionary()
for k, v in tuning.iteritems():
best_tune[k] = v
self.models.append(
Model(opt_clsf,
best_tune,
acc=acc,
dss=self.fv[self.maxi][self.maxj]['vector'].shape[0]))
#second classifier
alter_clsf = self.Cross_Validation_Plot(
self.fv[self.maxi][self.maxj]['vector'], self.sd.target[self.maxj],
self.fv[self.maxi][self.maxj]['info'])
alter_clsf_value = max(alter_clsf.iteritems(),
key=operator.itemgetter(1))[0] # fetch name
if alter_clsf_value is opt_clsf:
print 'Reached on agreement for Optimum Classifier'
else:
result = self.parameter_estimation(
self.fv[self.maxi][self.maxj]['vector'],
self.sd.target[self.maxj], alter_clsf_value)
best_tune = self.formTuneDictionary()
for k, v in result[alter_clsf_value][-1]['tune'].iteritems():
best_tune[k] = v
self.models.append(
Model(alter_clsf_value,
best_tune,
acc=result[alter_clsf_value][2][result[alter_clsf_value]
[0]['score']],
dss=self.fv[self.maxi][self.maxj]['vector'].shape[0]))
return
def Validation_Curve(self, X, y, model, display=False, filename=None):
"""
Plot the influence of a single hyperparameter on the training score and the
validation score to find out whether the estimator is overfitting or underfitting
for some hyperparameter values.
If the training score and the validation score are both low, the estimator
will be underfitting. If the training score is high and the validation score
is low, the estimator is overfitting and otherwise it is working very well.
A low training score and a high validation score is usually not possible
"""
from sklearn.model_selection import validation_curve
classifier = model.getClassifier()
if classifier is 'MLP':
return
tune = model.getTune()
scoring = "accuracy"
try:
if classifier == 'KNeighbor':
param_range = np.array(
filter(lambda x: x % 2 != 0, list(range(4, 30))))
param_name = "n_neighbors"
xtitle = 'neighbors'
train_scores, test_scores = validation_curve(
KNeighborsClassifier(weights=tune['weights'],
algorithm=tune['algorithm']),
X,
y,
param_name=param_name,
param_range=param_range,
cv=self.config.cv(),
scoring=scoring,
n_jobs=multiprocessing.cpu_count())
elif classifier == 'ADA':
param_range = np.array(
filter(lambda x: x % 2 != 0, list(range(4, 30))))
param_name = 'n_estimators'
xtitle = 'estimators'
train_scores, test_scores = validation_curve(
AdaBoostClassifier(algorithm=tune['algorithm']),
X,
y,
param_name=param_name,
param_range=param_range,
cv=self.config.cv(),
scoring=scoring,
n_jobs=multiprocessing.cpu_count())
elif classifier == 'SVM':
param_range = np.logspace(-6, -1, 5)
param_name = 'gamma'
xtitle = "$\gamma$"
train_scores, test_scores = validation_curve(
SVC(kernel=tune['kernel'], C=tune['C']),
X,
y,
param_name=param_name,
param_range=param_range,
cv=self.config.cv(),
scoring=scoring,
n_jobs=multiprocessing.cpu_count())
elif classifier == 'DTree':
param_range = np.array(
filter(lambda x: x % 5 == 0, list(range(4, 50))))
param_name = 'max_depth'
xtitle = 'depth'
train_scores, test_scores = validation_curve(
DecisionTreeClassifier(class_weight=tune['weights']),
X,
y,
param_name=param_name,
param_range=param_range,
cv=self.config.cv(),
scoring=scoring,
n_jobs=multiprocessing.cpu_count())
elif classifier == 'RForest':
param_range = np.array(
filter(lambda x: x % 2 != 0, list(range(4, 30))))
param_name = 'n_estimators'
xtitle = 'estimators'
train_scores, test_scores = validation_curve(
RandomForestClassifier(
min_samples_split=tune['min_samples_split'],
oob_score=tune['oob_score'],
class_weight=tune['class_weight']),
X,
y,
param_name=param_name,
param_range=param_range,
cv=self.config.cv(),
scoring=scoring,
n_jobs=multiprocessing.cpu_count())
except ValueError, err:
self.log.error(
"Cannot validate estimation graph for %s.\nReason:" %
classifier)
self.log.error(err)
return
train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
test_scores_mean = np.mean(test_scores, axis=1)
test_scores_std = np.std(test_scores, axis=1)
if display or filename:
self.plot_validation_curve(classifier, xtitle, "Score",
param_range, train_scores_mean,
train_scores_std, test_scores_std,
test_scores_mean)
if filename:
plt.savefig(self.generateFilename('ValidationCurve'))
if display:
plt.show()
if content != req:
with open("database.json", 'w', encoding='UTF-8') as file:
json.dump(req, file)
print("更新成功")
return 0
# 主程序
print("正更新字典内容中......")
status = update_database()
if status == 1:
print("更新失败")
input("按回车键继续或直接退出")
call("cls", shell=True)
# 读取配置文件
config = Config().get()
# 提取配置内容
__SLEEPTIME__ = int(float(config["sleep_time"]))
__ALERTSTAT__ = int(float(config["alert"]))
__ALERTPATH__ = str(config["alert_filepath"])
__LASTITEM__ = str(config["last_item"])
status_cn = {"offline": "离线", "online": "在线", "ingame": "在游戏中"}
# 记忆内容
if __LASTITEM__ != "Unknown":
# 转换为中文
LastitemCN = get_name("eTc", get_name("sTe", __LASTITEM__))
if LastitemCN != 1:
print("检测上一次您监控了 " + __LASTITEM__ + " (" + LastitemCN + ")")
else:
print("检测上一次您监控了 " + __LASTITEM__)
parser.add_argument('--data', action='store_true', help="prepare data")
args = parser.parse_args()
# args = parser.parse_args(['twitter'])
if (args.predict or args.data) and args.test:
print("can't test while predict or data prepare", file=sys.stderr)
exit(1)
dev_mode = args.train or args.test
base_dir = '../data/%s/' % args.socialnet
if args.socialnet == 'vk':
config = Config(base_dir=base_dir,
dev_mode=dev_mode,
degree_threshold=30,
degree_city_threshold=5,
city_freq_threshold=1000,
max_train_size=8 * 10**6,
steps=32_000,
test_every=500)
elif args.socialnet == 'facebook':
config = Config(base_dir=base_dir,
dev_mode=dev_mode,
directed=True,
degree_city_threshold=2,
city_freq_threshold=500,
nsample=[30, 5],
steps=5_000,
test_every=500)
config.part_dirs = [base_dir]
elif args.socialnet == 'twitter':
)
opts = parser.parse_args()
set_loglevel(opts)
if not opts.spec_path.is_dir():
logger.error(f"{opts.spec_path} is not a directory")
sys.exit(1)
with clone_if_missing(repo=Repo.ProblemSpecifications, directory=opts.spec_path):
result = True
buckets = {
TemplateStatus.MISSING: [],
TemplateStatus.INVALID: [],
TemplateStatus.TEST_FAILURE: [],
}
config = Config.load()
for exercise in filter_exercises(config.exercises.all()):
status = get_status(exercise, opts.spec_path)
if status == TemplateStatus.OK:
logger.info(f"{exercise.slug}: {status.name}")
else:
buckets[status].append(exercise.slug)
result = False
if opts.stop_on_failure:
logger.error(f"{exercise.slug}: {status.name}")
break
if not opts.quiet and not opts.stop_on_failure:
for status, bucket in sorted(buckets.items()):
if bucket:
print(f"The following exercises have status '{status.name}'")