def classification_user(user_name):
classificationDemon = dm.Demon("classification_" + user_name + ".pid")
trainingDemon = dm.Demon("training.pid")
try:
#
QIlogger.info("------------------------------------------------------")
QIlogger.info("1 - Classification Script")
QIlogger.info("2 - Check Demon Existence")
# Check dell'esistenza di un pid file della classificazione. Nel caso esista usciamo
if (classificationDemon.checkPidDemon()):
ctime = classificationDemon.getCreationTime()
today = classificationDemon.getCurrentTimeStamp()
days = classificationDemon.findDaysBetweenTimeStamps(today, ctime)
if days < 1:
QIlogger.info("-- Pid Exist. EXIT.")
return False
else:
QIlogger.info("-- The past demon has been created " +
str(days) +
" days ago. Delete it to prevent stall")
classificationDemon.deletePidDemon()
#
QIlogger.info("3 - Create Classification PID file")
classificationDemon.createPidDemon()
#
QIlogger.info("4 - Load Configuration")
config = cg.Config()
dir_path = os.path.dirname(os.path.realpath(__file__))
config.configFromFile(dir_path + "/config/config.json")
#
users = config.users
#
for user in users:
if user["name"] == user_name:
QIlogger.info("++++++++++++++++++++++++++++++++++++++++++++")
QIlogger.info("5 - User Selected : " + str(user["name"]))
classification = Classification(user)
#clLog.configureLogger(QIlogger, user["name"], "classification")
dir_path = os.path.dirname(os.path.realpath(
__file__)) + "/../ticket_classification/config/"
QIlogger.info("6 - Open Classification Script For user : "******"name"]))
ct.classifyNextTicket(dir_path, user["name"])
QIlogger.info("++++++++++++++++++++++++++++++++++++++++++++\n")
#
QIlogger.info("7 - Delete PID File")
classificationDemon.deletePidDemon()
QIlogger.info(
"------------------------------------------------------\n")
#
except Exception as e:
classificationDemon.deletePidDemon()
QIlogger.error("Error in classification method " + str(e))
def classification():
classificationDemon = dm.Demon("classification.pid")
trainingDemon = dm.Demon("training.pid")
try:
#
QIlogger.info("------------------------------------------------------")
QIlogger.info("1 - Classification Script")
QIlogger.info("2 - Check Demon Existence")
#
while (classificationDemon.checkPidDemon()):
ctime = classificationDemon.getCreationTime()
today = classificationDemon.getCurrentTimeStamp()
days = classificationDemon.findDaysBetweenTimeStamps(today, ctime)
if days < 1:
r = randint(1, 30)
QIlogger.info("-- Pid Exist. Wait " + str(r) + " sec")
time.sleep(r)
else:
QIlogger.info("-- The past demon has been created " +
str(days) +
" days ago. Delete it to prevent stall")
classificationDemon.deletePidDemon()
#
QIlogger.info("3 - Create Classification PID file")
classificationDemon.createPidDemon()
#
QIlogger.info("4 - Load Configuration")
config = cg.Config()
dir_path = os.path.dirname(os.path.realpath(__file__))
config.configFromFile(dir_path + "/config/config.json")
#
users = config.users
#
for user in users:
QIlogger.info("++++++++++++++++++++++++++++++++++++++++++++")
QIlogger.info("5 - User Selected : " + str(user["name"]))
classification = Classification(user)
#clLog.configureLogger(QIlogger, user["name"], "classification")
dir_path = os.path.dirname(os.path.realpath(
__file__)) + "/../ticket_classification/config/"
QIlogger.info("6 - Open Classification Script For user : "******"name"]))
ct.classifyNextTicket(dir_path, user["name"])
QIlogger.info("++++++++++++++++++++++++++++++++++++++++++++\n")
#
QIlogger.info("7 - Delete PID File")
classificationDemon.deletePidDemon()
QIlogger.info(
"------------------------------------------------------\n")
#
except Exception as e:
classificationDemon.deletePidDemon()
QIlogger.error("Error in classification method " + str(e))
"""##########################################################################"""
def apply_demon(g, subsize=1000):
print 'COMPUTING DEMON SCORE'
g = g.to_undirected()
iterations = dict()
for eps in [0.25, 0.50, 0.75]:
d = dm.Demon(graph=g, min_community_size=3, epsilon=eps)
coms_demon = d.execute()
max_len = max([len(c) for c in coms_demon])
min_len = min([len(c) for c in coms_demon])
max_community = [c for c in coms_demon if len(c) == max_len][0]
print 'FOR EPSILON ' + str(eps) + ' GREATEST COMMUNITY COMPOSED BY ' \
+ str(max_len) + ' NODES'
extract_info({
'community': max_community,
'fname': './results_ita/demon/' + str(eps) + '_',
'ncommunities': len(coms_demon),
'maxcomlen': max_len,
'mincomlen': min_len
})
evaluate_partition({
'alg': 'demon',
'network': g,
'epsilon': eps,
'partition': coms_demon
})
iterations[eps] = coms_demon
return iterations
def test_quality_functions(self):
g = nx.karate_club_graph()
D = d.Demon(graph=g, epsilon=0.3)
coms = D.execute()
res = pquality_summary(g, coms)
for _, v in res.items():
print(v)
def test_demon_lib(self):
g = nx.karate_club_graph()
nx.write_edgelist(g, "test.csv", delimiter=" ")
D = d.Demon("test.csv", file_output=False, epsilon=0.3)
coms = D.execute()
self.assertEqual(len(coms), 2)
D = d.Demon("test.csv", file_output=True, epsilon=0.3)
D.execute()
f = open("demon_0.3_communities.txt")
count = 0
for _ in f:
count += 1
self.assertEqual(count, 2)
os.remove("test.csv")
os.remove("demon_0.3_communities.txt")
def test_demon_lib(self):
g = nx.karate_club_graph()
nx.write_edgelist(g, "test.csv", delimiter=" ")
D = d.Demon(network_filename="test.csv", epsilon=0.3)
coms = D.execute()
print(coms)
self.assertEqual(len(coms), 2)
D = d.Demon(graph=g, file_output="communities.txt", epsilon=0.3)
D.execute()
f = open("communities.txt")
count = 0
for _ in f:
count += 1
self.assertEqual(count, 2)
os.remove("test.csv")
os.remove("communities.txt")
def demon_alg(g, epsilon, min_community_size):
import demon as d
dm = d.Demon(graph=g,
epsilon=epsilon,
min_community_size=min_community_size)
results = dm.execute()
# remove 'communities' attribute
for n in g.nodes(data=True):
n[1].pop('communities', None)
c = []
for c_name, c_nodes in enumerate(results):
for n in c_nodes:
c.append({'user_id': n, 'community': c_name})
return pd.DataFrame(c)
def get_reacts_for_demon(self):
"""
формирует словарь команд на основании класса ReactFunction
:return: словарь методов
"""
localCon = ReactFunction(self._demon)
react_dict = {}
for react in dir(localCon):
if react[0:1] != '_':
react_dict[react] = getattr(localCon, react)
return react_dict
if __name__ == "__main__":
daemon = demon.Demon(confsb.PID_FILE_PATH, confsb.QUEUE_NAME,
confsb.LOG_NAME, 'sb')
config = DaemonConfigurator(daemon)
react_dict = config.get_reacts_for_demon()
if len(sys.argv) > 1:
if sys.argv[1] in react_dict.keys():
try:
react_dict[sys.argv[1]](*sys.argv[2:len(sys.argv)])
sys.exit(0)
except TypeError, error:
print error
sys.exit(2)
else:
print "usage: %s %s" % (sys.argv[0], react_dict)
sys.exit(2)
def demon(graph, epsilon):
dm = d.Demon(graph=graph, epsilon=epsilon, min_community_size=3)
coms = dm.execute()
dcoms = list(coms)
return dcoms
def training():
classificationDemon = dm.Demon("classification.pid")
trainingDemon = dm.Demon("training.pid")
try:
#
QIlogger.info("------------------------------------------------------")
QIlogger.info("1 - Training Script")
QIlogger.info("2 - Check Demon Existence")
#
while (trainingDemon.checkPidDemon()):
ctime = trainingDemon.getCreationTime()
today = trainingDemon.getCurrentTimeStamp()
days = trainingDemon.findDaysBetweenTimeStamps(today, ctime)
if days < 3:
r = randint(1, 30)
QIlogger.info("-- Pid Exist. Wait " + str(r) + " sec")
time.sleep(r)
else:
QIlogger.info("-- The past demon has been created " +
str(days) +
" days ago. Delete it to prevent stall")
trainingDemon.deletePidDemon()
#
#
QIlogger.info("3 - Create Training PID file")
trainingDemon.createPidDemon()
#
QIlogger.info("4 - Load Configuration")
config = cg.Config()
dir_path = os.path.dirname(os.path.realpath(__file__))
config.configFromFile(dir_path + "/config/config.json")
#
users = config.users
#
lastDateClass = tl.TimeLogger("last_date.json")
for user in users:
QIlogger.info("++++++++++++++++++++++++++++++++++++++++++++")
QIlogger.info("5 - User Selected : " + str(user["name"]))
training = Training(user, config)
for model_type in config.model_types:
QIlogger.info("6 - Model Type : " + str(model_type))
training.setModelType(model_type)
QIlogger.info("7 - Check last date from the log date file")
lastDate = ""
if lastDateClass.checkFileExistence():
QIlogger.info(" -- File exists. Get data from there")
lastDate = lastDateClass.getLastDate()
else:
lastDateClass.createTimeFile()
QIlogger.info(
" -- File does not exist. It will be created")
lastDate = datetime.datetime.today().strftime('%Y-%m-%d')
#
training.setLastDate(lastDate)
QIlogger.info("8 - Create directories for new model")
training.createNewModelDirs()
#
QIlogger.info(
"9 - Copy data dir from old model to the new one")
training.copyDataFromModelToNewModel()
#
from_date = training.getLastDate()
from_date_array = from_date.split("-")
from_date = from_date_array[0] + from_date_array[
1] + from_date_array[2]
to_date = training.getCurrentDate()
to_date_array = to_date.split("-")
to_date = to_date_array[0] + to_date_array[1] + to_date_array[2]
#
#clLog.configureLogger(QIUserLogger, user["name"], "training")
dir_path = os.path.dirname(os.path.realpath(
__file__)) + "/../ticket_classification/config/"
QIlogger.info("10 - Open Training Script For user : "******"name"]) + " --------------------- ")
#ct.classifyNextTicket(dir_path, user["name"])
try:
tm.training_model(
training.getNewModelDir() + "/", model_type, dir_path +
"/" + user["name"] + "/" + model_type + "_config.json",
from_date, to_date, user["name"])
except Exception as e:
#trainingDemon.deletePidDemon()
QIlogger.error("Error in training model " + str(e))
#
while (classificationDemon.checkPidDemon()):
r = randint(1, 30)
QIlogger.info("-- Classification Pid Exist. Wait " +
str(r) + " sec")
time.sleep(r)
QIlogger.info("11 - Evaluate new model and old model")
dir_path = os.path.dirname(os.path.realpath(__file__))
config_path = dir_path + "/../ticket_classification/config/" + user[
"name"] + "/" + model_type + "_config.json"
model_dir = training.getModelDir() + "/"
new_model_dir = training.getNewModelDir() + "/"
skipgramModelPath = config.skipgram_model_path
eval = ev.Evaluator(user, config_path, model_dir,
new_model_dir, skipgramModelPath)
res = eval.runEval()
if res == 'new_model':
QIlogger.info(
" -- EVALUATION : New Model has better performance")
QIlogger.info(
" -- Create a Classification DEMON to ensure that classification not start now"
)
classificationDemon.createPidDemon()
QIlogger.info("11 - Remove old model directory")
training.removeDir(training.old_model_dir)
#
QIlogger.info(
"12 - Move New model as Model and Model as old model")
os.mkdir(training.getOldModelDir(), 0o775)
training.moveFileToDir(training.getModelDir(),
training.getOldModelDir())
#delete Model directory
training.removeDir(training.model_dir)
#create Model directory
os.mkdir(training.getModelDir(), 0o775)
training.moveFileToDir(training.getNewModelDir(),
training.getModelDir())
training.removeDir(training.new_model_dir)
QIlogger.info(
"13 - String replace nel file del checkpoint")
training.replaceDataInCheckpoint()
QIlogger.info(" -- Destroy classification demon")
classificationDemon.deletePidDemon()
else:
QIlogger.info(
" -- EVALUATION : OLD Model has better performance")
QIlogger.info(
"11 - Rename tickets and targets in model dir")
#rename tickets and targets as old_tickets and old_targets in model
shutil.move(
training.getModelDir() + "/data/tickets.txt",
training.getModelDir() + "/data/old_tickets.txt")
shutil.move(
training.getModelDir() + "/data/targets.txt",
training.getModelDir() + "/data/old_targets.txt")
#copy the new tickets and targets from new_model
QIlogger.info(
"12 - Move tickets and targets from new model to old model"
)
training.moveSpecificFilesToDir(
training.getNewModelDir() + "/data",
training.getModelDir() + "/data",
["tickets.txt", "targets.txt"])
# delete new Model directory
QIlogger.info("13 - Remove new model directory")
training.removeDir(training.getNewModelDir())
#clLog.removeHandlers(QIUserLogger)
QIlogger.info("++++++++++++++++++++++++++++++++++++++++++++")
#
QIlogger.info("14 - Update date")
lastDateClass.deleteFile()
lastDateClass.createTimeFile()
QIlogger.info("15 - Delete PID File")
trainingDemon.deletePidDemon()
QIlogger.info(
"------------------------------------------------------\n")
#
except Exception as e:
trainingDemon.deletePidDemon()
QIlogger.error("Error in training method " + str(e))
def get_reacts_for_demon(self):
"""
формирует словарь команд на основании класса ReactFunction
:return: словарь методов
"""
localCon = ReactFunction(self._demon)
react_dict = {}
for react in dir(localCon):
if react[0:1] != '_':
react_dict[react] = getattr(localCon, react)
return react_dict
if __name__ == "__main__":
daemon = demon.Demon(conf.PID_FILE_PATH, conf.QUEUE_NAME, conf.LOG_NAME, 'ab')
config = DaemonConfigurator(daemon)
react_dict = config.get_reacts_for_demon()
if len(sys.argv) > 1:
if sys.argv[1] in react_dict.keys():
try:
react_dict[sys.argv[1]](*sys.argv[2:len(sys.argv)])
sys.exit(0)
except TypeError, error:
print error
sys.exit(2)
else:
print "usage: %s %s" % (sys.argv[0], react_dict)
sys.exit(2)
def add_demon_community_attribute(G):
H = G.copy()
dm = d.Demon(graph=H, epsilon=0.5, min_community_size=6)
communities = dm.execute()
set_community_attribute(G, communities, 'demon')
def get_demon(graph):
dm = d.Demon(graph=graph, epsilon=0.25, min_community_size=3)
communities = dm.execute()
return communities