def run_monitoring(setting_id, *args, **kwargs):
monitoring_run = MonitoringRun.objects.create()
try:
setting = MonitoringSetting.objects.get(pk=setting_id)
except (ObjectDoesNotExist, MultipleObjectsReturned):
print(f'Could not retrieve setting with id {setting_id}')
return
metadatas = setting.metadatas.all()
for metadata in metadatas:
try:
monitor = Monitor(
metadata=metadata,
monitoring_run=monitoring_run,
)
monitor.run_checks()
monitoring_logger.debug(f'Health checks completed for {metadata}')
except Exception as e:
monitoring_logger.exception(e, exc_info=True, stack_info=True)
end_time = timezone.now()
duration = end_time - monitoring_run.start
monitoring_run.end = end_time
monitoring_run.duration = duration
monitoring_run.save()
def main():
#------------------------------ Processing input arguments
try:
opts, args = getopt.getopt(
sys.argv[1:], "c:d:e:f:q:",
["command=", "database=", "experiment=", "file=", "query="])
except getopt.GetoptError as err:
print(err)
sys.exit(2)
command = database = experiment = query = fileName = None
data = ""
for o, a in opts:
if o in ("-c", "--command"):
command = a
elif o in ("-d", "--database"):
database = a
elif o in ("-e", "--experiment"):
experiment = a
elif o in ("-f", "--file"):
fileName = a
elif o in ("-q", "--query"):
query = a
else:
assert False, "unhandled option."
#------------------------------ Generate CSV file
m = Monitoring()
if query and (command or database or experiment):
print("WARN: Parameter query was set, other will be ignored.")
if not query:
query = m.getReportQuery(command, database, experiment)
m.copyToCSV(query, fileName)
def run_manual_monitoring(monitoring_run, *args, **kwargs):
monitoring_run = MonitoringRun.objects.get(pk=monitoring_run)
monitoring_run.start = timezone.now()
for metadata in monitoring_run.metadatas.all():
try:
monitor = Monitor(
metadata=metadata,
monitoring_run=monitoring_run,
)
monitor.run_checks()
monitoring_logger.debug(f'Health checks completed for {metadata}')
except Exception as e:
monitoring_logger.error(
msg=_(f'Something went wrong while monitoring {metadata}'))
monitoring_logger.exception(
e,
exc_info=True,
stack_info=True,
)
end_time = timezone.now()
duration = end_time - monitoring_run.start
monitoring_run.end = end_time
monitoring_run.duration = duration
monitoring_run.save()
def run_manual_service_monitoring(monitoring_run, *args, **kwargs):
monitoring_run = MonitoringRun.objects.get(pk=monitoring_run)
monitoring_run.start = timezone.now()
for metadata in monitoring_run.metadatas.all():
if current_task:
current_task.update_state(
state=states.STARTED,
meta={
'current': 0,
'total': 100,
'phase': f'start monitoring checks for {metadata}',
}
)
try:
monitor = Monitor(metadata=metadata, monitoring_run=monitoring_run, )
monitor.run_checks()
monitoring_logger.debug(f'Health checks completed for {metadata}')
except Exception as e:
monitoring_logger.error(msg=_(f'Something went wrong while monitoring {metadata}'))
monitoring_logger.exception(e, exc_info=True, stack_info=True, )
end_time = timezone.now()
duration = end_time - monitoring_run.start
monitoring_run.end = end_time
monitoring_run.duration = duration
monitoring_run.save()
return {'msg': 'Done. Service(s) successfully monitored.',
'id': str(monitoring_run.pk),
'absolute_url': monitoring_run.get_absolute_url(),
'absolute_url_html': f'<a href={monitoring_run.get_absolute_url()}>{monitoring_run.__str__()}</a>'}
def generateCreate(experiment, fileName):
m = Monitoring()
data = dict(m.getGraphDataCreate(experiment))
plt.bar(range(len(data)), data.values(), align='center')
plt.xticks(range(len(data)), data.keys())
plt.title("Type of experiment: CREATE ({})".format(experiment))
plt.xlabel("Graph databases")
plt.ylabel("Size of new database in KB")
plt.savefig(fileName)
print("INFO: PNG file was created - {}".format(fileName))
def generateInsert(experiment, fileName):
m = Monitoring()
data = dict(m.getGraphDataInsert(experiment))
plt.bar(range(len(data)), data.values(), align='center')
plt.xticks(range(len(data)), data.keys())
plt.title("Type of experiment: INSERT ({})".format(experiment))
plt.xlabel("Graph databases")
plt.ylabel("Insert time in seconds")
plt.savefig(fileName)
print("INFO: PNG file was created - {}".format(fileName))
def generateSelect(databases, experiment, fileName):
m = Monitoring()
for db in databases:
data = m.getGraphDataSelect(experiment, db)
plt.plot(data, label=db)
plt.title("Type of experiment: SELECT ({})".format(experiment))
plt.xlabel("Number of iteration")
plt.ylabel("Run time in seconds")
plt.legend()
plt.savefig(fileName)
print("INFO: PNG file was created - {}".format(fileName))
def generateImportExport(experiments, fileName):
m = Monitoring()
data = dict()
for experiment in experiments:
data = {k:[v, data[k]] if k in data else v for (k,v) in m.getGraphDataImportExport(experiment)}
labels=('import','export')
dim = 2
w = 0.75
dimw = w / dim
x = pylab.arange(len(data.values()))
for i in range(dim) :
y = [d[i] for d in data.values()]
b = plt.bar(x + i * dimw, y, dimw, bottom=0.001, label=labels[i])
plt.xticks(range(len(data.values())), data.keys())
plt.title("Type of experiment: IMPORT & EXPORT \n{}".format(experiments))
plt.xlabel("Graph databases")
plt.ylabel("Run time in seconds")
plt.legend()
plt.savefig(fileName)
print("INFO: PNG file was created - {}".format(fileName))
def run_monitoring(setting_id, *args, **kwargs):
monitoring_run = MonitoringRun.objects.create()
try:
setting = MonitoringSetting.objects.get(pk=setting_id)
except (ObjectDoesNotExist, MultipleObjectsReturned):
print(f'Could not retrieve setting with id {setting_id}')
return
metadatas = setting.metadatas.all()
for metadata in metadatas:
if current_task:
current_task.update_state(
state=states.STARTED,
meta={
'current': 0,
'total': 100,
'phase': f'start monitoring checks for {metadata}',
}
)
try:
monitor = Monitor(metadata=metadata, monitoring_run=monitoring_run, )
monitor.run_checks()
monitoring_logger.debug(f'Health checks completed for {metadata}')
except Exception as e:
monitoring_logger.exception(e, exc_info=True, stack_info=True)
end_time = timezone.now()
duration = end_time - monitoring_run.start
monitoring_run.end = end_time
monitoring_run.duration = duration
monitoring_run.save()
return {'msg': 'Done. Service(s) successfully monitored.',
'id': str(monitoring_run.pk),
'absolute_url': monitoring_run.get_absolute_url(),
'absolute_url_html': f'<a href={monitoring_run.get_absolute_url()}>{monitoring_run.__str__()}</a>'}
import sys
import time
from monitoring.monitoring import Monitoring
db = sys.argv[1]
ver = sys.argv[2]
desc = sys.argv[3]
timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
if not db:
print("ERR: Choose graph database for install.")
sys.exit(2)
database = {'gdb_name': db, 'gdb_description': desc, 'gdb_version': ver , 'last_update': timestamp}
mon = Monitoring()
mon.insertDatabase(database)
import sys
from setupConf import Configuration
from monitoring.monitoring import Monitoring
dictionary = dict()
dictionary['select'] = ['run_time', 'size']
dictionary['create'] = ['run_time', 'size']
dictionary['drop'] = ['run_time', 'size']
dictionary['insert'] = ['run_time', 'size', 'size_after']
dictionary['delete'] = ['run_time', 'size', 'size_after']
dictionary['import'] = ['run_time', 'size']
dictionary['export'] = ['run_time', 'size']
directory = sys.argv[1]
conf_name = sys.argv[2]
file = directory + conf_name
mon = Monitoring()
if not mon.getId(mon.confTab, conf_name, 'conf_id', 'conf_name'):
exper = Configuration(file)
exper.setupConf()
type_ex = exper.get('experiment_type')
if type_ex:
conf_id = mon.insertConfiguration(conf_name)
for meas_name in dictionary[type_ex]:
meas_id = mon.getId(mon.measTab, meas_name, 'meas_id', 'meas_name')
if meas_id:
mon.insertTypes(type_ex, conf_id, meas_id)
def main():
#------------------------------ Zpracovani vstupnich argumentu
try:
opts, args = getopt.getopt(sys.argv[1:], "e:d:",
["experiment=", "database="])
except getopt.GetoptError as err:
print(err)
sys.exit(2)
experiment = database = None
for o, a in opts:
if o in ("-e", "--experiment"):
experiment = a
elif o in ("-d", "--database"):
database = a
else:
assert False, "unhandled option."
#------------------------------ Configuration
exper = Configuration(experiment)
exper.setupConf()
#monitoring = True if exper.get('monitoring')=='yes' else False
record = dict()
record['run_date'] = time.strftime("%Y-%m-%d %H:%M:%S")
record['conf_name'] = experiment[(experiment.rfind('/') + 1):]
record['gdb_name'] = exper.get('db_name')
record['gdb_server'] = database
record['type_name'] = exper.get('experiment_type')
record['status'] = 'ERR'
record['value'] = dict()
#------------------------------ Graph database
if database == "orientdb":
g = OrientDB(record['gdb_name'])
g.setup()
elif database == "arangodb":
g = ArangoDB(record['gdb_name'])
g.setup()
else:
g = MyGremlin(record['gdb_name'], record['gdb_server'])
#------------------------------ Run Experiment
if record['type_name'] in ['create', 'drop', 'import', 'export']:
if record['type_name'] == 'create':
start_time = time.time()
res = g.createDB()
end_time = (time.time() - start_time)
time.sleep(15)
size = g.sizedb()
if record['type_name'] == 'drop':
size = g.sizedb()
start_time = time.time()
time.sleep(6)
res = g.dropDB()
end_time = (time.time() - start_time)
if record['type_name'] == 'import':
start_time = time.time()
res = g.importJSON(exper.get('import_file'))
end_time = (time.time() - start_time)
size = g.sizedb()
if record['type_name'] == 'export':
size = g.sizedb()
start_time = time.time()
res = g.exportJSON(exper.get('export_path'))
end_time = (time.time() - start_time)
if res:
record['status'] = "OK"
record['value']['run_time'] = end_time
record['value']['size'] = size
mon = Monitoring()
record['iteration_count'] = 1
record['exper_id'] = mon.insertExperiment(record)
record['iter_timestamp'] = record['run_date']
record['iter_number'] = 1
record['iter_id'] = mon.insertIteration(record)
if 'run_time' in record['value']:
mon.insertValues(record)
if __debug__:
print(">>>> experiment: {}".format(record))
elif record['type_name'] in ['insert', 'delete']:
record['iteration_count'] = 1
record['commands'] = exper.get('commands').split("\n|")
mon = Monitoring()
record['exper_id'] = mon.insertExperiment(record)
record['value']['size'] = g.sizedb()
start_time = time.time()
if g.runCommands(record['commands']):
record['status'] = "OK"
record['value']['run_time'] = (time.time() - start_time)
record['iter_timestamp'] = time.strftime("%Y-%m-%d %H:%M:%S")
record['iter_number'] = 1
record['iter_id'] = mon.insertIteration(record)
if 'run_time' in record['value']:
record['value']['size_after'] = g.sizedb()
mon.insertValues(record)
if __debug__:
print(">>>> experiment: {}".format(record))
elif record['type_name'] == 'select':
record['iteration_count'] = exper.get('iteration')
record['commands'] = exper.get('commands').split("|")
mon = Monitoring()
record['exper_id'] = mon.insertExperiment(record)
record['value']['size'] = g.sizedb()
if __debug__:
print("----------")
for i in range(int(record['iteration_count'])):
if __debug__:
print(">>> " + str(i + 1) + ". SELECT")
record['iter_number'] = i + 1
runSelect(g, record, mon)
if __debug__:
print("----------")
else:
print("WARN: Not implemented.")
return 3
return 0
from monitoring.monitoring import Monitoring m = Monitoring() m.cleanDB()