-
Notifications
You must be signed in to change notification settings - Fork 0
/
phantomprobe.py
executable file
·272 lines (235 loc) · 10.5 KB
/
phantomprobe.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
#!/usr/bin/python3
import sys
import os
import shutil
import subprocess
import tarfile
import logging
import logging.config
import time
import datetime
import signal
import sqlite3
from probe.configuration import Configuration
from probe.phantomjsmanager import PJSLauncher, PhantomjsNotFoundError
from probe.activemeasure import ActiveMonitor
from probe.sender import JSONClient
from db.dbclient import DBClient
from diagnosis.localdiagnosis import LocalDiagnosisManager
from utils import utils
package_directory = os.path.dirname(os.path.abspath(__file__))
logging.config.fileConfig(os.path.join(package_directory, 'conf/logging.conf'))
class TstatManager():
def __init__(self, config):
self.start = config.get_tstat_configuration()['start']
self.stop = config.get_tstat_configuration()['stop']
self.tstatpath = os.path.join(config.get_tstat_configuration()['dir'], 'tstat/tstat')
self.interface = config.get_tstat_configuration()['netinterface']
self.netfile = config.get_tstat_configuration()['netfile']
self.outdir = config.get_tstat_configuration()['tstatout']
logger.info("Loaded TstatManager")
def start_capture(self):
cmd = "%s %s %s %s %s" % (self.start, self.tstatpath, self.interface, self.netfile, self.outdir)
subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
def stop_capture(self):
logger.debug("Stopping tstat...")
p = subprocess.Popen(self.stop, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False).wait()
if p > 0:
return False
return True
def check_tstat(self):
ps = subprocess.Popen(('ps', 'aux'), stdout=subprocess.PIPE)
output = subprocess.check_output(('grep', 'tstat'), stdin=ps.stdout)
ps.wait()
return int(output.split()[1])
class FlumeNotFoundError(FileNotFoundError):
pass
class FlumeManager():
def __init__(self, config):
self.confdir = config.get_flume_configuration()['confdir']
self.conffile = config.get_flume_configuration()['conffile']
self.agentname = config.get_flume_configuration()['agentname']
self.start = "{0}{1}".format(config.get_flume_configuration()['flumedir'], "/bin/flume-ng")
if not os.path.exists(self.start):
raise FlumeNotFoundError
def start_flume(self):
cmd = "{0} agent -c {1} -f {2} -n {3}".format(self.start, self.confdir, self.conffile, self.agentname)
subprocess.Popen(cmd.split())
def stop_flume(self, pid):
logger.debug("Stopping flume process {}".format(pid))
try:
os.kill(pid, signal.SIGKILL)
except ProcessLookupError:
logger.warning("Unable to find process")
def check_flume(self):
ps = subprocess.Popen(('ps', 'aux'), stdout=subprocess.PIPE)
output = subprocess.check_output(('grep', 'flume'), stdin=ps.stdout)
ps.wait()
return int(output.split()[1])
###mplane component needs to call this
logger = logging.getLogger('probe')
class PhantomProbe():
def __init__(self, conffile, url):
self.config = Configuration(conffile)
self.url = url
self.__prepare()
self.diagnosis = {}
self.flumepid = None
def __prepare(self):
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%Y%m%d-%H%M%S')
self.backup_dir = os.path.join(self.config.get_base_configuration()['backupdir'], st)
logger.info("Launching the probe...")
if not os.path.isdir(self.config.get_flume_configuration()['outdir']):
os.makedirs(self.config.get_flume_configuration()['outdir'])
if not os.path.isdir(self.backup_dir):
os.makedirs(self.backup_dir)
self.tstat_out_file = self.config.get_database_configuration()['tstatfile']
self.harfile = self.config.get_database_configuration()['harfile']
try:
self.launcher = PJSLauncher(self.config)
except PhantomjsNotFoundError:
logger.error("PhantomJS browser not found. Exiting.")
sys.exit(-1)
logger.debug('Backup dir set at: %s' % self.backup_dir)
try:
self.dbcli = DBClient(self.config)
self.dbcli.get_probe_id()
logger.info("Probe data already stored.")
except sqlite3.OperationalError:
self.loc_info = utils.get_location()
if not self.loc_info:
logger.warning("No info on location retrieved.")
else:
for k in ['city', 'region']:
self.loc_info.update({k: self.loc_info[k].replace("'", "''")})
self.dbcli = DBClient(self.config, self.loc_info, create=True)
try:
self.flumemanager = FlumeManager(self.config)
self.flumemanager.start_flume()
self.flumepid = self.flumemanager.check_flume()
logger.info("Flume started: pid = {}".format(self.flumepid))
except FlumeNotFoundError:
self.flumemanager = None
logger.warning("Flume not found, sending to server instead.")
self.pjs_config = self.config.get_phantomjs_configuration()
self.tstatmanager = TstatManager(self.config)
try:
self.tstatmanager.start_capture()
logger.info("Tstat.check_tstat. ret = {}".format(self.tstatmanager.check_tstat()))
logger.info("start.out process launched")
except AttributeError:
logger.error("Unable to start tstat process. Quit.")
sys.exit(-1)
logger.info("Ready")
def browse(self):
try:
stats = self.launcher.browse_url(self.url)
logger.info("Got data from browse_url")
except AttributeError:
logger.error("Problems in browser thread. Aborting session...")
logger.error("Forcing tstat to stop.")
if not self.tstatmanager.stop_capture():
logger.error("Unable to stop tstat.")
sys.exit("Problems in browser thread. Aborting session...")
if not stats:
logger.warning('Problem in session to [%s].. skipping' % self.url)
utils.clean_tmp_files(self.backup_dir, [self.tstat_out_file, self.harfile], self.url, True)
sys.exit("Problems in stats collecting. Quitting...")
if not os.path.exists(self.tstat_out_file):
logger.error('tstat outfile missing. Check your network configuration.')
sys.exit("tstat outfile missing. Check your network configuration.")
#testbed
#logger.debug("Sleeping 10 sec")
#time.sleep(10)
#end testbed
if not self.tstatmanager.stop_capture():
logger.error("Unable to stop tstat.")
else:
logger.info("tstat successfully stopped.")
inserted_sid = self.dbcli.load_to_db(stats)
logger.info('Ended browsing to %s' % self.url)
self.passive = self.dbcli.pre_process_raw_table()
if not self.passive:
logger.error("Unable to retrieve passive measurements.")
logger.error("Check if Tstat is running properly.")
logger.error("Quitting.")
return False
utils.clean_tmp_files(self.backup_dir, [self.tstat_out_file, self.harfile], self.url, False)
logger.debug('Saved backup files.')
return inserted_sid
def execute(self):
inserted_sid = self.browse()
if inserted_sid:
monitor = ActiveMonitor(self.config, self.dbcli)
self.active = monitor.run_active_measurement()
logger.debug('Ended Active probing to url %s' % (self.url))
for tracefile in [f for f in os.listdir('.') if f.endswith('.traceroute')]:
os.remove(tracefile)
l = LocalDiagnosisManager(self.dbcli, self.url)
self.diagnosis = l.run_diagnosis(inserted_sid)
self.send_results()
else:
self.diagnosis = {"Warning": "Unable to perform browsing"}
def send_results(self):
jc = JSONClient(self.config, self.dbcli)
measurements = jc.prepare_data()
to_update = [el['sid'] for el in measurements]
csv_path_fname_list = jc.save_csv_files(measurements)
if self.flumemanager:
self.flumepid = self.flumemanager.check_flume()
logger.info("Waiting for flume to stop...[{}]".format(self.flumepid))
time.sleep(5)
if self.flumepid:
self.flumemanager.stop_flume(self.flumepid)
else:
logger.error("Unable to stop flume")
else:
logger.info("Sending data to server...")
# FIXME
#try:
# jc.send_csv()
#except TimeoutError:
# logger.error("Timeout in server connection")
# pass
#finally:
# logger.info("Done.")
self.dbcli.update_sent(to_update)
try:
for csv_path_fname in csv_path_fname_list:
shutil.copyfile(csv_path_fname, os.path.join(self.backup_dir, os.path.basename(csv_path_fname)))
except FileNotFoundError:
pass
logger.info("Packing backups...")
for root, _, files in os.walk(self.backup_dir):
if len(files) > 0:
tar = tarfile.open("%s.tar.gz" % self.backup_dir, "w:gz")
tar.add(self.backup_dir)
tar.close()
logger.info('tar.gz backup file created.')
shutil.rmtree(self.backup_dir)
logger.info('Done.')
def get_result(self):
return self.diagnosis
if __name__ == '__main__':
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-c", "--conf", dest="conf_file", type="string", help="specify a configuration file", metavar="FILE")
parser.add_option("-u", "--url", dest="url", type="string", help="specify a url", metavar="URL")
(options, args) = parser.parse_args()
if not options.url:
print("Use -h for complete list of options")
print("Usage: {} -u url_to_browse [-c conf_file] ".format(__file__))
sys.exit(0)
url = options.url
if not options.conf_file:
conffile = os.path.join(package_directory, 'conf/firelog.conf')
else:
if not os.path.isfile(options.conf_file):
print("Use -h for complete list of options: wrong configuration file")
sys.exit(0)
conffile = options.conf_file
f = PhantomProbe(conffile, url)
f.execute()
print(str(f.get_result()))
print("Bye.")