def capture_live_traces():
# Start tcpdump
utils.init_temp_dir('traces')
tempdir = utils.get_temp_dir('traces')
logging.getLogger(__name__).debug('Dumping traces to temp dir: %s', tempdir)
tracefile = os.path.join(tempdir, '%F_%H-%M-%S_trace.pcap')
try:
p = subprocess.Popen('%s -i %s -G %i -w %s' % (TCPDUMP, args.interface, args.rotate_seconds, tracefile), shell=True)
except Exception as e:
logging.getLogger(__name__).error('Error starting tcpdump: %s', e)
sys.exit()
try:
stats = UserStats()
while True:
full_traces = os.listdir(utils.get_temp_dir('traces'))[0:-1] # don't start reading trace tcpdump is currently filling
if len(full_traces) == 0:
time.sleep(5)
elif len(full_traces) * int(args.rotate_seconds) > 300:
logging.getLogger(__name__).warning('Analyzer is more than 5 minutes behind (%d unprocessed trace files of %s seconds each)', len(full_traces), args.rotate_seconds)
for trace in full_traces:
trace_path = os.path.join(utils.get_temp_dir('traces'), trace)
stats = analyze_trace(trace_path, stats)
os.remove(trace_path)
except (KeyboardInterrupt, SystemExit), e:
p.terminate()
sys.exit()
def update_bgsub(self):
'''
this is only to read the current inputs and replot
'''
if not self.model.base_ptn_exist():
QtWidgets.QMessageBox.warning(self.widget, "Warning",
"Load a base pattern first.")
return
"""receive new bg parameters and update the graph"""
bg_params = [
self.widget.spinBox_BGParam0.value(),
self.widget.spinBox_BGParam1.value(),
self.widget.spinBox_BGParam2.value()
]
bg_roi = [
self.widget.doubleSpinBox_Background_ROI_min.value(),
self.widget.doubleSpinBox_Background_ROI_max.value()
]
if (bg_roi[0] <= self.model.base_ptn.x_raw.min()):
bg_roi[0] = self.model.base_ptn.x_raw.min()
self.widget.doubleSpinBox_Background_ROI_min.setValue(bg_roi[0])
if (bg_roi[1] >= self.model.base_ptn.x_raw.max()):
bg_roi[1] = self.model.base_ptn.x_raw.max()
self.widget.doubleSpinBox_Background_ROI_max.setValue(bg_roi[1])
self.model.base_ptn.subtract_bg(bg_roi, bg_params, yshift=0)
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
self.model.base_ptn.write_temporary_bgfiles(temp_dir=temp_dir)
if self.model.waterfall_exist():
print(
str(datetime.datetime.now())[:-7],
": BGfit and BGsub for waterfall patterns even if they are displayed.\n",
"Yes this is a bit of waste. Future fix needed.")
for pattern in self.model.waterfall_ptn:
pattern.subtract_bg(bg_roi, bg_params, yshift=0)
self.plot_new_graph()
def _add_patterns(self, files):
if files is not None:
for f in files:
filename = str(f)
wavelength = self.widget.doubleSpinBox_SetWavelength.value()
bg_roi = [
self.widget.doubleSpinBox_Background_ROI_min.value(),
self.widget.doubleSpinBox_Background_ROI_max.value()
]
bg_params = [
self.widget.spinBox_BGParam0.value(),
self.widget.spinBox_BGParam1.value(),
self.widget.spinBox_BGParam2.value()
]
if self.widget.checkBox_UseTempBGSub.isChecked():
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
else:
temp_dir = None
self.model.append_a_waterfall_ptn(filename,
wavelength,
bg_roi,
bg_params,
temp_dir=temp_dir)
self.waterfall_table_ctrl.update()
self._apply_changes_to_graph()
return
def add_base_pattern_to_waterfall(self):
if not self.model.base_ptn_exist():
QtWidgets.QMessageBox.warning(self.widget, "Warning",
"Pick a base pattern first.")
return
filename = self.model.get_base_ptn_filename()
if self.model.exist_in_waterfall(filename):
self.widget.pushButton_AddBasePtn.setChecked(True)
return
wavelength = self.widget.doubleSpinBox_SetWavelength.value()
bg_roi = [
self.widget.doubleSpinBox_Background_ROI_min.value(),
self.widget.doubleSpinBox_Background_ROI_max.value()
]
bg_params = [
self.widget.spinBox_BGParam0.value(),
self.widget.spinBox_BGParam1.value(),
self.widget.spinBox_BGParam2.value()
]
if self.widget.checkBox_UseTempBGSub.isChecked():
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
#temp_dir = os.path.join(self.model.chi_path, 'temporary_pkpo')
else:
temp_dir = None
self.model.append_a_waterfall_ptn(filename,
wavelength,
bg_roi,
bg_params,
temp_dir=temp_dir)
self.waterfall_table_ctrl.update()
self._apply_changes_to_graph()
def process_temp_cake(self):
"""
load cake through either temporary file or make a new cake
"""
if not self.model.associated_image_exists():
QtWidgets.QMessageBox.warning(
self.widget, "Warning",
"Image file for the base pattern does not exist.")
return
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
#temp_dir = os.path.join(self.model.chi_path, 'temporary_pkpo')
if self.widget.checkBox_UseTempCake.isChecked():
#if os.path.exists(temp_dir):
self._load_new_image()
success = self.model.diff_img.read_cake_from_tempfile(
temp_dir=temp_dir)
if success:
print(str(datetime.datetime.now())[:-7],
": Load cake image from temporary file.")
pass
else:
print(str(datetime.datetime.now())[:-7],
": Create new temporary file for cake image.")
self._update_temp_cake_files(temp_dir)
#else:
#os.makedirs(temp_dir)
#self._update_temp_cake_files(temp_dir)
else:
self._update_temp_cake_files(temp_dir)
def unpack_package(name):
package_source = get_config('package_source')
package_arch = get_config('package_arch')
package_options = get_config('package_options')
dir = tempfile.mkdtemp('', '', get_temp_dir())
os.chdir(dir)
try:
filename = download_from_apt(name, package_source, package_arch,
package_options)
result = re.match('([^_]+)_([^_]+)_([^.]+).deb', filename)
if not result:
raise Exception("\'" + name + "\' is not properly downloaded")
name = result.group(1)
version = result.group(2)
arch = result.group(3)
result = subprocess.call(["dpkg", "-x", filename, "."],
stdout=null_dev,
stderr=null_dev)
if result != 0:
raise Exception("Cannot unpack \'" + name + "\'")
except:
os.chdir(root_dir)
package.remove_dir(dir)
raise
os.chdir(root_dir)
return (dir, name, version)
def _plot_ma_data(self, nse_data, nse_data_arr):
'''
A function for plotting MA data
'''
try:
temp_dir = get_temp_dir(work_dir=self.work_dir)
os.chdir(temp_dir)
date_format = DateFormatter('%b %y')
filename = '{}_{}_{}_{}.png'.format(self.stock, self.date,
self.small_window,
self.large_window)
filename = os.path.join(temp_dir, filename)
title = '{}_{}_{}_{}'.format(self.stock, self.date,
self.small_window, self.large_window)
# plot data
plt.figure(figsize=(self.fig_width, self.fig_height))
fig, ax = plt.subplots()
ax.xaxis_date()
ax.xaxis.set_major_formatter(date_format)
mlp = candlestick_ohlc(ax,
nse_data_arr,
width=0.7,
colorup='g',
colordown='r')
nse_data['MA{}'.format(self.small_window)].plot(
style='-',
alpha=0.5,
color='orange',
label='MA{}'.format(self.small_window))
nse_data['MA{}'.format(self.large_window)].plot(
style='-',
alpha=0.5,
color='blue',
label='MA{}'.format(self.large_window))
nse_data['EMA{}'.format(self.small_window)].plot(
style='-',
alpha=0.5,
color='purple',
label='EMA{}'.format(self.small_window))
nse_data['EMA{}'.format(self.large_window)].plot(
style='-',
alpha=0.5,
color='lime',
label='EMA{}'.format(self.large_window))
handles, labels = ax.get_legend_handles_labels()
ax.legend(handles, labels, loc='best', fontsize=8)
plt.title(title)
plt.savefig(filename, bbox_inches='tight')
plt.close("all")
except Exception as e:
print('Error {}'.format(e))
clean_temp_dir(temp_dir=temp_dir)
else:
return filename, temp_dir
def _load_a_new_pattern(self, new_filename):
"""
load and process base pattern. does not signal to update_graph
"""
self.model.set_base_ptn(
new_filename, self.widget.doubleSpinBox_SetWavelength.value())
# self.widget.textEdit_DiffractionPatternFileName.setText(
# '1D Pattern: ' + self.model.get_base_ptn_filename())
self.widget.lineEdit_DiffractionPatternFileName.setText(
str(self.model.get_base_ptn_filename()))
print(
str(datetime.datetime.now())[:-7], ": Receive request to open ",
str(self.model.get_base_ptn_filename()))
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
if self.widget.checkBox_UseTempBGSub.isChecked():
if os.path.exists(temp_dir):
success = self.model.base_ptn.read_bg_from_tempfile(
temp_dir=temp_dir)
if success:
self._update_bg_params_in_widget()
print(
str(datetime.datetime.now())[:-7],
': Read temp chi successfully.')
else:
self._update_bgsub_from_current_values()
print(
str(datetime.datetime.now())[:-7],
': No temp chi file found. Force new bgsub fit.')
else:
os.makedirs(temp_dir)
self._update_bgsub_from_current_values()
print(
str(datetime.datetime.now())[:-7],
': No temp chi file found. Force new bgsub fit.')
else:
self._update_bgsub_from_current_values()
print(
str(datetime.datetime.now())[:-7],
': Temp chi ignored. Force new bgsub fit.')
filen_tif = self.model.make_filename('tif', original=True)
filen_tiff = self.model.make_filename('tiff', original=True)
filen_mar3450 = self.model.make_filename('mar3450', original=True)
filen_cbf = self.model.make_filename('cbf', original=True)
if not (os.path.exists(filen_tif) or \
os.path.exists(filen_tiff) or \
os.path.exists(filen_mar3450)) or \
os.path.exists(filen_cbf):
self.widget.checkBox_ShowCake.setChecked(False)
return
# self._update_bg_params_in_widget()
if self.widget.checkBox_ShowCake.isChecked() and \
(self.model.poni is not None):
self.cake_ctrl.process_temp_cake()
def del_temp_cake(self):
reply = QtWidgets.QMessageBox.question(
self.widget, 'Message',
'This can slow down PeakPo, but update PONI. Proceed?',
QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No,
QtWidgets.QMessageBox.Yes)
if reply == QtWidgets.QMessageBox.No:
return
if self._temporary_pkpo_exists():
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
temp_cake = os.path.join(temp_dir, '*.npy')
for f in glob.glob(temp_cake):
os.remove(f)
def _addremove_cake(self):
"""
add / remove cake
no signal to update_graph
"""
if not self.widget.checkBox_ShowCake.isChecked():
return True
if not self.model.poni_exist():
QtWidgets.QMessageBox.warning(
self.widget, 'Warning', 'Choose PONI file first.')
self.widget.checkBox_ShowCake.setChecked(False),
return False
# check if model.poni exist
if not os.path.exists(self.model.poni):
QtWidgets.QMessageBox.warning(
self.widget, 'Warning', 'The poni does not exist in the path.')
self.widget.checkBox_ShowCake.setChecked(False),
return False
if not self.model.base_ptn_exist():
QtWidgets.QMessageBox.warning(
self.widget, 'Warning', 'Choose CHI file first.')
self.widget.checkBox_ShowCake.setChecked(False)
return False
# check if model.poni is in temp_dir
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
poni_filen = extract_filename(self.model.poni) + '.' + \
extract_extension(self.model.poni)
if self.model.poni != os.path.join(temp_dir, poni_filen):
shutil.copy(self.model.poni, os.path.join(temp_dir, poni_filen))
self.model.poni = os.path.join(temp_dir, poni_filen)
self.widget.lineEdit_PONI.setText(self.model.poni)
filen_tif = self.model.make_filename('tif', original=True)
filen_tiff = self.model.make_filename('tiff', original=True)
filen_mar3450 = self.model.make_filename('mar3450', original=True)
filen_cbf = self.model.make_filename('cbf', original=True)
if not ((os.path.exists(filen_tif) or
os.path.exists(filen_tiff) or
os.path.exists(filen_mar3450)) or
os.path.exists(filen_cbf)):
QtWidgets.QMessageBox.warning(
self.widget, 'Warning',
'Cannot find image file: %s or %s or %s or %s.' %
(filen_tif, filen_tiff, filen_mar3450, filen_cbf))
self.widget.checkBox_ShowCake.setChecked(False)
return False
if self.model.diff_img_exist() and \
self.model.same_filename_as_base_ptn(
self.model.diff_img.img_filename):
return True
self.process_temp_cake()
return True
def save_xls(self):
"""
Export jlist to an excel file
"""
if not self.model.jcpds_exist():
return
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
filen_xls_t = make_filename(self.model.get_base_ptn_filename(),
'jlist.xls',
temp_dir=temp_dir)
filen_xls = dialog_savefile(self.widget, filen_xls_t)
if str(filen_xls) == '':
return
xls_jlist(filen_xls, self.model.jcpds_lst,
self.widget.doubleSpinBox_Pressure.value(),
self.widget.doubleSpinBox_Temperature.value())
def download_package_source(name, unpack=False):
package_source = get_config('package_source')
package_arch = get_config('package_arch')
package_options = get_config('package_options')
dir = tempfile.mkdtemp('', '', get_temp_dir())
os.chdir(dir)
try:
download_source_from_apt(name, package_source, package_arch,
package_options, unpack)
except:
os.chdir(root_dir)
package.remove_dir(dir)
raise
os.mkdir(dir + '/refs')
os.chdir(root_dir)
return dir
def _update_bgsub_from_current_values(self):
x_raw, y_raw = self.model.base_ptn.get_raw()
if (x_raw.min() >= self.widget.doubleSpinBox_Background_ROI_min.value()) or \
(x_raw.max() <= self.widget.doubleSpinBox_Background_ROI_min.value()):
self.widget.doubleSpinBox_Background_ROI_min.setValue(x_raw.min())
if (x_raw.max() <= self.widget.doubleSpinBox_Background_ROI_max.value()) or \
(x_raw.min() >= self.widget.doubleSpinBox_Background_ROI_max.value()):
self.widget.doubleSpinBox_Background_ROI_max.setValue(x_raw.max())
self.model.base_ptn.subtract_bg([
self.widget.doubleSpinBox_Background_ROI_min.value(),
self.widget.doubleSpinBox_Background_ROI_max.value()
], [
self.widget.spinBox_BGParam0.value(),
self.widget.spinBox_BGParam1.value(),
self.widget.spinBox_BGParam2.value()
],
yshift=0)
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
self.model.base_ptn.write_temporary_bgfiles(temp_dir)
def unpack_package(name):
package_source = get_config('package_source')
package_arch = get_config('package_arch')
package_options = get_config('package_options')
dir = tempfile.mkdtemp('', '', get_temp_dir())
os.chdir(dir)
try:
#filename = download_from_apt(name, package_source,
# package_arch, package_options)
#compile_package(name)
directoryname = "/filer/bin/gcc-nostripopt/" + name
filename1 = ''
for filename in (os.listdir(directoryname)):
result = re.match('([^_]+)_([^_]+)_([^.]+).deb', filename)
if not result:
raise Exception("\'" + name + "\' is not properly downloaded")
name1 = result.group(1)
version = result.group(2)
arch = result.group(3)
if name1 == name:
filename1 = filename
break
if filename1 != '':
result = subprocess.call(
["dpkg", "-x", directoryname + "/" + filename1, "."],
stdout=null_dev,
stderr=null_dev)
else:
raise Exception("File not found \'" + name + "\'")
if result != 0:
raise Exception("Cannot unpack \'" + name + "\'")
except:
os.chdir(root_dir)
package.remove_dir(dir)
raise
os.chdir(root_dir)
return (dir, name, version)
def get_poni(self):
"""
Opens a pyFAI calibration file
signal to update_graph
"""
filen = QtWidgets.QFileDialog.getOpenFileName(
self.widget, "Open a PONI File",
self.model.chi_path, "PONI files (*.poni)")[0]
filename = str(filen)
if os.path.exists(filename):
# copy the chose file to temp_dir
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
shutil.copy(filename, temp_dir)
filen = extract_filename(filename) + '.' + \
extract_extension(filename)
# set filename to that exists in temp_dir
self.model.poni = os.path.join(temp_dir, filen)
self.widget.lineEdit_PONI.setText(self.model.poni)
if self.model.diff_img_exist():
self.produce_cake()
self._apply_changes_to_graph()
def save_to_xls(self):
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
filen_xls = make_filename(self.model.get_base_ptn_filename(),
'peakfit.xls', temp_dir=temp_dir)
reply = QtWidgets.QMessageBox.question(
self.widget, 'Question',
'Do you want to save in default filename, %s ?' % filen_xls,
QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No,
QtWidgets.QMessageBox.Yes)
if reply == QtWidgets.QMessageBox.No:
filen_xls = QtWidgets.QFileDialog.getSaveFileName(
self.widget, "Save an Excel File", filen_xls, "(*.xls)")
else:
if os.path.exists(filen_xls):
reply = QtWidgets.QMessageBox.question(
self.widget, 'Question',
'The file already exist. Overwrite?',
QtWidgets.QMessageBox.Yes | QtWidgets.QMessageBox.No,
QtWidgets.QMessageBox.No)
if reply == QtWidgets.QMessageBox.No:
return
self.model.save_peak_fit_results_to_xls(filen_xls)
def monitor(interface, outfile=None, verbose=None, sniff=None):
tracefile = os.path.abspath("ld-pcap-dec.pcap")
def kill_handler(signum, frame):
"""Kill tcpdump and then analyze pcap."""
if tcpdump_proc:
tcpdump_proc.terminate()
# Analyze pcap with bro
run_bro('-r %s' % (tracefile), logdir)
analyze_logs(logdir, outfile=outfile)
# Now run leak detector
print "Analyzing captured network traffic"
utils.remove_temp_dir('bro_logs')
analyze.main(outfile, "%s.analyzed" % outfile)
signal.signal(signal.SIGTERM, kill_handler)
signal.signal(signal.SIGINT, kill_handler)
logging.basicConfig(
format = "%(levelname) -10s %(asctime)s %(module)s:%(lineno)s %(funcName) -26s %(message)s",
level = logging.DEBUG if verbose else logging.WARNING
)
# Make log directory for bro
utils.init_temp_dir('bro_logs')
logdir = utils.get_temp_dir('bro_logs')
# run sniff.sh
# tcpdump_proc = subprocess.Popen(["sleep", "256"]) testing
if sniff:
tcpdump_proc = subprocess.Popen(["bash", "sniff.sh", interface])
else:
tcpdump_proc = subprocess.Popen(['tcpdump', '-i', interface, '-w', tracefile])
tcpdump_proc.wait()
logging.getLogger(__name__).info('Analyzing trace: %s', tracefile)
from tarfile import TarFile
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
import plistlib
from configobj import ConfigObj
inside_packed_exe = getattr(sys, 'frozen', '')
CWD = os.getcwd()
TEMP_DIR = get_temp_dir()
DEFAULT_DOWNLOAD_PATH = os.path.join(CWD, 'files',
'downloads').replace('\\', '\\\\')
try:
os.makedirs(DEFAULT_DOWNLOAD_PATH)
except:
pass
def get_base_url():
url = None
try:
url = open(os.path.join(CWD, 'files', 'base_url.txt')).read().strip()
except (OSError, IOError):
url = 'http://dl.node-webkit.org/v{}/'
return url
with open(get_file('files/version.txt')) as f:
__version__ = f.read().strip()
ICON_PATH = 'files/images/icon.png'
WARNING_ICON = 'files/images/warning.png'
APP_SETTINGS_ICON = 'files/images/app_settings.png'
WINDOW_SETTINGS_ICON = 'files/images/window_settings.png'
EXPORT_SETTINGS_ICON = 'files/images/export_settings.png'
COMPRESS_SETTINGS_ICON = 'files/images/compress_settings.png'
DOWNLOAD_SETTINGS_ICON = 'files/images/download_settings.png'
FOLDER_OPEN_ICON = 'files/images/folder_open.png'
W2E_VER_FILE = 'files/version.txt'
TEMP_DIR = utils.get_temp_dir()
ERROR_LOG_FILE = 'files/error.log'
VER_FILE = 'files/nw-versions.txt'
SETTINGS_FILE = 'files/settings.cfg'
GLOBAL_JSON_FILE = 'files/global.json'
WEB2EXE_JSON_FILE = 'web2exe.json'
LAST_PROJECT_FILE = 'files/last_project_path.txt'
RECENT_FILES_FILE = 'files/recent_files.txt'
NW_BRANCH_FILE = 'files/nw-branch.txt'
UPX_WIN_PATH = 'files/compressors/upx-win.exe'
UPX_MAC_PATH = 'files/compressors/upx-mac'
UPX_LIN32_PATH = 'files/compressors/upx-linux-x32'
def update_cake(self):
if self.model.poni_exist():
self.produce_cake()
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
self.model.diff_img.write_temp_cakefiles(temp_dir=temp_dir)
self._apply_changes_to_graph()
with open(get_file('files/version.txt')) as f:
__version__ = f.read().strip()
ICON_PATH = 'files/images/icon.png'
WARNING_ICON = 'files/images/warning.png'
APP_SETTINGS_ICON = 'files/images/app_settings.png'
WINDOW_SETTINGS_ICON = 'files/images/window_settings.png'
EXPORT_SETTINGS_ICON = 'files/images/export_settings.png'
COMPRESS_SETTINGS_ICON = 'files/images/compress_settings.png'
DOWNLOAD_SETTINGS_ICON = 'files/images/download_settings.png'
FOLDER_OPEN_ICON = 'files/images/folder_open.png'
W2E_VER_FILE = 'files/version.txt'
TEMP_DIR = utils.get_temp_dir()
ERROR_LOG_FILE = 'files/error.log'
VER_FILE = 'files/nw-versions.txt'
SETTINGS_FILE = 'files/settings.cfg'
GLOBAL_JSON_FILE = 'files/global.json'
WEB2EXE_JSON_FILE = 'web2exe.json'
LAST_PROJECT_FILE = 'files/last_project_path.txt'
RECENT_FILES_FILE = 'files/recent_files.txt'
NW_BRANCH_FILE = 'files/nw-branch.txt'
UPX_WIN_PATH = 'files/compressors/upx-win.exe'
UPX_MAC_PATH = 'files/compressors/upx-mac'
UPX_LIN32_PATH = 'files/compressors/upx-linux-x32'
def perform_ucfit(self):
# get jcpds data in df. use display to choose data points
if self.model.section_lst == []:
QtWidgets.QMessageBox.warning(
self.widget, "Warning",
"No peak fitting result exist for this file.")
return
if self.phase == None:
QtWidgets.QMessageBox.warning(
self.widget, "Warning", "No phase has been chosen for fitting")
return
data_by_phase_df = self._get_all_peakfit_results_df()
data_to_fit_df = data_by_phase_df[self.phase].loc[data_by_phase_df[
self.phase]['display'] == True]
# number of data point check
n_data_points = len(data_to_fit_df.index)
if n_data_points < 2:
QtWidgets.QMessageBox.warning(self.widget, "Warning",
"You need at least 2 data points.")
return
# perform ucfit
text_output = self.phase + '\n\n'
text_output += 'Fitted unit cell parameters \n'
text_output += 'Crystal system = ' + \
self.widget.comboBox_Symmetry.currentText() + '\n'
wavelength = self.model.get_base_ptn_wavelength()
if self.widget.comboBox_Symmetry.currentText() == 'cubic':
a, s_a, v, s_v, res_lin, res_nlin = fit_cubic_cell(data_to_fit_df,
wavelength,
verbose=False)
cell_params = [a, a, a]
text_output += "a = {0:.5f} +/- {1:.5f} \n".format(a, s_a)
text_output += "V = {0:.5f} +/- {1:.5f} \n\n".format(v, s_v)
elif self.widget.comboBox_Symmetry.currentText() == 'tetragonal':
if n_data_points < 3:
QtWidgets.QMessageBox.warning(
self.widget, "Warning",
"You need at least 3 data points for tetragonal.")
return
a, s_a, c, s_c, v, s_v, res_lin, res_nlin = \
fit_tetragonal_cell(data_to_fit_df, wavelength,
verbose=False)
cell_params = [a, a, c]
text_output += "a = {0:.5f} +/- {1:.5f} \n".format(a, s_a)
text_output += "c = {0:.5f} +/- {1:.5f} \n".format(c, s_c)
text_output += "V = {0:.5f} +/- {1:.5f} \n\n".format(v, s_v)
elif self.widget.comboBox_Symmetry.currentText() == 'hexagonal':
if n_data_points < 3:
QtWidgets.QMessageBox.warning(
self.widget, "Warning",
"You need at least 3 data points for hexagonal.")
return
a, s_a, c, s_c, v, s_v, res_lin, res_nlin = \
fit_hexagonal_cell(data_to_fit_df, wavelength,
verbose=False)
cell_params = [a, a, c]
text_output += "a = {0:.5f} +/- {1:.5f} \n".format(a, s_a)
text_output += "c = {0:.5f} +/- {1:.5f} \n".format(c, s_c)
text_output += "V = {0:.5f} +/- {1:.5f} \n\n".format(v, s_v)
elif self.widget.comboBox_Symmetry.currentText() == 'orthorhombic':
if n_data_points < 4:
QtWidgets.QMessageBox.warning(
self.widget, "Warning",
"You need at least 4 data points for orthorhombic.")
return
a, s_a, b, s_b, c, s_c, v, s_v, res_lin, res_nlin = \
fit_orthorhombic_cell(data_to_fit_df, wavelength,
verbose=False)
cell_params = [a, b, c]
text_output += "a = {0:.5f} +/- {1:.5f} \n".format(a, s_a)
text_output += "b = {0:.5f} +/- {1:.5f} \n".format(b, s_b)
text_output += "c = {0:.5f} +/- {1:.5f} \n".format(c, s_c)
text_output += "V = {0:.5f} +/- {1:.5f} \n\n".format(v, s_v)
# output results
output_df = make_output_table(res_lin, res_nlin, data_to_fit_df)
text_output += 'Output table\n'
text_output += output_df[[
'h', 'k', 'l', 'twoth', 'dsp', 'twoth residue'
]].to_string()
text_output += '\n\nHat: influence for the fit result. \n'
text_output += ' 1 ~ large influence, 0 ~ no influence.\n'
text_output += output_df[['h', 'k', 'l', 'twoth', 'dsp',
'hat']].to_string()
text_output += '\n\nRstudent: how much the parameter would change' + \
' if deleted.\n'
text_output += output_df[['h', 'k', 'l', 'twoth', 'dsp',
'Rstudent']].to_string()
text_output += '\n\ndfFits: deletion diagnostic giving' + \
' the change in\n'
text_output += ' the predicted value twotheta.\n'
text_output += ' upon deletion of the data point as a ' + \
'multiple of\n'
text_output += ' the standard deviation for 1/d-spacing^2.\n'
text_output += output_df[['h', 'k', 'l', 'twoth', 'dsp',
'dfFits']].to_string()
text_output += '\n\ndfBetas: normalized residual\n'
text_output += output_df[['h', 'k', 'l', 'twoth', 'dsp',
'dfBetas']].to_string()
text_output += '\n\nNon-linear fit statistics \n'
text_output += lmfit.fit_report(res_nlin)
self.widget.plainTextEdit_UCFitOutput.setPlainText(text_output)
# save jcpds and save output file automatically.
# ask for filename. at the moment, simply overwrite
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
ext = "ucfit.jcpds"
#filen_t = self.model.make_filename(ext)
filen_t = make_filename(self.template_jcpds.file,
ext,
temp_dir=temp_dir)
filen_j = dialog_savefile(self.widget, filen_t)
if str(filen_j) == '':
return
self._write_to_jcpds(filen_j, cell_params)
# write to a textfile
ext = "ucfit.output"
#filen_t = self.model.make_filename(ext)
filen_t = make_filename(self.template_jcpds.file,
ext,
temp_dir=temp_dir)
filen_o = dialog_savefile(self.widget, filen_t)
if str(filen_o) == '':
return
with open(filen_o, "w") as f:
f.write(text_output)
from HttpConversationParser import *
from TCPAnalyzer import *
#with open('html_convo', 'r') as f:
# data = f.read()
#f.closed
#
#parser = HttpConversationParser(data)
#print len(parser.html_pages)
#print parser.html_pages[0]
utils.init_temp_dir('tcpflow')
utils.init_temp_dir('images')
t = TCPAnalyzer('imgtest.pcap')
for stream in t.http_streams:
if 63554 in stream.ports:
p = HttpConversationParser(stream.http_data)
p.save_images_to_dir(utils.get_temp_dir('images'))
for m in p.messages:
print m
print '\n\n'
#for page in p.html_pages:
# print '<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<'
# print page
# print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>'
#parser = HttpConversationParser(t.streams[26].data)
#print len(parser.html_pages)
#print parser.html_pages[0]
# spsfsync.py - syncs status for sp and sf sync
import sys
import utils
import os
import sfupdspstatus
from mailer import sendMail
import csv
import datetime
tempdir = utils.get_temp_dir()
valid_status = ["HELD", "NORMAL", "UNSUB", "BLOCKED", "OPTOUT"]
def __create_sf_updates(fin, workdir, rmap, debug_max = None):
splits = []
split_size = 500
logger = utils.createLogger("sp_get_status.py")
date = datetime.datetime.now()
sdate = "%02d-%02d-%04d"%(date.month, date.day, date.year)
if debug_max is not None:
debug_max = int(debug_max)
logger.debug("debug_max "+str(debug_max))
logger.debug("*** WORKDIR "+workdir)
def main(interface, outfile=None, tracefile=None, analyzeinterval=None, _filter=None, logdir=None, verbose=False, stdout=False):
"""Main function to run from command line."""
def kill_handler(signum, frame):
"""Kill bro."""
logging.info('Exiting...')
global bro_proc
global log_proc
if bro_proc:
bro_proc.terminate()
if not logdir:
utils.remove_temp_dir('bro_logs')
analyze_logs(logdir, outfile=outfile)
print "Analyzing captured network traffic (this may take a second...)"
analyze.main(outfile, "%s.analyzed"%outfile)
if tracefile:
tracefile = os.path.abspath(tracefile)
# Set up signal handlers
signal.signal(signal.SIGTERM, kill_handler)
signal.signal(signal.SIGINT, kill_handler)
# Set up logging
logging.basicConfig(
format = "%(levelname) -10s %(asctime)s %(module)s:%(lineno)s %(funcName) -26s %(message)s",
level = logging.DEBUG if verbose else logging.WARNING
)
# make a temp dir for bro logs if none was specified
if not logdir:
utils.init_temp_dir('bro_logs')
logdir = utils.get_temp_dir('bro_logs')
else:
if not os.path.isdir(logdir):
try:
os.makedirs(logdir)
except Exception as e:
logging.getLogger(__name__).error('Error creating logdir %s: %s', logdir, e)
if tracefile:
logging.getLogger(__name__).info('Analyzing trace: %s', tracefile)
run_bro('-r %s' % (tracefile), logdir)
analyze_logs(logdir, outfile=outfile)
elif interface:
logging.getLogger(__name__).info('Analyzing traffic on %s', interface)
if analyzeinterval:
analyze_thread = LiveLogAnalyzer(logdir, analyzeinterval, outfile)
analyze_thread.start()
run_bro('-i %s' % (interface), logdir)
analyze_thread.stop()
else:
run_bro('-i %s' % (interface), logdir)
analyze_logs(logdir, outfile=outfile)
elif logdir:
logging.getLogger(__name__).info('Analyzing Bro logs in %s', logdir)
analyze_logs(logdir, outfile=outfile)
else:
logging.getLogger(__name__).warn('Must provide either a packet trace, an interface to sniff, or a directory of existing Bro logs.')
sys.exit()
else:
# we are running in a normal Python environment
CWD = os.getcwd()
def get_file(path):
parts = path.split('/')
independent_path = utils.path_join(CWD, *parts)
return independent_path
__version__ = "v0.0.0"
with open(get_file('files/version.txt')) as f:
__version__ = f.read().strip()
TEMP_DIR = get_temp_dir()
DEFAULT_DOWNLOAD_PATH = get_data_path('files/downloads')
logger = logging.getLogger('W2E logger')
LOG_FILENAME = get_data_file_path('files/error.log')
if __name__ != '__main__':
logging.basicConfig(
filename=LOG_FILENAME,
format=("%(levelname) -10s %(asctime)s %(module)s.py: "
"%(lineno)s %(funcName)s - %(message)s"),
level=logging.DEBUG
)
logger = logging.getLogger('W2E logger')
handler = lh.RotatingFileHandler(LOG_FILENAME, maxBytes=100000, backupCount=2)
logger.addHandler(handler)
def _temporary_pkpo_exists(self):
temp_dir = get_temp_dir(self.model.get_base_ptn_filename())
return os.path.exists(temp_dir)
def __init__(self,
code: bytes = None,
read_only_access: list = None,
write_exec_access: list = None):
self._env_path = get_temp_dir()
def __init__(self, executable_path: str, read_only_access: list = None, write_exec_access: list = None):
self._executable = executable_path
self._read_only_access = read_only_access or []
self._write_exec_access = write_exec_access or []
self._temp_dir = get_temp_dir()
self._apparmor_profile = AppArmorSecProfile(self._executable)
t = TCPAnalyzer(trace)
# look for regex matches
regexes = {} #{'Phone Number':[r'\(?[2-9][0-8][0-9]\s*\W?\s*[2-9][0-9]{2}\s*\W?\s*[0-9]{4}']}
for s in t.streams:
match_dict = s.find_regexes(regexes)
stats.update_matched_regexes(match_dict)
logging.getLogger(__name__).info('Analyzing HTTP conversations...')
# process HTML pages
for html_file in t.html_files:
with open(html_file, 'r') as f:
html = f.read()
ha = HTMLAnalyzer(html)
stats.update_page_titles( ha.page_titles )
stats.update_amazon_products( ha.amazon_products )
f.closed
# save images to temp dir
for image in t.images:
if do_display_image(image):
new_image_path = os.path.join(utils.get_temp_dir('images'),\
os.path.basename(image))
shutil.copyfile(image, new_image_path)
stats.update_image_paths( set([new_image_path]) )
utils.remove_temp_dir('tcpflow')
return stats
