def mailer(num_files, dir_size):
print('[*] In Mailer!!')
#You will have to formal the email below properly
send_to = ['Nik Alleyne <nik alleyne at gmail dot com>']
send_from = email.utils.formataddr(
('IBM QRadar', '*****@*****.**'))
qradar_version = sp.check_output(['/opt/qradar/bin/getHostVersion.sh'
]).split('\n')[2]
# Read Qradar Device Hostname
msg_body = '[*] Running on host: {} \n'.format(hostname())
msg_body = msg_body + '\n[*] Current QRadar Version: {} \n'.format(
qradar_version.split('=')[1])
msg_body = msg_body + '\n[*] Persistent Folder: /store/persistent_queue/ecs-ec-ingress.ecs-ec-ingress/ \n'
msg_body = msg_body + '\n[*] Current Status as of {} \n'.format(
datetime.datetime.now())
msg_body = msg_body + '\n[*] Current Number of files: {} \n'.format(
num_files)
msg_body = msg_body + '\n[*] Current Directory Size in Bytes:{}B \n'.format(
dir_size)
msg_body = msg_body + '\n[*] Current Directory Size in MBs: {}M'.format(
int(dir_size) / 1000000)
msg_body = msg_body + '\n[*] Current Directory Size in Gigs: {}G'.format(
int(dir_size) / 1000000000)
msg_body = msg_body + '\n\n ***Powered By Sirius Computer Solutions *** \n\n'
print('[*] Preparing to send mail ... ')
msg = mt(msg_body)
msg['To'] = ','.join(send_to)
msg['From'] = send_from
if ((num_files < 10) and (int(dir_size) < 10000000000)):
msg['Subject'] = '[**] {} :: INFORMATIONAL - Monitoring of Persistent Queue [**]'.format(
hostname())
else:
msg['Subject'] = '[!!] {} :: POTENTIAL PROBLEM - Persistent Queue is growing [!!]'.format(
hostname())
send_mail = smtplib.SMTP('localhost')
try:
# once again, you will have to properly format the email
send_mail.sendmail(send_from, 'nikalleyne at gmail dot com',
msg.as_string())
print('[*] Mail sent successfully!')
except:
print(
'[!] Ooops! Looks like an error occurred while sending the mail.')
send_mail.quit()
def process_graph(G, E, noise, outname, asym=False):
root = max(G.items(), key=lambda x: len(x[1]))[0]
if not outname.startswith('belgrade/'):
outname = 'belgrade/' + outname
basename = '{}_{}'.format(outname, hostname())
suffix = '.asymres' if asym else '.myres'
if os.path.isfile(basename+'_perf'+suffix):
return
bfs = gs.perturbed_bfs(G, root)
gtx, _ = galaxy_maker(G, 50, short=True, output_name=outname)
stretch = None
binary_signs = {e: (1 if s else -1) for e, s in E.items()}
perf = []
for train_edges in [bfs, gtx]:
if asym:
perf.extend(run_asym(G, E, train_edges))
else:
tree = {}
for u, v in train_edges:
gs.add_edge(tree, u, v)
tags = pot.dfs_tagging(tree, binary_signs, root)
gold, pred = pot.make_pred(tree, tags, binary_signs)
tp, tn, fp, fn = confusion_number(gold, pred)
perf.extend([accuracy(tp, tn, fp, fn), f1_score(tp, tn, fp, fn),
mcc(tp, tn, fp, fn)])
if asym:
_, edges = pot.read_tree(outname+'_0.edges')
perf.extend(run_asym(G, E, edges))
perf.extend(run_asym(G, E, tree_edges=None))
else:
gold, pred, _ = pot.predict_edges(outname+'_0', all_signs=E,
degrees={root: 5})
tp, tn, fp, fn = confusion_number(gold, pred)
perf.extend([accuracy(tp, tn, fp, fn), f1_score(tp, tn, fp, fn),
mcc(tp, tn, fp, fn)])
if noise == 0 and not asym:
print(basename)
bfsst = average_strech(set(E.keys()), bfs)
persistent.save_var(basename+'_bfsst'+suffix, bfsst)
gtxst = average_strech(set(E.keys()), gtx)
persistent.save_var(basename+'_gtxst'+suffix, gtxst)
stretch = [bfsst, gtxst]
persistent.save_var(basename+'_perf'+suffix, perf)
return perf, stretch
import socket
s = socket.socket()
host = socket.hostname()
port = 4444
s.bind((host, port))
s.listen(5)
while True:
c.addr = s.accept()
print("connection from ", addr)
c.send("Success connected")
c.close()
def acs():
"""
Script to reduce ACS Cluster data
"""
import glob
import os
import drizzlepac
import numpy as np
import threedhst
import threedhst.prep_flt_astrodrizzle as init
from threedhst import catIO
import unicorn
import unicorn.interlace_acs
field = 'MACS*1149'
### First, in PREP_FLT, run "flt_info.sh"
### Make ACS associations
unicorn.candels.make_asn_files(uniquename=True, translate={'-ROT':''})
### Need to take out "_flc" extension from the ASN files
files=glob.glob('*asn.fits')
for asn_file in files:
asn = threedhst.utils.ASNFile(asn_file)
## print asn.exposures
for i in range(len(asn.exposures)):
asn.exposures[i] = asn.exposures[i].split('_flc')[0]
#
asn.write(asn_file)
#
#### Combine parallels
for par in [1,2]:
for filter in ['F775W','F850LP']:
files=glob.glob(field+'*-ACSPAR%d-[AB]*-%s*' %(par, filter))
print files
list = []
for file in files:
asn = threedhst.utils.ASNFile(file)
list.extend(asn.exposures)
os.remove(file)
asn.exposures = list
asn.product = 'MACS0416-ACSPAR%d-%s' %(par, filter)
asn.write('%s_asn.fits' %(asn.product))
files, radec = glob.glob('MACS0416-ACSPA*asn.fits'), 'subaru.radec'
files, radec = glob.glob('MACS0416-2403*asn.fits'), 'clash_radec.dat'
files, radec = glob.glob('*0717*F814W*asn.fits'), 'subaru_macs0717.radec'
files, radec = glob.glob(field+'*POS5-1?*814*asn.fits'), 'macs0717_subaru_radec.dat'
files, radec = glob.glob(field+'*PAR1*775*asn.fits'), 'macs0416_subaru_radec.dat'
files, radec = glob.glob(field+'*814*asn.fits'), 'macs1149_radec.dat'
refimage='hlsp_XXX'
files, radec = glob.glob('*asn.fits'), 'clash_radec.dat'
files, radec = glob.glob('*LENS*asn.fits'), 'clash_radec.dat'
from socket import gethostname as hostname
for asn_file in files:
#if os.path.exists(asn_file.replace('asn', 'drc_sci')):
#continue
#
init.prep_direct_grism_pair(direct_asn=asn_file, grism_asn=None, radec=radec, ACS=True, align_threshold=5)
if 'science' in hostname():
asn = threedhst.utils.ASNFile(asn_file)
for exp in asn.exposures:
os.remove('../RAW/%s_flc.fits.gz' %(exp))
##### Combine visits on the cluster to flag CRs
info = catIO.Table("files.info")
filters = np.unique(info['FILTER'])
for filter in filters:
#### Get all FLTs
flt_list = []
files=glob.glob('MACS*0416-2403-??-???-%s_asn.fits' %(filter))
#files=glob.glob('MACS*1149*%s_asn.fits' %(filter))
for asn_file in files:
asn = threedhst.utils.ASNFile(asn_file)
for exp in asn.exposures:
flt_list.append('%s_flc.fits' %(exp))
#
bits = 96 #+512
skyuser = '******'
#### Ignore MDRIZSKY
fp = open('ACS.skyfile','w')
fp.writelines(['%s 0.0\n' %(file) for file in flt_list])
fp.close()
#
#### Run AstroDrizzle to flag CRs on both orients
drizzlepac.astrodrizzle.AstroDrizzle(flt_list, output='MACS0717+3745-%s_total_comb' %(filter), clean=True, context=False, preserve=False, skysub=True, skyfile='ACS.skyfile', driz_separate=True, driz_sep_wcs=True, median=True, blot=True, driz_cr=True, driz_combine=True, final_wcs=True, final_refimage=None, final_scale=0.065, final_pixfrac=0.8, final_rot=0, final_kernel='square', resetbits=0, final_bits=bits, final_wht_type='IVM')
#
#drizzlepac.astrodrizzle.AstroDrizzle(flt_list, output='MACS0416-2403-30mas-%s' %(filter), clean=True, context=False, preserve=False, skysub=True, skyfile='ACS.skyfile', driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_combine=True, final_wcs=True, final_refimage='hlsp_clash_hst_acs-30mas_macs0416_f475w_v1_drz.fits', final_pixfrac=0.8, final_kernel='square', resetbits=0, final_bits=bits, final_wht_type='IVM')
#
#drizzlepac.astrodrizzle.AstroDrizzle(flt_list, output='MACS1149-2223-Mosaic-%s' %(filter), clean=True, context=False, preserve=False, skysub=True, skyfile='ACS.skyfile', driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_combine=True, final_wcs=True, final_scale=0.05, final_rot=0, final_pixfrac=0.8, final_kernel='square', resetbits=0, final_bits=bits, final_wht_type='IVM')
#### MACS0717 mosaic
for filter in filters:
flt_list = []
#files=glob.glob('MACS0416-2403-??-???-%s_asn.fits' %(filter))
files=glob.glob(field+'*%s_asn.fits' %(filter))
for asn_file in files:
asn = threedhst.utils.ASNFile(asn_file)
for exp in asn.exposures:
if os.path.exists('%s_flc.fits' %(exp)):
flt_list.append('%s_flc.fits' %(exp))
#
bits = 96 #+512
skyuser = '******'
#### Ignore MDRIZSKY
fp = open('ACS.skyfile','w')
fp.writelines(['%s 0.0\n' %(file) for file in flt_list])
fp.close()
#
#drizzlepac.astrodrizzle.AstroDrizzle(flt_list, output='MACS0717-Mosaic-%s' %(filter), clean=True, context=False, preserve=False, skysub=True, skyfile='ACS.skyfile', driz_separate=False, driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_combine=True, final_wcs=True, final_rot=0, final_pixfrac=0.8, final_kernel='square', resetbits=0, final_bits=bits, final_wht_type='IVM', final_ra=1.093574923885E+02, final_dec=3.778490152063E+01, final_scale=0.1798, final_outnx=4185, final_outny=4670)
### Here, specify the output image size, pixel scale, etc.
#drizzlepac.astrodrizzle.AstroDrizzle(flt_list, output='MACS0717-Mosaic-%s' %(filter), clean=True,
# context=False, preserve=False, skysub=True, skyfile='ACS.skyfile', driz_separate=False,
# driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_combine=True, final_wcs=True,
# final_rot=0, final_pixfrac=0.8, final_kernel='square', resetbits=0, final_bits=bits, final_wht_type='IVM',
# final_ra=1.093574923885E+02, final_dec=3.778490152063E+01, final_scale=0.1, final_outnx=7524, final_outny=8397)
### Here, force the output to match the FF mosaic
#drizzlepac.astrodrizzle.AstroDrizzle(flt_list, output='MACS1149-Mosaic-%s' %(filter), clean=True,
# context=False, preserve=False, skysub=True, skyfile='ACS.skyfile', driz_separate=False,
# driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_combine=True, final_wcs=True,
# final_rot=0, final_pixfrac=0.8, final_kernel='square', resetbits=0, final_bits=bits, final_wht_type='IVM',
# final_refimage=refimage)
drizzlepac.astrodrizzle.AstroDrizzle(flt_list, output='MACS1149-Mosaic-v2-%s' %(filter), clean=True,
context=True, preserve=False, skysub=True, skyfile='ACS.skyfile', driz_separate=False,
driz_sep_wcs=False, median=False, blot=False, driz_cr=False, driz_combine=True, final_wcs=True,
final_rot=0, final_pixfrac=0.8, final_kernel='square', resetbits=0, final_bits=bits, final_wht_type='IVM',
final_ra=177.405988, final_dec=22.343847, final_outnx=11658, final_outny=16355)
def main():
"""
Usage: python3 send_mail.py email1 email2 etc.
"""
recipients = sys.argv[1:]
host = hostname()
conn = dbConn()
filterOutUselessRecs(conn)
msg = MIMEMultipart()
msg['Subject'] = 'Eyra at {} has recorded {} utterances'.format(
host, countRecordings(conn))
msg['To'] = ', '.join(recipients)
msg['From'] = SENDER
body = """
SUMMARY:
Recorded utterances: {recordings}
Distinct session count: {sessions}
""".format(recordings=countRecordings(conn), sessions=countSessions(conn))
body += """
RECORDINGS BY USERNAME:
{lines}
""".format(lines='\n'.join(
['{:<4} {}'.format(count, user) for user, count in recsByUser(conn)]))
# TODO: by date (we have to parse text...)
#byDate
body += """
RECORDINGS BY GENDER:
{lines}
""".format(lines='\n'.join('{:<6} {}'.format(count, gender)
for gender, count in recsByGender(conn)))
body += """
RECORDINGS BY DAY:
{lines}
""".format(lines='\n'.join('{:<6} {}'.format(count, day)
for day, count in recsByDate(conn)))
body += """
RECORDINGS BY AGE GROUP:
{lines}
""".format(lines='\n'.join('{:<6} {}'.format(count, year)
for year, count in recsByAge(conn)))
body += """
RECORDINGS BY PHONE IMEI:
{lines}
""".format(lines='\n'.join('{:<4} {}'.format(count, imei)
for imei, count in recsByDevice(conn)))
body += """
RECORDINGS BY APK AND OS:
{lines}
""".format(lines='\n'.join('{:<6} {}'.format(count, apkAndOs)
for apkAndOs, count in recsByApkAndOs(conn)))
part = MIMEText('text', "plain")
part.set_payload(body)
msg.attach(part)
session = smtplib.SMTP(SMTP_SERVER, SMTP_PORT)
session.ehlo()
session.starttls()
session.ehlo
session.login(SENDER, PASSWORD)
session.sendmail(SENDER, recipients, msg.as_string())
session.quit()
sys.exit(0)
#!/reg/g/psdm/sw/conda/inst/miniconda2-prod-rhel7/envs/ana-1.3.9/bin/python
import Queue
import os
import sys
import subprocess
import threading
import argparse
import socket
thisHost = socket.hostname()
def sendSSHCommandQWrapper(q, commandString):
q.put(sendSSHCommand(commandString))
def sendSSHCommand(commandString):
ssh = subprocess.Popen(commandString.split(" "),
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
result = ssh.stdout.readlines()
error = ssh.stderr.readlines()
if ([] == result):
#for i in error:
print >> sys.stderr, "ERROR: %s" % error
else:
if ("There is" in str(result)):
#if(True):
resultString = commandString + " " + str(result)
print resultString
# Samples.DATA_samples [ 'DiCharm/Up' ] )
# psi_and_charm = lfn2disk (
# Samples.DATA_samples [ 'DiMuon&Charm/Down'] +
# Samples.DATA_samples [ 'DiMuon&Charm/Up' ]
# )
## lfns = Samples.DATA_samples [ 'DiMuon&Charm/Down']
## + Samples.DATA_samples [ 'DiMuon&Charm/Up' ]
# psi_and_psi = data [ '2xDiMuon/Up' ]
# psi_and_charm = data [ 'DiMuon&Charm/Up' ]
# charm_and_charm = data [ 'DiCharm/Up' ]
from socket import gethostname as hostname
runningAtUZH = 'physik.uzh.ch' in hostname()
if not runningAtUZH:
import shelve
db = shelve.open('/afs/cern.ch/user/i/ibelyaev/public/LFNs/lfns.db')
charm_ew = db['charm/EW-down-iso']
charm_ew += db['charm/EW-up-iso']
else:
from glob import glob
charm_ew = glob(
'/home/hep/bursche/wd/data/R12S17mDSTS/v/*/*/EW.CharmEW.mdst')
configure(charm_ew, [], True)
run(-1)
#!/usr/bin/env python
# encoding: utf-8
"""
__init__.py
Created by Gabriel Brammer on 2011-05-18.
"""
__version__ = "1.0"
import os
from socket import gethostname as hostname
if hostname().startswith('uni') or hostname().startswith('hyp'):
GRISM_HOME = '/3DHST/Spectra/Work/'
else:
GRISM_HOME = '/research/HST/GRISM/3DHST/'
if hostname().startswith('850dhcp8'):
GRISM_HOME = '/3DHST/Spectra/Work/'
#threedhst.sex.RUN_MODE='direct'
if hostname().lower().startswith('gabriel') | hostname().startswith('vpn-brammer'):
GRISM_HOME = '/Users/brammer/3DHST/Spectra/Work/'
GRISM_HOME = os.getenv('THREEDHST')
if GRISM_HOME[-1] != '/':
GRISM_HOME += '/'
import threedhst
from socket import gethostname as hostname
import subprocess as cli
import datetime
import time
import pytz
import filecmp
# from slackclient import SlackClient as slk
from shutil import copyfile as cpfile
import pwd
import grp
import logging
from logging.handlers import RotatingFileHandler
cwd = os.path.dirname(os.path.realpath(__file__))
full_hostname = hostname()
short_hostname = full_hostname.split('.')[0]
source_dir = cwd.strip('bin')
conf_dir = source_dir + 'conf/'
ignore_true = 'true'
global current_time
# calculate the offset taking into account daylight saving time
# utc_offset_sec = time.altzone if time.localtime().tm_isdst else time.timezone
# utc_offset = datetime.timedelta(seconds=-utc_offset_sec)
# current_time = datetime.datetime.now().replace(
# tzinfo=datetime.timezone(offset=utc_offset), microsecond=0).isoformat()
country = "America/Los_Angeles"
# Register atexit
atexit.register(cleanup, None)
########################################################################################################################
# Argument parsing
########################################################################################################################
argparser = argparse.ArgumentParser(
description='Run a docker container containing a keepalived Instance')
argparser.add_argument('--router-name',
'-n',
action='store',
nargs='?',
help='The name you want to call this VRRP',
default=hostname())
argparser.add_argument(
'--master',
'-m',
action='store_true',
help='Set if this keepalived should act as the master for this VRRP')
argparser.add_argument('--nopreempt',
action='store_true',
help='Set nopreempt')
argparser.add_argument('--unicast', action='store_true', help='Set unicast')
argparser.add_argument('--unicast-src-ip',
action='store',
nargs='?',
help='unicast-src-ip')
argparser.add_argument('--unicast-peer-ip',
action='store',
return proc.returncode
# Register atexit
atexit.register(cleanup,None)
########################################################################################################################
# ARGUMENT PARSER #
# This is where you put the Argument Parser lines #
########################################################################################################################
argparser = argparse.ArgumentParser(description='Run a docker container containing a keepalived Instance')
argparser.add_argument('--router-name','-n',
action='store',
nargs='?',
help='The name you want to call this VRRP',
default=hostname())
argparser.add_argument('--master','-m',
action='store_true',
help='Set if this keepalived should act as the master for this VRRP')
argparser.add_argument('--auth-pass','-p',
action='store',
nargs='?',
help='This is the password this VRRP should use for authentication.')
argparser.add_argument('--vrid','-v',
action='store',
type=int,
nargs='?',
help='This is the Virtual Router ID this VRRP should use.')
argparser.add_argument('--exclude','-x',
action='append',
nargs='?',
#!/usr/bin/env python
# encoding: utf-8
"""
__init__.py
Created by Gabriel Brammer on 2011-05-18.
"""
__version__ = "1.0"
import os
from socket import gethostname as hostname
if hostname().startswith('uni') or hostname().startswith('hyp'):
GRISM_HOME = '/3DHST/Spectra/Work/'
else:
GRISM_HOME = '/research/HST/GRISM/3DHST/'
if hostname().startswith('850dhcp8'):
GRISM_HOME = '/3DHST/Spectra/Work/'
#threedhst.sex.RUN_MODE='direct'
if hostname().lower().startswith('gabriel') | hostname().startswith(
'vpn-brammer'):
GRISM_HOME = '/Users/brammer/3DHST/Spectra/Work/'
GRISM_HOME = os.getenv('THREEDHST')
if GRISM_HOME[-1] != '/':
GRISM_HOME += '/'
c.fonts.monospace = '"xos4 Terminus", Terminus, Monospace, "DejaVu Sans Mono", Monaco, "Bitstream Vera Sans Mono", "Andale Mono", "Courier New", Courier, "Liberation Mono", monospace, Fixed, Consolas, Terminal'
c.fonts.prompts = '10pt sans-serif'
c.fonts.statusbar = '10pt monospace'
c.fonts.tabs = '10pt monospace'
c.fonts.web.family.cursive = ''
c.fonts.web.family.fantasy = ''
c.fonts.web.family.fixed = 'MesloLGL Nerd Font Mono'
c.fonts.web.family.sans_serif = 'SFNS Display'
c.fonts.web.family.serif = 'Linux Libertine'
c.fonts.web.family.standard = 'SFNS Display'
c.fonts.web.size.default_fixed = 13
c.fonts.web.size.minimum = 0
c.fonts.web.size.minimum_logical = 6
### device specific fonts config
if hostname() == "dauntless":
c.fonts.web.size.default = 16
if hostname() == "vanguard":
c.fonts.web.size.default = 18
## Hints
c.hints.auto_follow = 'always'
c.hints.auto_follow_timeout = 0
c.hints.chars = 'asdfghjkl'
c.hints.hide_unmatched_rapid_hints = True
c.hints.min_chars = 1
c.hints.mode = 'letter'
c.hints.next_regexes = [
'\\bnext\\b', '\\bmore\\b', '\\bnewer\\b', '\\b[>→≫]\\b', '\\b(>>|»)\\b',
'\\bcontinue\\b', '\\b[« Newer]\\b'
]
def main():
"""
Usage: python3 send_mail.py email1 email2 etc.
"""
recipients = sys.argv[1:]
host = hostname()
conn = dbConn()
filterOutUselessRecs(conn)
msg = MIMEMultipart()
msg['Subject'] = 'Eyra at {} has recorded {} utterances'.format(host, countRecordings(conn))
msg['To'] = ', '.join(recipients)
msg['From'] = SENDER
body = """
SUMMARY:
Recorded utterances: {recordings}
Distinct session count: {sessions}
""".format(recordings=countRecordings(conn), sessions=countSessions(conn))
body += """
RECORDINGS BY USERNAME:
{lines}
""".format(lines='\n'.join(['{:<4} {}'.format(count, user)
for user, count in recsByUser(conn)]))
# TODO: by date (we have to parse text...)
#byDate
body += """
RECORDINGS BY GENDER:
{lines}
""".format(lines='\n'.join('{:<6} {}'.format(count, gender)
for gender, count in recsByGender(conn)))
body += """
RECORDINGS BY DAY:
{lines}
""".format(lines='\n'.join('{:<6} {}'.format(count, day)
for day, count in recsByDate(conn)))
body += """
RECORDINGS BY AGE GROUP:
{lines}
""".format(lines='\n'.join('{:<6} {}'.format(count, year)
for year, count in recsByAge(conn)))
body += """
RECORDINGS BY PHONE IMEI:
{lines}
""".format(lines='\n'.join('{:<4} {}'.format(count, imei)
for imei, count in recsByDevice(conn)))
body += """
RECORDINGS BY APK AND OS:
{lines}
""".format(lines='\n'.join('{:<6} {}'.format(count, apkAndOs) for
apkAndOs, count in recsByApkAndOs(conn)))
part = MIMEText('text', "plain")
part.set_payload(body)
msg.attach(part)
session = smtplib.SMTP(SMTP_SERVER, SMTP_PORT)
session.ehlo()
session.starttls()
session.ehlo
session.login(SENDER, PASSWORD)
session.sendmail(SENDER, recipients, msg.as_string())
session.quit()
sys.exit(0)