def after_step(context, step):
# if BEHAVE_DEBUG_ON_ERROR and step.status == "failed":
if step.status == "failed":
# import ipdb
# ipdb.post_mortem(step.exc_traceback)
from subprocess import Popen as sub_popen
from subprocess import PIPE as sub_PIPE
ft_name = context.scenario.feature.name
scenario_type = context.scenario.keyword # e.g., u'Scenario Outline'
if ft_name == 'Verify Online Presence':
active_outline = context.active_outline
if context.timeout:
msg = "Unit Test Failed: %s [%s] did not load in %s"%tuple(active_outline.cells)
else: msg = "Unit Test Failed: %s -- msg(%s),code(%s)"%(active_outline[0],context.resp_msg,context.resp_code)
cmd = 'echo "%s" | mail -s "Unit-Test Failure" [email protected]'%msg
proc = sub_popen([''.join(cmd)], stdout=sub_PIPE, shell=True)
(t, err) = proc.communicate()
else:
msg = 'Unit Test Failed: %s'%ft_name
cmd = 'echo "%s" | mail -s "Unit-Test Failure" [email protected]'%msg
proc = sub_popen([''.join(cmd)], stdout=sub_PIPE, shell=True)
(t, err) = proc.communicate()
def exec_cmds(cmds,cmd_host,this_worker):
cmd = ' '.join(cmds)
if cmd_host==this_worker:
p = sub_popen(cmd,stdout=sub_PIPE,shell=True)
else:
cmd = "ssh %s '%s'" % (cmd_host,cmd)
p = sub_popen(cmd,stdout=sub_PIPE,shell=True)
return p.communicate()
def after_step(context, step):
ft_name = context.feature.name
scenario_name = context.scenario.name
step_name = step.name
if step.status == "failed" and [it for it in context.tags].count('debug')>0:
import ipdb
ipdb.post_mortem(step.exc_traceback)
elif step.status == "failed" and [BEHAVE_TXT_ON_ERROR].count(True)>0:
msg = ['Failure: %s --'%ft_name,
'Feature: %s'%ft_name,
'Scenario: %s'%scenario_name,
'STEPS:']
all_steps = context.scenario.steps
for it in all_steps:
s = it.name
if s==step_name:
msg.append('\tFAILED -->> '+s)
break
else:
msg.append('\t'+s)
msg.append('VARS:')
std_vars = ['scenario', 'tags', 'text', 'stdout_capture', 'feature', 'log_capture', 'table', 'stderr_capture']
current_vars = context._record.keys()
for it in current_vars:
if std_vars.count(it)==0:
try:
msg.append('\t'+it+': '+str(context.__getattr__(it)))
except:
pass
msg.append('-- END --')
pb_msg = '\n'.join(msg)
pb_url = make_pastebin(pb_msg,msg[0],'2W',1)
mail_msg = '\n'.join([msg[0],pb_url])
cmd = 'logger -t "Aprinto_Unit_Test" "%s"' % mail_msg.replace('\n',' ').replace('-','')
proc = sub_popen([''.join(cmd)], stdout=sub_PIPE, shell=True)
(t, err) = proc.communicate()
if BEHAVE_TXT_ON_ERROR==True:
cmd = 'echo "%s" | mail -t [email protected]'%mail_msg
proc = sub_popen([''.join(cmd)], stdout=sub_PIPE, shell=True)
(t, err) = proc.communicate()
def check_db_for_action():
cmd = """ select * from system_log
where operation='rotate_pgdump'
and stout='go';
"""
df = pd.read_sql( cmd,SYS_R.T.sys_eng)
if len(df):
# from ipdb import set_trace as i_trace; i_trace()
for idx,row in df.iterrows():
logs = row.parameters.split(',')
cmds = []
for f in logs:
cmds.append( 'rm %s;' % f.strip())
p = sub_popen(cmds,stdout=sub_PIPE,shell=True)
(_out, _err) = p.communicate()
assert not _out
assert _err is None
cmd = """ update system_log set ended=now()
where uid = %d;
""" % row.uid
SYS_R.T.conn.set_isolation_level( 0)
SYS_R.T.cur.execute( cmd)
SYS_R._growl( 'pgdump logrotate process finished')
def wrapper(func):
declared_args = getattr(func, ATTR_ARGS, [])
run_qry = True
if parsed_args:
run_qry, qry_vars = False, []
for it in parsed_args:
if in_args.count(it):
idx = in_args.index(it)
qry_vars.append(in_args[idx + 1])
if len(qry_vars) == len(parsed_args):
query = query % tuple(qry_vars)
run_qry = True
if not run_qry or query.count('%s'):
return []
cmd = 'curl --get "http://%s:9999/qry" --data-urlencode "qry=%s" 2> /dev/null ;'
_out, _err = sub_popen(cmd % (DB_HOST, query.replace('\n', '')),
stdout=sub_PIPE,
shell=True).communicate()
assert _err is None
try:
j_res = eval(_out)
except:
print 'ERROR -- dropping to ipy'
i_trace()
# for it in ['action','default','choices']:
# if declared_args.has_key(it):
# del declared_args[it]
setattr(func, 'res', sorted([it['res'] for it in j_res]))
return func
def wrapper(func):
declared_args = getattr(func, ATTR_ARGS, [])
run_qry = True
if parsed_args:
run_qry, qry_vars = False, []
for it in parsed_args:
if in_args.count(it):
idx = in_args.index(it)
qry_vars.append(in_args[idx + 1])
if len(qry_vars) == len(parsed_args):
query = query % tuple(qry_vars)
run_qry = True
if not run_qry or query.count('%s'):
return []
cmd = 'curl --get "http://%s:9999/qry" --data-urlencode "qry=%s" 2> /dev/null ;'
_out, _err = sub_popen(cmd % (DB_HOST, query.replace('\n', '')), stdout=sub_PIPE, shell=True).communicate()
assert _err is None
try:
j_res = eval(_out)
except:
print 'ERROR -- dropping to ipy'
i_trace()
# for it in ['action','default','choices']:
# if declared_args.has_key(it):
# del declared_args[it]
setattr(func, 'res', sorted([ it['res'] for it in j_res ]))
return func
def parse_choices_from_pgsql(query, parsed_args = []):
global pgsql_queries
query = re.sub('[\\s]{2,}', ' ', query).strip()
if pgsql_queries.has_key(query):
return pgsql_queries[query]
from system_settings import DB_HOST
run_qry = True
if parsed_args:
run_qry, qry_vars = False, []
for it in parsed_args:
if in_args.count(it):
idx = in_args.index(it)
qry_vars.append(in_args[idx + 1])
if len(qry_vars) == len(parsed_args):
query = query % tuple(qry_vars)
run_qry = True
if not run_qry or query.count('%s'):
return []
cmd = 'curl -s http://%s:9999/curl_query?qry="%s" 2> /dev/null;'
cmd = re.sub('[\\s]{2,}', ' ', cmd % (DB_HOST, query)).strip()
_out, _err = sub_popen(cmd, stdout=sub_PIPE, shell=True).communicate()
assert _err is None
try:
j_res = eval(_out)
this_res = sorted([ it['res'] for it in j_res ])
pgsql_queries[query] = this_res
return this_res
except:
print 'ERROR -- dropping to ipy'
import ipdb as I; I.set_trace()
def wrapper(func):
declared_args = getattr(func, ATTR_ARGS, [])
run_cmd = True
if parsed_args:
run_cmd, cmd_vars = False, []
for it in parsed_args:
if in_args.count(it):
idx = in_args.index(it)
cmd_vars.append(in_args[idx + 1])
if len(cmd_vars) == len(parsed_args):
cmd = cmd % tuple(cmd_vars)
run_cmd = True
if not run_cmd:
RES = []
# os.system( "logger -t 'PY_PARSE_EXEC_1' '%s'" % str(cmd))
_out, _err = sub_popen(cmd.replace('\n', ''),
stdout=sub_PIPE,
shell=True).communicate()
assert _err is None
RES = _out
# for it in ['action','default','choices']:
# if declared_args.has_key(it):
# del declared_args[it]
setattr(func, 'res', RES)
return func
def parse_choices_from_pgsql(query, parsed_args=[]):
global pgsql_queries
query = re.sub('[\\s]{2,}', ' ', query).strip()
if pgsql_queries.has_key(query):
return pgsql_queries[query]
from system_settings import DB_HOST
run_qry = True
if parsed_args:
run_qry, qry_vars = False, []
for it in parsed_args:
if in_args.count(it):
idx = in_args.index(it)
qry_vars.append(in_args[idx + 1])
if len(qry_vars) == len(parsed_args):
query = query % tuple(qry_vars)
run_qry = True
if not run_qry or query.count('%s'):
return []
cmd = 'curl -s http://%s:9999/curl_query?qry="%s" 2> /dev/null;'
cmd = re.sub('[\\s]{2,}', ' ', cmd % (DB_HOST, query)).strip()
_out, _err = sub_popen(cmd, stdout=sub_PIPE, shell=True).communicate()
assert _err is None
try:
j_res = eval(_out)
this_res = sorted([it['res'] for it in j_res])
pgsql_queries[query] = this_res
return this_res
except:
print 'ERROR -- dropping to ipy'
import ipdb as I
I.set_trace()
def check_db_for_action():
cmd = """ select * from system_log
where operation='rotate_pgdump'
and stout='go';
"""
df = pd.read_sql(cmd, SYS_R.T.sys_eng)
if len(df):
# from ipdb import set_trace as i_trace; i_trace()
for idx, row in df.iterrows():
logs = row.parameters.split(',')
cmds = []
for f in logs:
cmds.append('rm %s;' % f.strip())
p = sub_popen(cmds, stdout=sub_PIPE, shell=True)
(_out, _err) = p.communicate()
assert not _out
assert _err is None
cmd = """ update system_log set ended=now()
where uid = %d;
""" % row.uid
SYS_R.T.conn.set_isolation_level(0)
SYS_R.T.cur.execute(cmd)
SYS_R._growl('pgdump logrotate process finished')
def run_cmd(cmd):
p = sub_popen(cmd,stdout=sub_PIPE,
shell=True,
executable='/bin/bash')
(_out,_err) = p.communicate()
assert _err is None
return _out.rstrip('\n')
def get_console_output():
global tty
if not tty:
cmd = 'env ps -o tty,cmd -e | grep \'logrotate_console.py\' | grep -v grep | column -t | cut -d \' \' -f1;'
p = sub_popen(cmd,stdout=sub_PIPE,shell=True)
(_out, _err) = p.communicate()
tty = '/dev/%s' % _out.strip()
return tty
def get_console_output():
global tty
if not tty:
cmd = 'env ps -o tty,cmd -e | grep \'logrotate_console.py\' | grep -v grep | column -t | cut -d \' \' -f1;'
p = sub_popen(cmd, stdout=sub_PIPE, shell=True)
(_out, _err) = p.communicate()
tty = '/dev/%s' % _out.strip()
return tty
def post_to_wordpress(args):
"""convert markdown, clean up html, post to wordpress"""
convert_cmd = "grip --gfm --export --wide %s 2>&1"
post_cmd = "wp post create %s --post_title='%s'"
if hasattr(args,'input'):
(_out,_err) = sub_popen(convert_cmd % args.input,stdout=sub_PIPE,
shell=True).communicate()
assert _out.count('Exporting to ')
args.html_file = _out.replace('Exporting to ','').strip('\n')
res = cleanup_grip(args.html_file)
assert res==True
(_out,_err) = sub_popen(post_cmd % (args.html_file,args.title),
stdout=sub_PIPE,shell=True).communicate()
print 'Posted!'
def print_to_console(msg):
tty = get_console_output(obj_type)
cmd = 'printf \'\n%s\n\n\' "%s" > %s 2>&1' % ('%s',msg,tty)
p = sub_popen(cmd,stdout=sub_PIPE,shell=True)
(_out, _err) = p.communicate()
assert _out == ''
assert _err is None
return
def print_to_console(msg):
tty = get_console_output(obj_type)
cmd = 'printf \'\n%s\n\n\' "%s" > %s 2>&1' % ('%s', msg, tty)
p = sub_popen(cmd, stdout=sub_PIPE, shell=True)
(_out, _err) = p.communicate()
assert _out == ''
assert _err is None
return
def post_to_wordpress(args):
"""convert markdown, clean up html, post to wordpress"""
convert_cmd = "grip --gfm --export --wide %s 2>&1"
post_cmd = "wp post create %s --post_title='%s'"
if hasattr(args, 'input'):
(_out, _err) = sub_popen(convert_cmd % args.input,
stdout=sub_PIPE,
shell=True).communicate()
assert _out.count('Exporting to ')
args.html_file = _out.replace('Exporting to ', '').strip('\n')
res = cleanup_grip(args.html_file)
assert res == True
(_out, _err) = sub_popen(post_cmd % (args.html_file, args.title),
stdout=sub_PIPE,
shell=True).communicate()
print 'Posted!'
def parse_choices_from_exec(cmd, parsed_args = []):
run_cmd = True
if parsed_args:
run_cmd, cmd_vars = False, []
for it in parsed_args:
if in_args.count(it):
idx = in_args.index(it)
cmd_vars.append(in_args[idx + 1])
if len(cmd_vars) == len(parsed_args):
cmd = cmd % tuple(cmd_vars)
run_cmd = True
if not run_cmd:
return []
# os.system( "logger -t 'PY_PARSE_EXEC_1' '%s'" % str(cmd))
_out, _err = sub_popen(cmd.replace('\n', ''), stdout=sub_PIPE, shell=True).communicate()
assert _err is None
return _out
def parse_choices_from_exec(cmd, parsed_args=[]):
run_cmd = True
if parsed_args:
run_cmd, cmd_vars = False, []
for it in parsed_args:
if in_args.count(it):
idx = in_args.index(it)
cmd_vars.append(in_args[idx + 1])
if len(cmd_vars) == len(parsed_args):
cmd = cmd % tuple(cmd_vars)
run_cmd = True
if not run_cmd:
return []
# os.system( "logger -t 'PY_PARSE_EXEC_1' '%s'" % str(cmd))
_out, _err = sub_popen(cmd.replace('\n', ''), stdout=sub_PIPE,
shell=True).communicate()
assert _err is None
return _out
def before_scenario(context, scenario):
celery_tails = ['A manager prints a Client Contract',
'After a manager prints a Client Contract, the Client prints the same Contract',
'Check In Requests & Document Post Attempts Made to Aporo']
if scenario.name.find('Check In Requests')>=0:
t = str(scenario.steps[1])
t = t[t.find('"')+1:t.rfind('"')]
cred,machine_id = re_findall(r'\"(.+?)\"',t)
if cred=='manager':
t=pd_read_sql(""" INSERT INTO aprinto_access (date_created,machine_id,ip_addr,vend_name,known_user,type_admin)
VALUES ('now'::timestamp with time zone,'%s','%s','tmp','true','true')
RETURNING uid
""" % (machine_id,THIS_IP),aprinto_engine)['uid'][0]
context.row_created = t
elif cred=='vendor':
t=pd_read_sql(""" INSERT INTO aprinto_access (date_created,machine_id,ip_addr,vend_name,known_user,type_vendor)
VALUES ('now'::timestamp with time zone,'%s','%s','tmp','true','true')
RETURNING uid
""" % (machine_id,THIS_IP),aprinto_engine)['uid'][0]
context.row_created = t
if celery_tails.count(scenario.name)>0:
context.celery_tail = '/tmp/aprinto_celery_tail'
cmd = PY_PATH + '/tests/files/tail_celery.bash'
proc = sub_popen([''.join(cmd)], stdout=sub_PIPE, shell=True)
(t, err) = proc.communicate()
if hasattr(context,'processes'):
context.processes.append(str(t))
else:
context.processes = [str(t)]
# from ipdb import set_trace as i_trace; i_trace()
delay(2)
def wrapper(func):
declared_args = getattr(func, ATTR_ARGS, [])
run_cmd = True
if parsed_args:
run_cmd, cmd_vars = False, []
for it in parsed_args:
if in_args.count(it):
idx = in_args.index(it)
cmd_vars.append(in_args[idx + 1])
if len(cmd_vars) == len(parsed_args):
cmd = cmd % tuple(cmd_vars)
run_cmd = True
if not run_cmd:
RES = []
# os.system( "logger -t 'PY_PARSE_EXEC_1' '%s'" % str(cmd))
_out, _err = sub_popen(cmd.replace('\n', ''), stdout=sub_PIPE, shell=True).communicate()
assert _err is None
RES = _out
# for it in ['action','default','choices']:
# if declared_args.has_key(it):
# del declared_args[it]
setattr(func, 'res', RES)
return func
from subprocess import Popen as sub_popen
from subprocess import PIPE as sub_PIPE
from os.path import isdir
folders = ['/Volumes/mb/Users/admin',
'/Volumes/mbp1/Users/admin',
'/Volumes/mbp2/Users/admin']
for it in folders:
if isdir(it):
cmd = ['cp -R ~/.alias_shared %s/'%it]
proc = sub_popen(cmd, stdout=sub_PIPE, shell=True)
(t, err) = proc.communicate()
def main(args,kwargs):
if args.count('requests'): import requests
if args.count('urllib'): from urllib import quote_plus,unquote
import datetime as DT
import time
delay = time.sleep
from dateutil import parser as DU # e.g., DU.parse('some date as str') --> obj(datetime.datetime)
from re import findall as re_findall
from re import sub as re_sub # re_sub('patt','repl','str','cnt')
from re import search as re_search # re_search('patt','str')
import json
from subprocess import Popen as sub_popen
from subprocess import PIPE as sub_PIPE
from traceback import format_exc as tb_format_exc
from sys import exc_info as sys_exc_info
from types import NoneType
from uuid import uuid4 as get_guid
from py_classes import To_Class,To_Class_Dict,To_Sub_Classes
import pandas as pd
pd.set_option( 'expand_frame_repr', False)
pd.set_option( 'display.max_columns', None)
pd.set_option( 'display.max_colwidth', 250)
pd.set_option( 'display.max_rows', 1000)
pd.set_option( 'display.width', 1500)
pd.set_option( 'display.colheader_justify','left')
np = pd.np
np.set_printoptions( linewidth=1500,threshold=np.nan)
import logging
logger = logging.getLogger( 'sqlalchemy.dialects.postgresql')
logger.setLevel(logging.INFO)
if args.count('pgsql'):
from sqlalchemy import create_engine
from psycopg2 import connect as pg_connect
try:
eng = create_engine(r'postgresql://%(DB_USER)s:%(DB_PW)s@%(DB_HOST)s:%(DB_PORT)s/%(DB_NAME)s'
% self.T.pgsql,
encoding='utf-8',
echo=False)
conn = pg_connect("dbname='%(DB_NAME)s' host='%(DB_HOST)s' port=%(DB_PORT)s \
user='******' password='******' "
% self.T.pgsql);
cur = conn.cursor()
except:
from getpass import getpass
pw = getpass('Root password (to create DB:"%(DB_NAME)s" via CL): ' % self.T.pgsql)
p = sub_popen(" ".join(["echo '%s' | sudo -S prompt='' " % pw,
'su postgres -c "psql --cluster 9.4/main -c ',
"'create database %(DB_NAME)s;'" % self.T.pgsql,
'"']),
stdout=sub_PIPE,
shell=True)
(_out, _err) = p.communicate()
assert _err is None
import inspect, os
D = {'guid' : str(get_guid().hex)[:7],
'pg_classes_pwd' : os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
}
D.update( {'tmp_tbl' : 'tmp_'+D['guid'],
'current_filepath' : inspect.getfile(inspect.currentframe())})
T = To_Class_Dict( self,
dict_list=[D,locals()],
update_globals=True)
def run_method(illustrate=True):
"""
Every time log files are evaluated, keep a log file if:
(1) the log file falls on a marked day as graphed above, or
(2) there is no log file for a marked day "A" and
a log file "F" is the closest in time
between and including
the day before X
and
the day after the next oldest marked day "B",
i.e., B+1<= F <=A-1
GOAL: find log closest to ILD
"""
check_db_for_action()
df = make_ideal_log_spectrum()
# Drop non-log dates for now (and in practice, but will re-include for illustration)
idx = df[df.log_dates.isnull()].index
df = df.drop(idx, axis=0).reset_index(drop=True)
# Conform Data Type
df['log_dates'] = df.log_dates.map(lambda D: pd.to_datetime(D))
ideal_log_dates = df.log_dates.tolist()
# Get Logs
p = sub_popen(['ls ~/.pg_dump'], stdout=sub_PIPE, shell=True)
(_out, _err) = p.communicate()
logs = _out.strip('\n').split('\n')
log_dates = map(
lambda X: dt.datetime.strptime(
"%s/%s/%s" % re_findall(r'(\d{4})[_\.](\d{2})[_\.](\d{2})\.', X)[
0], "%Y/%m/%d"), logs)
log_dict = dict(zip(map(lambda D: pd.to_datetime(D), log_dates), logs))
lf = pd.DataFrame(data={'logs': pd.unique(log_dates)})
# Find Intersecting Values
initial_matches = pd.Series(
pd.np.intersect1d(df.log_dates.values, lf.logs.values))
# (1) the log file falls on a marked day
lf['keep'] = lf.logs.where(lf.logs.isin(initial_matches))
df['paired'] = df.log_dates.where(df.log_dates.isin(initial_matches))
# (2) What is left?
# A. Check by getting date bounds of unclaimed logs,
# then counting how many remaining ILDs are not yet paired with a log.
# B. Iterate these remaining ILDs to match up with log,
# then discard any unmatched logs.
# (A)
to_check = lf[lf.keep.isnull()]
oldest_log, latest_log = to_check.logs.min(), to_check.logs.max()
older_dates, earlier_dates = df[df.log_dates < oldest_log], df[
latest_log < df.log_dates]
assert len(older_dates) + len(earlier_dates) >= 2
next_older_date, prev_earlier_date = older_dates.iloc[
0, :], earlier_dates.iloc[-1, :]
idl_dates = df.ix[prev_earlier_date.name:next_older_date.name, :]
# (B)
pt, last_idx = 0, idl_dates.index.tolist()[-1]
for idx, row in idl_dates.iterrows():
if idx == last_idx:
break
if pd.isnull(row.paired):
A, B = row.log_dates, idl_dates.iloc[pt + 1, :].log_dates
possible_logs = lf[(lf.logs < A) & (B < lf.logs)]
if len(possible_logs):
res = possible_logs.sort('logs',
ascending=False).iloc[0, :].logs
D = row.to_dict()
D.update({'paired': res})
df[df.index == idx].update(D.values())
pt += 1
# Find Intersecting Values b/t Paired IDLs and Remaining Logs
final_matches = pd.Series(
pd.np.intersect1d(idl_dates.paired.values, lf.logs.values))
lf.keep.update(lf.logs.where(lf.logs.isin(final_matches)))
if illustrate:
# SHOW ME THE RESULTS: [ what did we want, what did we get, what did we do ]
# Plot these "IDLs" as blue vertical bars
# Then Overlay all logs in yellow
# Then Overlay all logs to be deleted in red
start, end = lf.logs.max(), lf.logs.min()
one_day = dt.timedelta(days=+1)
res = pd.DataFrame({
'dates':
[start - (i * one_day) for i in range((start - end).days)]
})
res['days'] = res.dates.map(lambda D: D.dayofyear)
ndf = make_ideal_log_spectrum()
ndf['log_dates'] = ndf.log_dates.map(lambda D: pd.to_datetime(D))
all_log_dates = ndf.log_dates.tolist()
res['IDLs'] = res.dates.map(lambda D: 0
if not all_log_dates.count(D) else 3)
logs_to_keep = lf[lf.keep.notnull()].logs.tolist()
logs_to_delete = lf[lf.keep.isnull()].logs.tolist()
res['Keep'] = res.dates.map(lambda D: 0
if not logs_to_keep.count(D) else 2)
res['Delete'] = res.dates.map(lambda D: 0
if not logs_to_delete.count(D) else 1)
# Make Plot
import pylab as plt
fig = plt.figure()
axes = fig.add_subplot(1, 1, 1)
res.plot(x='days',
y='IDLs',
ylim=(0, 6),
ax=axes,
kind='bar',
figsize=(107, 8),
color='b')
res.plot(x='days',
y='Keep',
ylim=(0, 6),
ax=axes,
kind='bar',
figsize=(107, 8),
color='y')
res.plot(x='days',
y='Delete',
ylim=(0, 6),
ax=axes,
kind='bar',
figsize=(107, 8),
color='r')
axes.invert_xaxis()
fig.savefig('/Volumes/mbp2/Users/admin/Desktop/plot.png')
log_files = res[res.Delete == 1].dates.map(log_dict).tolist()
cmd = """ insert into system_log (operation,started,parameters)
values ('rotate_pgdump',now(),'%s')
""" % str(log_files).strip('[]').replace("'", '').replace(
' ', '')
SYS_R.T.conn.set_isolation_level(0)
SYS_R.T.cur.execute(cmd)
SYS_R._growl(
'check desktop for intended logrotate actions and update pgsql')
def run_method(illustrate=True):
"""
Every time log files are evaluated, keep a log file if:
(1) the log file falls on a marked day as graphed above, or
(2) there is no log file for a marked day "A" and
a log file "F" is the closest in time
between and including
the day before X
and
the day after the next oldest marked day "B",
i.e., B+1<= F <=A-1
GOAL: find log closest to ILD
"""
check_db_for_action()
df = make_ideal_log_spectrum()
# Drop non-log dates for now (and in practice, but will re-include for illustration)
idx = df[df.log_dates.isnull()].index
df = df.drop(idx,axis=0).reset_index(drop=True)
# Conform Data Type
df['log_dates'] = df.log_dates.map(lambda D: pd.to_datetime(D))
ideal_log_dates = df.log_dates.tolist()
# Get Logs
p = sub_popen(['ls ~/.pg_dump'],stdout=sub_PIPE,shell=True)
(_out, _err) = p.communicate()
logs = _out.strip('\n').split('\n')
log_dates = map(lambda X: dt.datetime.strptime("%s/%s/%s" %
re_findall(r'(\d{4})[_\.](\d{2})[_\.](\d{2})\.',X)[0],
"%Y/%m/%d"),logs)
log_dict = dict(zip(map(lambda D: pd.to_datetime(D),log_dates),logs))
lf = pd.DataFrame(data={'logs':pd.unique(log_dates)})
# Find Intersecting Values
initial_matches = pd.Series(pd.np.intersect1d(df.log_dates.values,lf.logs.values))
# (1) the log file falls on a marked day
lf['keep'] = lf.logs.where(lf.logs.isin(initial_matches))
df['paired'] = df.log_dates.where(df.log_dates.isin(initial_matches))
# (2) What is left?
# A. Check by getting date bounds of unclaimed logs,
# then counting how many remaining ILDs are not yet paired with a log.
# B. Iterate these remaining ILDs to match up with log,
# then discard any unmatched logs.
# (A)
to_check = lf[lf.keep.isnull()]
oldest_log,latest_log = to_check.logs.min(),to_check.logs.max()
older_dates,earlier_dates = df[df.log_dates<oldest_log],df[latest_log<df.log_dates]
assert len(older_dates)+len(earlier_dates)>=2
next_older_date,prev_earlier_date = older_dates.iloc[0,:],earlier_dates.iloc[-1,:]
idl_dates = df.ix[prev_earlier_date.name:next_older_date.name,:]
# (B)
pt,last_idx = 0,idl_dates.index.tolist()[-1]
for idx,row in idl_dates.iterrows():
if idx==last_idx:
break
if pd.isnull(row.paired):
A,B=row.log_dates,idl_dates.iloc[pt+1,:].log_dates
possible_logs = lf[(lf.logs<A)&(B<lf.logs)]
if len(possible_logs):
res = possible_logs.sort('logs',ascending=False).iloc[0,:].logs
D=row.to_dict()
D.update({'paired':res})
df[df.index==idx].update(D.values())
pt+=1
# Find Intersecting Values b/t Paired IDLs and Remaining Logs
final_matches = pd.Series(pd.np.intersect1d(idl_dates.paired.values,lf.logs.values))
lf.keep.update(lf.logs.where(lf.logs.isin(final_matches)))
if illustrate:
# SHOW ME THE RESULTS: [ what did we want, what did we get, what did we do ]
# Plot these "IDLs" as blue vertical bars
# Then Overlay all logs in yellow
# Then Overlay all logs to be deleted in red
start,end=lf.logs.max(),lf.logs.min()
one_day = dt.timedelta(days=+1)
res = pd.DataFrame({'dates':[start-(i*one_day) for i in range( (start-end).days )] })
res['days'] = res.dates.map(lambda D: D.dayofyear)
ndf = make_ideal_log_spectrum()
ndf['log_dates'] = ndf.log_dates.map(lambda D: pd.to_datetime(D))
all_log_dates = ndf.log_dates.tolist()
res['IDLs'] = res.dates.map(lambda D: 0 if not all_log_dates.count(D) else 3)
logs_to_keep = lf[lf.keep.notnull()].logs.tolist()
logs_to_delete = lf[lf.keep.isnull()].logs.tolist()
res['Keep'] = res.dates.map(lambda D: 0 if not logs_to_keep.count(D) else 2)
res['Delete'] = res.dates.map(lambda D: 0 if not logs_to_delete.count(D) else 1)
# Make Plot
import pylab as plt
fig = plt.figure()
axes = fig.add_subplot(1,1,1)
res.plot(x='days',y='IDLs',ylim=(0,6),ax=axes,kind='bar',figsize=(107,8),color='b')
res.plot(x='days',y='Keep',ylim=(0,6),ax=axes,kind='bar',figsize=(107,8),color='y')
res.plot(x='days',y='Delete',ylim=(0,6),ax=axes,kind='bar',figsize=(107,8),color='r')
axes.invert_xaxis()
fig.savefig('/Volumes/mbp2/Users/admin/Desktop/plot.png')
log_files = res[res.Delete==1].dates.map(log_dict).tolist()
cmd = """ insert into system_log (operation,started,parameters)
values ('rotate_pgdump',now(),'%s')
""" % str(log_files).strip('[]').replace("'",'').replace(' ','')
SYS_R.T.conn.set_isolation_level( 0)
SYS_R.T.cur.execute( cmd)
SYS_R._growl('check desktop for intended logrotate actions and update pgsql')
def run_cmd(cmd):
p = sub_popen(cmd,stdout=sub_PIPE,shell=True,executable='/bin/zsh')
(_out,_err) = p.communicate()
assert _err is None
return _out.rstrip('\n')
def run_simulation(client_socket=None,run_cfgs=''):
global end_test
def signal_handler(signal,frame):
global end_test
print '\nSimulation Ending...\n'
end_test = True
SIGNAL.signal(SIGNAL.SIGINT,signal_handler)
if not run_cfgs:
run_cfgs = get_configs(client_socket=None,print_cfgs=False)
run_dir = os.environ['PWD'] + '/runtime'
cmds = [
'which logrotate;',
'env ps -ef -o tty,comm= | grep test_logrotate | column -t | cut -d \' \' -f1;',
'mkdir -p %s;' % run_dir,
]
p = sub_popen(' '.join(cmds),stdout=sub_PIPE,shell=True)
(_out, _err) = p.communicate()
assert _out
assert _err is None
_out = _out.split('\n')
T = {'_dir' : os.environ['PWD'] + '/mock_log_dir',
'f_path' : os.environ['PWD'] + '/mock_log_dir/transactions.log',
'lr_path' : _out[0],
'tty' : '/dev/%s' % _out[1],
'lr_cfg' : '%s/log_r_config' % run_dir,
'lr_status' : '%s/log_r_status' % run_dir,
'cfg_insert' : ''.join(['\t%s\n' % it for it in run_cfgs]),
}
runtime_cfg = [
'"%(_dir)s/*.log" {' % T,
T['cfg_insert'],
'}'
]
with open(T['lr_cfg'],'w') as f:
f.write( '\n'.join(runtime_cfg))
logrotate_prep="""
#!/bin/bash
mkdir -p %(_dir)s
rm -f %(_dir)s/*
:> %(f_path)s
""" % T
## Setup Simulation
args = shlex.split('bash -lc \'%s\'' % logrotate_prep)
lp = sub_popen(args)
print 'Press ctrl+c to end simulation\n'
time.sleep( 1)
list_dir_cmd = shlex.split('bash -lc "ls -lAtr %(_dir)s | tail -n +2"' % T)
lr_cmd = shlex.split('bash -lc " %(lr_path)s --state %(lr_status)s %(lr_cfg)s"' % T)
while not end_test:
with open(T['f_path'],"a+") as f:
f.write( "TEST\n")
p = sub_popen(list_dir_cmd,stdout=sub_PIPE)
(_out,_err) = p.communicate()
print '\n',_out.strip('\n'),'\n'
p = sub_popen(lr_cmd,stdout=sub_PIPE)
(_out,_err) = p.communicate()
time.sleep( 1)
if end_test: break
end_test = False
return
def run_simulation(client_socket=None, run_cfgs=''):
global end_test
def signal_handler(signal, frame):
global end_test
print '\nSimulation Ending...\n'
end_test = True
SIGNAL.signal(SIGNAL.SIGINT, signal_handler)
if not run_cfgs:
run_cfgs = get_configs(client_socket=None, print_cfgs=False)
run_dir = os.environ['PWD'] + '/runtime'
cmds = [
'which logrotate;',
'env ps -ef -o tty,comm= | grep test_logrotate | column -t | cut -d \' \' -f1;',
'mkdir -p %s;' % run_dir,
]
p = sub_popen(' '.join(cmds), stdout=sub_PIPE, shell=True)
(_out, _err) = p.communicate()
assert _out
assert _err is None
_out = _out.split('\n')
T = {
'_dir': os.environ['PWD'] + '/mock_log_dir',
'f_path': os.environ['PWD'] + '/mock_log_dir/transactions.log',
'lr_path': _out[0],
'tty': '/dev/%s' % _out[1],
'lr_cfg': '%s/log_r_config' % run_dir,
'lr_status': '%s/log_r_status' % run_dir,
'cfg_insert': ''.join(['\t%s\n' % it for it in run_cfgs]),
}
runtime_cfg = ['"%(_dir)s/*.log" {' % T, T['cfg_insert'], '}']
with open(T['lr_cfg'], 'w') as f:
f.write('\n'.join(runtime_cfg))
logrotate_prep = """
#!/bin/bash
mkdir -p %(_dir)s
rm -f %(_dir)s/*
:> %(f_path)s
""" % T
## Setup Simulation
args = shlex.split('bash -lc \'%s\'' % logrotate_prep)
lp = sub_popen(args)
print 'Press ctrl+c to end simulation\n'
time.sleep(1)
list_dir_cmd = shlex.split('bash -lc "ls -lAtr %(_dir)s | tail -n +2"' % T)
lr_cmd = shlex.split(
'bash -lc " %(lr_path)s --state %(lr_status)s %(lr_cfg)s"' % T)
while not end_test:
with open(T['f_path'], "a+") as f:
f.write("TEST\n")
p = sub_popen(list_dir_cmd, stdout=sub_PIPE)
(_out, _err) = p.communicate()
print '\n', _out.strip('\n'), '\n'
p = sub_popen(lr_cmd, stdout=sub_PIPE)
(_out, _err) = p.communicate()
time.sleep(1)
if end_test: break
end_test = False
return
def __init__(self,**kwargs):
"""
pgSQL(db_settings=[DB_NAME, DB_USER, DB_PW, DB_HOST, DB_PORT])
"""
def run_cmd(cmd):
p = sub_popen(cmd,stdout=sub_PIPE,
shell=True,
executable='/bin/bash')
(_out,_err) = p.communicate()
assert _err is None
return _out.rstrip('\n')
def download_file(url,save_path):
import os
_dir = save_path[:save_path.rfind('/')]
if not os.path.exists(_dir):
os.makedirs(_dir)
with open(save_path, 'wb') as handle:
response = self.T.requests.get( url, stream=True)
if not response.ok:
# Something went wrong
print 'error'
for block in response.iter_content(1024):
if not block:
break
handle.write(block)
handle.flush()
return True
def read_json_from_url_response(url):
r = self.T.requests.get(url)
assert r.status_code=='200'
# print r.text
g = r.text
g = g.replace('true',"'true'")
a = eval(g)
return a
def to_sql(cmd):
self.T.conn.set_isolation_level( 0)
self.T.cur.execute( cmd)
def redirect_logs_to_file(file_desc='/dev/pts/0',msg_form="%(asctime)s - %(levelname)s - %(message)s"):
# print T.logger.__dict__
# print T.logger.manager.__dict__
# for it in dir(logger):
# print it,getattr(logger,it)
for it in self.T.logger.handlers:
self.T.logger.removeHandler(it)
for it in self.T.logger.parent.handlers:
self.T.logger.parent.removeHandler(it)
for it in self.T.logger.root.handlers:
self.T.logger.root.removeHandler(it)
# print logger.manager.__dict__
del_these = ['IPKernelApp','basic_logger']
for it in del_these:
if self.T.logger.manager.__dict__['loggerDict'].has_key(it):
del self.T.logger.manager.__dict__['loggerDict'][it]
for k in self.T.logger.manager.__dict__['loggerDict'].keys():
if k.count('sqlalchemy') or k.count('pandas'):
del self.T.logger.manager.__dict__['loggerDict'][k]
self.T.logging.basicConfig(filename=file_desc, level=self.T.logging.DEBUG, format=msg_form)
return
def custom_geoseries_plot(s,figsize=(8,8)):
# s=T.gd.GeoSeries(A)
colormap='Set1'
axes=None
linewidth=1.0
import matplotlib.pyplot as plt
if axes is None:
fig, ax = plt.subplots(figsize=figsize)
ax.set_aspect('equal')
else:
ax = axes
ax.get_xaxis().get_major_formatter().set_scientific(False)
ax.get_xaxis().get_major_formatter().set_useOffset(False)
plt.xticks(rotation='vertical')
ax.get_yaxis().get_major_formatter().set_scientific(False)
ax.get_yaxis().get_major_formatter().set_useOffset(False)
color = T.gd.plotting.gencolor(len(s), colormap=colormap)
for geom in s:
if geom.type == 'Polygon' or geom.type == 'MultiPolygon':
T.gd.plotting.plot_multipolygon(ax, geom, facecolor=next(color), linewidth=linewidth)
elif geom.type == 'LineString' or geom.type == 'MultiLineString':
T.gd.plotting.plot_multilinestring(ax, geom, color=next(color), linewidth=linewidth)
elif geom.type == 'Point':
T.gd.plotting.plot_point(ax, geom)
plt.ticklabel_format(style='plain')
plt.grid()
plt.draw()
def _load_connectors():
eng = create_engine(r'postgresql://%(DB_USER)s:%(DB_PW)s@%(DB_HOST)s:%(DB_PORT)s/%(DB_NAME)s'
% T,
encoding='utf-8',
echo=False)
conn = pg_connect("dbname='%(DB_NAME)s' host='%(DB_HOST)s' port=%(DB_PORT)s \
user='******' password='******' "
% T);
cur = conn.cursor()
return eng,conn,cur
import datetime as DT
dt = DT
from dateutil import parser as DU # e.g., DU.parse('some date as str') --> obj(datetime.datetime)
import time
delay = time.sleep
from urllib import quote_plus,unquote
import re
from re import findall as re_findall
from re import sub as re_sub # re_sub('patt','repl','str','cnt')
from re import search as re_search # re_search('patt','str')
import json
from subprocess import Popen as sub_popen
from subprocess import PIPE as sub_PIPE
from traceback import format_exc as tb_format_exc
from sys import exc_info as sys_exc_info
from types import NoneType
from time import sleep as delay
from uuid import uuid4 as get_guid
import requests
from py_classes.py_classes import To_Sub_Classes,To_Class,To_Class_Dict
T = To_Class()
T.config = To_Class(kwargs,recursive=True)
if hasattr(T,'config') and hasattr(T.config,'pgsql'):
T.update( T.config.pgsql.__dict__)
else:
T.update( T.config.__dict__)
db_vars = ['DB_NAME','DB_HOST','DB_PORT','DB_USER','DB_PW']
db_vars = [it for it in db_vars if not T._has_key(it)]
if not db_vars:
pass
elif locals().keys().count('db_settings'):
DB_NAME,DB_USER,DB_PW,DB_HOST,DB_PORT = db_settings
for it in db_vars:
eval('T["%s"] = %s' % (it,it))
else:
z = eval("__import__('db_settings')")
for it in db_vars:
T[it] = getattr(z,it)
import pandas as pd
pd.set_option( 'expand_frame_repr', False)
pd.set_option( 'display.max_columns', None)
pd.set_option( 'display.max_colwidth', 250)
pd.set_option( 'display.max_rows', 1000)
pd.set_option( 'display.width', 1500)
pd.set_option( 'display.colheader_justify','left')
np = pd.np
np.set_printoptions( linewidth=1500,threshold=np.nan)
# import geopandas as gd
import logging
logger = logging.getLogger( 'sqlalchemy.dialects.postgresql')
logger.setLevel(logging.INFO)
from sqlalchemy import create_engine
from psycopg2 import connect as pg_connect
try:
eng,conn,cur = _load_connectors()
except:
from getpass import getpass
pw = getpass('Root password (to create DB:"%(DB_NAME)s" via CL): ' % pgsql)
p = sub_popen(" ".join(["echo '%s' | sudo -S prompt='' " % pw,
'su postgres -c "psql --cluster 9.4/main -c ',
"'create database %(DB_NAME)s;'" % T,
'"']),
stdout=sub_PIPE,
shell=True)
(_out, _err) = p.communicate()
assert _err is None
eng,conn,cur = _load_connectors()
import inspect, os
D = {'guid' : str(get_guid().hex)[:7],
'pg_classes_pwd' : os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
}
D.update( {'tmp_tbl' : 'tmp_'+D['guid'],
'current_filepath' : inspect.getfile(inspect.currentframe())})
self.T = To_Class_Dict( self,
dict_list=[T.__dict__,D,locals()],
update_globals=True)
self.Functions = pgSQL_Functions(self)
self.Triggers = pgSQL_Triggers(self)
self.Tables = pgSQL_Tables(self)
self.Databases = pgSQL_Databases(self)
self.Types = pgSQL_Types(self)
# if hasattr(T,'project_sql_files') and T.project_sql_files:
# self.F.functions_create_from_command_line(one_directory=T.project_sql_files)
# if hasattr(T,'base_sql_files') and T.base_sql_files:
# self.F.functions_create_from_command_line(one_directory=T.base_sql_files)
if hasattr(T,'initial_check') and T.initial_check:
self.__initial_check__()
if hasattr(T,'temp_options') and T.temp_options:
self.__temp_options__()