def handle_data(self, data):
"""Function for work with data between tags"""
global title
global raw_content
try:
if data == None or data == '':
return
### Title tag ###
if is_title == 1:
# title = self.mod_data.split()
# print("Title: '", ' ""'.join(title),"'"" '", self.mod_data)
title = data + "\n(" + header_date + ")\n" + ('-' * 31) + "\n\n"
else:
if skip == 1:
return
if is_body == 1 and data and data != ' ':
raw_content += data
except Exception as e:
sys.stderr.write("ERROR ({},{}): Unable to handle data:\n\t{}: {}\n".
format(getframeinfo(currentframe()).filename,
getframeinfo(currentframe()).lineno,
str(sys.exc_info()[0]), e))
return 1
def logFunctionAndArgs(self):
frame = inspect.getouterframes(inspect.currentframe())[1][0]
args, _, _, values = inspect.getargvalues(frame)
frameinfo = inspect.getframeinfo(frame)
functionName=inspect.getframeinfo(frame)[2]
output = ""
for arg in args[1:]: #[1:] skip the first argument 'self'
value = values[arg]
if isinstance(value, str):
#add apostrophes for string values
value = "\'"+value+"\'"
elif isinstance(value, int):
value = ''.join('%02X' % value)
else:
newValue = ""
for i in value:
if isinstance(i, int):
newValue += '%02X' % i
else:
newValue += str(i)
value = newValue
output += arg + '=' + value
if arg != args[-1]:
#add comma if not the last element
output +=','
#do not print "\n' as a new line
output = output.replace("\n","\\n")
self.logging.info("--> "+functionName+'('+output+')')
def hello():
print 'hello'
frame_me = inspect.currentframe()
print inspect.currentframe()
print inspect.getframeinfo(frame_me)
print inspect.stack()[0]
print globals()['hello']
def find_nondebug_frame(obj, f):
"""Find the first frame that isn't a debugger frame.
Generally we want traceback information without polluting
it with debugger information.
"""
if obj.dbg_pydb: return f
f = obj.is_in_dbg(f)
### FIXME: would like a routine like is_in_threaddb_dispatch
### but works with threading instead. Decorating or subclassing
### threadding might do the trick.
(filename, line_no, routine) = inspect.getframeinfo(f)[0:3]
(path, basename)=os.path.split(filename)
while (basename.startswith('threading.py') or
basename.startswith('gdb.py') or
basename.startswith('threaddbg.py') or
basename.startswith('subcmd.py') or
basename.startswith('pydb.py') or
routine == 'trace_dispatch_gdb') and f.f_back:
f = f.f_back
(filename, line_no, routine) = \
inspect.getframeinfo(f)[0:3]
(path, basename)=os.path.split(filename)
return f
def run(self):
"""
Initialization of the connections and accepting incomming communications
"""
self.slave_connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM);
self.cmd_connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM);
self.slave_connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1);
self.cmd_connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1);
self.slave_connection.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1);
self.cmd_connection.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1);
s_port = self._parser.getValueFromSection(MASTER_CONF_LISTEN_SECTION, MASTER_CONF_LISTEN_PORT_SLAVE_ENTRY);
c_port = self._parser.getValueFromSection(MASTER_CONF_LISTEN_SECTION, MASTER_CONF_LISTEN_PORT_CMD_ENTRY);
if not s_port:
frameinfo = getframeinfo(currentframe());
self.logger.error('in run: No slave listening port defined in '+MASTER_CONF_FILE);
sys.exit(1);
if not c_port:
frameinfo = getframeinfo(currentframe());
self.logger.error('in run: No command listening port defined in '+MASTER_CONF_FILE);
sys.exit(1);
self.slave_connection.bind(('', int(s_port)));
self.slave_connection.listen(MAX_SLAVES);
self.cmd_connection.bind(('', int(c_port)));
self.cmd_connection.listen(MAX_CMDS);
self.loop();
def f1():
myid = mpi_comm_rank(MPI_COMM_WORLD)
program_state_stack(locals(), getframeinfo(currentframe()), "my_state.json")
for local_var_i in range(2):
for local_var_j in range(2):
if program_state_stack(locals(), getframeinfo(currentframe())):
my_s = "%s_f1_%d_%d_%03d_%03d.txt" % (
mydir,
local_var_i,
local_var_j,
myid,
getframeinfo(currentframe()).lineno,
)
f = open(my_s, "w")
f.write(my_s[5:])
f.flush()
f.close()
if program_state_stack(locals(), getframeinfo(currentframe())):
my_s = "%s_f1_%d_%d_%03d_%03d.txt" % (
mydir,
local_var_i,
local_var_j,
myid,
getframeinfo(currentframe()).lineno,
)
f = open(my_s, "w")
f.write(my_s[5:])
f.flush()
f.close()
if program_state_stack(locals(), getframeinfo(currentframe())):
my_s = "%s_f1_%d_%d_%03d_%03d.txt" % (
mydir,
local_var_i,
local_var_j,
myid,
getframeinfo(currentframe()).lineno,
)
f = open(my_s, "w")
f.write(my_s[5:])
f.flush()
f.close()
a(local_var_i, local_var_j)
if program_state_stack(locals(), getframeinfo(currentframe())):
my_s = "%s_f1_%d_%d_%03d_%03d.txt" % (
mydir,
local_var_i,
local_var_j,
myid,
getframeinfo(currentframe()).lineno,
)
f = open(my_s, "w")
f.write(my_s[5:])
f.flush()
f.close()
program_state_stack(locals(), getframeinfo(currentframe()), last_call="LastCall")
mpi_finalize()
def test_mocking_inspect_getframeinfo(self): orig = inspect.getframeinfo when(inspect).getframeinfo.then_call(orig) # just check this doesn't cause infinite recursion f = inspect.currentframe() inspect.getframeinfo(f)
def set_next_execution(self):
"""
Set self.next_execution to the next time the import is going to be run. If run_immediately
is set, time is set to now()
"""
debug_enter(who_am_i(), who_is_parent(), getframeinfo(currentframe()).lineno)
time_adds = {
self.EVERY_TWO_MINUTES: timedelta(seconds=120), # used for testing
self.HOURLY: timedelta(hours=1),
self.EVERY_SIX_HOURS: timedelta(hours=6),
self.DAILY: timedelta(days=1),
self.EVERY_THREE_DAYS: timedelta(days=3),
self.WEEKLY: timedelta(weeks=1),
self.BI_WEEKLY: timedelta(weeks=2),
self.EVERY_FOUR_WEEKS: timedelta(weeks=4),
}
if self.frequency:
if not self.next_execution:
self.next_execution = datetime.now() + time_adds[self.frequency]
else:
self.next_execution += time_adds[self.frequency]
else:
self.next_execution = None
self.save()
debug_exit(who_am_i(), getframeinfo(currentframe()).lineno)
def formatCode(frame, stream):
_updatecache = linecache.updatecache
def updatecache(*args):
# linecache.updatecache looks in the module search path for
# files that match the module name. This is a problem if you
# have a file without source with the same name as a python
# standard library module. We'll just check to see if the file
# exists first and require exact path matches.
if not os.access(args[0], os.R_OK):
return []
return _updatecache(*args)
linecache.updatecache = updatecache
try:
try:
frameInfo = inspect.getframeinfo(frame, context=1)
except:
frameInfo = inspect.getframeinfo(frame, context=0)
fileName, lineNo, funcName, text, idx = frameInfo
stream.write(' File "%s", line %d, in %s\n' %
(fileName, lineNo, funcName))
if text is not None and len(text) > idx:
# If the source file is not available, we may not be able to get
# the line
stream.write(' %s\n' % text[idx].strip())
finally:
linecache.updatecache = _updatecache
def handler(signum, frame):
import inspect
print inspect.getframeinfo(frame)
print inspect.trace()
while 1:
s = raw_input(\
"""
Enter sense switch.
sol: Print current best solution.
cont: Continue calculation.
call: Executes sigint_callback [%s].
exit: Exits with current best solution.
>>> """ % sigint_callback)
if s.lower() == 'sol':
print self.bestSolution
elif s.lower() == 'cont':
return
elif s.lower() == 'call':
# sigint call_back
if sigint_callback is not None:
sigint_callback(self.bestSolution)
elif s.lower() == 'exit':
self._EARLYEXIT = True
return
else:
print "unknown option : %s" % s
return
def logout(request):
# *************************************************************************
# *************************************************************************
# *************************************************************************
#
#
# ログアウトのアクション
#
#
# *************************************************************************
# *************************************************************************
# *************************************************************************
logger.info('<' + __name__ + '.' + inspect.getframeinfo(inspect.currentframe()).function + '()> $$$ start $$$');
# =========================================================================
# setup
# =========================================================================
# =========================================================================
# validation
# =========================================================================
# =========================================================================
# process
# =========================================================================
_logout(request)
# =========================================================================
# contents
# =========================================================================
logger.info('<' + __name__ + '.' + inspect.getframeinfo(inspect.currentframe()).function + '()> --- end ---');
return django.http.HttpResponseRedirect('/')
def set_up_data(self, data_gross=None, data_costs=None, weight_matrix=None):
if weight_matrix is not None:
setattr(self, "data", weight_matrix.ffill())
return None
log=self.log
frequency=self.frequency
equalise_gross = self.equalise_gross
cost_multiplier = self.cost_multiplier
annualisation = self.annualisation
period_target_SR = self.period_target_SR
data_gross = [data_item.cumsum().resample(frequency, how="last").diff() for
data_item in data_gross]
data_costs = [data_item.cumsum().resample(frequency, how="last").diff() for
data_item in data_costs]
data_gross=df_from_list(data_gross)
data_costs=df_from_list(data_costs)
if equalise_gross:
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Setting all gross returns to be identical - optimisation driven only by costs")
if cost_multiplier!=1.0:
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Using cost multiplier on optimisation of %.2f" % cost_multiplier)
data = work_out_net(data_gross, data_costs, annualisation=annualisation,
equalise_gross=equalise_gross, cost_multiplier=cost_multiplier,
period_target_SR=period_target_SR)
setattr(self, "data", data)
def show(request):
###########################################################################
# netstat の状態を表示するビュー
###########################################################################
logger.info('<' + __name__ + '.' + inspect.getframeinfo(inspect.currentframe()).function + '()> $$$ start $$$');
# =========================================================================
# setup
# =========================================================================
# =========================================================================
# validation
# =========================================================================
# =========================================================================
# process
# =========================================================================
# =========================================================================
# contents
# =========================================================================
fields = {}
util.fill_menu_items(request, fields)
context = django.template.RequestContext(request, fields)
template = django.template.loader.get_template('listeners/show.html')
logger.info('<' + __name__ + '.' + inspect.getframeinfo(inspect.currentframe()).function + '()> --- end ---');
return django.http.HttpResponse(template.render(context))
def loop(self):
"""
Main loop. Waits for new connections.
"""
self.run = True;
while self.run:
try:
rlist, wlist, elist = select.select([self.slave_connection], [], [], SELECT_TIMEOUT);
for connection in rlist:
self.accept_new_slave_connection(connection);
rlist, wlist, elist = select.select([self.cmd_connection], [], [], SELECT_TIMEOUT);
for connection in rlist:
self.accept_new_cmd_connection(connection);
except KeyboardInterrupt as e:
frameinfo = getframeinfo(currentframe());
self.logger.info('in loop: Keyboard interrupt: leaving program');
print("[ MASTER DAEMON " + frameinfo.filename + ":" + str(frameinfo.lineno) + " ]: Keyboard Interrupt");
self.stop();
sys.exit(0);
except ValueError as e:
frameinfo = getframeinfo(currentframe());
self.logger.error('in loop: Value error: ' + str(e));
print("[ MASTER DAEMON " + frameinfo.filename + ":" + str(frameinfo.lineno) + "]: Value Error");
print(e);
pass;
def get_catch_exception_paraname( self ):
paraname = ''
lines = self.ori_block.split( self.split_by )
for line in lines:
if SSW( line , "}" ) and SEW( line , "{" ) \
and line.find( "catch" ) > 0 :
int_l = line.find( "(" )
int_r = line.find( ")" )
if int_l > 0 and int_r > int_l :
str_a = line[ int_l + 1 : int_r ]
str_a=str_a.replace( "final" , "" )
str_a = RemoveJavaCommentsInLine( str_a )
str_a = str_a.strip()
str_a = MSTOO( str_a )
strs = str_a.split( ' ' )
if 2 != len( strs ):
print " Catch parameters variable is not right "
print self.ori_block
print line
frameinfo = getframeinfo(currentframe())
print frameinfo.filename, frameinfo.lineno
sys.exit( 0 )
paraname = strs[ 1 ]
if '' == paraname:
print " Catch parameters variable is not right "
print self.ori_block
frameinfo = getframeinfo(currentframe())
print frameinfo.filename, frameinfo.lineno
sys.exit( 0 )
return paraname
def connect_to_masters(self):
"""
Stored every device on network which have his hostname beginning by "MD3" and stores it
in the self.connected_masters dict(), with hostnames as keys and sockets freshly open as values.
"""
hostname = socket.gethostname()
self.connected_masters = {};
for host in self._hostlist:
if MASTER_NAME_PREFIX in host._Hostname or str(host._IpAddr) == '127.0.0.1':
port = self._parser.getValueFromSection(SLAVE_CONF_CONNECT_SECTION, SLAVE_CONF_CONNECT_PORT_ENTRY);
if not port:
self.logger.error('in connect_to_masters: No ' + SLAVE_CONF_CONNECT_PORT_ENTRY + ' in ' + SLAVE_CONF_CONNECT_SECTION + ' section or maybe no such ' + SLAVE_CONF_CONNECT_SECTION + ' defined');
sys.exit(1);
try:
self.logger.debug('Connecting to ' + str(host._IpAddr) + ':' + str(port));
sock = socket.create_connection((host._IpAddr, port));
hostname = host._Hostname.split('.')[0];
self.connected_masters[host._Hostname] = sock;
except Exception as e:
frameinfo = getframeinfo(currentframe());
self.logger.error('in connect_to_masters: ' + str(e));
pass;
if SLAVE_NAME_PREFIX in hostname:
port = self._parser.getValueFromSection(SLAVE_CONF_CONNECT_SECTION, SLAVE_CONF_CONNECT_PORT_ENTRY);
if not port:
self.logger.error('in connect_to_masters: No ' + SLAVE_CONF_CONNECT_PORT_ENTRY + ' in ' + SLAVE_CONF_CONNECT_SECTION + ' section or maybe no such ' + SLAVE_CONF_CONNECT_SECTION + ' defined');
sys.exit(1);
try:
self.logger.debug('Connecting to 127.0.0.1:' + str(port));
sock = socket.create_connection(('127.0.0.1', port));
self.connected_masters[hostname] = sock;
except Exception as e:
frameinfo = getframeinfo(currentframe());
self.logger.error('in connect_to_masters: ' + str(e));
pass;
def parse_module_or_class(mod_class, parser, in_args, parse_type = 'class'):
if DEBUG_PARSING: print 'LINE:',inspect.getframeinfo(inspect.currentframe()).lineno,'parse_module_or_class'
global single_function
p = parser
non_help_in_args = [ it for it in in_args if not it == '--help' ]
# function is being pulled from command line
in_args = in_args if in_args == ['--help'] else in_args[1:]
THIS_ARG = '' if not non_help_in_args else non_help_in_args[0]
p.description = mod_class.__doc__
p.usage = p.format_usage().rstrip(' [--help] \n').replace('usage: ', '') + ' ' + THIS_ARG + ' [--help]\n'
if parse_type == 'class':
fxs = [ name for name, fx in inspect.getmembers(mod_class, inspect.ismethod) if hasattr(fx, 'argh_args') ]
elif parse_type == 'module':
fxs = [ name for name, fx in inspect.getmembers(mod_class, inspect.isfunction) if hasattr(fx, 'argh_args') ]
# fxs = [name for name,fx in inspect.getmembers(mod_class,inspect.isfunction)]
# If single-class, clean usage
if no_class or single_class:
p.usage = p.usage.replace(' %s ' % THIS_ARG, ' ')
# If Class only has one function, this function is called by default
if len(fxs) == 1:
single_function = fxs[0]
if not in_args == ['--help']:
in_args.insert(0, single_function)
else:
single_function = False
# Parsing Class Ends Here
if not in_args or in_args == ['--help']:
sp = p.add_subparsers()
for it in fxs:
sp.add_parser(it, help=getattr(mod_class, it).__doc__)
# if os.environ.has_key('COMP_LINE'):
# os.system( "logger -t 'PY_PARSE_C1' '%s'" % str('>>'+os.environ['COMP_LINE']+'<<'))
# else:
# os.system( "logger -t 'PY_PARSE_C1' '%s'" % str('>>'+str(in_args)+'<<'))
p.usage = '\n\t' + p.format_usage().replace('usage: ', '', 1).replace('[--help]', '').replace(' ... ', '').rstrip(' \n')
sp.metavar = 'FUNCTION'
p.usage += ' %s [--help]' % sp.metavar
argcomplete.autocomplete(p)
if DEBUG_PARSING: print 'LINE:',inspect.getframeinfo(inspect.currentframe()).lineno
return p.parse_args()
# Unrecognized Parse instruction
if in_args and not fxs.count(in_args[0]):
argcomplete.autocomplete(p)
print 'Unrecognized <function>'
print in_args
p.print_help()
if DEBUG_PARSING: print 'LINE:',inspect.getframeinfo(inspect.currentframe()).lineno
exit(1)
# Execute Parse instruction
else:
# if os.environ.has_key('COMP_LINE'):
# os.system( "logger -t 'PY_PARSE_C3' '%s'" % str('>>'+os.environ['COMP_LINE']+'<<'))
# else:
# os.system( "logger -t 'PY_PARSE_C3' '%s'" % str('>>'+str(in_args)+'<<'))
# os.system( "logger -t 'PY_PARSE_C3' '%s'" % str('>>'+str(p.usage)+'<<'))
p_help = argparse.SUPPRESS if [ it for it in in_args if not it == '--help' ] else mod_class.__doc__
p.add_argument('_func', choices=fxs, help=p_help)
if DEBUG_PARSING: print 'LINE:',inspect.getframeinfo(inspect.currentframe()).lineno
return parse_function(getattr(mod_class, in_args[0]), p, in_args)
def main ():
setup_logging()
_monkeypatch_console()
try:
if doLaunch():
post_startup()
core.goUp()
else:
return
except SystemExit:
return
except:
import traceback
traceback.print_exc()
return
if cli:
print "This program comes with ABSOLUTELY NO WARRANTY. This program is " \
"free software,"
print "and you are welcome to redistribute it under certain conditions."
print "Type 'help(pox.license)' for details."
import pox.license
import time
time.sleep(1)
import code
import sys
sys.ps1 = "POX> "
sys.ps2 = " ... "
l = dict(locals())
l['core'] = core
code.interact('Ready.', local=l)
else:
try:
import traceback
import time
import sys
import inspect
while True:
if 'deadlock' in globals()['options'] and globals()['options']['deadlock']:
frames = sys._current_frames()
for key in frames:
frame = frames[key]
print inspect.getframeinfo(frame)
outer_frames = inspect.getouterframes(frame)
for i in range(0, len(outer_frames)):
print " " + str(inspect.getframeinfo(outer_frames[i][0]))
time.sleep(5)
except:
if 'deadlock' in globals()['options'] and globals()['options']['deadlock']:
traceback.print_exc(file=sys.stdout)
#core.scheduler._thread.join() # Sleazy
try:
pox.core.core.quit()
except:
pass
def optimise(self, ann_SR_costs=None):
"""
Optimise weights over some returns data
"""
log=self.log
date_method = self.date_method
rollyears = self.rollyears
optimiser = self.optimiser
cleaning = self.cleaning
apply_cost_weight = self.apply_cost_weight
data=getattr(self, "data", None)
if data is None:
log.critical("You need to run .set_up_data() before .optimise()")
fit_dates = generate_fitting_dates(data, date_method=date_method, rollyears=rollyears)
setattr(self, "fit_dates", fit_dates)
## Now for each time period, estimate weights
## create a list of weight vectors
weight_list=[]
## create a class object for each period
opt_results=[]
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Optimising...")
for fit_period in fit_dates:
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Optimising for data from %s to %s" % (str(fit_period.period_start), str(fit_period.period_end)))
## Do the optimisation for one period, using a particular optimiser instance
results_this_period=optSinglePeriod(self, data, fit_period, optimiser, cleaning)
opt_results.append(results_this_period)
weights=results_this_period.weights
## We adjust dates slightly to ensure no overlaps
dindex=[fit_period.period_start+datetime.timedelta(days=1),
fit_period.period_end-datetime.timedelta(days=1)]
## create a double row to delineate start and end of test period
weight_row=pd.DataFrame([weights]*2, index=dindex, columns=data.columns)
weight_list.append(weight_row)
## Stack everything up
raw_weight_df=pd.concat(weight_list, axis=0)
if apply_cost_weight:
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Applying cost weighting to optimisation results")
weight_df = apply_cost_weighting(raw_weight_df, ann_SR_costs)
else:
weight_df =raw_weight_df
setattr(self, "results", opt_results)
setattr(self, "weights", weight_df)
setattr(self, "raw_weights", raw_weight_df)
def display_function_info():
caller = inspect.getframeinfo(inspect.currentframe().f_back)[2]
print ""
print term_color.OKGREEN+"{0}:{1}".format(inspect.getframeinfo(inspect.currentframe().f_back)[0], caller) +term_color.ENDC
return
def __init__(self, a_log_flag=False):
self.log_flag = a_log_flag
callerframerecord = inspect.stack()[1]
frame = callerframerecord[0]
self.file = inspect.getframeinfo(frame).filename.split("/").pop()
self.func = inspect.getframeinfo(frame).function
self.line = inspect.getframeinfo(frame).lineno
def get_caller(self):
fm = inspect.currentframe()
fm_info = inspect.getframeinfo(fm)
curr_filename = fm_info.filename
while fm_info.filename == curr_filename:
fm = fm.f_back
fm_info = inspect.getframeinfo(fm)
return (fm, fm_info)
def trace_calls_and_returns(frame, event, arg):
co = frame.f_code
filename = co.co_filename
func_name = co.co_name
code = inspect.getframeinfo(frame)[3]
global current_trace
'''if sys==None:
return
if func_name == 'write':
# Ignore write() calls from print statements
return
'''
if filename=='/usr/lib/python2.7/socket.py': # use socket.__file__ to get this
#print filename + '/' + func_name
#traceback.print_stack(frame)
previous_frames = inspect.getouterframes(frame)
for parent in previous_frames:
frame_details = inspect.getframeinfo(parent[0])
if frame_details[2] in functions_being_traced:
#print frame_details
#TODO if already in the list, don't add a duplicate
current_trace.setdefault(frame_details[2],{}).setdefault('per_line_dependencies',{}).setdefault(frame_details[1],[]).append('network')
if code is not None:
if string.find(' '.join(code),' open(')!=-1:
previous_frames = inspect.getouterframes(frame)
for parent in previous_frames:
frame_details = inspect.getframeinfo(parent[0])
if frame_details[2] in functions_being_traced:
#print frame_details
current_trace.setdefault(frame_details[2],{}).setdefault('per_line_dependencies',{}).setdefault(frame_details[1],[]).append('filesystem')
#TODO want to trace child functions as well... but for that, need to know which files to ignore/not ignore
#figure out based on analysis of previous_frames
#return
#'kw'
#line_no = frame.f_lineno
# ignore everything outside of these files, as built-in or third-party
if func_name in functions_being_traced:#filename in files_to_trace and
'''print "%s: %s %s[%s]" % (
event,
frame.f_code.co_name,
frame.f_code.co_filename,
frame.f_lineno,
)'''
if event=='call':
#get the input parameters
#print str(frame.f_locals)
current_trace.setdefault(func_name,{})['function_name'] = func_name
current_trace.setdefault(func_name,{})['arg'] = copy.copy(frame.f_locals)
#current_trace.setdefault(str(f.__name__),{})['kw'] = kw
return trace_calls_and_returns
elif event=='line': # also gets called for return events, too!
return trace_lines
else:
return
def Test_Start(message=""):
print "_______________________________________________"
caller = inspect.getframeinfo(inspect.currentframe().f_back)[2]
print "Test Harness:" + inspect.getframeinfo(inspect.currentframe().f_back)[0]
print term_color.OKGREEN + "Entering:" + caller + term_color.ENDC
if message != "":
print "Notes:"+message
print "------------------------------------------------"
return
def handle_stackframe_without_leak():
frame = inspect.currentframe()
print "stack:"
try:
while frame!=None:
print inspect.getframeinfo(frame)
frame=frame.f_back
finally:
del frame
sys.stdout.flush()
def bootstrap_portfolio(subset_data, moments_estimator,
cleaning, must_haves,
monte_runs=100, bootstrap_length=50,
**other_opt_args):
"""
Given dataframe of returns; returns_to_bs, performs a bootstrap optimisation
We run monte_carlo numbers of bootstraps
Each one contains monte_length days drawn randomly, with replacement
(so *not* block bootstrapping)
The other arguments are passed to the optimisation function markosolver
:param subset_data: The data to optimise over
:type subset_data: pd.DataFrame TxN
:param cleaning: Should we clean correlations so can use incomplete data?
:type cleaning: bool
:param must_haves: The indices of things we must have weights for when cleaning
:type must_haves: list of bool
:param monte_runs: The number of bootstraps to do
:type monte_runs: list of float
:param bootstrap_length: Number of periods in each bootstrap
:type bootstrap_length: int
*_params passed through to data estimation functions
**other_opt_args passed to single period optimiser
:returns: float
"""
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" + "bootstrap_length=" + str(bootstrap_length))
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" + "bootstrap_length=" + str(type(moments_estimator)))
all_results=[bs_one_time(subset_data, moments_estimator,
cleaning, must_haves,
bootstrap_length,
**other_opt_args)
for unused_index in range(monte_runs)]
### We can take an average here; only because our weights always add up to 1. If that isn't true
### then you will need to some kind of renormalisation
weightlist=np.array([x[0] for x in all_results], ndmin=2)
diaglist=[x[1] for x in all_results]
theweights_mean=list(np.mean(weightlist, axis=0))
diag=dict(bootstraps=diaglist)
return (theweights_mean, diag)
def _caller(depth=1):
''' Get caller of current frame '''
cf = inspect.currentframe()
for i in range(depth + 1):
cf = cf.f_back
try:
return '[%s] ' % inspect.getframeinfo(cf).function
except AttributeError:
return '[%s] ' % inspect.getframeinfo(inspect.currentframe()).function
def ParseCommandLine():
parser = argparse.ArgumentParser(description='A tool to compare to directories and move files')
parser.add_argument('-debug',help='Enable debugging',action='store_true')
args = parser.parse_args()
if args.debug:
import pdb
pdb.set_trace()
Logger('debug is on', getframeinfo(currentframe()))
else:
Logger('debug is off', getframeinfo(currentframe()))
def handle_stackframe_without_leak():
cat = ["meow"]
frame = sys.getframe(0)
print frame
try:
print inspect.getframeinfo( frame )
# do something with the frame
finally:
del frame
def reportError(msg):
if not jcl_islogging:
return
frame = inspect.currentframe()
# TODO: Need a more reliable approach to get the external caller's frame
while inspect.getframeinfo(frame).filename.endswith('jclib.py'):
frame = frame.f_back
print(msg)
while frame:
frameinfo = inspect.getframeinfo(frame)
print("\t" + frameinfo.filename + "(" + str(frameinfo.lineno) + ")")
frame = frame.f_back
isBlock = isBlockDevice(args["device"])
if (isBlock):
if (args["image"] != "") and (args[""]):
None
else:
print(
"'" + args["device"] +
"' is not a valid device, check if it exists", 2)
print("could not find: " + args["device"], -1)
# declaring usefull global variables
home = os.path.expanduser("~")
name = os.path.basename(sys.argv[0])
fileAndPath = inspect.getframeinfo(inspect.currentframe()).filename
path = os.path.dirname(os.path.abspath(fileAndPath))
prePend = "[ " + name + " ] "
description = name + "; " + "Python script entry point for RavenNucleon\
so that archlinux may reign supreme on whatever you choose for Nucleon\
to install it on."
dependancies = ["https://github.com/DreamingRaven/RavenPythonLib"]
# capture arguments in dict then put into json for bash
args = argz(sys.argv[1:], description=description)
args_json = json.loads(json.dumps(args))
# setting fallback logger here pre-update
log = Log(logLevel=args["loglevel"])
def _current_postion_info():
cf = currentframe()
frameinfo = " (at {}:{})".format(
os.path.basename(getframeinfo(cf).filename), cf.f_back.f_lineno
)
return frameinfo
def LogCaller():
fi = getframeinfo(currentframe().f_back.f_back)
msg = '[{}] Called from: {}:{}'.format(g.__plugin__, opb(fi.filename),
fi.lineno)
xbmc.log(msg, Log.NOTICE)
rospy.init_node(name='lidar', log_level=rospy.DEBUG)
while True:
countLog += 1
if (countLog % 5 == 1):
rospy.logdebug("This is a debug message " + str(countLog))
if (countLog % 5 == 2):
try:
a = 1 / 0
except ZeroDivisionError:
rospy.loginfo("Cannot divide by zero lineno - " +
str(getframeinfo(currentframe()).lineno) +
" count -" + str(countLog))
if (countLog % 5 == 3):
try:
raise IOError
except IOError:
rospy.logerr("IO Error has occurred lineno - " +
str(getframeinfo(currentframe()).lineno) +
" count -" + str(countLog))
if (countLog % 5 == 4):
try:
raise EOFError
except EOFError:
rospy.logwarn("Reached End-Of-File warning lineno - " +
def check_total_irradiation_other_source(self,
column,
other_radiation,
threshold_pct,
label_other='_other'):
name_check_function = inspect.getframeinfo(
inspect.currentframe()).function
other_radiation_copy = other_radiation.copy()
other_radiation_copy.name += label_other
column_other = other_radiation_copy.name
df_joined = self.df.join(other_radiation_copy, how='inner')
irradiation = mc_solar.daily_irradiation(
df_joined[column], samples_per_hour=self.samples_per_hour)
irradiation_other = mc_solar.daily_irradiation(
df_joined[column_other], samples_per_hour=self.samples_per_hour)
if irradiation < DAILY_IRRADIATION_THRESHOLD: # Avoids future errors
return None
diff_radiation = abs(irradiation -
irradiation_other) / irradiation * 100
condition_list = diff_radiation < threshold_pct
buffer = None
if not (condition_list):
plt.figure()
df_joined[column].plot(style='k.')
df_joined[column_other].plot(style='r.')
plt.legend([column, column_other])
plt.title(name_check_function + ':' + column)
plt.suptitle(self.type_data_station, fontsize=18)
buffer = io.BytesIO()
plt.savefig(buffer)
buffer.seek(0)
irrad_filt = self.df[column][lambda m: m > DNI_RADIATION_THRESHOLD]
num_radiation_transitions_value = mc_solar.num_radiation_transitions(
irrad_filt)
if num_radiation_transitions_value < NUM_RADIATION_TRANSITIONS_THRESHOLD:
self.assertion_base(
condition=condition_list,
error_message=
'Total irradiation from {} is different to {} in more than {}%. It is {:.1f}% while DAILY_IRRADIATION_THRESHOLD is {:.2} kWh/(m2·day)'
.format(column, column_other, threshold_pct, diff_radiation,
DAILY_IRRADIATION_THRESHOLD),
check_type=name_check_function,
figure=buffer)
else:
self.assertion_base(
condition=False,
error_message=
'Comparison of total irradiations {} and {} not checked because the number of cloudy moments={} [with a DRADIATION_DT={}] is higher than threshold={}'
.format(column, column_other, num_radiation_transitions_value,
DRADIATION_DT, NUM_RADIATION_TRANSITIONS_THRESHOLD),
error_level='INFO',
check_type=name_check_function,
figure=buffer)
def check_radiation_other_source(self,
column,
other_radiation,
threshold_pct,
radiation_threshold=None,
label_other='_other'):
name_check_function = inspect.getframeinfo(
inspect.currentframe()).function
if radiation_threshold is None:
radiation_threshold = DNI_RADIATION_THRESHOLD
other_radiation_copy = other_radiation.copy()
other_radiation_copy.name += label_other
column_other = other_radiation_copy.name
df_joined = self.df.join(other_radiation_copy, how='inner')
df_filt = df_joined[df_joined[column] > radiation_threshold]
if len(df_filt) == 0: # Avoids future errors
return None
condition_list = (df_filt[column] - df_filt[column_other]
).abs() / df_filt[column_other] * 100 < threshold_pct
buffer = None
if not condition_list.all():
plt.figure()
df_filt[column].plot(style='.')
df_filt[column_other].plot(style='.')
df_filt[column][~condition_list].plot(style='rP')
plt.legend([column, column_other])
plt.title(name_check_function + ':' + column)
plt.suptitle(self.type_data_station, fontsize=18)
buffer = io.BytesIO()
plt.savefig(buffer)
buffer.seek(0)
irrad_filt = self.df[column][lambda m: m > DNI_RADIATION_THRESHOLD]
num_radiation_transitions_value = mc_solar.num_radiation_transitions(
irrad_filt)
if num_radiation_transitions_value < NUM_RADIATION_TRANSITIONS_THRESHOLD:
self.assertion_base(
condition=condition_list.all(),
error_message=
'No coherence between {} and {} radiation sources considering a percentage THRESHOLD of {} % in {}'
.format(column, column_other, threshold_pct,
df_filt[column][~condition_list].index),
check_type=name_check_function,
figure=buffer)
else:
self.assertion_base(
condition=False,
error_message=
'Comparison of radiation {} and {} from different sources not checked because the number of cloudy moments={} [with a DRADIATION_DT={}] is higher than threshold={}'
.format(column, column_other, num_radiation_transitions_value,
DRADIATION_DT, NUM_RADIATION_TRANSITIONS_THRESHOLD),
error_level='INFO',
check_type=name_check_function,
figure=buffer)
def reset_default_config(self):
# This is the only variable that we should change, the top directory
topdir = self.get_config(param=Config.PARAM_TOPDIR)
if not os.path.isdir(topdir):
errmsg = str(self.__class__) + ' ' + str(getframeinfo(currentframe()).lineno) \
+ ': Fatal error initializing config, "' + str(topdir) + '" is not a directory!'
lg.Log.critical(errmsg)
raise Exception(errmsg)
param_values_to_set = {
# Top directory
Config.PARAM_TOPDIR:
Config.DEFVAL_TOPDIR,
# Logging
Config.PARAM_LOG_LEVEL:
Config.DEFVAL_LOGLEVEL,
#######################################################################
# Intent Server Stuff
#######################################################################
Config.PARAM_DO_PROFILING:
Config.DEFVAL_DO_PROFILING,
#######################################################################
# Models Stuff
#######################################################################
Config.PARAM_MODEL_DIR:
Config.DEFVAL_MODEL_DIR,
Config.PARAM_MODEL_NAME:
Config.DEFVAL_MODEL_NAME,
Config.PARAM_MODEL_LANG:
Config.DEFVAL_MODEL_LANG,
Config.PARAM_MODEL_IDENTIFIER:
Config.DEFVAL_MODEL_IDENTIFIER,
#######################################################################
# NLP Stuff
#######################################################################
# Word lists
Config.PARAM_NLP_DIR_WORDLIST:
Config.DEFVAL_NLP_DIR_WORDLIST,
Config.PARAM_NLP_DIR_APP_WORDLIST:
Config.DEFVAL_NLP_DIR_APP_WORDLIST,
Config.PARAM_NLP_POSTFIX_WORDLIST:
Config.DEFVAL_NLP_POSTFIX_WORDLIST,
Config.PARAM_NLP_POSTFIX_APP_WORDLIST:
Config.DEFVAL_NLP_POSTFIX_APP_WORDLIST,
# Synonym lists
Config.PARAM_NLP_DIR_SYNONYMLIST:
Config.DEFVAL_NLP_DIR_SYNONYMLIST,
Config.PARAM_NLP_POSTFIX_SYNONYMLIST:
Config.DEFVAL_NLP_POSTFIX_SYNONYMLIST,
# Stopwords lists (to be outdated)
Config.PARAM_NLP_POSTFIX_STOPWORDS:
Config.DEFVAL_NLP_POSTFIX_STOPWORDS,
Config.PARAM_NLP_DIR_APP_STOPWORDS:
Config.DEFVAL_NLP_DIR_APP_STOPWORDS,
Config.PARAM_NLP_POSTFIX_APP_STOPWORDS:
Config.DEFVAL_NLP_POSTFIX_APP_STOPWORDS,
# NLTK or whatever download dir
Config.PARAM_NLP_DIR_NLP_DOWNLOAD:
Config.DEFVAL_NLP_DIR_NLP_DOWNLOAD,
#######################################################################
# NLP Conversation Model
#######################################################################
Config.PARAM_NLP_DAEHUA_DIR:
Config.DEFVAL_DAEHUA_DIR,
Config.PARAM_NLP_DAEHUA_PATTERN_JSON_FILE:
Config.DEFVAL_DAEHUA_PATTERN_JSON_FILE,
#######################################################################
# Model Backtesting
#######################################################################
Config.PARAM_MODEL_BACKTEST_DETAILED_STATS:
Config.DEFVAL_MODEL_BACKTEST_DETAILED_STATS
}
for param in param_values_to_set.keys():
default_value = param_values_to_set[param]
self.set_default_value_if_not_exist(param=param,
default_value=default_value)
return
def display_memory_info():
import inspect, os
f = inspect.currentframe()
fileline = inspect.getframeinfo(f.f_back)
fileline = f"{os.path.basename(fileline.filename)}:{fileline.lineno}"
core.display_memory_info(fileline)
def compile_for_single_resource_type(
self, query, resource_type, mapping=None, root_replacer=None
):
"""
:param: query
:param: mapping: Elasticsearch mapping for FHIR resources.
:root_replacer: Path´s root replacer:
Could be mapping name or index name in zope´s ZCatalog context
"""
body_structure = ElasticSearchDialect.create_structure()
conditional_terms = [
w
for w in query.get_where()
if (
not INonFhirTerm.providedBy(w)
and w.path.context.resource_type == resource_type
)
or INonFhirTerm.providedBy(w)
]
for term in conditional_terms:
q, unary_operator = self.resolve_term(term, mapping, root_replacer)
if unary_operator == OPERATOR.neg:
container = body_structure["query"]["bool"]["must_not"]
elif unary_operator == OPERATOR.pos:
container = body_structure["query"]["bool"]["filter"]
else:
# xxx: if None may be should?
from inspect import currentframe, getframeinfo
frameinfo = getframeinfo(currentframe())
raise NotImplementedError(
f"File: {frameinfo.filename} Line: {frameinfo.lineno + 1}"
)
container.append(q)
# if not searching on all resources, add a predicate to filter on resourceType
if resource_type != "Resource":
ElasticSearchDialect.apply_from_constraint(
query, body_structure, resource_type, root_replacer=root_replacer
)
# Sorting
ElasticSearchDialect.apply_sort(
query.get_sort(), body_structure, root_replacer=root_replacer
)
# Limit
ElasticSearchDialect.apply_limit(query.get_limit(), body_structure)
# ES source_
ElasticSearchDialect.apply_source_filter(
query, body_structure, root_replacer=root_replacer
)
ElasticSearchDialect.clean_up(body_structure)
if "should" in body_structure["query"]["bool"]:
if "minimum_should_match" not in body_structure["query"]["bool"]:
body_structure["query"]["bool"]["minimum_should_match"] = 1
return body_structure
def varname(p):
for line in inspect.getframeinfo(inspect.currentframe().f_back)[3]:
m = re.search(r'\bvarname\s*\(\s*([A-Za-z_][A-Za-z0-9_]*)\s*\)', line)
if m:
return m.group(1)
def log_frame(self):
if self.chooser.dirspec_log:
stack = inspect.stack()
frame, _, _, _, _, _ = stack[1]
args, _, _, values = inspect.getargvalues(frame)
print(">>> %s: args=%s" % (inspect.getframeinfo(frame)[2], [(i, values[i]) for i in args[1:]]))
'develop-eggs',
'downloads',
'eggs',
'etc',
'parts',
'python',
'sql_dumps',
]
TAG = '%%s(PROJECT_NAME)%%s' %% ('$', '$') #make sure the tag definition does not replace it self
UTAG = '$(USERNAME)$'
#SKELETON_PATH = 'odoo/skeleton'
SKELETON_NAME = 'skeleton'
DEBUG = True
filename = inspect.getframeinfo(inspect.currentframe()).filename
# PROJECT_HOME is the folder in which the project is created
PROJECT_HOME = os.path.split(os.path.dirname(os.path.abspath(filename)))[0]
# PROJECT_LIST_DIR the folder that houses all projects
PROJECT_LIST_DIR = os.path.split(os.path.split(PROJECT_HOME)[0])[0]
class bcolors:
"""
"""
HEADER = '\033[95m'
OKBLUE = '\033[94m'
OKGREEN = '\033[92m'
WARNING = '\033[93m'
FAIL = '\033[91m'
ENDC = '\033[0m'
BOLD = '\033[1m'
def __init__(self):
self.frame = getframeinfo(currentframe())
self.establish_connection()
def loc(self, c: Any) -> Tuple[str, str, int]:
(f, line, name, lines, index) = inspect.getframeinfo(c)
""" Returns location information of the caller """
return (f, line, name, lines, index)
def _trace(self, text, tabs=0, level=0, severity='info'):
cf = currentframe()
calframe = getouterframes(cf, 2)
logger = self._setLoggingFile()
try:
if self.traceOutput == 'un-defined':
pass
except:
print(
'traceOutput not set defaulting to terminal and loggingLevel of 5'
)
self.traceOutput = 'terminal'
self.loggingLevel = 5
if self.traceOutput == 'un-defined':
self._error(self._errors['error_002'],
'err-002',
getframeinfo(cf).filename,
str(cf.f_lineno),
sys._getframe().f_code.co_name,
'terminate',
notify='void')
if self.traceOutput not in {'both', 'terminal', 'none', 'file'}:
self._error(self._errors['error_003'],
'err-003',
getframeinfo(cf).filename,
str(cf.f_lineno),
sys._getframe().f_code.co_name,
'terminate',
notify='void')
if severity not in {'info', 'debug', 'critical', 'error', 'warning'}:
self._error(self._errors['error_004'],
'err-004',
getframeinfo(cf).filename,
str(cf.f_lineno),
sys._getframe().f_code.co_name,
'terminate',
notify='void')
if self.traceOutput == 'terminal' or self.traceOutput == 'both':
if level < self.loggingLevel:
for i in range(1, tabs + 1):
print("\t", end='')
print('% ' + str(os.path.basename(calframe[1][1])) + '[' +
str(calframe[1][2]) + '] : ' + str(text))
if self.traceOutput == 'file' or self.traceOutput == 'both':
if level < self.loggingLevel:
optStr = 'logger.setLevel(logging.' + severity.upper() + ')'
eval(optStr)
cmdStr = 'logger.' + severity + '(str(text))'
eval(cmdStr)
# set back to default of INFO
logger.setLevel(logging.INFO)
def main():
program_name = os.path.basename(sys.argv[0])
usage = program_name + """ input_image_path output_directory --selection_list=selection_list --wn=CTF_WINDOW_SIZE --apix=PIXEL_SIZE --Cs=CS --voltage=VOLTAGE --ac=AMP_CONTRAST --f_start=FREA_START --f_stop=FREQ_STOP --vpp --kboot=KBOOT --overlap_x=OVERLAP_X --overlap_y=OVERLAP_Y --edge_x=EDGE_X --edge_y=EDGE_Y --check_consistency --stack_mode --debug_mode
Automated estimation of CTF parameters with error assessment.
All Micrographs Mode - Process all micrographs in a directory:
Specify a list of input micrographs using a wild card (*), called here input micrographs path pattern.
Use the wild card to indicate the place of variable part of the file names (e.g. serial number, time stamp, and etc).
Running from the command line requires enclosing the string by single quotes (') or double quotes (").
sxgui.py will automatically adds single quotes to the string.
BDB files can not be selected as input micrographs.
Then, specify output directory where all outputs should be saved.
In this mode, all micrographs matching the path pattern will be processed.
mpirun -np 16 sxcter.py './mic*.hdf' outdir_cter --wn=512 --apix=2.29 --Cs=2.0 --voltage=300 --ac=10.0
Selected Micrographs Mode - Process all micrographs in a selection list file:
In addition to input micrographs path pattern and output directry arguments,
specify a name of micrograph selection list text file using --selection_list option
(e.g. output of sxgui_unblur.py or sxgui_cter.py). The file extension must be ".txt".
In this mode, only micrographs in the selection list which matches the file name part of the pattern (ignoring the directory paths) will be processed.
If a micrograph name in the selection list does not exists in the directory specified by the micrograph path pattern, processing of the micrograph will be skipped.
mpirun -np 16 sxcter.py './mic*.hdf' outdir_cter --selection_list=mic_list.txt --wn=512 --apix=2.29 --Cs=2.0 --voltage=300 --ac=10.0
Single Micrograph Mode - Process a single micrograph:
In addition to input micrographs path pattern and output directry arguments,
specify a single micrograph name using --selection_list option.
In this mode, only the specified single micrograph will be processed.
If this micrograph name does not matches the file name part of the pattern (ignoring the directory paths), the process will exit without processing it.
If this micrograph name matches the file name part of the pattern but does not exists in the directory which specified by the micrograph path pattern, again the process will exit without processing it.
Use single processor for this mode.
sxcter.py './mic*.hdf' outdir_cter --selection_list=mic0.hdf --wn=512 --apix=2.29 --Cs=2.0 --voltage=300 --ac=10.0
Stack Mode - Process a particle stack (Not supported by SPHIRE GUI))::
Use --stack_mode option, then specify the path of particle stack file (without wild card "*") and output directory as arguments.
This mode ignores --selection_list, --wn --overlap_x, --overlap_y, --edge_x, and --edge_y options.
Use single processor for this mode. Not supported by SPHIRE GUI (sxgui.py).
sxcter.py bdb:stack outdir_cter --apix=2.29 --Cs=2.0 --voltage=300 --ac=10.0 --stack_mode
"""
parser = optparse.OptionParser(usage, version=sp_global_def.SPARXVERSION)
parser.add_option("--selection_list", type="string", default=None, help="Micrograph selecting list: Specify path of a micrograph selection list text file for Selected Micrographs Mode. The file extension must be \'.txt\'. Alternatively, the file name of a single micrograph can be specified for Single Micrograph Mode. (default none)")
parser.add_option("--wn", type="int", default=512, help="CTF window size [pixels]: The size should be slightly larger than particle box size. This will be ignored in Stack Mode. (default 512)")
parser.add_option("--apix", type="float", default=-1.0, help="Pixel size [A/Pixels]: The pixel size of input micrograph(s) or images in input particle stack. (default -1.0)")
parser.add_option("--Cs", type="float", default=2.0, help="Microscope spherical aberration (Cs) [mm]: The spherical aberration (Cs) of microscope used for imaging. (default 2.0)")
parser.add_option("--voltage", type="float", default=300.0, help="Microscope voltage [kV]: The acceleration voltage of microscope used for imaging. (default 300.0)")
parser.add_option("--ac", type="float", default=10.0, help="Amplitude contrast [%]: The typical amplitude contrast is in the range of 7% - 14%. The value mainly depends on the thickness of the ice embedding the particles. (default 10.0)")
parser.add_option("--f_start", type="float", default=-1.0, help="Lowest resolution [A]: Lowest resolution to be considered in the CTF estimation. Determined automatically by default. (default -1.0)")
parser.add_option("--f_stop", type="float", default=-1.0, help="Highest resolution [A]: Highest resolution to be considered in the CTF estimation. Determined automatically by default. (default -1.0)")
parser.add_option("--kboot", type="int", default=16, help="Number of CTF estimates per micrograph: Used for error assessment. (default 16)")
parser.add_option("--overlap_x", type="int", default=50, help="X overlap [%]: Overlap between the windows in the x direction. This will be ignored in Stack Mode. (default 50)")
parser.add_option("--overlap_y", type="int", default=50, help="Y overlap [%]: Overlap between the windows in the y direction. This will be ignored in Stack Mode. (default 50)")
parser.add_option("--edge_x", type="int", default=0, help="Edge x [pixels]: Defines the edge of the tiling area in the x direction. Normally it does not need to be modified. This will be ignored in Stack Mode. (default 0)")
parser.add_option("--edge_y", type="int", default=0, help="Edge y [pixels]: Defines the edge of the tiling area in the y direction. Normally it does not need to be modified. This will be ignored in Stack Mode. (default 0)")
parser.add_option("--check_consistency",action="store_true", default=False, help="Check consistency of inputs: Create a text file containing the list of inconsistent Micrograph ID entries (i.e. inconsist_mic_list_file.txt). (default False)")
parser.add_option("--stack_mode", action="store_true", default=False, help="Use stack mode: Use a stack as the input. Please set the file path of a stack as the first argument and output directory for the second argument. This is advanced option. Not supported by sxgui. (default False)")
parser.add_option("--debug_mode", action="store_true", default=False, help="Enable debug mode: Print out debug information. (default False)")
parser.add_option("--vpp", action="store_true", default=False, help="Volta Phase Plate - fit smplitude contrast. (default False)")
parser.add_option("--defocus_min", type="float", default=0.3, help="Minimum defocus search [um] (default 0.3)")
parser.add_option("--defocus_max", type="float", default=9.0, help="Maximum defocus search [um] (default 9.0)")
parser.add_option("--defocus_step", type="float", default=0.1, help="Step defocus search [um] (default 0.1)")
parser.add_option("--phase_min", type="float", default=5.0, help="Minimum phase search [degrees] (default 5.0)")
parser.add_option("--phase_max", type="float", default=175.0, help="Maximum phase search [degrees] (default 175.0)")
parser.add_option("--phase_step", type="float", default=5.0, help="Step phase search [degrees] (default 5.0)")
parser.add_option("--pap", action="store_true", default=False, help="Use power spectrum for fitting. (default False)")
(options, args) = parser.parse_args(sys.argv[1:])
# ====================================================================================
# Prepare processing
# ====================================================================================
# ------------------------------------------------------------------------------------
# Set up MPI related variables
# ------------------------------------------------------------------------------------
# Detect if program is running under MPI
RUNNING_UNDER_MPI = "OMPI_COMM_WORLD_SIZE" in os.environ
main_mpi_proc = 0
if RUNNING_UNDER_MPI:
####mpi.mpi_init( 0, [] )
my_mpi_proc_id = mpi.mpi_comm_rank(mpi.MPI_COMM_WORLD)
n_mpi_procs = mpi.mpi_comm_size(mpi.MPI_COMM_WORLD)
sp_global_def.MPI = True
else:
my_mpi_proc_id = 0
n_mpi_procs = 1
# ------------------------------------------------------------------------------------
# Set up SPHIRE global definitions
# ------------------------------------------------------------------------------------
if sp_global_def.CACHE_DISABLE:
pass#IMPORTIMPORTIMPORT from sp_utilities import disable_bdb_cache
sp_utilities.disable_bdb_cache()
# Change the name log file for error message
original_logfilename = sp_global_def.LOGFILE
sp_global_def.LOGFILE = os.path.splitext(program_name)[0] + '_' + original_logfilename + '.txt'
# ------------------------------------------------------------------------------------
# Check error conditions of arguments and options, then prepare variables for arguments
# ------------------------------------------------------------------------------------
input_image_path = None
output_directory = None
# not a real while, an if with the opportunity to use break when errors need to be reported
error_status = None
# change input unit
freq_start = -1.0
freq_stop = -1.0
if options.f_start >0.0:
if options.f_start <=0.5:
sp_global_def.ERROR( "f_start should be in Angstrom" ) # exclude abs frequencies and spatial frequencies
else:
freq_start = 1./options.f_start
if options.f_stop >0.0:
if options.f_stop <=0.5:
sp_global_def.ERROR( "f_stop should be in Angstrom" ) # exclude abs frequencies and spatial frequencies
else:
freq_stop = 1./options.f_stop
while True:
# --------------------------------------------------------------------------------
# Check the number of arguments. If OK, then prepare variables for them
# --------------------------------------------------------------------------------
if len(args) != 2:
error_status = ("Please check usage for number of arguments.\n Usage: " + usage + "\n" + "Please run %s -h for help." % (program_name), inspect.getframeinfo(inspect.currentframe()))
break
# NOTE: 2015/11/27 Toshio Moriya
# Require single quotes (') or double quotes (") when input micrograph pattern is give for input_image_path
# so that sys.argv does not automatically expand wild card and create a list of file names
#
input_image_path = args[0]
output_directory = args[1]
# --------------------------------------------------------------------------------
# NOTE: 2016/03/17 Toshio Moriya
# cter_mrk() will take care of all the error conditions
# --------------------------------------------------------------------------------
break
sp_utilities.if_error_then_all_processes_exit_program(error_status)
# Toshio, please see how to make it informative
assert input_image_path != None, " directory missing input_image_path"
assert output_directory != None, " directory missing output_directory"
if options.vpp == False :
wrong_params = False
pass#IMPORTIMPORTIMPORT import string as str
vpp_options = ["--defocus_min","--defocus_max","--defocus_step","--phase_min","--phase_max","--phase_step"]
for command_token in sys.argv:
for vppo in vpp_options:
if str.find(command_token, vppo) > -1 : wrong_params = True
if wrong_params: break
if wrong_params: break
if wrong_params:
sp_global_def.ERROR( "Some options are valid only for Volta Phase Plate command s" % command_token, myid=my_mpi_proc_id )
if my_mpi_proc_id == main_mpi_proc:
command_line = ""
for command_token in sys.argv:
command_line += command_token + " "
sp_global_def.sxprint(" ")
sp_global_def.sxprint("Shell line command:")
sp_global_def.sxprint(command_line)
if options.vpp:
vpp_options = [options.defocus_min, options.defocus_max, options.defocus_step, options.phase_min, options.phase_max, options.phase_step]
pass#IMPORTIMPORTIMPORT from sp_morphology import cter_vpp
result = sp_morphology.cter_vpp(input_image_path, output_directory, options.selection_list, options.wn, \
options.apix, options.Cs, options.voltage, options.ac, freq_start, freq_stop, \
options.kboot, options.overlap_x, options.overlap_y, options.edge_x, options.edge_y, \
options.check_consistency, options.stack_mode, options.debug_mode, program_name, vpp_options, \
RUNNING_UNDER_MPI, main_mpi_proc, my_mpi_proc_id, n_mpi_procs)
elif options.pap:
pass#IMPORTIMPORTIMPORT from sp_morphology import cter_pap
result = sp_morphology.cter_pap(input_image_path, output_directory, options.selection_list, options.wn, \
options.apix, options.Cs, options.voltage, options.ac, freq_start, freq_stop, \
options.kboot, options.overlap_x, options.overlap_y, options.edge_x, options.edge_y, \
options.check_consistency, options.stack_mode, options.debug_mode, program_name, \
RUNNING_UNDER_MPI, main_mpi_proc, my_mpi_proc_id, n_mpi_procs)
else:
pass#IMPORTIMPORTIMPORT from sp_morphology import cter_mrk
result = sp_morphology.cter_mrk(input_image_path, output_directory, options.selection_list, options.wn, \
options.apix, options.Cs, options.voltage, options.ac, freq_start, freq_stop, \
options.kboot, options.overlap_x, options.overlap_y, options.edge_x, options.edge_y, \
options.check_consistency, options.stack_mode, options.debug_mode, program_name, \
RUNNING_UNDER_MPI, main_mpi_proc, my_mpi_proc_id, n_mpi_procs)
if RUNNING_UNDER_MPI:
mpi.mpi_barrier(mpi.MPI_COMM_WORLD)
if main_mpi_proc == my_mpi_proc_id:
if options.debug_mode:
sp_global_def.sxprint("Returned value from cter_mrk() := ", result)
sp_global_def.sxprint(" ")
sp_global_def.sxprint("DONE!!!")
sp_global_def.sxprint(" ")
# ====================================================================================
# Clean up
# ====================================================================================
# ------------------------------------------------------------------------------------
# Reset SPHIRE global definitions
# ------------------------------------------------------------------------------------
sp_global_def.LOGFILE = original_logfilename
# ------------------------------------------------------------------------------------
# Clean up MPI related variables
# ------------------------------------------------------------------------------------
if RUNNING_UNDER_MPI:
mpi.mpi_barrier( mpi.MPI_COMM_WORLD )
sys.stdout.flush()
return
def get_frame_info(level):
caller_frame = inspect.stack()[level]
info = inspect.getframeinfo(caller_frame[0])
return info.filename + ':' + str(info.lineno) + ': '
def check_coherence_radiation(self,
threshold_pct,
dni,
ghi,
dhi,
radiation_threshold=None):
# Check radiation coherence between GHI and DNI&DHI
# THRESHOLD is in percentage
name_check_function = inspect.getframeinfo(
inspect.currentframe()).function
if radiation_threshold is None:
radiation_threshold = GHI_RADIATION_THRESHOLD
df_filt = self.df[self.df[ghi] > GHI_RADIATION_THRESHOLD]
if len(df_filt) == 0: # Avoids future errors
return None
_, Zz = mc_solar.solpos(df_filt.index)
ghi_model = (df_filt[dhi] + df_filt[dni] * np.cos(Zz))
condition_list = (((df_filt[ghi] - ghi_model).abs()) / df_filt[ghi] *
100 < threshold_pct)
buffer = None
if not condition_list.all():
plt.figure()
df_filt[ghi].plot(style='.')
df_filt[ghi][~condition_list].plot(style='rP')
# plt.legend()
plt.title(name_check_function)
plt.suptitle(self.type_data_station, fontsize=18)
buffer = io.BytesIO()
plt.savefig(buffer)
buffer.seek(0)
num_radiation_transitions_value = mc_solar.num_radiation_transitions(
self.df[ghi])
if num_radiation_transitions_value < NUM_RADIATION_TRANSITIONS_THRESHOLD:
self.assertion_base(
condition=condition_list.all(),
error_message=
'No coherence between radiations considering a percentage threshold of GHI {}% in {}'
.format(threshold_pct, df_filt[~condition_list].index),
check_type=name_check_function,
figure=buffer)
else:
self.assertion_base(
condition=False,
error_message=
'Radiation coherence based on GHI not checked because the number of cloudy moments={} [with a DRADIATION_DT={}] is higher than threshold={}'
.format(num_radiation_transitions_value, DRADIATION_DT,
NUM_RADIATION_TRANSITIONS_THRESHOLD),
error_level='INFO',
check_type=name_check_function,
figure=buffer)
def run_and_report(func: Any) -> Any:
try:
return func()
except Exception as ex:
if _is_env_set("HYDRA_FULL_ERROR"):
raise ex
else:
if isinstance(ex, CompactHydraException):
sys.stderr.write(str(ex) + os.linesep)
if isinstance(ex.__cause__, OmegaConfBaseException):
sys.stderr.write(str(ex.__cause__) + os.linesep)
else:
# Custom printing that strips the Hydra related stack frames from the top
# And any omegaconf frames from the bottom.
# It is possible to add additional libraries to sanitize from the bottom later,
# maybe even make it configurable.
tb: Any = ex.__traceback__
search_max = 10
# strip Hydra frames from start of stack
# will strip until it hits run_job()
while search_max > 0:
if tb is None:
break
frame = tb.tb_frame
tb = tb.tb_next
search_max = search_max - 1
if inspect.getframeinfo(frame).function == "run_job":
break
if search_max == 0 or tb is None:
# could not detect run_job, probably a runtime exception before we got there.
# do not sanitize the stack trace.
print_exc()
sys.exit(1)
# strip OmegaConf frames from bottom of stack
end = tb
num_frames = 0
while end is not None:
frame = end.tb_frame
mdl = inspect.getmodule(frame)
assert mdl is not None
name = mdl.__name__
if name.startswith("omegaconf."):
break
end = end.tb_next
num_frames = num_frames + 1
@dataclass
class FakeTracebackType:
tb_next: Any = None # Optional[FakeTracebackType]
tb_frame: Optional[FrameType] = None
tb_lasti: Optional[int] = None
tb_lineno: Optional[int] = None
iter_tb = tb
final_tb = FakeTracebackType()
cur = final_tb
added = 0
while True:
cur.tb_lasti = iter_tb.tb_lasti
cur.tb_lineno = iter_tb.tb_lineno
cur.tb_frame = iter_tb.tb_frame
if added == num_frames - 1:
break
added = added + 1
cur.tb_next = FakeTracebackType()
cur = cur.tb_next
iter_tb = iter_tb.tb_next
print_exception(etype=None, value=ex,
tb=final_tb) # type: ignore
sys.stderr.write(
"\nSet the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.\n"
)
sys.exit(1)
def api_remove_ml_backend():
project = project_get_or_create()
ml_backend_name = request.json['name']
project.remove_ml_backend(ml_backend_name)
project.analytics.send(getframeinfo(currentframe()).function)
return make_response(jsonify('Deleted!'), 204)
def _insert_alias_assertions(tn, parsed, aliases_dict, aliases_case_sensitive):
time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
# convert aliases_dict into dictionary of dataframes
aliases = {
k: _read_file_from_path_or_read_str_as_buffer(v,
pd.read_csv,
encoding='utf8',
skipinitialspace=True)
for k, v in aliases_dict.items()
if k in tn.node_types['node_type'].array
}
# Get relevant aliases out of each set of aliases
aliases = {
k: _select_aliases(v, parsed, k, aliases_case_sensitive)
for k, v in aliases.items()
}
if not aliases_case_sensitive:
low_keys = {
k: v.apply(lambda x: x.str.casefold() if x.name == 'key' else x)
for k, v in aliases.items()
}
low_values = {
k: v.apply(lambda x: x.str.casefold() if x.name == 'value' else x)
for k, v in aliases.items()
}
aliases = {
k: pd.concat([v, low_keys[k], low_values[k]]).drop_duplicates()
for k, v in aliases.items()
}
# get integer IDs for the relevant node types
node_type_ids = {
k: tn.id_lookup('node_types', k, column_label='node_type')
for k in aliases.keys()
}
# add any new strings we found to the textnet strings frame
new_strings = [
pd.DataFrame({
'string': v.stack().array,
'node_type_id': node_type_ids[k],
'date_inserted': time_string,
'date_modified': pd.NA
}) for k, v in aliases.items()
]
new_strings = pd.concat(new_strings).drop_duplicates()
# print(new_strings.string, '\n\n\n')
# Add a node type for alias reference strings and add the alias
# reference strings to the new strings
tn.insert_node_type('alias_reference')
alias_ref_node_id = tn.id_lookup('node_types', 'alias_reference')
alias_ref_strings = pd.DataFrame({
'string': aliases_dict.values(),
'node_type_id': alias_ref_node_id,
'date_inserted': time_string,
'date_modified': pd.NA
})
new_strings = pd.concat([new_strings, alias_ref_strings])
# drop any strings already present in tn.strings
new_strings = new_strings.loc[~new_strings['string'].
isin(tn.strings['string'])]
# Insert the new strings in tn.strings
new_strings = shnd.util.normalize_types(new_strings, tn.strings)
tn.strings = pd.concat([tn.strings, new_strings])
# make maps between string values and integer IDs relevant to each
# set of aliases
string_maps = {
k: bg.util.get_string_values(tn, node_type_subset=v)
for k, v in node_type_ids.items()
}
if not aliases_case_sensitive:
low_maps = {k: v.str.casefold() for k, v in string_maps.items()}
case_alias = {k: (v != low_maps[k]) for k, v in string_maps.items()}
new_aliases = {
k: pd.concat([v[case_alias[k]], low_maps[k][case_alias[k]]],
axis='columns',
ignore_index=True)
for k, v in string_maps.items()
}
new_aliases = {
k: v.rename(columns={
0: 'key',
1: 'value'
})
for k, v in new_aliases.items()
}
# print(new_aliases['actor'], '\n\n\n')
aliases = {
k: pd.concat([v, new_aliases[k]])
for k, v in aliases.items()
}
# print(string_maps['actor'], '\n\n\n')
# print(aliases['actor'], '\n\n\n')
string_maps = {
k: pd.Series(v.index, index=v.array)
for k, v in string_maps.items()
}
'''# convert alias string values to integer string IDs
aliases = {
k: v.apply(lambda x: x.map(string_maps[k]))
for k, v in aliases.items()
}'''
# convert alias keys to integer string IDs
def merge_keys_and_values(aliases, string_map):
k = aliases.merge(string_map.rename('k'),
left_on='key',
right_index=True)
v = aliases.merge(string_map.rename('v'),
left_on='value',
right_index=True)
output = k[['k', 'key']].merge(v[['v', 'key']])[['k', 'v']]
output = output.rename(columns={'k': 'key', 'v': 'value'})
return output.drop_duplicates()
aliases = {
k: merge_keys_and_values(v, string_maps[k])
for k, v in aliases.items()
}
# print(
# aliases['actor'].apply(lambda x: x.map(tn.strings['string'])),
# '\n\n\n'
# )
# Start building an assertions frame out of string values
new_assertions = {
k: pd.DataFrame({
'src_string_id': aliases[k]['key'],
'tgt_string_id': aliases[k]['value'],
'ref_string_id': tn.id_lookup('strings', v)
})
for k, v in aliases_dict.items()
}
new_assertions = pd.concat(new_assertions.values(), ignore_index=True)
# make a string value representing the current python function
frame = inspect.currentframe()
current_module = inspect.getframeinfo(frame).filename
current_module = Path(current_module).stem.split('.')[0]
current_function = inspect.getframeinfo(frame).function
inp_string = '.'.join(['bibliograph', current_module, current_function])
# Insert a strings row for the input string
new_string = {
'string': inp_string,
'node_type_id': tn.id_lookup('node_types', 'python_function'),
'date_inserted': time_string,
'date_modified': pd.NA
}
new_string = shnd.util.normalize_types(new_string, tn.strings)
tn.strings = pd.concat([tn.strings, new_string])
# Add a link type for aliases if it doesn't already exist
tn.insert_link_type('alias')
alias_link_type_id = tn.id_lookup('link_types', 'alias')
# make a dataframe with the remaining data for the assertions
# frame and then concat it with the partial assertions frame
assertion_metadata = pd.DataFrame(
{
'inp_string_id': tn.strings.index[-1],
'link_type_id': alias_link_type_id,
'date_inserted': time_string,
'date_modified': pd.NA
},
index=new_assertions.index)
new_assertions = pd.concat([new_assertions, assertion_metadata],
axis='columns')
# put the new assertions columns in the right order and then
# add them to the textnet assertions
new_assertions = new_assertions[tn.assertions.columns]
new_assertions = shnd.util.normalize_types(new_assertions, tn.assertions)
tn.assertions = pd.concat([tn.assertions, new_assertions])
def api_instruction():
""" Instruction for annotators
"""
project = project_get_or_create()
project.analytics.send(getframeinfo(currentframe()).function)
return make_response(project.config['instruction'], 200)
from array import array
from time import time
from inspect import currentframe,getframeinfo
frameinfo=getframeinfo(currentframe())
def brute_force(arr: array)->int:
numInv=0
for i in range(0,len(arr)-1):
for j in range(i+1,len(arr)):
if arr[i]>arr[j]:
numInv+=1
return numInv
def merge_and_count_splitinv(arr1: array, arr2: array)->(array,int):
#if arr2[0]==0:
# arr2.pop(0)
n = len(arr1)+len(arr2)
arr_fin= array("i",[])
i,j,splitinv=0,0,0
for k in range(n):
#checking if one of the iterators is at the end
if i==int(len(arr1)):
#print("inside 1st i")
#print(locals())
arr_fin.extend(arr2[j:])
def api_project_storage_settings():
project = project_get_or_create()
# GET: return selected form, populated with current storage parameters
if request.method == 'GET':
# render all forms for caching in web
all_forms = {'source': {}, 'target': {}}
for storage_for in all_forms:
for name, description in project.get_available_storage_names(
storage_for).items():
current_type = project.config.get(storage_for,
{'type': ''})['type']
current = name == current_type
form_class = get_storage_form(name)
form = form_class(data=project.get_storage(
storage_for).get_params()) if current else form_class()
all_forms[storage_for][name] = {
'fields': [serialize_class(field) for field in form],
'type':
name,
'current':
current,
'description':
description,
'path':
getattr(project, storage_for + '_storage').readable_path
}
# generate data key automatically
if project.data_types.keys():
for field in all_forms[storage_for][name]['fields']:
if field['name'] == 'data_key' and not field['data']:
field['data'] = list(project.data_types.keys())[0]
project.analytics.send(getframeinfo(currentframe()).function,
method=request.method)
return make_response(jsonify(all_forms), 200)
# POST: update storage given filled form
if request.method == 'POST':
selected_type = request.args.get('type', '')
storage_for = request.args.get('storage_for')
current_type = project.config.get(storage_for, {'type': ''})['type']
selected_type = selected_type if selected_type else current_type
form = get_storage_form(selected_type)(data=request.json)
project.analytics.send(getframeinfo(currentframe()).function,
method=request.method,
storage=selected_type,
storage_for=storage_for)
if form.validate_on_submit():
storage_kwargs = dict(form.data)
storage_kwargs['type'] = request.json['type'] # storage type
try:
project.update_storage(storage_for, storage_kwargs)
except Exception as e:
traceback = tb.format_exc()
logger.error(str(traceback))
return make_response(
jsonify(
{'detail': 'Error while storage update: ' + str(e)}),
400)
else:
return make_response(jsonify({'result': 'ok'}), 201)
else:
logger.error('Errors: ' + str(form.errors) + ' for request body ' +
str(request.json))
return make_response(jsonify({'errors': form.errors}), 400)
def save(self):
super(Metadata, self).save()
# Execute only if the caller is not in watch. (See above)
caller = inspect.getframeinfo(sys._getframe(1), context=0)[2]
try:
if not caller == 'f':
# Save overlapping Keyword / Metadata fields
k = Keyword.objects.get(pk=self.image.pk)
k.cright = self.copyright
k.subject = self.subject
k.source = self.source
k.keywords = self.keywords
k.save()
# Save overlapping Image / Metadata fields
i = Image.objects.get(pk=self.image.pk)
i.group_status = self.headline
i.save()
m = metadata.Metadata()
path = os.path.join(settings.APP_CONTENT_ROOT,
i.image_real_path)
cmdDict = {
'source':
self.source,
'captionwriter':
self.caption_writer,
'subject':
self.keywords,
'xmp:keywords':
self.keywords, # revision 182, changed to xmp keywords
'description':
self.description,
'location':
self.location,
'city':
self.city,
'province-state':
self.provincestate,
'country':
self.country,
'instructions':
self.instructions,
'title':
self.subject,
'creatortool':
self.creator_tool,
'creator':
self.creator,
'author':
self.author,
'credit':
self.credit,
'xmp:headline':
self.
headline, # the xmp prefix should accomodate illustrator and other pdf-based file formats
'album':
self.album,
'documentname':
self.subject,
'copyright':
'yes' if self.copyright == 1 else
'no' if self.copyright == 0 else 'unknown',
'orientation':
self.orientation
}
if self.document:
cmdDict['ManagedFromFilePath'] = self.document.path
m.exifWriteAll(cmdDict, path)
else:
logging.info(
"Caller was %s so no models.Metadata super save()" %
caller)
finally:
del caller
def main(path=None,
browser=None,
report=None,
title="Seldom Test Report",
description="Test case execution",
debug=False,
rerun=0,
save_last_run=False,
driver_path=None,
grid_url=None):
"""
runner test case
:param path:
:param browser:
:param report:
:param title:
:param description:
:param debug:
:param rerun:
:param save_last_run:
:param driver_path:
:param grid_url:
:return:
"""
if path is None:
stack_t = inspect.stack()
ins = inspect.getframeinfo(stack_t[1][0])
file_dir = os.path.dirname(os.path.abspath(ins.filename))
file_path = ins.filename
if "\\" in file_path:
this_file = file_path.split("\\")[-1]
elif "/" in file_path:
this_file = file_path.split("/")[-1]
else:
this_file = file_path
suits = unittest.defaultTestLoader.discover(file_dir, this_file)
else:
if len(path) > 3:
if path[-3:] == ".py":
if "/" in path:
path_list = path.split("/")
path_dir = path.replace(path_list[-1], "")
suits = unittest.defaultTestLoader.discover(path_dir, pattern=path_list[-1])
else:
suits = unittest.defaultTestLoader.discover(os.getcwd(), pattern=path)
else:
suits = unittest.defaultTestLoader.discover(path)
else:
suits = unittest.defaultTestLoader.discover(path)
if browser is None:
BrowserConfig.name = "chrome"
else:
BrowserConfig.name = browser
BrowserConfig.grid_url = grid_url
if driver_path is not None:
ret = os.path.exists(driver_path)
if ret is False:
raise ValueError("Browser - driven path error，Please check if the file exists. => {}".format(driver_path))
BrowserConfig.driver_path = driver_path
"""
Global launch browser
"""
Seldom.driver = b(BrowserConfig.name, BrowserConfig.driver_path, BrowserConfig.grid_url)
if debug is False:
for filename in os.listdir(os.getcwd()):
if filename == "reports":
break
else:
os.mkdir(os.path.join(os.getcwd(), "reports"))
if report is None:
now = time.strftime("%Y_%m_%d_%H_%M_%S")
report = os.path.join(os.getcwd(), "reports", now + "_result.html")
BrowserConfig.report_path = report
with(open(report, 'wb')) as fp:
runner = HTMLTestRunner(stream=fp, title=title, description=description)
log.info(seldom_str)
runner.run(suits, rerun=rerun, save_last_run=save_last_run)
print("generated html file: file:///{}".format(report))
else:
runner = unittest.TextTestRunner(verbosity=2)
log.info("A run the test in debug mode without generating HTML report!")
log.info(seldom_str)
runner.run(suits)
"""
Close browser globally
"""
Seldom.driver.quit()
import os.path
import logging
import sys
import inspect
import mitmproxy.websocket
# This file is executed by `mitmdump' with `execfile'. Therefore, in
# order to import submodules under the directory where this file exists,
# we need some tricks. See http://stackoverflow.com/questions/3718657
# for the details of the tricks used in the following lines.
THIS_FILENAME = inspect.getframeinfo(inspect.currentframe()).filename
THIS_DIR_PATH = os.path.dirname(os.path.abspath(THIS_FILENAME))
sys.path.append(THIS_DIR_PATH)
import mahjongsoul_sniffer.logging as logging_
from mahjongsoul_sniffer.redis_mirroring import RedisMirroring
logging_.initialize(module_name='game_detail_crawler', service_name='sniffer')
_REDIS_MIRRORING_CONFIG = {
'websocket': {
'.lq.Lobby.loginBeat': {
'request_direction': 'outbound',
'action': {
'command': 'SET',
'key': 'login-beat'
}
},
'.lq.Lobby.fetchGameRecord': {
'request_direction': 'outbound',
'action': {
'command': 'SET',
import unittest import os import inspect import sys proj_dir = os.path.abspath(os.path.join(inspect.getframeinfo(inspect.currentframe()).filename, '../../')) sys.path.append(os.path.join(proj_dir, 'src')) data_dir = os.path.join(proj_dir, 'data') tmp_dir = os.path.join(proj_dir, 'tmp') if not os.path.exists(tmp_dir): os.mkdir(tmp_dir) import polca fnames_ascii = '''E.coli a.txt aaa.txt alice29.txt alphabet.txt asyoulik.txt bib bible.txt book1 book2 download.sh fields.c grammar.lsp lcet10.txt news
def check_coherence_isotypes(self,
dni,
top,
mid,
bot,
threshold_pct,
radiation_threshold=None):
# Check radiation coherence between DNI and isotypes
# THRESHOLD is in percentage
name_check_function = inspect.getframeinfo(
inspect.currentframe()).function
if radiation_threshold is None:
radiation_threshold = DNI_RADIATION_THRESHOLD
df_filt = self.df[self.df[dni] > DNI_RADIATION_THRESHOLD]
if len(df_filt) == 0: # Avoids future errors
return None
dni_model = (df_filt[top] * 0.51 + df_filt[mid] * 0.10 +
df_filt[bot] * 0.39)
condition_list = (((df_filt[dni] - dni_model).abs()) / df_filt[dni] *
100 < threshold_pct)
buffer = None
if not condition_list.all():
plt.figure()
df_filt[dni].plot(style='k.')
df_filt[top].plot(style='.')
df_filt[mid].plot(style='.')
df_filt[bot].plot(style='.')
df_filt[dni][~condition_list].plot(marker='P',
markersize=8,
color='darkred',
markeredgecolor='yellow',
markeredgewidth=2)
plt.legend([top, mid, bot])
plt.title(name_check_function)
plt.suptitle(self.type_data_station, fontsize=18)
buffer = io.BytesIO()
plt.savefig(buffer)
buffer.seek(0)
irrad_filt = self.df[dni][lambda m: m > DNI_RADIATION_THRESHOLD]
num_radiation_transitions_value = mc_solar.num_radiation_transitions(
irrad_filt)
if num_radiation_transitions_value < NUM_RADIATION_TRANSITIONS_THRESHOLD:
self.assertion_base(
condition=condition_list.all(),
error_message=
'No coherence between DNI radiation and isotypes considering a percentage threshold of {} % in {}'
.format(threshold_pct, df_filt[dni][~condition_list].index),
check_type=name_check_function,
figure=buffer)
else:
self.assertion_base(
condition=False,
error_message=
'DNI vs isotypes comparison not checked because the number of cloudy moments={} [with a DRADIATION_DT={}] is higher than threshold={}'
.format(num_radiation_transitions_value, DRADIATION_DT,
NUM_RADIATION_TRANSITIONS_THRESHOLD),
error_level='INFO',
check_type=name_check_function,
figure=buffer)
def test_model_against_training_data(
self,
include_detailed_accuracy_stats = False
):
start_get_td_time = pf.Profiling.start()
lg.Log.info(
str(self.__class__) + ' ' + str(getframeinfo(currentframe()).lineno)
+ '. Start Load Training Data: ' + str(start_get_td_time)
)
# Get training data to improve LeBot intent/command detection
self.model.load_training_data_from_storage()
td = self.model.training_data
lg.Log.debug(
str(self.__class__) + ' ' + str(getframeinfo(currentframe()).lineno)
+ ': TD x_name, shape=' + str(td.get_x_name().shape) + ': ' + str(td.get_x_name())
+ '\n\rTD shape=' + str(td.get_x().shape)
+ '\n\rTD[0:10] =' + str(td.get_x()[0:10])
)
stop_get_td_time = pf.Profiling.stop()
lg.Log.info(
str(self.__class__) + ' ' + str(getframeinfo(currentframe()).lineno)
+ '. Stop Load Training Data: '
+ str(pf.Profiling.get_time_dif_str(start_get_td_time, stop_get_td_time)))
start_test_time = pf.Profiling.start()
lg.Log.info(
str(self.__class__) + ' ' + str(getframeinfo(currentframe()).lineno)
+ '. Start Testing of Training Data from DB Time : ' + str(start_get_td_time)
)
#
# Read from chatbot training files to compare with LeBot performance
#
self.reset_test_stats()
self.test_stats[ModelBackTest.KEY_STATS_START_TEST_TIME] = start_test_time
x_name = td.get_x_name()
x = td.get_x()
y = td.get_y()
for i in range(0, x.shape[0], 1):
y_expected = y[i]
v = nputil.NumpyUtil.convert_dimension(arr=x[i], to_dim=2)
x_features = x_name[v[0]>0]
df_match_details = self.predict_top_x(
v = v,
y_expected = y_expected,
x_features = x_features
)
self.update_test_stats(
df_match_details = df_match_details,
y_expected = y_expected,
x_features = x_features,
include_detailed_accuracy_stats = include_detailed_accuracy_stats
)
stop_test_time = pf.Profiling.stop()
lg.Log.important('. Stop Testing of Training Data from DB Time : '
+ str(pf.Profiling.get_time_dif_str(start_test_time, stop_test_time)))
lg.Log.important(
str(self.test_stats[ModelBackTest.KEY_STATS_RESULT_WRONG]) + ' wrong results from '
+ str(self.test_stats[ModelBackTest.KEY_STATS_RESULT_WRONG]) + ' total tests.'
)
for q in (0.0, 0.05, 0.1, 0.25, 0.50, 0.75, 0.9, 0.95):
lg.Log.important(
'Score Quantile (' + str(q) + '): '
+ str(self.test_stats[ModelBackTest.KEY_STATS_DF_SCORES]['Score'].quantile(q))
)
return