def thread_func(testclass, mainTransaction, funcToRun):
self.assertIsNone(LockingTransaction.get())
LockingTransaction._transactions.local = LockingTransaction()
LockingTransaction._transactions.local._info = Info(TransactionState.Running, LockingTransaction._transactions.local._startPoint)
funcToRun(testclass, mainTransaction)
LockingTransaction._transactions = thread_local()
self.assertIsNone(LockingTransaction.get())
def __init__(self):
self.__queue = Queue()
self.__lock = Lock()
# Mapping of LID to deque (so can serialise messages for same LID). Appended to right, removed from left.
self.__lid_mapping = {}
self.__local = thread_local()
self.__new_msg = Event()
def thread_func(testclass, mainTransaction, funcToRun):
self.assertIsNone(LockingTransaction.get())
LockingTransaction._transactions.local = LockingTransaction()
LockingTransaction._transactions.local._info = Info(
TransactionState.Running,
LockingTransaction._transactions.local._startPoint)
funcToRun(testclass, mainTransaction)
LockingTransaction._transactions = thread_local()
self.assertIsNone(LockingTransaction.get())
def _cache_thread_local(func):
@wraps(func)
def wrapper(*args, **kwargs):
if not hasattr(wrapper.cache, 'result'):
wrapper.cache.result = func(*args, **kwargs)
return wrapper.cache.result
wrapper.cache = thread_local()
return wrapper
def running_transaction(thetest):
# Fake a running transaction
LockingTransaction._transactions.local = LockingTransaction()
LockingTransaction._transactions.local._info = Info(TransactionState.Running, LockingTransaction._transactions.local._startPoint)
LockingTransaction.ensureGet()._readPoint = -1
LockingTransaction.transactionCounter = count()
LockingTransaction.ensureGet()._startPoint = time()
yield
# Clean up and remove LockingTransaction we created
LockingTransaction._transactions = thread_local()
thetest.assertIsNone(LockingTransaction.get())
def running_transaction(thetest):
# Fake a running transaction
LockingTransaction._transactions.local = LockingTransaction()
LockingTransaction._transactions.local._info = Info(
TransactionState.Running,
LockingTransaction._transactions.local._startPoint)
LockingTransaction.ensureGet()._readPoint = -1
LockingTransaction.transactionCounter = count()
LockingTransaction.ensureGet()._startPoint = time()
yield
# Clean up and remove LockingTransaction we created
LockingTransaction._transactions = thread_local()
thetest.assertIsNone(LockingTransaction.get())
def setUp(self):
# Poor-man's version of a threading.Barrier (which was added in 3.2). Make a
# new threading.Lock and immediately acquire it. Spawn the sync_server thread,
# then acquire the lock again. The second acquire() call will block until the
# sync server thread is initialized -- it calls release() before entering the
# serve_forever loop. This isn't totally free of race conditions -- a test
# method could conceivably be run between sync_server's calls to
# self.lock.release() and server.serve_forever. We just need our test runs
# to behave as deterministically as possible, so hopefully this is good
# enough for now.
# TODO: use a real threading.Barrier when we drop support for Python 2
self.lock = Lock()
self.lock.acquire()
self.thread_local = thread_local()
self.server_thread = Thread(target=self.sync_server)
self.server_thread.start()
self.lock.acquire()
self.remote_sync_server = ServerProxy(LOCAL_SYNC_SERVER_URL, allow_none=True)
self.lock.release()
def setUp(self):
# Poor-man's version of a threading.Barrier (which was added in 3.2). Make a
# new threading.Lock and immediately acquire it. Spawn the sync_server thread,
# then acquire the lock again. The second acquire() call will block until the
# sync server thread is initialized -- it calls release() before entering the
# serve_forever loop. This isn't totally free of race conditions -- a test
# method could conceivably be run between sync_server's calls to
# self.lock.release() and server.serve_forever. We just need our test runs
# to behave as deterministically as possible, so hopefully this is good
# enough for now.
# TODO: use a real threading.Barrier when we drop support for Python 2
self.lock = Lock()
self.lock.acquire()
self.thread_local = thread_local()
self.server_thread = Thread(target=self.sync_server)
self.server_thread.start()
self.lock.acquire()
self.remote_sync_server = ServerProxy(LOCAL_SYNC_SERVER_URL,
allow_none=True)
self.lock.release()
def __new__(mcls, name, bases, dct):
dct.setdefault('__singleton_value__', thread_local())
return super(ThreadLocalSingletonType,
mcls).__new__(mcls, name, bases, dct)
def local(self):
return thread_local()
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
from threading import Thread, local as thread_local
from Queue import Queue
from cloudify import amqp_client
thread_storage = thread_local()
class AMQPWrappedThread(Thread):
"""
creates an amqp client before calling the target method.
This thread is always set as a daemon.
"""
def __init__(self, target, *args, **kwargs):
def wrapped_target():
client = amqp_client.create_client()
self.started_amqp_client.put_nowait(True)
thread_storage.amqp_client = client
try:
def __init__ ( self ):
""" Initializes the object.
"""
self.facets_logger = None
self.main_thread = None
self.thread_local = thread_local()
def __init__ ( self ):
self.traits_logger = None
self.main_thread = None
self.thread_local = thread_local()
def local(self): return thread_local()
def queue(self, max_size=None):
from threading import local as thread_local
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker, Session
from core.config import get_env
_thread_local = thread_local()
def set_current_test_db_session(db_session: scoped_session) -> None:
""" スレッドローカルにテスト用のDBセッションをセット
Args:
db_session (scoped_session): テスト用のDBセッション
"""
setattr(_thread_local, 'db_session', db_session)
def get_current_test_db_session() -> scoped_session:
""" スレッドローカルからテスト用のDBセッションを取得
Returns:
scoped_session: テスト用のDBセッション
"""
return getattr(_thread_local, 'db_session')
class TestingDBSession(Session):
""" commit()の挙動を変えるため、Sessionクラスをオーバーライド
"""
def __init__(self):
super(ThreadLocalSecurityContext, self).__init__(thread_local())
class ExecutionController(Controller):
"""
The Controller object has two functions: it holds the state of the script
thread, and it controls execution of scripts via a C trace function.
The `script_thread_entry` method is called in a separate thread, which then
processes the requests put into `self.queue` by the server thread.
When `set_break` is called, execution is suspended and `_breakfunc` is
called which waits until somebody calls `set_continue` or `set_stop`. Since
`break_only_in_filename` is given at creation time, a break request will
only break when the execution frame is one from the toplevel script.
The attributes `currentframe` and `lineno` correspond to the current
execution frame and the current line number in the toplevel script.
See the docstring of pyctl.Controller for more details on the inner
working of the Controller object and the trace function.
"""
thread_data = thread_local()
def __init__(self, log, eventfunc, startupsetup, simmode, autosim):
self.log = log # daemon logger object
self.eventfunc = eventfunc # event emitting callback
self.setup = startupsetup # first setup on start
# start in simulation mode?
self.simmode = simmode and SIMULATION or SLAVE
self.autosim = autosim # simulate script when running it?
self.queue = ScriptQueue() # user scripts get put here
self.current_script = None # currently executed script
# namespaces in which scripts execute
self.namespace = session.namespace
# completer for the namespaces
self.completer = NicosCompleter(self.namespace)
self.watchexprs = set() # watch expressions to evaluate
self.watchlock = Lock() # lock for watch expression list modification
self.estop_functions = [] # functions to run on emergency stop
self.thread = None # thread executing scripts
self.reqid_work = None # ID of the last executing request
self.debugger = None # currently running debugger (Rpdb)
self.last_handler = None # handler of current exec/eval
# only one user or admin can issue non-read-only commands
self.controlling_user = None
Controller.__init__(self, break_only_in_filename='<script>')
self.set_observer(self._observer)
def _setup(self):
# this code is executed as the first thing when the daemon starts
session.handleInitialSetup(self.setup, self.simmode)
# remove this function from the user namespace after completion
self.namespace.pop('NicosSetup', None)
def _observer(self, status, lineno):
if status != STATUS_INBREAK:
# Do not go into pause state immediately, since we will skip
# many breakpoints if not on the highest level.
self.eventfunc('status', (status, lineno))
def _breakfunc(self, frame, flag):
# check level of breakpoint reached
fn = frame.f_code.co_filename
if fn.startswith('<break>'): # '<break>n' means stoplevel n
bplevel = int(fn[7:])
else:
bplevel = BREAK_AFTER_LINE
# flag is a tuple (mode, required stoplevel, user name)
if flag[1] < bplevel or flag[0] in ('stop', 'emergency stop'):
# don't pause/stop here...
self.set_continue(flag)
else:
self.log.info('script paused in %s', self.current_location())
session.log.info('Script paused by %s', flag[2])
self.eventfunc('status', (STATUS_INBREAK, self.lineno))
new_flag = self.wait_for_continue()
# new_flag is either a flag coming from Handler.stop(), from
# Handler.continue() or the old one from above
if new_flag[0] == 'continue':
# level is ignored here
self.log.info('paused script continued')
session.log.info('Script continued by %s', new_flag[2])
elif new_flag[1] < bplevel:
# didn't pause/stop here, try again on next breakpoint
self.set_break(new_flag)
elif new_flag[0] in ('stop', 'emergency stop'):
# we can stop here, do it
self.log.info('paused script now stopped: %s', new_flag)
self.set_stop(new_flag)
def current_location(self, verbose=False):
frame = self.currentframe
if verbose:
return '\n' + ''.join(traceback.format_stack(frame))
else:
return traceback.format_stack(frame, 1)[0].strip()[5:]. \
replace('\n ', ': ')
def new_request(self, request, notify=True):
assert isinstance(request, Request)
# first send the notification, otherwise the request could be processed
# (resulting in a "processing" event) before the "request" event is
# even sent
if notify:
self.eventfunc('request', request.serialize())
# put the script on the queue (will not block)
self.queue.put(request)
return request.reqid
def block_all_requests(self):
deleted = self.queue.delete_all()
self.eventfunc('blocked', deleted)
def block_requests(self, requests):
for req in requests:
self.queue.delete_one(req)
self.eventfunc('blocked', [req])
def script_stop(self, level, user, message=None):
"""High-level "stop" routine."""
if self.status == STATUS_STOPPING:
return
elif self.status == STATUS_RUNNING:
self.log.info('script stop request while running')
suffix = user.name
if message:
suffix += ': ' + message
if level == BREAK_AFTER_LINE:
session.log.info('Stop after command requested by %s', suffix)
else:
session.log.info('Stop requested by %s', suffix)
self.block_all_requests()
self.set_break(('stop', level, user.name))
if level >= BREAK_NOW:
session.countloop_request = ('pause',
'Stopped by %s' % user.name)
else:
self.log.info('script stop request while in break')
self.block_all_requests()
self.set_continue(('stop', level, user.name))
def script_immediate_stop(self, user, message=None):
"""High-level "immediate stop"/estop routine."""
if self.status in (STATUS_IDLE, STATUS_IDLEEXC):
# only execute emergency stop functions
self.new_request(EmergencyStopRequest(user))
return
elif self.status == STATUS_STOPPING:
return
suffix = user.name
if message:
suffix += ': ' + message
session.log.warning('Immediate stop requested by %s', suffix)
self.block_all_requests()
if self.status == STATUS_RUNNING:
self.set_break(('emergency stop', 5, user.name))
else:
# in break
self.set_continue(('emergency stop', 5, user.name))
def get_queue(self):
return self.queue.serialize_queue()
def get_current_handler(self):
# both of these attributes are weakrefs, so we have to call them to get
# either the handler or None
if self.last_handler:
return self.last_handler()
elif self.current_script and self.current_script.handler:
return self.current_script.handler()
return None
def exec_script(self, code, user, handler):
# execute code in the script namespace (this is called not from
# the script thread, but from a handle thread)
self.thread_data.user = user
temp_request = ScriptRequest(code, None, user)
temp_request.parse()
session.log.log(INPUT, formatScript(temp_request, '---'))
self.last_handler = weakref.ref(handler)
try:
for block in temp_request.code:
exec(block, self.namespace)
finally:
self.last_handler = None
def eval_expression(self, expr, handler, stringify=False):
self.last_handler = weakref.ref(handler)
ns = {'session': session, 'config': config}
ns.update(self.namespace)
try:
ret = eval(expr, ns)
if stringify:
return repr(ret)
return ret
except Exception as err:
if stringify:
return '<cannot be evaluated: %s>' % err
return err
finally:
self.last_handler = None
def simulate_script(self, uuid, code, name, user):
req = ScriptRequest(code, name, user)
# use a uuid provided from client
if uuid:
req.reqid = uuid
self.simulate_request(req)
def simulate_request(self, request, quiet=False):
code, _ = parseScript(request.text, request.name, compilecode=False)
session.runSimulation(code[0], request.reqid, wait=False, quiet=quiet)
def add_watch_expression(self, val):
self.watchlock.acquire()
try:
self.watchexprs.add(val)
finally:
self.watchlock.release()
def remove_watch_expression(self, val):
self.watchlock.acquire()
try:
self.watchexprs.discard(val)
finally:
self.watchlock.release()
def remove_all_watch_expressions(self, group):
self.watchlock.acquire()
try:
for expr in self.watchexprs.copy():
if expr.endswith(group):
self.watchexprs.remove(expr)
finally:
self.watchlock.release()
def eval_watch_expressions(self):
ret = {}
self.watchlock.acquire()
try:
vals = list(self.watchexprs)
finally:
self.watchlock.release()
for val in vals:
try:
expr = val.partition(':')[0]
ret[val] = repr(eval(expr, self.namespace))
except Exception as err:
ret[val] = '<cannot be evaluated: %s>' % err
return ret
def debug_start(self, request):
# remote debugging support: tell the Controller that we want the
# debugger called immediately; we then wait for debugger commands from
# the client and feed them to the Rpdb stdin queue
self.debugger = Rpdb(self.debug_end)
request.settrace = self.debugger.set_trace
self.new_request(request)
# let clients know we're debugging and expect commands via debug_input
self.eventfunc('debugging', True)
def debug_running(self):
# remote debugging support of running script: set_debug() calls the
# given set_trace during the next call to the controller's C trace
# function, and then the debugger takes over; we then wait for debugger
# commands from the client and feed them to the Rpdb stdin queue
self.debugger = Rpdb(self.debug_end)
self.set_debug(self.debugger.set_trace)
# let clients know we're debugging and expect commands via debug_input
self.eventfunc('debugging', True)
def debug_input(self, line):
# some debugger commands arrived from a client
if self.debugger:
self.debugger.stdin.put(line)
def debug_end(self, tracing=True):
# this is called by the debugger when a command such as "continue" or
# "quit" is entered, which means that debugging is finished
self.debugger = None
self.eventfunc('debugging', False)
# set our own trace function again (Pdb replaced it)
if tracing:
self.reset_trace()
else:
sys.settrace(None)
def complete_line(self, line, lastword):
if not session._spmode:
return self.completer.get_matches(lastword, line)
spmatches = session._spmhandler.complete(lastword, line)
return [m + ' ' for m in spmatches]
def add_estop_function(self, func, args):
if not callable(func):
raise TypeError('emergency stop function must be a callable')
if not isinstance(args, tuple):
raise TypeError('emergency stop function args must be a tuple')
self.estop_functions.append((func, args))
return len(self.estop_functions)
def execute_estop(self, user):
self.log.warning('emergency stop caught, executing ESFs')
session.log.info('Stopping devices for immediate stop')
# now execute emergency stop functions
for (func, args) in self.estop_functions:
try:
self.log.info('executing ESF: %s%s', func, args)
func(*args)
except Exception:
self.log.exception('ESF raised error')
else:
self.log.info('ESF finished')
def rearrange_queue(self, ids):
"""Rearrange the queued scripts according to the given id sequence."""
def match_ids(client_ids, queued_ids):
"""Checks consistency between the ID sequences."""
queued_idset = set(queued_ids)
client_idset = set(client_ids)
if queued_idset ^ client_idset:
if client_idset - queued_idset:
temp = client_idset - queued_idset
if set(client_ids[:len(temp)]) ^ temp:
raise RequestError('Inconsistency between clients '
'script IDs and queued IDs')
# remove already executed scripts from clients id sequence
client_ids = client_ids[len(temp):]
if queued_idset - client_idset:
temp = queued_idset - client_idset
if set(queued_ids[:-len(temp)]) ^ temp:
raise RequestError('Inconsistency between clients '
'script IDs and queued IDs')
with self.queue as qop:
match_ids(ids, qop.get_ids())
for new_index, script_id in enumerate(ids):
qop.move_item(script_id, new_index)
self.eventfunc('rearranged', qop.get_ids())
def update_script(self, reqid, newcode, reason, user):
"""The desired update can be either for the executed script or a
queued script.
"""
with self.queue as qop:
# check if currently executed script needs update
if reqid == self.current_script.reqid or reqid is None:
self.current_script.update(newcode, reason, self, user)
self.log.info('running script updated by %s', user.name)
if session.cache and self.autosim:
self.simulate_request(self.current_script, quiet=True)
self.current_script.setSimstate('running')
return
# update queued script with newuser and code
qop.update(reqid, newcode, user)
self.log.info('queued script %s updated by %s', reqid, user.name)
self.eventfunc('updated', qop.get_item(reqid).serialize())
def start_script_thread(self, *args):
if self.thread:
raise RuntimeError('script thread already started')
self.thread = createThread('daemon script_thread',
self.script_thread_entry,
args=args)
def script_thread_entry(self):
"""The script execution thread entry point. This thread executes setup
code, then waits for scripts on self.queue. The script is then
executed in the context of self.namespace, using the controller (self)
to watch execution.
"""
self.log.debug('script_thread (re)started')
session.script_thread_id = current_thread().ident
try:
self.namespace['NicosSetup'] = self._setup
# and put it in the queue as the first request
request = ScriptRequest('NicosSetup()',
'setting up NICOS',
system_user,
quiet=True,
format='py')
self.new_request(request, notify=False)
while 1:
# get a script (or other request) from the queue
request = self.queue.get()
self.thread_data.user = request.user
if isinstance(request, EmergencyStopRequest):
self.log.info('executing estop request from %s',
request.user.name)
self.execute_estop(request.user.name)
continue
elif not isinstance(request, ScriptRequest):
self.log.error('unknown request: %s', request)
continue
self.log.info('processing script %s by %s', request.reqid,
request.user.name)
self.reqid_work = request.reqid
if session.cache and self.autosim:
self.simulate_request(request, quiet=True)
request.setSimstate('running')
# notify clients that we're processing this request now
self.eventfunc('processing', request.serialize())
# parse the script and split it into blocks
try:
self.current_script = request
self.current_script.parse()
except Exception:
session.log.log(INPUT, formatScript(request))
session.logUnhandledException(cut_frames=1)
continue
try:
self.current_script.execute(self)
except ControlStop as err:
if err.args[0] == 'emergency stop':
# block all pending requests (should have been done
# already, but to be on the safe side do it here again)
self.block_all_requests()
self.execute_estop(err.args[2])
else:
# in this case, we have already blocked all scripts
# queued before the "stop" command was given; scripts
# that are queued after that should be executed, so
# we don't block requests here
session.log.info('Script stopped by %s', err.args[2])
except BdbQuit as err: # pylint: disable=bad-except-order
session.log.error('Script stopped through debugger')
except Exception as err: # pylint: disable=bad-except-order
# the topmost two frames are still in the
# daemon, so don't display them to the user
# perhaps also send an error notification
try:
session.scriptEvent('exception', sys.exc_info())
except Exception:
# last resort: do not exit script thread even if we
# can't handle this exception
pass
if self.debugger:
self.debug_end(tracing=False)
session.clearActions()
session.scriptEvent('finish', None)
except Exception:
self.log.exception('unhandled exception in script thread')
session.log.error('internal error in NICOS daemon, please restart')
finally:
self.thread = None
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
from threading import Thread, local as thread_local
from Queue import Queue
from cloudify import amqp_client
thread_storage = thread_local()
class AMQPWrappedThread(Thread):
"""
creates an amqp client before calling the target method.
This thread is always set as a daemon.
"""
def __init__(self, target, *args, **kwargs):
def wrapped_target():
client = amqp_client.create_client()
self.started_amqp_client.put_nowait(True)
thread_storage.amqp_client = client
try:
self.target_method()
finally:
def __init__ ( self ):
self.traits_logger = None
self.main_thread = None
self.thread_local = thread_local()
"context",
"current_step",
"StepTiming",
"Counters",
]
COLLECT_TIMING = False
TIMING_PATH = "timing.csv"
TIMING_MODE = "a"
TIMING_BUFFERING = 1
TIMING_FILE: Optional[IO] = None
LOG_ALL_STEPS = False
THREAD_LOCAL = thread_local()
COUNTED_THREADS = 0
@utd.expand_doc()
def collect_timing(
collect: bool,
path: str = TIMING_PATH, # pylint: disable=used-prior-global-declaration
mode: str = TIMING_MODE, # pylint: disable=used-prior-global-declaration
*,
buffering: int = TIMING_BUFFERING, # pylint: disable=used-prior-global-declaration
) -> None:
"""
Specify whether, where and how to collect timing information.
By default, we do not. Override this by setting the ``METACELLS_COLLECT_TIMING`` environment
