def list_roles(): roles = getattr(threading.local(), "roles", None) if roles is None: roleapi = RoleAPI() roles = dict([(r, roleapi.info(r)) for r in roleapi.list()]) setattr(threading.local(), "roles", roles) return roles
def run(self): global threadLock s = threading.local() self.__req = self.__victim + '/contact-info' self.__headers = {'Referer': self.__req, 'Content-Type': 'multipart/form-data; boundary=---------------------------29713827018367436031035745563'} # Use a backoff sleep time to avoid all threads starting at once time.sleep(random.random()) session = threading.local() session = requests.Session() while self.__running: self.__randstr = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) self.__data = '-----------------------------29713827018367436031035745563\x0d\x0a' \ 'Content-Disposition: form-data; name=\"form.widgets.sender_fullname\"\x0d\x0a\x0d\x0aspammer ' + self.__randstr + '\x0d\x0a' \ '-----------------------------29713827018367436031035745563\x0d\x0a' \ 'Content-Disposition: form-data; name=\"form.widgets.sender_from_address\"\x0d\x0a\x0d\x0aspam-' + self.__randstr + '@nowhere.org\x0d\x0a' \ '-----------------------------29713827018367436031035745563\x0d\x0a' \ 'Content-Disposition: form-data; name=\"form.widgets.subject\"\x0d\x0a\x0d\x0aspam ' + self.__randstr + '\x0d\x0a' \ '-----------------------------29713827018367436031035745563\x0d\x0a' \ 'Content-Disposition: form-data; name=\"form.widgets.message\"\x0d\x0a\x0d\x0ajust_spam_' + self.__randstr + '\x0d\x0a' \ '-----------------------------29713827018367436031035745563\x0d\x0a' \ 'Content-Disposition: form-data; name=\"form.buttons.send\"\x0d\x0a\x0d\x0aSenden\x0d\x0a' \ '-----------------------------29713827018367436031035745563--\x0d\x0a' try: self.__r = session.post(self.__req, headers=self.__headers, data=self.__data, verify = False) self.__e = self.__r.status_code self.__h = self.__r.headers self.__c = self.__r.content if self.__e >= 500: s = "5" else: s = "." # No need for thread safety here... Hrrhrrhrr :) sys.stdout.write(s) except (requests.ConnectionError): sys.stdout.write("E") pass
def list_filters(): filters = getattr(threading.local(), "filters", None) if filters is None: filterapi = FilterAPI() filters = dict([(f['id'], f) for f in filterapi.filters()]) setattr(threading.local(), "filters", filters) return filters
def test_create_local_subclass_init_args(self): with self.assertRaisesRegex(TypeError, "Initialization arguments are not supported"): local("foo") with self.assertRaisesRegex(TypeError, "Initialization arguments are not supported"): local(kw="foo")
def NewCursor(): global connection if not connection: try: connection = threading.local().connection except: connection = NewConnection() threading.local().connection = connection return connection.cursor()
def add_request(self,req_dict): if hasattr(threading.local(), 'ds_request'): local_req = getattr(threading.local(),'ds_request') else: local_req = {'cache':True} if 'cache' in req_dict and str(req_dict['cache']).lower() == 'false': local_req['cache'] = False setattr(threading.local(), 'ds_request', local_req)
def _browser(): """ Returns a thread-bound Browser object. The browser is initialized when created. """ if hasattr(threading.local(), "browser"): return threading.local().browser else: browser = mechanize.Browser() _initialize(browser) threading.local().browser = browser return browser
def trapExit( code ): """ Cherrypy likes to call os._exit() which causes the interpreter to bomb without a chance of catching it. This sucks. This function will replace os._exit() and throw an exception instead """ if hasattr( threading.local(), "isMain" ) and threading.local().isMain: # The main thread should raise an exception sys.stderr.write("*******EXIT WAS TRAPPED**********\n") raise RuntimeError, "os._exit() was called in the main thread" else: # os._exit on child threads should just blow away the thread raise SystemExit, "os._exit() was called in a child thread. " +\ "Protecting the interpreter and trapping it"
def run(self): serverip = threading.local() _timeout = threading.local() serverip = self.kwargs['server'] _timeout = self.kwargs['timeout'] _lock = self.kwargs['lock'] with _lock: print 'monitoring ' + serverip + ' with timeout ' +\ str(_timeout) if syslog: syslog.syslog('monitoring ' + serverip) count = 0 while (1): for domain in config.domains(): startTime = datetime.datetime.now() try: req = threading.local() req = DNS.Request(domain, qtype='A', server=serverip, timeout=_timeout).req() endTime = datetime.datetime.now() duration = threading.local() duration = _convert_milliseconds(endTime - startTime) if req.header['status'] == 'NOERROR': if config.verbose(): with _lock: print domain + '@' + serverip + ' OK' statusQueue.put('OK' + ' ' + serverip + ' ' + domain + ' ' + str(duration)) except Exception: endTime = datetime.datetime.now() duration = _convert_milliseconds(endTime - startTime) statusQueue.put('BAD ' + serverip + ' ' + domain + ' ' + str(duration)) sleep(config.frequency()) if (config.cycles()): count += 1 cycles = config.cycles() if (count >= cycles): break
def get_repos(reload=False): if reload: repos = None else: repos = getattr(threading.local(), "repos", None) if repos is None: # this looks inefficient, and it is, but repos has to be fully # loaded before we can call _load_repo_extras(), so we have to # do this in two separate loops repoapi = RepositoryAPI() repos = dict([(r["id"], r) for r in repoapi.repositories(dict())]) for repo in repos.values(): _load_repo_extras(repo, repos=repos) setattr(threading.local(), "repos", repos) return repos
def __init__(self, servers=None, timeout=None, ca_cert=None, verify_ssl_cert=False, error_trace=False, cert_file=None, key_file=None): if not servers: servers = [self.default_server] else: servers = _to_server_list(servers) self._active_servers = servers self._inactive_servers = [] self._http_timeout = timeout pool_kw = _pool_kw_args(ca_cert, verify_ssl_cert) pool_kw['cert_file'] = cert_file pool_kw['key_file'] = key_file self.server_pool = {} self._update_server_pool(servers, timeout=timeout, **pool_kw) self._pool_kw = pool_kw self._lock = threading.RLock() self._local = threading.local() self.path = self.SQL_PATH if error_trace: self.path += '?error_trace=1'
def __init__(self, databases=None): """ databases is an optional dictionary of database definitions (structured like settings.DATABASES). """ self._databases = databases self._connections = local()
def __init__(self, preprocessors=None): self.use_default_preprocessors = False if isinstance(preprocessors, Iterable): self.preprocessors = tuple(preprocessors) elif preprocessors: self.preprocessors = (preprocessors,) self.__tls = threading.local()
def __init__(self, locks): if not isinstance(locks, tuple): locks = tuple(locks) if len(locks) <= 0: raise ValueError("Zero locks requested") self._locks = locks self._local = threading.local()
def reset_translations(): import gettext from django.utils.translation import trans_real gettext._translations = {} trans_real._translations = {} trans_real._default = None trans_real._active = threading.local()
def __init__(self): self._key_to_registration = dict() self._singleton_instances = dict() self._singleton_instances_lock = threading.Lock() self._weak_references = WeakValueDictionary() self._weak_references_lock = threading.Lock() self._thread_local = threading.local()
def _load_tile(self, tile): # global db context cannot be shared across threads. ctx = threading.local() if not hasattr(ctx, "db"): ctx.db = sqlite3.connect(self.filename) # get the right tile c = ctx.db.cursor() c.execute( ("SELECT tile_data FROM tiles WHERE " "zoom_level=? AND tile_column=? AND tile_row=?"), (tile.zoom, tile.tile_x, tile.tile_y)) # print "fetch", tile.zoom, tile.tile_x, tile.tile_y row = c.fetchone() if not row: tile.state = "done" return # no-file loading try: data = io.BytesIO(row[0]) except: # android issue, "buffer" does not have the buffer interface # ie row[0] buffer is not compatible with BytesIO on Android?? data = io.BytesIO(bytes(row[0])) im = CoreImage(data, ext='png', filename="{}.{}.{}.png".format(tile.zoom, tile.tile_x, tile.tile_y)) if im is None: tile.state = "done" return return self._load_tile_done, (tile, im, )
def __init__(self, size, **kwargs): if not isinstance(size, int): raise TypeError("Pool 'size' arg must be an integer") if not size > 0: raise ValueError("Pool 'size' arg must be greater than zero") logger.debug( "Initializing connection pool with %d connections", size) self._lock = threading.Lock() self._queue = Queue.LifoQueue(maxsize=size) self._thread_connections = threading.local() connection_kwargs = kwargs connection_kwargs['autoconnect'] = False for i in xrange(size): connection = Connection(**connection_kwargs) self._queue.put(connection) # The first connection is made immediately so that trivial # mistakes like unresolvable host names are raised immediately. # Subsequent connections are connected lazily. with self.connection(): pass
def setUp(self): """ Clear cached appstore connection """ tank.util.shotgun.connection._g_sg_cached_connections = threading.local() tank.set_authenticated_user(None) # Prevents from connecting to Shotgun. self._server_caps_mock = patch("tank_vendor.shotgun_api3.Shotgun.server_caps") self._server_caps_mock.start() self.addCleanup(self._server_caps_mock.stop) # Avoids crash because we're not in a pipeline configuration. self._get_api_core_config_location_mock = patch( "tank.util.shotgun.connection.__get_api_core_config_location", return_value="unused_path_location" ) self._get_api_core_config_location_mock.start() self.addCleanup(self._get_api_core_config_location_mock.stop) # Mocks app store script user credentials retrieval self._get_app_store_key_from_shotgun_mock = patch( "tank.descriptor.io_descriptor.appstore.IODescriptorAppStore." "_IODescriptorAppStore__get_app_store_key_from_shotgun", return_value=("abc", "123") ) self._get_app_store_key_from_shotgun_mock.start() self.addCleanup(self._get_app_store_key_from_shotgun_mock.stop)
def __init__( self, parent=None, **kwargs ): self._construction_complete = False self._database = kwargs.pop('database', not_set) explicit_kwargs = list(kwargs) defaults = parent or settings.default if defaults is not None: for setting in all_settings.values(): if kwargs.get(setting.name, not_set) is not_set: kwargs[setting.name] = getattr(defaults, setting.name) elif setting.validator: kwargs[setting.name] = setting.validator( kwargs[setting.name]) if self._database is not_set: self._database = defaults.database for name, value in kwargs.items(): if name not in all_settings: raise InvalidArgument( 'Invalid argument %s' % (name,)) setattr(self, name, value) self.storage = threading.local() self._construction_complete = True for k in explicit_kwargs: deprecation = all_settings[k].deprecation if deprecation: note_deprecation(deprecation, self)
def __init__(self, hub): self.badge_rules = [] self.hub = hub super(FedoraBadgesConsumer, self).__init__(hub) self.consume_delay = int(self.hub.config.get('badges.consume_delay', self.consume_delay)) self.delay_limit = int(self.hub.config.get('badges.delay_limit', self.delay_limit)) # Five things need doing at start up time # 0) Set up a request local to hang thread-safe db sessions on. # 1) Initialize our connection to the tahrir DB and perform some # administrivia. # 2) Initialize our connection to the datanommer DB. # 3) Load our badge definitions and rules from YAML. # 4) Initialize fedmsg so that those listening to us can handshake. # Thread-local stuff self.l = threading.local() # Tahrir stuff. self._initialize_tahrir_connection() # Datanommer stuff self._initialize_datanommer_connection() # Load badge definitions directory = hub.config.get("badges.yaml.directory", "badges_yaml_dir") self.badge_rules = self._load_badges_from_yaml(directory)
def getConf(): _data = threading.local() if hasattr(_data, 'conf'): log.debug(_("Returning thread local configuration")) return _data.conf return conf
def _get_dict(): if not Timing.context: import threading Timing.context = threading.local() if not hasattr(Timing.context, 'dict'): Timing.context.dict = {} return Timing.context.dict
def __init__( self, ns, logical_home_folder, inherits_from=None, inheritable_folders=None): """Creates a new instance of the datastore-backed file system. Args: ns: A datastore namespace to use for storing all data and metadata. logical_home_folder: A logical home dir of all files (/a/b/c/...). inherits_from: A file system to use for the inheritance. inheritable_folders: A list of folders that support inheritance. Returns: A new instance of the object. Raises: Exception: if invalid inherits_from is given. """ if inherits_from and not isinstance( inherits_from, LocalReadOnlyFileSystem): raise Exception('Can only inherit from LocalReadOnlyFileSystem.') self._ns = ns self._logical_home_folder = AbstractFileSystem.normpath( logical_home_folder) self._inherits_from = inherits_from self._inheritable_folders = [] self._cache = threading.local() if inheritable_folders: for folder in inheritable_folders: self._inheritable_folders.append(AbstractFileSystem.normpath( folder))
def _reset(): ''' Reset the thread local. ''' import django_mobile del django_mobile._local django_mobile._local = threading.local()
def __init__(self, url, schema=None, reflectMetadata=True, engine_kwargs=None): if engine_kwargs is None: engine_kwargs = {} if url.startswith("postgres"): engine_kwargs.setdefault("poolclass", NullPool) self.lock = threading.RLock() self.local = threading.local() if "?" in url: url, query = url.split("?", 1) query = parse_qs(query) if schema is None: # le pop schema_qs = query.pop("schema", query.pop("searchpath", [])) if len(schema_qs): schema = schema_qs.pop() if len(query): url = url + "?" + urlencode(query, doseq=True) self.schema = schema self.engine = create_engine(url, **engine_kwargs) self.url = url self.metadata = MetaData(schema=schema) self.metadata.bind = self.engine if reflectMetadata: self.metadata.reflect(self.engine) self._tables = {}
def _blank_copy(self, c): super(FullFrontend, self)._blank_copy(c) c._track = self._track c._solver_backend = self._solver_backend c.timeout = self.timeout c._tls = threading.local() c._to_add = []
def __init__(self, solver_backend, timeout=None, track=False, **kwargs): ConstrainedFrontend.__init__(self, **kwargs) self._track = track self._solver_backend = solver_backend self.timeout = timeout if timeout is not None else 300000 self._tls = threading.local() self._to_add = []
def __init__(self, sqluri, standard_collections=False, **dbkwds): self.sqluri = sqluri self.dbconnector = DBConnector(sqluri, **dbkwds) # There doesn't seem to be a reliable cross-database way to set the # initial value of an autoincrement column. Fake it by inserting # a row into the table at the desired start id. self.standard_collections = standard_collections if self.standard_collections and dbkwds.get("create_tables", False): zeroth_id = FIRST_CUSTOM_COLLECTION_ID - 1 with self.dbconnector.connect() as connection: params = {"collectionid": zeroth_id, "name": ""} try: connection.query("INSERT_COLLECTION", params) except IntegrityError: pass # A local in-memory cache for the name => collectionid mapping. self._collections_by_name = {} self._collections_by_id = {} if self.standard_collections: for id, name in STANDARD_COLLECTIONS: self._collections_by_name[name] = id self._collections_by_id[id] = name # A thread-local to track active sessions. self._tldata = threading.local()
def _ana_setstate(self, s): backend_name, self.timeout, self._track, base_state = s self._solver_backend = backends._backends_by_type[backend_name] # self._tls = None self._tls = threading.local() self._to_add = [] ConstrainedFrontend._ana_setstate(self, base_state)
def __init__(self): self.local_res = threading.local()
from tensorflow.python.eager import execute from tensorflow.python.eager import tape from tensorflow.python.eager.graph_only_ops import graph_placeholder from tensorflow.python.framework import c_api_util from tensorflow.python.framework import constant_op from tensorflow.python.framework import dtypes as dtypes_module from tensorflow.python.framework import errors from tensorflow.python.framework import ops from tensorflow.python.ops import gradients_impl from tensorflow.python.util import compat from tensorflow.python.util import nest from tensorflow.python.util import tf_decorator # Thread-local storage for tfe Tensors which are referenced while evaluating a # graph-mode function. _scoped_captures = threading.local() # _scoped_captures.tensors is either None or a map from Tensor id to a pair # of a tfe tensor and its corresponding placeholder to pass as a function # argument. The value should be None unless we're in function definition # context. _scoped_captures.tensors = None @contextlib.contextmanager def capture_tensors(captures): old = _scoped_captures.__dict__.get("tensors", None) try: _scoped_captures.tensors = captures yield finally: _scoped_captures.tensors = old
from __future__ import print_function import collections import inspect import linecache import sys import threading # Names for indices into TF traceback tuples. TB_FILENAME = 0 TB_LINENO = 1 TB_FUNCNAME = 2 TB_CODEDICT = 3 # Dictionary of Python interpreter state. stacks = threading.local() def _source_mappers(): if not hasattr(stacks, 'source_mapper'): stacks.source_mapper = [] return stacks.source_mapper def _source_filters(): if not hasattr(stacks, 'source_filter'): stacks.source_filter = [] return stacks.source_filter class StackTraceMapper(object):
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== """Contains AutoCastVariable, a variable which automatically casts itself.""" import tensorflow.compat.v2 as tf import threading from keras.distribute import distributed_training_utils # _autocast_dtype.dtype is the dtype AutoCastVariables should be cast to, or # None if AutoCastVariables should not be cast. _autocast_dtype = threading.local() def numpy_text(tensor, is_repr=False): """Human readable representation of a tensor's numpy value.""" if tensor.dtype.is_numpy_compatible: # pylint: disable=protected-access text = repr(tensor._numpy()) if is_repr else str(tensor._numpy()) # pylint: enable=protected-access else: text = '<unprintable>' if '\n' in text: text = '\n' + text return text
except ImportError: have_minify = False try: import simplejson as sj #external installed library except: try: import json as sj #standard installed library except: import contrib.simplejson as sj #pure python library regex_session_id = re.compile('^([\w\-]+/)?[\w\-\.]+$') __all__ = ['Request', 'Response', 'Session'] current = threading.local() # thread-local storage for request-scope globals css_template = '<link href="%s" rel="stylesheet" type="text/css" />' js_template = '<script src="%s" type="text/javascript"></script>' coffee_template = '<script src="%s" type="text/coffee"></script>' typescript_template = '<script src="%s" type="text/typescript"></script>' less_template = '<link href="%s" rel="stylesheet/less" type="text/css" />' css_inline = '<style type="text/css">\n%s\n</style>' js_inline = '<script type="text/javascript">\n%s\n</script>' def copystream_progress(request, chunk_size=10**5): """ copies request.env.wsgi_input into request.body and stores progress upload status in cache_ram X-Progress-ID:length and X-Progress-ID:uploaded
_ITEMS_PROCESSOR_ARGS_LEN = 0 """Number of arguments accepted by items processor.""" _I18N_TREES = [] """Stores aliases of trees supporting internationalization.""" _DYNAMIC_TREES = {} """Holds trees dynamically loaded from project apps.""" _IDX_ORPHAN_TREES = 'orphans' """Dictionary index in `_DYNAMIC_TREES` for orphaned trees list.""" _IDX_TPL = '%s|:|%s' """Name template used as dictionary index in `_DYNAMIC_TREES`.""" _THREAD_LOCAL = local() _THREAD_SITETREE = 'sitetree' _CONTEXT_FLATTEN = VERSION >= (1, 11) _UNSET = set() # Sentinel def get_sitetree(): """Returns SiteTree (thread-singleton) object, implementing utility methods. :rtype: SiteTree """ sitetree = getattr(_THREAD_LOCAL, _THREAD_SITETREE, None) if sitetree is None:
from logging.config import dictConfig from connect.config import Config from connect.logger import logger # Configure logging if os.path.exists('/etc/cloudblue-connector/config-logging.json'): with open( '/etc/cloudblue-connector/config-logging.json') as config_log_file: settings = json.load(config_log_file) dictConfig(settings['logging']) # Set connect log level / default level ERROR logger.setLevel('DEBUG') context_data = threading.local() context_data.request_id = None class ContextFilter(logging.Filter): """ This is a filter which injects contextual information into the log record. """ def filter(self, record): if context_data.request_id: record.name += "." + context_data.request_id return True class PasswordFilter(logging.Filter): """
from django.utils.http import urlquote from django.utils.importlib import import_module from django.utils.module_loading import module_has_submodule from django.utils.regex_helper import normalize from django.utils import six from django.utils.translation import get_language _resolver_cache = {} # Maps URLconf modules to RegexURLResolver instances. _ns_resolver_cache = {} # Maps namespaces to RegexURLResolver instances. _callable_cache = { } # Maps view and url pattern names to their view functions. # SCRIPT_NAME prefixes for each thread are stored here. If there's no entry for # the current thread (which is the only one we ever access), it is assumed to # be empty. _prefixes = local() # Overridden URLconfs for each thread are stored here. _urlconfs = local() class ResolverMatch(object): def __init__(self, func, args, kwargs, url_name=None, app_name=None, namespaces=None): self.func = func self.args = args
def unoconv_client(self): if not hasattr(self, '_unoconv_client'): self._unoconv_client = threading.local() if not hasattr(self._unoconv_client, 'session'): self._unoconv_client.session = requests.Session() return self._unoconv_client.session
import threading import time from functools import partial from django.apps import apps from django.conf import settings from django.db.models.signals import pre_save from django.utils.deprecation import MiddlewareMixin from auditlog.models import LogEntry threadlocal = threading.local() class AuditlogMiddleware(MiddlewareMixin): """ Middleware to couple the request's user to log items. This is accomplished by currying the signal receiver with the user from the request (or None if the user is not authenticated). """ def process_request(self, request): """ Gets the current user from the request and prepares and connects a signal receiver with the user already attached to it. """ # Initialize thread local storage threadlocal.auditlog = { 'signal_duid': (self.__class__, time.time()), 'remote_addr': request.META.get('REMOTE_ADDR'), }
import threading current_user_store = threading.local() class GlobalUserMiddleware(object): def process_request(self, request): if request.user: current_user_store.user = request.user return None def process_response(self, request, response): if hasattr(current_user_store, 'user'): del current_user_store.user return response def process_exception(self, request, exception): if hasattr(current_user_store, 'user'): del current_user_store.user return None
return # Under eager mode. Eagerly execute check_numerics op. for slot, output in enumerate(outputs): if (output.dtype.is_floating and (op_type_bytes, slot) not in IGNORE_OP_OUTPUTS): array_ops.check_numerics( output, get_check_numerics_error_message( slot, len(outputs), op_type, output, inputs, stack_height_limit=_state.config.stack_height_limit, path_length_limit=_state.config.path_length_limit)) CheckNumericsConfig = collections.namedtuple( "CheckNumericsConfig", "stack_height_limit path_length_limit") _state = threading.local() @tf_export("debugging.enable_check_numerics") def enable_check_numerics(stack_height_limit=30, path_length_limit=50): r"""Enable tensor numerics checking in an eager/graph unified fashion. The numerics checking mechanism will cause any TensorFlow eager execution or graph execution to error out as soon as an op's output tensor contains infinity or NaN. This method is idempotent. Calling it multiple times has the same effect as calling it once. This method takes effect only on the thread in which it is called.
class KlioContext(object): """Context related to the currently running job. Available to transforms via one of the :ref:`KlioContext decorators <klio-context-decorators>`. """ _thread_local = threading.local() def __init__(self): self.__transform_name = None def _create_klio_job_obj(self): klio_job = klio_pb2.KlioJob() klio_job.job_name = self.config.job_name klio_job.gcp_project = self.config.pipeline_options.project klio_job_str = klio_job.SerializeToString() return klio_job_str def _get_metrics_registry(self): native_metrics_client = native_metrics.NativeMetricsClient(self.config) clients = [native_metrics_client] use_logger, use_shumway = None, None metrics_config = self.config.job_config.metrics # use_logger/use_shumway could be False (turn off), # None (use default config), or a dict of configured values use_logger = metrics_config.get("logger") use_shumway = metrics_config.get("shumway") # TODO: set runner in OS environment (via klio-exec), since # the runner defined in config could be overwritten via # `--direct-runner`. # i.e.: runner = os.getenv("BEAM_RUNNER", "").lower() runner = self.config.pipeline_options.runner if kvars.KlioRunner.DIRECT_RUNNER != runner: if use_logger is None: use_logger = False # use shumway when running on DirectGKERunner unless it's explicitly # turned off/set to False. Don't set it to True if it's set to False # or it's a dictionary (aka has some configuration) if kvars.KlioRunner.DIRECT_GKE_RUNNER == runner: if use_shumway is None: use_shumway = True # shumway only works on DirectGKERunner, so we explicitly set it # to False else: use_shumway = False if use_logger is not False: logger_client = metrics_logger.MetricsLoggerClient(self.config) clients.append(logger_client) if use_shumway is not False: shumway_client = shumway.ShumwayMetricsClient(self.config) clients.append(shumway_client) return metrics_client.MetricsRegistry( clients, transform_name=self._transform_name) @property def config(self): """A ``KlioConfig`` instance representing the job's configuration.""" return RunConfig.get() @property def job(self): """An instance of :ref:`kliojob` of the current job.""" klio_job = getattr(self._thread_local, "klio_job", None) if not klio_job: self._thread_local.klio_job = self._create_klio_job_obj() return self._thread_local.klio_job @property def logger(self): """A namespaced logger. Equivalent to ``logging.getLogger("klio")``. """ klio_logger = getattr(self._thread_local, "klio_logger", None) if not klio_logger: self._thread_local.klio_logger = logging.getLogger("klio") return self._thread_local.klio_logger @property def metrics(self): """A metrics registry instance. See :ref:`metrics <metrics>` for more information.""" metrics_registry = getattr(self._thread_local, "klio_metrics", None) if not metrics_registry: self._thread_local.klio_metrics = self._get_metrics_registry() return self._thread_local.klio_metrics # <-- private/internal attributes --> @property def _transform_name(self): return self.__transform_name @_transform_name.setter def _transform_name(self, name): self.__transform_name = name
class action: """@action(...) is a decorator for functions to be exposed as actions""" current = threading.local() registered = set() app_name = "_default" def __init__(self, path, **kwargs): self.path = path self.kwargs = kwargs @staticmethod def uses(*fixtures_in): """Find all fixtures, including dependencies, topologically sorted""" fixtures = [] reversed_fixtures = [] stack = list(fixtures_in) while stack: fixture = stack.pop() reversed_fixtures.append(fixture) for other in getattr(fixture, "__prerequisites__", []): stack.append(other) for fixture in reversed(reversed_fixtures): if isinstance(fixture, str): fixture = Template(fixture) if not fixture in fixtures: fixtures.append(fixture) def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): # data shared by all fixtures in the piplined for each request shared_data = {"template_context": {}} try: [obj.on_request() for obj in fixtures] ret = func(*args, **kwargs) for obj in fixtures: ret = obj.transform(ret, shared_data) [obj.on_success() for obj in fixtures] return ret except HTTP: [obj.on_success() for obj in fixtures] raise except Exception: [obj.on_error() for obj in fixtures] raise return wrapper return decorator @staticmethod def requires(*requirements): """Enforces requirements or calls bottle.abort(401)""" def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): for requirement in requirements: if not requirement(): bottle.abort(401) return func(*args, **kwargs) return wrapper return decorator @staticmethod def catch_errors(app_name, func): """Catches and logs errors in an action; also sets request.app_name""" @functools.wraps(func) def wrapper(*func_args, **func_kwargs): try: request.app_name = app_name ret = func(*func_args, **func_kwargs) if isinstance(ret, dict): response.headers["Content-Type"] = "application/json" ret = dumps(ret) return ret except HTTP as http: response.status = http.status return "" except bottle.HTTPResponse: raise except Exception: logging.error(traceback.format_exc()) try: ticket = ErrorStorage().log(request.app_name, get_error_snapshot()) except Exception: logging.error(traceback.format_exc()) ticket = "unknown" return error_page(500, button_text=ticket, href="/_dashboard/ticket/" + ticket) return wrapper def __call__(self, func): """Building the decorator""" app_name = action.app_name path = ("/" if app_name == "_default" else "/%s/" % app_name) + self.path # the _default app has no prefix if not func in self.registered: func = action.catch_errors(app_name, func) func = bottle.route(path, **self.kwargs)(func) if path.endswith("/index"): # /index is always optional func = bottle.route(path[:-6] or "/", **self.kwargs)(func) self.registered.add(func) return func
from bugbug import bugzilla, db, repository from bugbug.models.regressor import ( BUG_FIXING_COMMITS_DB, BUG_INTRODUCING_COMMITS_DB, TOKENIZED_BUG_INTRODUCING_COMMITS_DB, ) from bugbug.utils import ( ThreadPoolExecutorResult, download_and_load_model, zstd_compress, ) basicConfig(level=INFO) logger = getLogger(__name__) thread_local = threading.local() MAX_MODIFICATION_NUMBER = 50 RELATIVE_START_DATE = relativedelta(years=2, months=6) # Only needed because mercurial<->git mapping could be behind. RELATIVE_END_DATE = relativedelta(days=7) IGNORED_COMMITS_DB = "data/ignored_commits.json" db.register( IGNORED_COMMITS_DB, "https://s3-us-west-2.amazonaws.com/communitytc-bugbug/data/ignored_commits.json.zst", 1, ) class RegressorFinder(object):
# is same as ThreadLocal in Java, used to save variant by thread,a variant is available in thread import threading local_school = threading.local() def process_student(): print 'Hello, %s (in %s)' % (local_school.student, threading.current_thread().name) def process_thread(name): # bind a student to Threadlocal local_school.student = name process_student() t1 = threading.Thread(target=process_thread, args=('Alice', ), name='Thread-A') t2 = threading.Thread(target=process_thread, args=('Bob', ), name='Thread-B') t1.start() t2.start() t1.join() t2.join() # you can understander a ThreadLocal obj is a dict,and the key is a thread obj,but you can't get the value of thread A in thread B
"""Local storage of variables using weak references""" import threading import weakref class WeakLocal(threading.local): def __getattribute__(self, attr): rval = super(WeakLocal, self).__getattribute__(attr) if rval: # pragma: no cover # NOTE(mikal): this bit is confusing. What is stored is a weak # reference, not the value itself. We therefore need to lookup # the weak reference and return the inner value here. rval = rval() return rval def __setattr__(self, attr, value): value = weakref.ref(value) return super(WeakLocal, self).__setattr__(attr, value) # NOTE(mikal): the name "store" should be deprecated in the future store = WeakLocal() # A "weak" store uses weak references and allows an object to fall out of scope # when it falls out of scope in the code that uses the thread local storage. A # "strong" store will hold a reference to the object so that it never falls out # of scope. weak_store = WeakLocal() strong_store = threading.local()
# This paramstyle uses '%s' parameters. paramstyle = 'format' # --- private global constants # Values of seconds are with 3 decimal places in SF, but they are rounded to # whole seconds for the most of fields. SALESFORCE_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%f+0000' # --- request_count = 0 # global counter connect_lock = threading.Lock() thread_connections = threading.local() class RawConnection(object): """ parameters: settings_dict: like settings.SADABASES['salesforce'] in Django alias: important if the authentication should be shared for more thread errorhandler: function with following signature ``errorhandler(connection, cursor, errorclass, errorvalue)`` use_introspection: bool """ # pylint:disable=too-many-instance-attributes Error = Error InterfaceError = InterfaceError
import os import threading from textwrap import wrap from typing import Any, Iterator, Optional, TextIO, Union import rich import rich.box import rich.console import rich.rule import rich.segment import rich.table import rich.text from flexget.options import ArgumentParser local_context = threading.local() class _Console(rich.console.Console): def __init__(self, *args, **kwargs): if "PYCHARM_HOSTED" in os.environ: kwargs.setdefault('color_system', 'truecolor') super().__init__(*args, **kwargs) def __call__(self, text: Any, *args, **kwargs) -> None: """ Print to console safely. Output is able to be captured by different streams in different contexts. Any plugin wishing to output to the user's console should use this function instead of print so that output can be redirected when FlexGet is invoked from another process.
def _setup_stdlib_logging(config: "HomeServerConfig", log_config_path: Optional[str], logBeginner: LogBeginner) -> None: """ Set up Python standard library logging. """ if log_config_path is None: log_format = ( "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s" " - %(message)s") logger = logging.getLogger("") logger.setLevel(logging.INFO) logging.getLogger("synapse.storage.SQL").setLevel(logging.INFO) formatter = logging.Formatter(log_format) handler = logging.StreamHandler() handler.setFormatter(formatter) logger.addHandler(handler) else: # Load the logging configuration. _load_logging_config(log_config_path) # We add a log record factory that runs all messages through the # LoggingContextFilter so that we get the context *at the time we log* # rather than when we write to a handler. This can be done in config using # filter options, but care must when using e.g. MemoryHandler to buffer # writes. log_context_filter = LoggingContextFilter() log_metadata_filter = MetadataFilter( {"server_name": config.server.server_name}) old_factory = logging.getLogRecordFactory() def factory(*args: Any, **kwargs: Any) -> logging.LogRecord: record = old_factory(*args, **kwargs) log_context_filter.filter(record) log_metadata_filter.filter(record) return record logging.setLogRecordFactory(factory) # Route Twisted's native logging through to the standard library logging # system. observer = STDLibLogObserver() threadlocal = threading.local() @implementer(ILogObserver) def _log(event: dict) -> None: if "log_text" in event: if event["log_text"].startswith( "DNSDatagramProtocol starting on "): return if event["log_text"].startswith("(UDP Port "): return if event["log_text"].startswith("Timing out client"): return # this is a workaround to make sure we don't get stack overflows when the # logging system raises an error which is written to stderr which is redirected # to the logging system, etc. if getattr(threadlocal, "active", False): # write the text of the event, if any, to the *real* stderr (which may # be redirected to /dev/null, but there's not much we can do) try: event_text = eventAsText(event) print("logging during logging: %s" % event_text, file=sys.__stderr__) except Exception: # gah. pass return try: threadlocal.active = True return observer(event) finally: threadlocal.active = False logBeginner.beginLoggingTo([_log], redirectStandardIO=False)
import threading import traceback from .signal import Signal local_tasks = threading.local() class TaskFailure(Exception): def __init__(self, message): Exception.__init__(self, message) self.message = message class TaskStopped(Exception): def __init__(self): Exception.__init__(self) class Task(object): Failure = TaskFailure Stopped = TaskStopped def __init__(self, name: str) -> None: self.__name = name self.stop_flag = False self.started = Signal() self.succeeded = Signal() self.progressed = Signal() self.failed = Signal() self.stopped = Signal()
class PredictionDoFn(beam.DoFn): """A DoFn class loading the model to create session and performing prediction. The input PCollection consists of a list of strings from the input files. The DoFn first loads model from a given path where meta graph data and checkpoint data are exported to. Then if the there is only one string input tensor or the model needs to preprocess the input, it directly passes the data to prediction. Otherwise, it tries to load the data into JSON. Then it batches the inputs of each instance into one feed_dict. After that, it runs session and predicts the interesting values for all the instances. Finally it emits the prediction result for each individual instance. """ class _ModelState(object): """Atomic representation of the in-memory state of the model.""" def __init__(self, model_dir, skip_preprocessing): self.model_dir = model_dir session, signature = mlprediction.load_model(model_dir) client = mlprediction.SessionClient(session, signature) self.model = mlprediction.DefaultModel.from_client( client, model_dir, skip_preprocessing=skip_preprocessing) # TODO(b/33746781): Get rid of this and instead use self._model_state for # all initialization detection. _thread_local = threading.local() def __init__(self, aggregator_dict=None, cloud_logger=None, skip_preprocessing=False, target="", config=None): """Constructor of Prediction beam.DoFn class. Args: aggregator_dict: A dict of aggregators containing maps from counter name to the aggregator. cloud_logger: The cloud logging client to send logs to. skip_preprocessing: bool whether to skip preprocessing even when the metadata.yaml/metadata.json file exists. target: The execution engine to connect to. See target in tf.Session(). In most cases, users should not set the target. config: A ConfigProto proto with configuration options. See config in tf.Session() Side Inputs: model_dir: The directory containing the model to load and the checkpoint files to restore the session. """ self._target = target self._skip_preprocessing = skip_preprocessing self._config = config self._aggregator_dict = aggregator_dict self._cloud_logger = cloud_logger self._model_state = None # Metrics. self._model_load_seconds_distribution = beam.metrics.Metrics.distribution( self.__class__, "model_load_seconds") def _create_snippet(self, input_data): """Truncate the input data to create a snippet.""" try: input_snippet = "\n".join(str(x) for x in input_data) return unicode(input_snippet[:LOG_SIZE_LIMIT], errors="replace") except Exception: # pylint: disable=broad-except logging.warning("Failed to create snippet from input: [%s].", traceback.format_exc()) return "Input snippet is unavailable." # TODO(user): Remove the try catch after sdk update def process(self, element, model_dir): try: element = element.element except AttributeError: pass try: if self._model_state is None: if (getattr(self._thread_local, "model_state", None) is None or self._thread_local.model_state.model_dir != model_dir): start = datetime.datetime.now() self._thread_local.model_state = self._ModelState( model_dir, self._skip_preprocessing) self._model_load_seconds_distribution.update( int((datetime.datetime.now() - start).total_seconds())) self._model_state = self._thread_local.model_state else: assert self._model_state.model_dir == model_dir # Try to load it. if (self._model_state.model.is_single_string_input() or self._model_state.model.need_preprocess()): loaded_data = element else: loaded_data = [json.loads(d) for d in element] instances = mlprediction.decode_base64(loaded_data) inputs, predictions = self._model_state.model.predict(instances) predictions = list(predictions) predictions = mlprediction.encode_base64( predictions, self._model_state.model.outputs_type_map()) if self._aggregator_dict: aggr = self._aggregator_dict.get( aggregators.AggregatorName.ML_PREDICTIONS, None) if aggr: aggr.inc(len(predictions)) for i, p in zip(inputs, predictions): yield i, p except mlprediction.PredictionError as e: logging.error("Got a known exception: [%s]\n%s", e.error_message, traceback.format_exc()) if self._cloud_logger: # TODO(user): consider to write a sink to buffer the logging events. It # also eliminates the restarting/duplicated running issue. self._cloud_logger.write_error_message( e.error_message, self._create_snippet(element)) # reraise failure to load model as permanent exception to end dataflow job if e.error_code == mlprediction.PredictionError.FAILED_TO_LOAD_MODEL: raise beam.utils.retry.PermanentException(e.error_message) yield beam.pvalue.SideOutputValue("errors", (e.error_message, element)) except Exception as e: # pylint: disable=broad-except logging.error("Got an unknown exception: [%s].", traceback.format_exc()) if self._cloud_logger: self._cloud_logger.write_error_message( str(e), self._create_snippet(element)) yield beam.pvalue.SideOutputValue("errors", (str(e), element))
class ProxyListenerDynamoDB(ProxyListener): thread_local = threading.local() def __init__(self): self._table_ttl_map = {} def forward_request(self, method, path, data, headers): if path.startswith('/shell'): return True data = json.loads(to_str(data)) ddb_client = aws_stack.connect_to_service('dynamodb') if random.random() < config.DYNAMODB_ERROR_PROBABILITY: return error_response_throughput() action = headers.get('X-Amz-Target') if action == '%s.CreateTable' % ACTION_PREFIX: # Check if table exists, to avoid error log output from DynamoDBLocal table_names = ddb_client.list_tables()['TableNames'] if to_str(data['TableName']) in table_names: return 200 elif action in ('%s.PutItem' % ACTION_PREFIX, '%s.UpdateItem' % ACTION_PREFIX, '%s.DeleteItem' % ACTION_PREFIX): # find an existing item and store it in a thread-local, so we can access it in return_response, # in order to determine whether an item already existed (MODIFY) or not (INSERT) ProxyListenerDynamoDB.thread_local.existing_item = find_existing_item( data) elif action == '%s.DescribeTable' % ACTION_PREFIX: # Check if table exists, to avoid error log output from DynamoDBLocal table_names = ddb_client.list_tables()['TableNames'] if to_str(data['TableName']) not in table_names: response = error_response( message='Cannot do operations on a non-existent table', error_type='ResourceNotFoundException') fix_headers_for_updated_response(response) return response elif action == '%s.DeleteTable' % ACTION_PREFIX: # Check if table exists, to avoid error log output from DynamoDBLocal table_names = ddb_client.list_tables()['TableNames'] if to_str(data['TableName']) not in table_names: response = error_response( message='Cannot do operations on a non-existent table', error_type='ResourceNotFoundException') fix_headers_for_updated_response(response) return response elif action == '%s.BatchWriteItem' % ACTION_PREFIX: existing_items = [] for table_name in sorted(data['RequestItems'].keys()): for request in data['RequestItems'][table_name]: for key in ['PutRequest', 'DeleteRequest']: inner_request = request.get(key) if inner_request: existing_items.append( find_existing_item(inner_request, table_name)) ProxyListenerDynamoDB.thread_local.existing_items = existing_items elif action == '%s.TransactWriteItems' % ACTION_PREFIX: existing_items = [] for item in data['TransactItems']: for key in ['Put', 'Update', 'Delete']: inner_item = item.get(key) if inner_item: existing_items.append(find_existing_item(inner_item)) ProxyListenerDynamoDB.thread_local.existing_items = existing_items elif action == '%s.UpdateTimeToLive' % ACTION_PREFIX: # TODO: TTL status is maintained/mocked but no real expiry is happening for items response = Response() response.status_code = 200 self._table_ttl_map[data['TableName']] = { 'AttributeName': data['TimeToLiveSpecification']['AttributeName'], 'Status': data['TimeToLiveSpecification']['Enabled'] } response._content = json.dumps( {'TimeToLiveSpecification': data['TimeToLiveSpecification']}) fix_headers_for_updated_response(response) return response elif action == '%s.DescribeTimeToLive' % ACTION_PREFIX: response = Response() response.status_code = 200 if data['TableName'] in self._table_ttl_map: if self._table_ttl_map[data['TableName']]['Status']: ttl_status = 'ENABLED' else: ttl_status = 'DISABLED' response._content = json.dumps({ 'TimeToLiveDescription': { 'AttributeName': self._table_ttl_map[data['TableName']] ['AttributeName'], 'TimeToLiveStatus': ttl_status } }) else: # TTL for dynamodb table not set response._content = json.dumps({ 'TimeToLiveDescription': { 'TimeToLiveStatus': 'DISABLED' } }) fix_headers_for_updated_response(response) return response elif action == '%s.TagResource' % ACTION_PREFIX or action == '%s.UntagResource' % ACTION_PREFIX: response = Response() response.status_code = 200 response._content = '' # returns an empty body on success. fix_headers_for_updated_response(response) return response elif action == '%s.ListTagsOfResource' % ACTION_PREFIX: response = Response() response.status_code = 200 response._content = json.dumps({ 'Tags': [] }) # TODO: mocked and returns an empty list of tags for now. fix_headers_for_updated_response(response) return response return True def return_response(self, method, path, data, headers, response): if path.startswith('/shell'): return data = json.loads(to_str(data)) # update table definitions if data and 'TableName' in data and 'KeySchema' in data: TABLE_DEFINITIONS[data['TableName']] = data if response._content: # fix the table and latest stream ARNs (DynamoDBLocal hardcodes "ddblocal" as the region) content_replaced = re.sub( r'("TableArn"|"LatestStreamArn"|"StreamArn")\s*:\s*"arn:aws:dynamodb:' + 'ddblocal:([^"]+)"', r'\1: "arn:aws:dynamodb:%s:\2"' % aws_stack.get_local_region(), to_str(response._content)) if content_replaced != response._content: response._content = content_replaced fix_headers_for_updated_response(response) action = headers.get('X-Amz-Target') if not action: return record = { 'eventID': '1', 'eventVersion': '1.0', 'dynamodb': { 'StreamViewType': 'NEW_AND_OLD_IMAGES', 'SizeBytes': -1 }, 'awsRegion': config.DEFAULT_REGION, 'eventSource': 'aws:dynamodb' } records = [record] if action == '%s.UpdateItem' % ACTION_PREFIX: if response.status_code == 200: updated_item = find_existing_item(data) if not updated_item: return record['eventName'] = 'MODIFY' record['dynamodb']['Keys'] = data['Key'] record['dynamodb']['OldImage'] = self._thread_local( 'existing_item') record['dynamodb']['NewImage'] = updated_item record['dynamodb']['SizeBytes'] = len(json.dumps(updated_item)) elif action == '%s.BatchWriteItem' % ACTION_PREFIX: records = self.prepare_batch_write_item_records(record, data) elif action == '%s.TransactWriteItems' % ACTION_PREFIX: records = self.prepare_transact_write_item_records(record, data) elif action == '%s.PutItem' % ACTION_PREFIX: if response.status_code == 200: existing_item = self._thread_local('existing_item') record[ 'eventName'] = 'INSERT' if not existing_item else 'MODIFY' keys = dynamodb_extract_keys(item=data['Item'], table_name=data['TableName']) if isinstance(keys, Response): return keys record['dynamodb']['Keys'] = keys record['dynamodb']['NewImage'] = data['Item'] record['dynamodb']['SizeBytes'] = len(json.dumps(data['Item'])) if existing_item: record['dynamodb']['OldImage'] = existing_item elif action == '%s.GetItem' % ACTION_PREFIX: if response.status_code == 200: content = json.loads(to_str(response.content)) # make sure we append 'ConsumedCapacity', which is properly # returned by dynalite, but not by AWS's DynamoDBLocal if 'ConsumedCapacity' not in content and data.get( 'ReturnConsumedCapacity') in ('TOTAL', 'INDEXES'): content['ConsumedCapacity'] = { 'CapacityUnits': 0.5, # TODO hardcoded 'TableName': data['TableName'] } response._content = json.dumps(content) fix_headers_for_updated_response(response) elif action == '%s.DeleteItem' % ACTION_PREFIX: if response.status_code == 200: old_item = self._thread_local('existing_item') record['eventName'] = 'REMOVE' record['dynamodb']['Keys'] = data['Key'] record['dynamodb']['OldImage'] = old_item elif action == '%s.CreateTable' % ACTION_PREFIX: if 'StreamSpecification' in data: create_dynamodb_stream(data) event_publisher.fire_event( event_publisher.EVENT_DYNAMODB_CREATE_TABLE, payload={'n': event_publisher.get_hash(data['TableName'])}) return elif action == '%s.DeleteTable' % ACTION_PREFIX: event_publisher.fire_event( event_publisher.EVENT_DYNAMODB_DELETE_TABLE, payload={'n': event_publisher.get_hash(data['TableName'])}) return elif action == '%s.UpdateTable' % ACTION_PREFIX: if 'StreamSpecification' in data: create_dynamodb_stream(data) return else: # nothing to do return if len(records) > 0 and 'eventName' in records[0]: if 'TableName' in data: records[0]['eventSourceARN'] = aws_stack.dynamodb_table_arn( data['TableName']) forward_to_lambda(records) forward_to_ddb_stream(records) def prepare_batch_write_item_records(self, record, data): records = [] i = 0 for table_name in sorted(data['RequestItems'].keys()): for request in data['RequestItems'][table_name]: put_request = request.get('PutRequest') if put_request: existing_item = self._thread_local('existing_items')[i] keys = dynamodb_extract_keys(item=put_request['Item'], table_name=table_name) if isinstance(keys, Response): return keys new_record = clone(record) new_record[ 'eventName'] = 'INSERT' if not existing_item else 'MODIFY' new_record['dynamodb']['Keys'] = keys new_record['dynamodb']['NewImage'] = put_request['Item'] if existing_item: new_record['dynamodb']['OldImage'] = existing_item new_record[ 'eventSourceARN'] = aws_stack.dynamodb_table_arn( table_name) records.append(new_record) delete_request = request.get('DeleteRequest') if delete_request: keys = delete_request['Key'] if isinstance(keys, Response): return keys new_record = clone(record) new_record['eventName'] = 'REMOVE' new_record['dynamodb']['Keys'] = keys new_record['dynamodb']['OldImage'] = self._thread_local( 'existing_items')[i] new_record[ 'eventSourceARN'] = aws_stack.dynamodb_table_arn( table_name) records.append(new_record) i += 1 return records def prepare_transact_write_item_records(self, record, data): records = [] for i, request in enumerate(data['TransactItems']): put_request = request.get('Put') if put_request: existing_item = self._thread_local('existing_items')[i] table_name = put_request['TableName'] keys = dynamodb_extract_keys(item=put_request['Item'], table_name=table_name) if isinstance(keys, Response): return keys new_record = clone(record) new_record[ 'eventName'] = 'INSERT' if not existing_item else 'MODIFY' new_record['dynamodb']['Keys'] = keys new_record['dynamodb']['NewImage'] = put_request['Item'] if existing_item: new_record['dynamodb']['OldImage'] = existing_item new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn( table_name) records.append(new_record) update_request = request.get('Update') if update_request: table_name = update_request['TableName'] keys = update_request['Key'] if isinstance(keys, Response): return keys updated_item = find_existing_item(update_request, table_name) if not updated_item: return new_record = clone(record) new_record['eventName'] = 'MODIFY' new_record['dynamodb']['Keys'] = keys new_record['dynamodb']['OldImage'] = self._thread_local( 'existing_items')[i] new_record['dynamodb']['NewImage'] = updated_item new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn( table_name) records.append(new_record) delete_request = request.get('Delete') if delete_request: table_name = delete_request['TableName'] keys = delete_request['Key'] if isinstance(keys, Response): return keys new_record = clone(record) new_record['eventName'] = 'REMOVE' new_record['dynamodb']['Keys'] = keys new_record['dynamodb']['OldImage'] = self._thread_local( 'existing_items')[i] new_record['eventSourceARN'] = aws_stack.dynamodb_table_arn( table_name) records.append(new_record) return records def _thread_local(self, name, default=None): try: return getattr(ProxyListenerDynamoDB.thread_local, name) except AttributeError: return default
from apache_beam.runners import common from apache_beam.utils.counters import Counter from apache_beam.utils.counters import CounterFactory from apache_beam.utils.counters import CounterName try: from apache_beam.runners.worker import statesampler_fast as statesampler_impl # type: ignore FAST_SAMPLER = True except ImportError: from apache_beam.runners.worker import statesampler_slow as statesampler_impl FAST_SAMPLER = False if TYPE_CHECKING: from apache_beam.metrics.execution import MetricsContainer _STATE_SAMPLERS = threading.local() def set_current_tracker(tracker): _STATE_SAMPLERS.tracker = tracker def get_current_tracker(): try: return _STATE_SAMPLERS.tracker except AttributeError: return None _INSTRUCTION_IDS = threading.local()
def __init__(self, *args): self._thread_local = threading.local() super(MockOsEnv, self).__init__(*args)
# BruteBuster by Cyber Security Consulting (www.csc.bg) """ Brutebuster needs access to the REMOTE_IP of the incoming request. We're doing this by adding the request object to the thread_local space """ try: from threading import local except ImportError: from django.utils.threading_local import local _thread_locals = local() def get_request(): return getattr(_thread_locals, 'request', None) class RequestMiddleware(object): """Provides access to the request object via thread locals""" def __init__(self, get_response): self.get_response = get_response # One-time configuration and initialization. def __call__(self, request): _thread_locals.request = request # Code to be executed for each request before # the view (and later middleware) are called. response = self.get_response(request)
class Registry(object): """Holds all registered properties and their various overrides.""" registered = {} test_overrides = {} db_items = {} db_overrides = {} names_with_draft = {} last_update_time = 0 update_index = 0 threadlocal = threading.local() REENTRY_ATTR_NAME = 'busy' @classmethod def get_overrides(cls, force_update=False): """Returns current property overrides, maybe cached.""" now = long(time.time()) age = now - cls.last_update_time max_age = UPDATE_INTERVAL_SEC.get_value(db_overrides=cls.db_overrides) # do not update if call is reentrant or outer db transaction exists busy = hasattr(cls.threadlocal, cls.REENTRY_ATTR_NAME) or (db.is_in_transaction()) if (not busy) and (force_update or age < 0 or age >= max_age): # Value of '0' disables all datastore overrides. if UPDATE_INTERVAL_SEC.get_value() == 0: cls.db_overrides = {} return cls.db_overrides # Load overrides from a datastore. setattr(cls.threadlocal, cls.REENTRY_ATTR_NAME, True) try: old_namespace = namespace_manager.get_namespace() try: namespace_manager.set_namespace( appengine_config.DEFAULT_NAMESPACE_NAME) cls._load_from_db() finally: namespace_manager.set_namespace(old_namespace) except Exception as e: # pylint: disable=broad-except logging.error('Failed to load properties from a database: %s.', str(e)) finally: delattr(cls.threadlocal, cls.REENTRY_ATTR_NAME) # Avoid overload and update timestamp even if we failed. cls.last_update_time = now cls.update_index += 1 return cls.db_overrides @classmethod def _load_from_db(cls): """Loads dynamic properties from db.""" items = {} overrides = {} drafts = set() for item in ConfigPropertyEntity.all().fetch(1000): items[item.key().name()] = item cls._set_value(item, overrides, drafts) cls.db_items = items cls.db_overrides = overrides cls.names_with_draft = drafts @classmethod def _config_property_entity_changed(cls, item): cls._set_value(item, cls.db_overrides, cls.names_with_draft) @classmethod def _set_value(cls, item, overrides, drafts): name = item.key().name() target = cls.registered.get(name, None) if not target: logging.warning('Property is not registered (skipped): %s', name) return if item.is_draft: if name in overrides: del overrides[name] drafts.add(name) else: if name in drafts: drafts.remove(name) # Enforce value type. try: value = transforms.string_to_value(item.value, target.value_type) except Exception: # pylint: disable=broad-except logging.error( 'Property %s failed to cast to a type %s; removing.', target.name, target.value_type) return # Enforce value validator. if target.validator: errors = [] try: target.validator(value, errors) except Exception as e: # pylint: disable=broad-except errors.append('Error validating property %s.\n%s', (target.name, e)) if errors: logging.error('Property %s has invalid value:\n%s', target.name, '\n'.join(errors)) return overrides[name] = value
BACKENDS = { 'multiprocessing': MultiprocessingBackend, 'threading': ThreadingBackend, 'sequential': SequentialBackend, 'loky': LokyBackend, } # name of the backend used by default by Parallel outside of any context # managed by ``parallel_backend``. DEFAULT_BACKEND = 'loky' DEFAULT_N_JOBS = 1 DEFAULT_THREAD_BACKEND = 'threading' # Thread local value that can be overridden by the ``parallel_backend`` context # manager _backend = threading.local() VALID_BACKEND_HINTS = ('processes', 'threads', None) VALID_BACKEND_CONSTRAINTS = ('sharedmem', None) def get_active_backend(prefer=None, require=None, verbose=0): """Return the active default backend""" if prefer not in VALID_BACKEND_HINTS: raise ValueError("prefer=%r is not a valid backend hint, " "expected one of %r" % (prefer, VALID_BACKEND_HINTS)) if require not in VALID_BACKEND_CONSTRAINTS: raise ValueError("require=%r is not a valid backend constraint, " "expected one of %r" % (require, VALID_BACKEND_CONSTRAINTS))
import os import torch import traceback import warnings import threading from torch._six import raise_from from ._utils import _get_device_index import torch._C try: from torch._C import _cudart except ImportError: _cudart = None _initialized = False _tls = threading.local() _initialization_lock = threading.Lock() _queued_calls = [] # don't invoke these until initialization occurs _is_in_bad_fork = getattr(torch._C, "_cuda_isInBadFork", lambda: False) def is_available(): r"""Returns a bool indicating if CUDA is currently available.""" if (not hasattr(torch._C, '_cuda_isDriverSufficient') or not torch._C._cuda_isDriverSufficient()): return False return torch._C._cuda_getDeviceCount() > 0 def _sleep(cycles): torch._C._cuda_sleep(cycles)
class _AngleParser: """ Parses the various angle formats including: * 01:02:30.43 degrees * 1 2 0 hours * 1°2′3″ * 1d2m3s * -1h2m3s * 1°2′3″N This class should not be used directly. Use `parse_angle` instead. """ # For safe multi-threaded operation all class (but not instance) # members that carry state should be thread-local. They are stored # in the following class member _thread_local = threading.local() def __init__(self): # TODO: in principle, the parser should be invalidated if we change unit # system (from CDS to FITS, say). Might want to keep a link to the # unit_registry used, and regenerate the parser/lexer if it changes. # Alternatively, perhaps one should not worry at all and just pre- # generate the parser for each release (as done for unit formats). # For some discussion of this problem, see # https://github.com/astropy/astropy/issues/5350#issuecomment-248770151 if '_parser' not in _AngleParser._thread_local.__dict__: (_AngleParser._thread_local._parser, _AngleParser._thread_local._lexer) = self._make_parser() @classmethod def _get_simple_unit_names(cls): simple_units = set( u.radian.find_equivalent_units(include_prefix_units=True)) simple_unit_names = set() # We filter out degree and hourangle, since those are treated # separately. for unit in simple_units: if unit != u.deg and unit != u.hourangle: simple_unit_names.update(unit.names) return sorted(simple_unit_names) @classmethod def _make_parser(cls): from astropy.extern.ply import lex, yacc # List of token names. tokens = ( 'SIGN', 'UINT', 'UFLOAT', 'COLON', 'DEGREE', 'HOUR', 'MINUTE', 'SECOND', 'SIMPLE_UNIT', 'EASTWEST', 'NORTHSOUTH' ) # NOTE THE ORDERING OF THESE RULES IS IMPORTANT!! # Regular expression rules for simple tokens def t_UFLOAT(t): r'((\d+\.\d*)|(\.\d+))([eE][+-−]?\d+)?' # The above includes Unicode "MINUS SIGN" \u2212. It is # important to include the hyphen last, or the regex will # treat this as a range. t.value = float(t.value.replace('−', '-')) return t def t_UINT(t): r'\d+' t.value = int(t.value) return t def t_SIGN(t): r'[+−-]' # The above include Unicode "MINUS SIGN" \u2212. It is # important to include the hyphen last, or the regex will # treat this as a range. if t.value == '+': t.value = 1.0 else: t.value = -1.0 return t def t_EASTWEST(t): r'[EW]$' t.value = -1.0 if t.value == 'W' else 1.0 return t def t_NORTHSOUTH(t): r'[NS]$' # We cannot use lower-case letters otherwise we'll confuse # s[outh] with s[econd] t.value = -1.0 if t.value == 'S' else 1.0 return t def t_SIMPLE_UNIT(t): t.value = u.Unit(t.value) return t t_SIMPLE_UNIT.__doc__ = '|'.join( f'(?:{x})' for x in cls._get_simple_unit_names()) t_COLON = ':' t_DEGREE = r'd(eg(ree(s)?)?)?|°' t_HOUR = r'hour(s)?|h(r)?|ʰ' t_MINUTE = r'm(in(ute(s)?)?)?|′|\'|ᵐ' t_SECOND = r's(ec(ond(s)?)?)?|″|\"|ˢ' # A string containing ignored characters (spaces) t_ignore = ' ' # Error handling rule def t_error(t): raise ValueError( f"Invalid character at col {t.lexpos}") lexer = parsing.lex(lextab='angle_lextab', package='astropy/coordinates') def p_angle(p): ''' angle : sign hms eastwest | sign dms dir | sign arcsecond dir | sign arcminute dir | sign simple dir ''' sign = p[1] * p[3] value, unit = p[2] if isinstance(value, tuple): p[0] = ((sign * value[0],) + value[1:], unit) else: p[0] = (sign * value, unit) def p_sign(p): ''' sign : SIGN | ''' if len(p) == 2: p[0] = p[1] else: p[0] = 1.0 def p_eastwest(p): ''' eastwest : EASTWEST | ''' if len(p) == 2: p[0] = p[1] else: p[0] = 1.0 def p_dir(p): ''' dir : EASTWEST | NORTHSOUTH | ''' if len(p) == 2: p[0] = p[1] else: p[0] = 1.0 def p_ufloat(p): ''' ufloat : UFLOAT | UINT ''' p[0] = p[1] def p_colon(p): ''' colon : UINT COLON ufloat | UINT COLON UINT COLON ufloat ''' if len(p) == 4: p[0] = (p[1], p[3]) elif len(p) == 6: p[0] = (p[1], p[3], p[5]) def p_spaced(p): ''' spaced : UINT ufloat | UINT UINT ufloat ''' if len(p) == 3: p[0] = (p[1], p[2]) elif len(p) == 4: p[0] = (p[1], p[2], p[3]) def p_generic(p): ''' generic : colon | spaced | ufloat ''' p[0] = p[1] def p_hms(p): ''' hms : UINT HOUR | UINT HOUR ufloat | UINT HOUR UINT MINUTE | UINT HOUR UFLOAT MINUTE | UINT HOUR UINT MINUTE ufloat | UINT HOUR UINT MINUTE ufloat SECOND | generic HOUR ''' if len(p) == 3: p[0] = (p[1], u.hourangle) elif len(p) in (4, 5): p[0] = ((p[1], p[3]), u.hourangle) elif len(p) in (6, 7): p[0] = ((p[1], p[3], p[5]), u.hourangle) def p_dms(p): ''' dms : UINT DEGREE | UINT DEGREE ufloat | UINT DEGREE UINT MINUTE | UINT DEGREE UFLOAT MINUTE | UINT DEGREE UINT MINUTE ufloat | UINT DEGREE UINT MINUTE ufloat SECOND | generic DEGREE ''' if len(p) == 3: p[0] = (p[1], u.degree) elif len(p) in (4, 5): p[0] = ((p[1], p[3]), u.degree) elif len(p) in (6, 7): p[0] = ((p[1], p[3], p[5]), u.degree) def p_simple(p): ''' simple : generic | generic SIMPLE_UNIT ''' if len(p) == 2: p[0] = (p[1], None) else: p[0] = (p[1], p[2]) def p_arcsecond(p): ''' arcsecond : generic SECOND ''' p[0] = (p[1], u.arcsecond) def p_arcminute(p): ''' arcminute : generic MINUTE ''' p[0] = (p[1], u.arcminute) def p_error(p): raise ValueError parser = parsing.yacc(tabmodule='angle_parsetab', package='astropy/coordinates') return parser, lexer def parse(self, angle, unit, debug=False): try: found_angle, found_unit = self._thread_local._parser.parse( angle, lexer=self._thread_local._lexer, debug=debug) except ValueError as e: if str(e): raise ValueError(f"{str(e)} in angle {angle!r}") from e else: raise ValueError( f"Syntax error parsing angle {angle!r}") from e if unit is None and found_unit is None: raise u.UnitsError("No unit specified") return found_angle, found_unit