class GRRSeleniumTest(test_lib.GRRBaseTest, acl_test_lib.AclTestMixin): """Baseclass for selenium UI tests.""" # Default duration (in seconds) for WaitUntil. duration = 5 # Time to wait between polls for WaitUntil. sleep_time = 0.2 # This is the global selenium handle. driver = None # Base url of the Admin UI base_url = None @staticmethod def _TearDownSelenium(): """Tear down Selenium session.""" try: if GRRSeleniumTest.driver: GRRSeleniumTest.driver.quit() except Exception as e: # pylint: disable=broad-except logging.exception(e) @staticmethod def _SetUpSelenium(port): """Set up Selenium session.""" atexit.register(GRRSeleniumTest._TearDownSelenium) GRRSeleniumTest.base_url = ("http://localhost:%s" % port) options = webdriver.ChromeOptions() prefs = { "profile.content_settings.exceptions.clipboard": { f"{GRRSeleniumTest.base_url},*": { "setting": 1, }, }, } options.add_experimental_option("prefs", prefs) if flags.FLAGS.chrome_binary_path: options.binary_location = flags.FLAGS.chrome_binary_path options.add_argument("--disable-notifications") if flags.FLAGS.use_headless_chrome: options.add_argument("--headless") options.add_argument("--window-size=1400,1080") if flags.FLAGS.disable_chrome_sandboxing: options.add_argument("--no-sandbox") # pylint: disable=unreachable os.environ.pop("http_proxy", None) if flags.FLAGS.chrome_driver_path: GRRSeleniumTest.driver = webdriver.Chrome( flags.FLAGS.chrome_driver_path, chrome_options=options) else: GRRSeleniumTest.driver = webdriver.Chrome(chrome_options=options) # TODO(user): Hack! This is needed to allow downloads in headless mode. # Remove this code when upstream Python ChromeDriver implementation has # send_command implemented. # # See # https://stackoverflow.com/questions/45631715/downloading-with-chrome-headless-and-selenium # and the code in setUp(). # pylint: disable=protected-access GRRSeleniumTest.driver.command_executor._commands["send_command"] = ( "POST", "/session/$sessionId/chromium/send_command") # pylint: enable=protected-access # pylint: enable=unreachable _selenium_set_up_lock = threading.RLock() _selenium_set_up_done = False # Cached jQuery source to be injected into pages on Open (for selectors # support). _jquery_source = None @classmethod def setUpClass(cls): super(GRRSeleniumTest, cls).setUpClass() with GRRSeleniumTest._selenium_set_up_lock: if not GRRSeleniumTest._selenium_set_up_done: port = portpicker.pick_unused_port() logging.info("Picked free AdminUI port %d.", port) # Start up a server in another thread GRRSeleniumTest._server_trd = wsgiapp_testlib.ServerThread( port, name="SeleniumServerThread") GRRSeleniumTest._server_trd.StartAndWaitUntilServing() GRRSeleniumTest._SetUpSelenium(port) jquery_path = package.ResourcePath( "grr-response-test", "grr_response_test/test_data/jquery_3.1.0.min.js") with open(jquery_path, mode="r", encoding="utf-8") as fd: GRRSeleniumTest._jquery_source = fd.read() GRRSeleniumTest._selenium_set_up_done = True def InstallACLChecks(self): """Installs AccessControlManager and stubs out SendEmail.""" acrwac = api_call_router_with_approval_checks # Clear the cache of the approvals-based router. acrwac.ApiCallRouterWithApprovalChecks.ClearCache() name = compatibility.GetName(acrwac.ApiCallRouterWithApprovalChecks) config_overrider = test_lib.ConfigOverrider({"API.DefaultRouter": name}) config_overrider.Start() self.addCleanup(config_overrider.Stop) # Make sure ApiAuthManager is initialized with this configuration setting. api_auth_manager.InitializeApiAuthManager() def _CheckJavascriptErrors(self): errors = self.driver.execute_script( "return (() => {const e = window.grrInterceptedJSErrors_ || []; " "window.grrInterceptedJSErrors_ = []; return e;})();") msgs = [] for e in errors: msg = "[javascript]: %s" % e logging.error(msg) msgs.append(msg) if msgs: self.fail("Javascript error encountered during test: %s" % "\n\t".join(msgs)) def DisableHttpErrorChecks(self): self.ignore_http_errors = True return DisabledHttpErrorChecksContextManager(self) def GetHttpErrors(self): return self.driver.execute_script( "return (() => {const e = window.grrInterceptedHTTPErrors_ || []; " "window.grrInterceptedHTTPErrors_ = []; return e;})();") def _CheckHttpErrors(self): if self.ignore_http_errors: return msgs = [] for e in self.GetHttpErrors(): try: msg = e["data"]["traceBack"] except (TypeError, KeyError): msg = "[http]: {!r}".format(e) logging.error(msg) msgs.append(msg) if msgs: self.fail("HTTP request failed during test: %s" % "\n\t".join(msgs)) def CheckBrowserErrors(self): self._CheckJavascriptErrors() self._CheckHttpErrors() def WaitUntil(self, condition_cb, *args): self.CheckBrowserErrors() for _ in range(int(self.duration / self.sleep_time)): try: res = condition_cb(*args) if res: return res # Raise in case of a test-related error (i.e. failing assertion). except self.failureException: raise # The element might not exist yet and selenium could raise here. (Also # Selenium raises Exception not StandardError). except Exception as e: # pylint: disable=broad-except logging.warning("Selenium raised %s", utils.SmartUnicode(e)) self.CheckBrowserErrors() time.sleep(self.sleep_time) self.fail( "condition %s %s not met, body is: %s" % (condition_cb, args, self.driver.find_element_by_tag_name("body").text)) def _FindElements(self, selector): selector_type, effective_selector = selector.split("=", 1) if selector_type != "css": raise ValueError( "Only CSS selector is supported for querying multiple elements.") elems = self.driver.execute_script( "return $(\"" + effective_selector.replace("\"", "\\\"") + "\");") return [e for e in elems if e.is_displayed()] def _FindElement(self, selector): try: selector_type, effective_selector = selector.split("=", 1) except ValueError: effective_selector = selector selector_type = None if selector_type == "css": elems = self.driver.execute_script( "return $(\"" + effective_selector.replace("\"", "\\\"") + "\");") elems = [e for e in elems if e.is_displayed()] if not elems: raise exceptions.NoSuchElementException() else: return elems[0] elif selector_type == "link": links = self.driver.find_elements_by_partial_link_text(effective_selector) for l in links: if l.text.strip() == effective_selector: return l raise exceptions.NoSuchElementException() elif selector_type == "xpath": return self.driver.find_element_by_xpath(effective_selector) elif selector_type == "id": return self.driver.find_element_by_id(effective_selector) elif selector_type == "name": return self.driver.find_element_by_name(effective_selector) elif selector_type is None: if effective_selector.startswith("//"): return self.driver.find_element_by_xpath(effective_selector) else: return self.driver.find_element_by_id(effective_selector) else: raise ValueError("unknown selector type %s" % selector_type) @SeleniumAction def Open(self, url): # In GRR Selenium tests calling Open() implies page refresh. # We make sure that browser/webdriver is not confused by the fact that # only the fragment part of the URL (after the '#' symbol) changes. # It's important to not confuse WebDriver since it tends to get stuck # when confused. self.driver.get("data:.") self.driver.get(self.base_url + url) jquery_present = self.driver.execute_script( "return window.$ !== undefined;") if not jquery_present: self.driver.execute_script(GRRSeleniumTest._jquery_source) @SeleniumAction def Refresh(self): self.driver.refresh() @SeleniumAction def Back(self): self.driver.back() @SeleniumAction def Forward(self): self.driver.forward() def WaitUntilNot(self, condition_cb, *args): self.WaitUntil(lambda: not condition_cb(*args)) def GetPageTitle(self): return self.driver.title def IsElementPresent(self, target): try: self._FindElement(target) return True except exceptions.NoSuchElementException: return False def GetCurrentUrlPath(self): url = urlparse.urlparse(self.driver.current_url) result = url.path if url.fragment: result += "#" + url.fragment return result def GetElement(self, target): try: return self._FindElement(target) except exceptions.NoSuchElementException: return None def GetVisibleElement(self, target): try: element = self._FindElement(target) if element.is_displayed(): return element except exceptions.NoSuchElementException: pass return None def IsTextPresent(self, text): return self.AllTextsPresent([text]) def AllTextsPresent(self, texts): body = self.driver.find_element_by_tag_name("body").text for text in texts: if utils.SmartUnicode(text) not in body: return False return True def IsVisible(self, target): element = self.GetElement(target) return element and element.is_displayed() def GetText(self, target): element = self.WaitUntil(self.GetVisibleElement, target) return element.text.strip() def GetValue(self, target): return self.GetAttribute(target, "value") def GetAttribute(self, target, attribute): element = self.WaitUntil(self.GetVisibleElement, target) return element.get_attribute(attribute) def GetClipboard(self): return self.GetJavaScriptValue( "return await navigator.clipboard.readText();") def IsUserNotificationPresent(self, contains_string): self.Click("css=#notification_button") self.WaitUntil(self.IsElementPresent, "css=grr-user-notification-dialog") self.WaitUntilNot( self.IsElementPresent, "css=grr-user-notification-dialog:contains('Loading...')") notifications_text = self.GetText("css=grr-user-notification-dialog") self.Click("css=grr-user-notification-dialog button:contains('Close')") return contains_string in notifications_text def GetJavaScriptValue(self, js_expression): return self.driver.execute_script(js_expression) def _WaitForAjaxCompleted(self): self.WaitUntilEqual( [], self.GetJavaScriptValue, "return (window.$ && $('body') && $('body').injector && " "$('body').injector().get('$http').pendingRequests) || []") @SeleniumAction def Type(self, target, text, end_with_enter=False): element = self.WaitUntil(self.GetVisibleElement, target) element.clear() element.send_keys(text) if end_with_enter: element.send_keys(keys.Keys.ENTER) # We experienced that Selenium sometimes swallows the last character of the # text sent. Raising an exception here will just retry in that case. if not end_with_enter: if text != self.GetValue(target): raise exceptions.WebDriverException("Send_keys did not work correctly.") @SeleniumAction def Click(self, target): # Selenium clicks elements by obtaining their position and then issuing a # click action in the middle of this area. This may lead to misclicks when # elements are moving. Make sure that they are stationary before issuing # the click action (specifically, using the bootstrap "fade" class that # slides dialogs in is highly discouraged in combination with .Click()). # Since Selenium does not know when the page is ready after AJAX calls, we # need to wait for AJAX completion here to be sure that all event handlers # are attached to their respective DOM elements. self._WaitForAjaxCompleted() element = self.WaitUntil(self.GetVisibleElement, target) element.click() @SeleniumAction def MoveMouseTo(self, target): self._WaitForAjaxCompleted() element = self.WaitUntil(self.GetVisibleElement, target) action_chains.ActionChains(self.driver).move_to_element(element).perform() @SeleniumAction def ScrollToBottom(self): self.driver.execute_script( "window.scrollTo(0, document.body.scrollHeight);") @SeleniumAction def DoubleClick(self, target): # Selenium clicks elements by obtaining their position and then issuing a # click action in the middle of this area. This may lead to misclicks when # elements are moving. Make sure that they are stationary before issuing # the click action (specifically, using the bootstrap "fade" class that # slides dialogs in is highly discouraged in combination with # .DoubleClick()). # Since Selenium does not know when the page is ready after AJAX calls, we # need to wait for AJAX completion here to be sure that all event handlers # are attached to their respective DOM elements. self._WaitForAjaxCompleted() element = self.WaitUntil(self.GetVisibleElement, target) action_chains.ActionChains(self.driver).double_click(element).perform() @SeleniumAction def Select(self, target, label): element = self.WaitUntil(self.GetVisibleElement, target) select.Select(element).select_by_visible_text(label) def GetSelectedLabel(self, target): element = self.WaitUntil(self.GetVisibleElement, target) return select.Select(element).first_selected_option.text.strip() def IsChecked(self, target): return self.WaitUntil(self.GetVisibleElement, target).is_selected() def GetCssCount(self, target): if not target.startswith("css="): raise ValueError("invalid target for GetCssCount: " + target) return len(self._FindElements(target)) def WaitUntilEqual(self, target, condition_cb, *args): condition_value = None for _ in range(int(self.duration / self.sleep_time)): try: condition_value = condition_cb(*args) if condition_value == target: return True # Raise in case of a test-related error (i.e. failing assertion). except self.failureException: raise # The element might not exist yet and selenium could raise here. (Also # Selenium raises Exception not StandardError). except Exception as e: # pylint: disable=broad-except logging.warning("Selenium raised %s", utils.SmartUnicode(e)) time.sleep(self.sleep_time) self.fail("condition %s(%s) not met (expected=%r, got_last_time=%r)" % (condition_cb, args, target, condition_value)) def WaitUntilContains(self, target, condition_cb, *args): data = "" target = utils.SmartUnicode(target) for _ in range(int(self.duration / self.sleep_time)): try: data = condition_cb(*args) if target in data: return True # Raise in case of a test-related error (i.e. failing assertion). except self.failureException: raise # The element might not exist yet and selenium could raise here. except Exception as e: # pylint: disable=broad-except logging.warning("Selenium raised %s", utils.SmartUnicode(e)) time.sleep(self.sleep_time) self.fail("condition not met. got: %r, does not contain: %s" % (data, target)) def setUp(self): super(GRRSeleniumTest, self).setUp() # Used by CheckHttpErrors self.ignore_http_errors = False self.token.username = u"gui_user" webauth.WEBAUTH_MANAGER.SetUserName(self.token.username) # Make the user use the advanced gui so we can test it. data_store.REL_DB.WriteGRRUser( self.token.username, ui_mode=api_user.GUISettings.UIMode.ADVANCED) artifact_patcher = ar_test_lib.PatchDatastoreOnlyArtifactRegistry() artifact_patcher.start() self.addCleanup(artifact_patcher.stop) self.InstallACLChecks() if flags.FLAGS.use_headless_chrome: params = { "cmd": "Page.setDownloadBehavior", "params": { "behavior": "allow", "downloadPath": self.temp_dir } } result = self.driver.execute("send_command", params) if result["status"] != 0: raise RuntimeError("can't set Page.setDownloadBehavior: %s" % result) def tearDown(self): self.CheckBrowserErrors() super(GRRSeleniumTest, self).tearDown() def WaitForNotification(self, username): sleep_time = 0.2 iterations = 50 for _ in range(iterations): try: pending_notifications = data_store.REL_DB.ReadUserNotifications( username, state=rdf_objects.UserNotification.State.STATE_PENDING) if pending_notifications: return except IOError: pass time.sleep(sleep_time) self.fail("Notification for user %s never sent." % username)
areg_2dim_formula = re.compile('(.+)\*(.)(.+)\*(.)(.*)') reset_value_pattern = re.compile('^\s*(\S+)\s+RESET_VALUE\s+(\S+).*$') field_reset_value_pattern = re.compile( '^\s*(\S+)\s+FLD_RESET_VALUE\s+(\S+).*$') # ---------------------------------------------------------------------------- # General global variables # ---------------------------------------------------------------------------- symbols = {} silent = False warnings = 0 saved_warnings = None print_arg = '' allow_duplicate_field_values = True lock = threading.RLock() # ============================================================================ # LogInfo # # Prints a message unless "silent" mode is in use. # ============================================================================ def LogInfo(log): if not silent: print log # ============================================================================ # LogWarning
# Sync up with the node def sync_with_ping(self, timeout=60): self.send_message(msg_ping(nonce=self.ping_counter)) test_function = lambda: self.last_message.get( "pong") and self.last_message["pong"].nonce == self.ping_counter wait_until(test_function, timeout=timeout, lock=mininode_lock) self.ping_counter += 1 # One lock for synchronizing all data access between the network event loop (see # NetworkThread below) and the thread running the test logic. For simplicity, # P2PConnection acquires this lock whenever delivering a message to a P2PInterface. # This lock should be acquired in the thread running the test logic to synchronize # access to any data shared with the P2PInterface or P2PConnection. mininode_lock = threading.RLock() class NetworkThread(threading.Thread): network_event_loop = None def __init__(self): super().__init__(name="NetworkThread") # There is only one event loop and no more than one thread must be created assert not self.network_event_loop NetworkThread.network_event_loop = asyncio.new_event_loop() def run(self): """Start the network thread.""" self.network_event_loop.run_forever()
def __init__(self, max_size=0): self._init(max_size) self.max_size = max_size self._lock = threading.RLock()
def lock(self, name): name = 'lock_' + name if not hasattr(self, name): setattr(self, name, threading.RLock()) return getattr(self, name)
def __init__(self, interval=0.5, max_tries=3, initial_configs=None, event_loop=None, logger_log_level=logging.WARNING): """Create a LoggerRunner. ``` interval - number of seconds to sleep between checking/updating loggers max_tries - number of times to try a dead logger config. If zero, then never stop retrying. initial_configs - optional dict of configs to start up on creation. event_loop - Optional event loop, if we are instantiated in a thread that does not have its own. logger_log_level - At what logging level our component loggers should operate. ``` """ logging.info('Starting LoggerRunner') # Map logger name to config, process running it, and any errors self.logger_configs = {} self.processes = {} self.errors = {} self.num_tries = {} self.failed_loggers = set() # We want to remember that we've shut down and erased a logger so # that the next time we're asked for a status update, we can add # one last notice that it's not running. Without it, the most # recent status report for it will be its previous state. self.disappeared_loggers = set() self.interval = interval self.max_tries = max_tries self.logger_log_level = logger_log_level self.quit_flag = False # If we've had an event loop passed in, use it self.event_loop = event_loop or asyncio.get_event_loop() if event_loop: asyncio.set_event_loop(event_loop) # Set the signal handler so that an external break will get # translated into a KeyboardInterrupt. But signal only works if # we're in the main thread - catch if we're not, and just assume # everything's gonna be okay and we'll get shut down with a proper # "quit()" call othewise. try: signal.signal(signal.SIGTERM, kill_handler) except ValueError: logging.info('LoggerRunner not running in main thread; ' 'shutting down with Ctl-C may not work.') # Don't let other threads mess with data while we're # messing. Re-entrant so that we don't have to worry about # re-acquiring when, for example set_configs() calls set_config(). self.config_lock = threading.RLock() # Only let one call to check_loggers() proceed at a time self.check_loggers_lock = threading.Lock() # If we were given any initial configs, set 'em up if initial_configs: self.set_configs(initial_configs)
def __init__(self): self.lock = threading.RLock() global log log = logging.getLogger("cloudscheduler") pass
from mistral import context from mistral.db.sqlalchemy import base as b from mistral.db.sqlalchemy import model_base as mb from mistral.db.sqlalchemy import sqlite_lock from mistral.db import utils as m_dbutils from mistral.db.v2.sqlalchemy import filters as db_filters from mistral.db.v2.sqlalchemy import models from mistral import exceptions as exc from mistral.services import security from mistral import utils from mistral.workflow import states CONF = cfg.CONF LOG = logging.getLogger(__name__) _SCHEMA_LOCK = threading.RLock() _initialized = False def get_backend(): """Consumed by openstack common code. The backend is this module itself. :return: Name of db backend. """ return sys.modules[__name__] def setup_db(): global _initialized
def __init__(self): """Initialize logging for modRana Logging is quite important as a one never knows when a bug shows up and it is important to know what was modRana doing at that time. """ self._log_folder_path = None self._file_handler = None self._log_file_enabled = False self._log_file_enabled_lock = threading.RLock() self._log_file_path = None self._log_file_compression = False self._compressed_log_file = None self._log_file_override = False # create main modRana logger (root logger) self._root_modrana_logger = logging.getLogger('') self._root_modrana_logger.setLevel(logging.DEBUG) # name is undocumented and might blow up in the future ^_- self._root_modrana_logger.name = "" # create log for the core modules self._core_logger = logging.getLogger('core') self._core_logger.setLevel(logging.DEBUG) # create a log for non-core/standalone/feature specific modules # (BaseModule subclasses) self._mod_logger = logging.getLogger('mod') self._mod_logger.setLevel(logging.DEBUG) # as we set the root logger to accept even debug messages, # we need to explicitly tell urllib3 to skip debug level # messages urllib3_logger = logging.getLogger("urllib3") urllib3_logger.setLevel(logging.ERROR) # create console handler that prints everything to stdout # (as was done previously by just using print) self._console_handler = logging.StreamHandler() self._console_handler.setLevel(logging.DEBUG) # create formatters and add them to the handlers # omit some stuff when printing to console to make the logs fit to terminal windows console_formatter = logging.Formatter('%(levelname)s %(name)s: %(message)s') # file viewers should be able to handle longer lines self._console_handler.setFormatter(console_formatter) # also, the file log can't be created and opened at once as modRana needs to load # and consult the persistent settings database first, but we don't want to loose # any early log messages # * solution -> MemoryLogHandler stores the log messages and is either flushed to the # log file or discarded when modRana discovers that logging to file is disabled self._memory_handler = logging.handlers.MemoryHandler(capacity = 1024*10) self._memory_handler.setLevel(logging.DEBUG) # now we attach the console and memory handlers to the root modRana logger # * like this all messages should arrive in the handlers self._root_modrana_logger.addHandler(self._console_handler) self._root_modrana_logger.addHandler(self._memory_handler) # check if we are running on Android - if we do (by checking qrc usage), check # if the Android debug log folder exists - if it does, enable the log file at once # This is done Android being such a mess needs the possibility to easily enable early # logging to debug all the breakage. :) if qrc.is_qrc: # check for the debug log folder (note that this is on purpose folder is different # from the folder used to store debug logs enabled by the user) if os.path.isdir(ANDROID_SPECIAL_LOG_FOLDER): self._root_modrana_logger.debug("special Android log folder (%s) detected", ANDROID_SPECIAL_LOG_FOLDER) self._root_modrana_logger.debug("enabling early Android log file") self.log_folder_path = ANDROID_SPECIAL_LOG_FOLDER self.enable_log_file() self._log_file_override = True self._root_modrana_logger.debug("early Android log file enabled")
self.timeout_int = timeout_int def __repr__(self): return "<User timeout '%s' in %s>" % (self.user_id, self.chat_id) Welcome.__table__.create(checkfirst=True) WelcomeButtons.__table__.create(checkfirst=True) GoodbyeButtons.__table__.create(checkfirst=True) CleanServiceSetting.__table__.create(checkfirst=True) WelcomeSecurity.__table__.create(checkfirst=True) UserRestrict.__table__.create(checkfirst=True) WelcomeTimeout.__table__.create(checkfirst=True) AllowedChat.__table__.create(checkfirst=True) INSERTION_LOCK = threading.RLock() WELC_BTN_LOCK = threading.RLock() LEAVE_BTN_LOCK = threading.RLock() CS_LOCK = threading.RLock() WS_LOCK = threading.RLock() UR_LOCK = threading.RLock() TO_LOCK = threading.RLock() ALLOWCHATLOCK = threading.RLock() CHAT_USERRESTRICT = {} CHAT_TIMEOUT = {} WHITELIST = set() def add_to_userlist(chat_id, user_id, is_clicked):
def __init__(self, filename, mode="r", compression=ZIP_STORED, allowZip64=False, compresslevel=None, strict=True): """Open the ZIP file with mode read "r", write "w" or append "a".""" # Mostly from zipfile.py if mode not in ("r", "w", "a"): raise RuntimeError('ZipFile() requires mode "r", "w", or "a"') if compression == self.ZIP_STORED: pass elif compression == self.ZIP_DEFLATED: if not zlib: raise RuntimeError("Compression requires the (missing) zlib module") else: raise RuntimeError("That compression method is not supported") self._allowZip64 = allowZip64 self._didModify = False self.debug = 0 # Level of printing: 0 through 3 self.NameToInfo = {} # Find file info given name self.filelist = [] # List of ZipInfo instances for archive self.compression = compression # Method of compression self.compresslevel = compresslevel self.mode = key = mode.replace('b', '')[0] self.pwd = None self._comment = b'' self.is_zip = True self._fileRefCnt = 1 self._lock = threading.RLock() self._seekable = True self._writing = False # Check if we were passed a file-like object if isinstance(filename, str): self._filePassed = 0 self.filename = filename mode_dict = {'r': 'rb', 'w': 'wb', 'a': 'r+b'} try: self.fp = open(filename, mode_dict[mode]) except IOError: if mode == 'a': mode = key = 'w' self.fp = open(filename, mode_dict[mode]) else: raise else: self._filePassed = 1 self.fp = filename self.filename = getattr(filename, 'name', None) # noinspection PyBroadException try: if key == 'r': self.fp.seek(0) if stringFileHeader not in self.fp.read(1024): self.is_zip = False self._RealGetContents() elif key == 'w': # set the modified flag so central directory gets written # even if no files are added to the archive self._didModify = True try: self.start_dir = self.fp.tell() except (AttributeError, OSError): # self.fp = _Tellable(self.fp) self.start_dir = 0 self._seekable = False else: # Some file-like objects can provide tell() but not seek() try: self.fp.seek(self.start_dir) except (AttributeError, OSError): self._seekable = False elif key == 'a': try: # See if file is a zip file self._RealGetContents() # seek to start of directory and overwrite self.fp.seek(self.start_dir, 0) except BadZipfile: # file is not a zip file, just append self.fp.seek(0, 2) # set the modified flag so central directory gets written # even if no files are added to the archive self._didModify = True else: raise RuntimeError('Mode must be "r", "w" or "a"') self.broken = False except Exception: if strict: if not self._filePassed: self.fp.close() self.fp = None raise else: self.broken = True
def __init__(self, min_proxies=200): self._bad_proxies = {} self._minProxies = min_proxies self.lock = threading.RLock() self.get_list()
return "<Fed {} subscribes for {}>".format(self.fed_id, self.fed_subs) # Dropping db # Federations.__table__.drop() # ChatF.__table__.drop() # BansF.__table__.drop() # FedSubs.__table__.drop() Federations.__table__.create(checkfirst=True) ChatF.__table__.create(checkfirst=True) BansF.__table__.create(checkfirst=True) FedsUserSettings.__table__.create(checkfirst=True) FedSubs.__table__.create(checkfirst=True) FEDS_LOCK = threading.RLock() CHAT_FEDS_LOCK = threading.RLock() FEDS_SETTINGS_LOCK = threading.RLock() FEDS_SUBSCRIBER_LOCK = threading.RLock() FEDERATION_BYNAME = {} FEDERATION_BYOWNER = {} FEDERATION_BYFEDID = {} FEDERATION_CHATS = {} FEDERATION_CHATS_BYID = {} FEDERATION_BANNED_FULL = {} FEDERATION_BANNED_USERID = {} FEDERATION_NOTIFICATION = {}
def run(config=None): global logger, broker3, broker5, brokerSN, server logger = logging.getLogger('MQTT broker') logger.setLevel(logging.INFO) logger.addFilter(filter) lock = threading.RLock() # shared lock persistence = False if persistence: connection, sharedData = setup_persistence("sharedData") # location for data shared between brokers - subscriptions for example else: sharedData = {} options = {} if config != None: servers_to_create, options, bridges_to_create = process_config(config) broker3 = MQTTV3Brokers(options=options, lock=lock, sharedData=sharedData) broker5 = MQTTV5Brokers(options=options, lock=lock, sharedData=sharedData) brokerSN = MQTTSNBrokers(lock=lock, sharedData=sharedData) brokers = [broker3, broker5, brokerSN] broker3.setBroker5(broker5) broker5.setBroker3(broker3) brokerSN.setBroker3(broker3) brokerSN.setBroker5(broker5) servers = [] bridges = [] UDPListeners.setBroker(brokerSN) TCPListeners.setBrokers(broker3, broker5) HTTPListeners.setBrokers(broker3, broker5, brokerSN) HTTPListeners.setSharedData(lock, sharedData) try: if config == None: # TCPBridges.setBroker5(broker5) # TCPBridges.create("bridge",1883,"172.16.0.4") servers.append(TCPListeners.create(1883, serve_forever=True)) else: logger.debug("Starting bridges") for bridge in bridges_to_create: bridge[0].setBroker5(broker5) bridges.append(bridge[0].create(**bridge[1])) logger.debug("Starting servers") for server in servers_to_create: servers.append(server[0].create(**server[1])) except KeyboardInterrupt: pass except: logger.exception("startBroker") # Stop incoming communications import socket for server in servers: try: logger.info("Stopping listener %s", str(server)) server.shutdown() except: traceback.print_exc() logger.info("Shutdown brokers") for broker in brokers: try: logger.info("Stopping broker %s", str(broker)) broker.shutdown() except: traceback.print_exc() filter.measure() logger.debug("Ending sharedData %s", sharedData) if persistence: sharedData._p_changed = True import transaction transaction.commit() connection.close()
"""Decorator for a cached property.""" def __init__(self, fn): self._fn = fn self.__doc__ = getattr(fn, '__doc__') def __get__(self, obj, objtype=None): value = self._fn(obj) setattr(obj, self._fn.__name__, value) return value # application lock/globals are global to one application # server locks lock the whole server # server globals are pickled in /tmp _app_lock = threading.RLock() def app_lock(name=''): return _app_lock _app_globals: dict = {} def get_app_global(name, init_fn): if name in _app_globals: return _app_globals[name] with app_lock(name): if name not in _app_globals:
def initialize(self, dictionary=None): if dictionary: abs_path = _get_abs_path(dictionary) if self.dictionary == abs_path and self.initialized: return else: self.dictionary = abs_path self.initialized = False else: abs_path = self.dictionary with self.lock: try: with DICT_WRITING[abs_path]: pass except KeyError: pass if self.initialized: return default_logger.debug("Building prefix dict from %s ..." % (abs_path or 'the default dictionary')) t1 = time.time() if self.cache_file: cache_file = self.cache_file # default dictionary elif abs_path == DEFAULT_DICT: cache_file = "jieba.cache" # custom dictionary else: cache_file = "jieba.u%s.cache" % md5( abs_path.encode('utf-8', 'replace')).hexdigest() cache_file = os.path.join(self.tmp_dir or tempfile.gettempdir(), cache_file) # prevent absolute path in self.cache_file tmpdir = os.path.dirname(cache_file) load_from_cache_fail = True if os.path.isfile(cache_file) and (abs_path == DEFAULT_DICT or os.path.getmtime(cache_file) > os.path.getmtime(abs_path)): default_logger.debug("Loading model from cache %s" % cache_file) try: with open(cache_file, 'rb') as cf: self.FREQ, self.total = pickle.load(cf) load_from_cache_fail = False except Exception: load_from_cache_fail = True if load_from_cache_fail: wlock = DICT_WRITING.get(abs_path, threading.RLock()) DICT_WRITING[abs_path] = wlock with wlock: self.FREQ, self.total = self.gen_pfdict( self.get_dict_file()) default_logger.debug("Dumping model to file cache %s" % cache_file) try: # prevent moving across different filesystems fd, fpath = tempfile.mkstemp(dir=tmpdir) with os.fdopen(fd, 'wb') as temp_cache_file: pickle.dump((self.FREQ, self.total), temp_cache_file) _replace_file(fpath, cache_file) except Exception: default_logger.exception("Dump cache file failed.") try: del DICT_WRITING[abs_path] except KeyError: pass self.initialized = True default_logger.debug("Loading model cost %.3f seconds." % (time.time() - t1)) default_logger.debug("Prefix dict has been built succesfully.")
def __init__(self): self.kernel = pyspec.framework.PySpecKernel() self.spec_modules = [] self.selected = [] self.project = pyspec.wxui.project.WxPySpecProjectManager() self.load_lock = threading.RLock()
def __init__(self, std, is_alive): threading.Thread.__init__(self) self.std = std self.is_alive = is_alive self.data = '' self.data_lock = threading.RLock()
def __init__(self): dbg("creating MembersLock") self._lock = threading.RLock() dbg("created MembersLock")
def __init__(self): self.mutex = threading.RLock()
name = Column(UnicodeText, nullable=False) url = Column(UnicodeText, nullable=False) same_line = Column(Boolean, default=False) def __init__(self, chat_id, keyword, name, url, same_line=False): self.chat_id = str(chat_id) self.keyword = keyword self.name = name self.url = url self.same_line = same_line CustomFilters.__table__.create(checkfirst=True) Buttons.__table__.create(checkfirst=True) CUST_FILT_LOCK = threading.RLock() BUTTON_LOCK = threading.RLock() CHAT_FILTERS = {} def get_all_filters(): try: return SESSION.query(CustomFilters).all() finally: SESSION.close() def add_filter(chat_id, keyword, reply, is_sticker=False,
# Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the # specific language governing permissions and limitations under the License. from obs import const LOCK_COUNT = 16 lock_list = None if const.IS_WINDOWS: lock_list = [] import threading for i in range(LOCK_COUNT): lock_list.append(threading.RLock()) else: import multiprocessing lock0 = multiprocessing.RLock() lock1 = multiprocessing.RLock() lock2 = multiprocessing.RLock() lock3 = multiprocessing.RLock() lock4 = multiprocessing.RLock() lock5 = multiprocessing.RLock() lock6 = multiprocessing.RLock() lock7 = multiprocessing.RLock() lock8 = multiprocessing.RLock() lock9 = multiprocessing.RLock() lock10 = multiprocessing.RLock() lock11 = multiprocessing.RLock() lock12 = multiprocessing.RLock()
def __init__(self, user_id, chat_id, chat_name, conn_time): self.user_id = user_id self.chat_id = str(chat_id) self.chat_name = str(chat_name) self.conn_time = int(conn_time) def __repr__(self): return "<connection user {} history {}>".format( self.user_id, self.chat_id) ChatAccessConnectionSettings.__table__.create(checkfirst=True) Connection.__table__.create(checkfirst=True) ConnectionHistory.__table__.create(checkfirst=True) CHAT_ACCESS_LOCK = threading.RLock() CONNECTION_INSERTION_LOCK = threading.RLock() CONNECTION_HISTORY_LOCK = threading.RLock() HISTORY_CONNECT = {} def allow_connect_to_chat(chat_id: Union[str, int]) -> bool: try: chat_setting = SESSION.query(ChatAccessConnectionSettings).get( str(chat_id)) if chat_setting: return chat_setting.allow_connect_to_chat return False finally: SESSION.close()
import sublime from .jsonclient import AsynClient from .decorators import auto_project_switch from .vagrant import VagrantStatus, VagrantIPAddress from .builder.python_builder import AnacondaSetPythonBuilder from .helpers import ( get_settings, get_traceback, project_name, create_subprocess, active_view ) logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler(sys.stdout)) logger.setLevel(logging.WARNING) WORKERS = {} WORKERS_LOCK = threading.RLock() LOOP_RUNNING = False class BaseWorker(object): """Base class for different worker interfaces """ def __init__(self): self.available_port = None self.reconnecting = False self.green_light = True self.last_error = None self.process = None self.client = None
def locked(self, lock=None): if not lock: lock = threading.RLock() yield lock.acquire() lock.release()
def __init__(self, size): self.size = size self._quence = [] self._mutex = threading.RLock() self._full = threading.Condition(self._mutex) self._empty = threading.Condition(self._mutex)
from lte.protos.mobilityd_pb2 import IPAddress, GWInfo, IPBlock from magma.pipelined.tests.app.start_pipelined import ( TestSetup, PipelinedController, ) from magma.pipelined.bridge_util import BridgeTools from magma.pipelined.tests.pipelined_test_util import (start_ryu_app_thread, stop_ryu_app_thread, create_service_manager, SnapshotVerifier) from magma.pipelined.app import inout gw_info_map = {} gw_info_lock = threading.RLock() # re-entrant locks def mocked_get_mobilityd_gw_info() -> List[GWInfo]: global gw_info_map global gw_info_lock with gw_info_lock: return gw_info_map.values() def mocked_set_mobilityd_gw_info(ip: IPAddress, mac: str, vlan: str): global gw_info_map global gw_info_lock with gw_info_lock:
def __init__(self): super(LambdaExecutorReuseContainers, self).__init__() # keeps track of each function arn and the last time it was invoked self.function_invoke_times = {} # locking thread for creation/destruction of docker containers. self.docker_container_lock = threading.RLock()
def __init__(self): super(GdbDebugger, self).__init__() self._is_running = False self._is_running_lock = threading.RLock() self._child_exited_event = threading.Event() self._signals_reset_event = threading.Event()
def __init__(self): dict.__init__(self) self._lock = threading.RLock( ) # reentrant lock allows multiple calls per thread self._set_times = 0