def __init__(self, filepath):
UserDict.__init__(self)
self.synchronized_mutex = RLock()
self.filepath = filepath
def __init__(self, type, connector_class, parallel_server=None):
self.type = type
self.connector_class = connector_class
self.parallel_server = parallel_server
self.connectors = {} # type: dict_[str, Connector]
self.lock = RLock()
def __init__(self):
self.alarms = {}
self.lock = RLock()
def _create_lock(self):
return RLock()
def __init__(self, type, connector_class):
self.type = type
self.connector_class = connector_class
self.connectors = {}
self.lock = RLock()
API_URL = "/swagger.yaml"
swaggerui_blueprint = get_swaggerui_blueprint(
SWAGGER_URL, API_URL, config={"app_name": "Rhasspy API"})
app.register_blueprint(swaggerui_blueprint, url_prefix=SWAGGER_URL)
# -----------------------------------------------------------------------------
# WebSocket API
# -----------------------------------------------------------------------------
WS_EVENT_INTENT = 0
WS_EVENT_LOG = 1
ws_queues: List[Dict[Any, GQueue]] = [{}, {}]
ws_locks: List[RLock] = [RLock(), RLock()]
def add_ws_event(event_type: int, text: str):
with ws_locks[event_type]:
for queue in ws_queues[event_type].values():
queue.put(text)
logging.root.addHandler(
FunctionLoggingHandler(lambda msg: add_ws_event(WS_EVENT_LOG, msg)))
class WebSocketObserver(RhasspyActor):
"""Observes the dialogue manager and outputs intents to the websocket."""
def in_started(self, message: Any, sender: RhasspyActor) -> None:
from interface.objects import SecurityToken, TokenTypeEnum
DEFAULT_WEB_SERVER_HOSTNAME = ""
DEFAULT_WEB_SERVER_PORT = 4000
DEFAULT_SESSION_TIMEOUT = 600
DEFAULT_GATEWAY_PREFIX = "/service"
CFG_PREFIX = "process.ui_server"
# Initialize the main Flask app
app = Flask("ui_server", static_folder=None, template_folder=None)
oauth = OAuth2Provider(app)
ui_instance = None
client_cache = {}
_token_lock = RLock()
class UIServer(SimpleProcess):
"""
Process to start a generic UI server that can be extended with content and service gateway
"""
def on_init(self):
# Retain a pointer to this object for use in routes
global ui_instance
ui_instance = self
# Main references to basic components (if initialized)
self.http_server = None
self.socket_io = None
self.service_gateway = None
def __init__(self, log_path):
self._log_path = log_path
self._started = False
self._lock = RLock()
self._process = None
self._datas = deque()
def empty(self):
return not self._shelf.get('devices')
def add_device(self, device):
assert isinstance(device, Device)
d = self._shelf.setdefault('devices', {})
d[device.name] = device
@property
def devices(self):
try:
return self._shelf.setdefault('devices', {}).itervalues()
except ImportError:
self._shelf.clear()
return self.devices
_CACHE_LOCK = RLock()
@contextmanager
def get_cache():
ensure_directory(in_home('.wemo'))
filename = in_home('.wemo', 'cache')
_CACHE_LOCK.acquire(blocking=True)
try:
with closing(shelve.open(filename, writeback=True)) as cache:
yield Cache(cache)
finally:
_CACHE_LOCK.release()
class GRobot(object):
_loop = None
_liveRobot = 0
_app = None
_kill_loop=None
exit_lock = RLock()
def __init__(self, user_agent=default_user_agent, operate_timeout=10, loading_timeout=60, log_level=logging.WARNING,
display=False, viewport_size=(1024, 768), accept_language='en,*', ignore_ssl_errors=True,
cache_dir=os.path.join(tempfile.gettempdir(), "GRobot"),
image_enabled=True, plugins_enabled=False, java_enabled=False, javascript_enabled=True,
plugin_path=None, develop=False, proxy=None, sleep=0.5, jquery_namespace='GRobot'):
"""GRobot manages a QWebPage.
@param user_agent: The default User-Agent header.
@param operate_timeout: Operation timeout.
@param loading_timeout: The page loading timeout.
@param log_level: The optional logging level.
@param display: A boolean that tells GRobot to displays UI.
@param viewport_size: A tupple that sets initial viewport size.
@param accept_language: Set the webkit accept language.
@param ignore_ssl_errors: A boolean that forces ignore ssl errors.
@param cache_dir: A directory path where to store cache datas.
@param image_enabled: Enable images.
@param plugins_enabled: Enable plugins (like Flash).
@param java_enabled: Enable Java JRE.
@param javascript_enabled: Enable Javascript.
@param plugin_path: Array with paths to plugin directories (default ['/usr/lib/mozilla/plugins'])
@param develop: Enable the Webkit Inspector.
@param proxy: Set a Socks5,HTTP{S} Proxy
@param sleep: Sleep `sleep` second,after operate
@param jquery_namespace: Set the jQuery namespace.
"""
GRobot.exit_lock.acquire()
if GRobot._kill_loop:
gevent.kill(GRobot._kill_loop)
GRobot._kill_loop=None
logger.setLevel(log_level)
plugin_path = plugin_path or ['/usr/lib/mozilla/plugins', ]
GRobot._liveRobot += 1
self.develop = develop
self.inspector = None
self.plugin = False
self.exitLoop = False
self._deleted = False
self.set_proxy(proxy)
self.sleep = sleep
self.jquery_namespace = jquery_namespace
self.popup_messages = None
self.accept_language = accept_language
self._loaded = True
self._confirm_expected = None
self._prompt_expected = None
self._upload_file = None
self._alert = None
self.http_resources = []
self.user_agent = user_agent
self.loading_timeout = loading_timeout
self.operate_timeout = operate_timeout
self.ignore_ssl_errors = ignore_ssl_errors
if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ \
and not hasattr(GRobot, 'xvfb'):
try:
os.environ['DISPLAY'] = ':99'
GRobot.xvfb = subprocess.Popen(['Xvfb', ':99'])
except OSError:
raise Exception('Xvfb is required to a GRobot run oustside ' + \
'an X instance')
self.display = display
if not GRobot._app:
GRobot._app = QApplication.instance() or QApplication(['GRobot'])
if plugin_path:
for p in plugin_path:
GRobot._app.addLibraryPath(p)
self.page = GRobotWebPage(self, GRobot._app)
QtWebKit.QWebSettings.setMaximumPagesInCache(0)
QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0)
QtWebKit.QWebSettings.globalSettings().setAttribute(QtWebKit.QWebSettings.LocalStorageEnabled, True)
self.page.setForwardUnsupportedContent(True)
# Page signals
self.page.loadFinished.connect(self._page_loaded)
self.page.loadStarted.connect(self._page_load_started)
self.page.unsupportedContent.connect(self._unsupported_content)
self.manager = self.page.networkAccessManager()
#TODO:Think about how to handle the network accessible signal
#self.manager.networkAccessibleChanged.connect()
self.manager.finished.connect(self._request_ended)
self.manager.sslErrors.connect(self._on_manager_ssl_errors)
# Cache
self.cache = QNetworkDiskCache()
self.cache.setCacheDirectory(cache_dir)
self.manager.setCache(self.cache)
# Cookie jar
self.cookie_jar = QNetworkCookieJar()
self.manager.setCookieJar(self.cookie_jar)
# User Agent
self.page.setUserAgent(self.user_agent)
self.page.networkAccessManager().authenticationRequired \
.connect(self._authenticate)
self.page.networkAccessManager().proxyAuthenticationRequired \
.connect(self._authenticate)
self.main_frame = self.page.mainFrame()
self.webview = None
self.viewport_size = viewport_size
self.webview = QtWebKit.QWebView()
self.webview.setPage(self.page)
self.webview.show() if display else self.webview.hide()
self.set_viewport_size(*viewport_size)
self.page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, plugins_enabled)
self.page.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, java_enabled)
self.page.settings().setAttribute(QWebSettings.DeveloperExtrasEnabled, self.develop)
self.enable_image = image_enabled
self.enable_javascript = javascript_enabled
#always open link in current window instead of new window
self.page.setLinkDelegationPolicy(QWebPage.DelegateAllLinks)
self.page.linkClicked.connect(self._link_clicked)
#start the qt main loop
GRobot._loop = QtMainLoop(GRobot._app)
GRobot._loop.start()
GRobot.exit_lock.release()
@property
def popup_messages(self):
return self._popup_messages
@popup_messages.setter
def popup_messages(self, value):
self._popup_messages = str(value)
@property
def url(self):
return str(self.main_frame.url().toString())
def content(self):
"""Returns current frame HTML as a string."""
return str(self.main_frame.toHtml())
@property
def cookies(self):
"""Returns all cookies."""
return self.cookie_jar.allCookies()
@property
def enable_image(self):
"""Disable the page images can speed up page loading.
"""
return self._enable_image
@enable_image.setter
def enable_image(self, value):
self.page.settings().setAttribute(QWebSettings.AutoLoadImages, value)
self._enable_image = value
#TODO:It seems not work?
# @enable_image.deleter
# def enable_image(self):
# raise NotImplemented
@property
def enable_javascript(self):
"""Disable the page javascript can speed up page loading.
"""
return self._enable_javascript
@enable_javascript.setter
def enable_javascript(self, value):
self.page.settings().setAttribute(QWebSettings.JavascriptEnabled, value)
self._enable_javascript = value
def open(self, address, method='get', headers=None, auth=None, body=None,
default_popup_response=None):
"""Opens a web page.
@param address: The resource URL.
@param method: The Http method.
@param headers: An optional dict of extra request hearders.
@param auth: An optional tupple of HTTP auth (username, password).
@param body: An optional string containing a payload.
@param default_popup_response: the default response for any confirm/
alert/prompt popup from the Javascript (replaces the need for the with
blocks)
"""
headers = headers or {}
body = body or QByteArray()
try:
method = getattr(QNetworkAccessManager,
"%sOperation" % method.capitalize())
except AttributeError:
raise Exception("Invalid http method %s" % method)
request = QNetworkRequest(QUrl(address))
request.CacheLoadControl = 0
for header in headers:
request.setRawHeader(header, headers[header])
self._auth = auth
self._auth_attempt = 0 # Avoids reccursion
self.page.mainFrame().load(request, method, body)
self._loaded = False
if default_popup_response is not None:
self._prompt_expected = (default_popup_response, None)
self._confirm_expected = (default_popup_response, None)
return self.wait_for_page_loaded()
def set_viewport_size(self, width, height):
"""Sets the page viewport size.
@param width: An integer that sets width pixel count.
@param height: An integer that sets height pixel count.
"""
if self.display:
self.webview.resize(QSize(width, height))
self.page.setViewportSize(QSize(width, height))
def set_proxy(self, proxy=None):
"""Set the proxy or using system configuration as None,supported socks5 http{s}.
@param proxy: Example:socks5://username:[email protected]:7070
"""
proxy_type = None
if proxy:
parse = urlparse(proxy)
scheme = parse.scheme
hostname = parse.hostname
port = parse.port
username = parse.username or ''
password = parse.password or ''
if scheme == 'socks5':
proxy_type = QNetworkProxy.Socks5Proxy
elif scheme in ('http', 'https'):
proxy_type = QNetworkProxy.HttpProxy
if proxy_type:
self.page.networkAccessManager().setProxy(
QNetworkProxy(proxy_type, hostname, port, username, password)
)
else:
QNetworkProxyFactory.setUseSystemConfiguration(True)
def first_element_position(self, selector):
try:
return self.elements_position(selector)[0]
except IndexError:
logger.warning("Can't locate selector " + selector)
return None
def elements_position(self, selector):
"""Get the position of elements whose match selector
@param selector:
@return: position of QPoint
"""
attr, pattern, val = self.parser_selector(selector, attr='identifier')
strip = lambda v: v.strip()
if pattern:
val = locals()[pattern](val)
def identifier(query):
return id(query) or name(query)
def name(query):
return css("*[name='%s']" % query)
def id(query):
return css('#' + query)
def link(query):
return xpath("//a[text()='%s']" % query.replace("\'", "\\'"))
def css(query):
result = []
for ele in self.main_frame.findAllElements(query):
if not ele.isNull():
result.append(ele.geometry().center())
return result
def xpath(query):
positions = self.evaluate("""
function GetAbsoluteLocationEx(element)
{
if ( arguments.length != 1 || element == null )
{
return null;
}
var elmt = element;
var offsetTop = elmt.offsetTop;
var offsetLeft = elmt.offsetLeft;
var offsetWidth = elmt.offsetWidth;
var offsetHeight = elmt.offsetHeight;
while( elmt = elmt.offsetParent )
{
// add this judge
if ( elmt.style.position == 'absolute' || elmt.style.position == 'relative'
|| ( elmt.style.overflow != 'visible' && elmt.style.overflow != '' ) )
{
break;
}
offsetTop += elmt.offsetTop;
offsetLeft += elmt.offsetLeft;
}
return { absoluteTop: offsetTop, absoluteLeft: offsetLeft,
offsetWidth: offsetWidth, offsetHeight: offsetHeight };
}
result=[];
for (var r = document.evaluate('%s', document, null, 5, null), n; n = r.iterateNext();) {
pos=GetAbsoluteLocationEx(n)
result.push([pos.absoluteLeft+pos.offsetWidth/2.0,pos.absoluteTop+pos.offsetHeight/2.0]);
}
result
""" % query.replace("\'", "\\'"))
return [QPoint(*tuple(x)) for x in positions]
return locals()[attr](val)
def _move_page_center_to(self, qpoint):
size = self.page.viewportSize()
self.main_frame.setScrollPosition(qpoint - QPoint(size.width(), size.height()) / 2)
def reload(self):
"""Reload page.
@return:
"""
self.trigger_action('Reload', expect_loading=True)
def back(self):
self.trigger_action('Back')
def forward(self):
self.trigger_action('Forward')
@can_load_page
def trigger_action(self, action):
"""Trigger QWebPage::WebAction
@param action:
"""
self.page.triggerAction(getattr(QWebPage, action))
def parser_selector(self, selector, attr=None, pattern=None, val=None):
index = selector.find('=')
if index <= 0:
val = selector
else:
attr = selector[:index]
value_ = selector[index + 1:]
index = value_.find(':')
if index > 0:
pattern = value_[:index]
val = value_[index + 1:]
return attr, pattern, val
@can_load_page
@have_a_break
def click(self, selector):
qpoint = self.first_element_position(selector)
if qpoint:
return self.qpoint_to_tuple(self._click_position(qpoint))
@can_load_page
@have_a_break
def test(self):
return self.qpoint_to_tuple(QPoint(1, 2))
@can_load_page
def _click_position(self, qpoint):
self._move_page_center_to(qpoint)
self.webview.repaint()
pos = qpoint - self.main_frame.scrollPosition()
self._move_to_position(pos)
QTest.mouseClick(self.webview, Qt.LeftButton, pos=pos)
gevent.sleep(1)
return pos
def qpoint_to_tuple(self, qpoint):
return qpoint.x(), qpoint.y()
@have_a_break
def move_to(self, selector):
qpoint = self.first_element_position(selector)
if qpoint:
self._move_to_position(qpoint)
return qpoint_to_tuple(qpoint)
def move_at(self, x, y):
return self._move_to_position(QPoint(x, y))
def _move_to_position(self, qpoint):
QTest.mouseMove(self.webview, pos=qpoint)
return qpoint
@can_load_page
@have_a_break
def click_at(self, x, y):
return self._click_position(QPoint(x, y))
@have_a_break
def key_clicks(self, selector, text):
if selector:
self.click(selector)
QTest.keyClicks(self.webview, text, delay=50)
@have_a_break
def type(self, selector, text):
position = self.click(selector)
ele = self._hit_element_from(QPoint(*position))
ele.setFocus()
ele.evaluateJavaScript(
"""
core.events.setValue(this, '%s')
""" % (text.replace("\n", "\\n").replace("\'", "\\'"))
)
logger.debug('type %s %s' % (selector, text))
def _hit_element_from(self, position):
return self.main_frame.hitTestContent(position).element()
def first_element(self, selector):
position = self.first_element_position(selector)
if position:
return self.main_frame.hitTestContent(position).element(), position
def wait_forever(self):
self.wait_for(lambda: False, time_for_stop=-1)
@have_a_break
def check(self, selector, checked=True):
ele, position = self.first_element(selector)
if ele and ele.tagName() == 'INPUT':
if ele.attribute('type') in ['checkbox', 'radio']:
ele_checked = ele.attribute('checked') == 'checked' or False
if ele_checked != checked:
self._click_position(position)
else:
raise ValueError("%s is not a checkbox or radio" % selector)
@have_a_break
def select(self, selector, value):
def _select(query, select_by, select):
select.evaluateJavaScript("""
triggerEvent(this, 'focus', false);
var changed = false;
var optionToSelect = '%s';
for (var i = 0; i < this.options.length; i++) {
var option = this.options[i];
if (option.selected && option.%s != optionToSelect) {
option.selected = false;
changed = true;
}
else if (!option.selected && option.%s == optionToSelect) {
option.selected = true;
changed = true;
}
}
if (changed) {
triggerEvent(this, 'change', true);
}
""" % ( query.replace("\'", "\\'"), select_by, select_by))
def _add_selection(query, select_by, select, selected):
select.evaluateJavaScript("""
triggerEvent(this, 'focus', false);
var optionToSelect = '%s';
for (var i = 0; i < this.options.length; i++) {
var option = this.options[i];
if (option.%s == optionToSelect)
{
option.selected = %s;
triggerEvent(this, 'change', true);
}
}
""" % ( query.replace("\'", "\\'"), select_by, selected and 'true' or 'false'))
ele, position = self.first_element(selector)
if ele and ele.tagName() == 'SELECT':
ele.setFocus()
if ele.attribute('multiple') == 'multiple':
assert isinstance(value, list)
for value_, selected in value:
attr, pattern, val = self.parser_selector(value_, attr='text')
_add_selection(val, attr, ele, selected)
else:
attr, pattern, val = self.parser_selector(value, attr='text')
_select(val, attr, ele)
def choose_file(self, selector, file):
self._upload_file = file
self.click(selector)
self._upload_file = None
def capture(self, selector=None):
"""Capture the images of selector.
@param selector: Css selector.
@return: Images
"""
elements = self.main_frame.documentElement().findAll(selector)
imgs = []
for element in elements:
geo = element.geometry()
img = QImage(geo.width(), geo.height(), QImage.Format_ARGB32)
painter = QPainter(img)
element.render(painter)
painter.end()
imgs.append(img)
return imgs
def capture_to(self, path, selector=None):
"""Capture the images of selector to files.
@param path: File path with index suffix.
@param selector: Css selector.
@return: The paths of saving.
"""
_, ext = os.path.splitext(path)
ext = ext[1:]
imgs = self.capture(selector)
result = []
for index, img in enumerate(imgs):
filepath = '%s.%s' % (path, index)
if img.save(filepath, ext.upper()):
result.append(filepath)
return result
def capture_to_buf(self, selector=None):
"""capture the images of selector to StringIO
@param selector: Css selector.
@return: The StringIO list.
"""
images = self.capture(selector)
result = []
for image in images:
ba = QByteArray()
buf = QBuffer(ba)
buf.open(QIODevice.ReadWrite)
image.save(buf, 'jpg')
stream = StringIO(str(buf.buffer()))
result.append(stream)
return result
@can_load_page
def evaluate(self, script):
"""Evaluates script in page frame.
@param script: The script to evaluate.
"""
result = self.main_frame.evaluateJavaScript("%s" % script)
# if isinstance(result,QString):
# result=unicode(result)
return result
def evaluate_js_file(self, path, encoding='utf-8'):
"""Evaluates javascript file at given path in current frame.
Raises native IOException in case of invalid file.
@param path: The path of the file.
@param encoding: The file's encoding.
"""
self.evaluate(codecs.open(path, encoding=encoding).read())
def __del__(self):
"""Depend on the CG of Python.
"""
self.exit()
def delete_cookies(self):
"""Deletes all cookies."""
self.cookie_jar.setAllCookies([])
def exists(self, selector):
"""Checks if element exists for given selector.
@param string: The element selector.
"""
return not self.main_frame.findFirstElement(selector).isNull()
#TODO: Still not work.
# def remove_css(self):
# """Remore the css,speed up page loading.
#
# @return:
# """
#
# return self.evaluate("""var targetelement="link";//determine element type to create nodelist from
# var targetattr="href"//determine corresponding attribute to test for
# var allsuspects=document.getElementsByTagName(targetelement)
# for (var i=allsuspects.length; i>=0; i--){ //search backwards within nodelist for matching elements to remove
# if (allsuspects[i] && allsuspects[i].getAttribute(targetattr)!=null )
# allsuspects[i].parentNode.removeChild(allsuspects[i]); //remove element by calling parentNode.removeChild()
# }
# """)
def filter_resources(self, pattern):
"""Filter resources with pattern.
@param pattern: Match pattern.
@param resources:
@return: @raise:
"""
if isinstance(pattern, str):
is_match = lambda x: pattern == x
elif isinstance(pattern, _pattern_type):
is_match = lambda x: pattern.match(x)
elif hasattr(pattern, '__call__'):
is_match = pattern
else:
raise TypeError('pattern must be one of str,re.compile,callable')
return filter(lambda x: is_match(x.request_url), self.http_resources)[:]
def save(self, path):
"""Save current page content to the path.
@param path: The path to save.
"""
f = open(path, 'w')
f.write(self.content().encode('utf-8'))
f.close()
def global_exists(self, global_name):
"""Checks if javascript global exists.
@param global_name: The name of the global.
"""
return self.evaluate('!(typeof %s === "undefined");' %
global_name)
def load_cookies( self, cookie_storage, keep_old=False ):
"""load from cookielib's CookieJar or Set-Cookie3 format text file.
@param cookie_storage: file location string on disk or CookieJar instance.
@param keep_old: Don't reset, keep cookies not overridden.
"""
def toQtCookieJar( PyCookieJar, QtCookieJar ):
allCookies = QtCookieJar.cookies if keep_old else []
for pc in PyCookieJar:
qc = toQtCookie(pc)
allCookies.append(qc)
QtCookieJar.setAllCookies(allCookies)
def toQtCookie(PyCookie):
qc = QNetworkCookie(PyCookie.name, PyCookie.value)
qc.setSecure(PyCookie.secure)
if PyCookie.path_specified:
qc.setPath(PyCookie.path)
if PyCookie.domain != "":
qc.setDomain(PyCookie.domain)
if PyCookie.expires != 0:
t = QDateTime()
t.setTime_t(PyCookie.expires)
qc.setExpirationDate(t)
# not yet handled(maybe less useful):
# py cookie.rest / QNetworkCookie.setHttpOnly()
return qc
if cookie_storage.__class__.__name__ == 'str':
cj = LWPCookieJar(cookie_storage)
cj.load()
toQtCookieJar(cj, self.cookie_jar)
elif cookie_storage.__class__.__name__.endswith('CookieJar'):
toQtCookieJar(cookie_storage, self.cookie_jar)
else:
raise ValueError('unsupported cookie_storage type.')
def save_cookies(self, cookie_storage):
"""Save to cookielib's CookieJar or Set-Cookie3 format text file.
@param cookie_storage: file location string or CookieJar instance.
"""
def toPyCookieJar(QtCookieJar, PyCookieJar):
for c in QtCookieJar.allCookies():
PyCookieJar.set_cookie(toPyCookie(c))
def toPyCookie(QtCookie):
port = None
port_specified = False
secure = QtCookie.isSecure()
name = str(QtCookie.name())
value = str(QtCookie.value())
v = str(QtCookie.path())
path_specified = bool(v != "")
path = v if path_specified else None
v = str(QtCookie.domain())
domain_specified = bool(v != "")
domain = v
domain_initial_dot = v.startswith('.') if domain_specified else None
v = int(QtCookie.expirationDate().toTime_t())
# Long type boundary on 32bit platfroms; avoid ValueError
expires = 2147483647 if v > 2147483647 else v
rest = {}
discard = False
return Cookie(0, name, value, port, port_specified, domain
, domain_specified, domain_initial_dot, path, path_specified
, secure, expires, discard, None, None, rest)
if cookie_storage.__class__.__name__ == 'str':
cj = LWPCookieJar(cookie_storage)
toPyCookieJar(self.cookie_jar, cj)
cj.save()
elif cookie_storage.__class__.__name__.endswith('CookieJar'):
toPyCookieJar(self.cookie_jar, cookie_storage)
else:
raise ValueError('unsupported cookie_storage type.')
def wait_for_confirm(self, confirm=True, callback=None):
"""Statement that tells GRobot how to deal with javascript confirm().
@param confirm: A bollean that confirm.
@param callable: A callable that returns a boolean for confirmation.
"""
self._robot._confirm_expected = (confirm, callback)
self._robot.wait_for(lambda: self._robot._confirm_expected is None)
return self.popup_messages
def wait_for_text(self, text, time_for_stop=None):
"""Waits until given text appear on main frame.
@param text: The text to wait for.
@return:
"""
logger.debug("Wait for text %s" % text)
self.wait_for(lambda: text in self.content(),
"Can\'t find '%s' in current frame" % text, time_for_stop=time_for_stop)
return self.wait_for_page_loaded()
def wait_for_xpath(self, expression, time_for_stop=None):
self.wait_for(lambda: XPath(self.content()).execute(expression),
"Can't find xpath=%s in current frame" % expression, time_for_stop=time_for_stop)
return self.wait_for_page_loaded()
def wait_for_selector(self, selector):
"""Waits until selector match an element on the frame.
@param selector: The selector to wait for.
"""
self.wait_for(lambda: self.exists(selector),
'Can\'t find element matching "%s"' % selector)
def wait_for_page_loaded(self, time_for_stop=None):
"""Waits until page is loaded, assumed that a page as been requested.
"""
return self.wait_for(lambda: self._loaded,
'Unable to load requested page', time_for_stop=time_for_stop)
def wait_for(self, condition, timeout_message='', time_for_stop=None):
"""Waits until condition is True.
@param condition: A callable that returns the condition.
@param timeout_message: The exception message on timeout.-1 means never timeout.
"""
if self._loaded:
time_for_stop = time_for_stop or self.operate_timeout
else:
time_for_stop = time_for_stop or self.loading_timeout
started_at = time.time()
while not condition():
if time_for_stop != -1 and time.time() > (started_at + time_for_stop):
if self._loaded:
raise OperateTimeout(timeout_message)
else:
# raise LoadingTimeout, timeout_message
self.trigger_action('Stop') #QWebPage::Stop
self._loaded = True
logger.warning("Page loading timeout.Force to stop the page")
break
gevent.sleep(2)
def wait_for_alert(self):
"""Waits for main frame alert().
"""
self.wait_for(lambda: self._alert is not None,
'User has not been alerted.')
msg, self._alert = self._alert, None
return msg
def _release_last_resources(self):
"""Releases last loaded resources.
:return: The released resources.
"""
last_resources, self.http_resources = self.http_resources[:], []
return last_resources
def _page_loaded(self, success):
if self.develop and self.display:
if self.inspector is None:
self.inspector = QWebInspector()
self.inspector.setPage(self.page)
self.inspector.show()
scripts = [
'atoms.js',
'htmlutils.js',
]
if self.jquery_namespace:
scripts.append('jquery-1.9.1.min.js', )
for script in scripts:
self.evaluate_js_file(os.path.dirname(__file__) + '/javascripts/' + script)
if self.jquery_namespace:
self.evaluate("%s=jQuery.noConflict();" % self.jquery_namespace)
self._loaded = True
# self.cache.clear()
logger.debug("Page load finished")
def _page_load_started(self):
logger.debug("Start load page")
self._loaded = False
def _unsupported_content(self, reply):
"""Adds an HttpResource object to http_resources with unsupported
content.
@param reply: The QNetworkReply object.
"""
if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute):
self.http_resources.append(HttpResource(reply, self.cache,
reply.readAll()))
def _link_clicked(self, href):
"""Contorl the page link clicked event,forbid open new window.
@param href: The href attribute of a tag.
"""
self.main_frame.load(href)
def _request_ended(self, reply):
"""Adds an HttpResource object to http_resources.
@param reply: The QNetworkReply object.
"""
if reply.attribute(QNetworkRequest.HttpStatusCodeAttribute):
self.http_resources.append(HttpResource(reply, self.cache))
def _authenticate(self, mix, authenticator):
"""Called back on basic / proxy http auth.
@param mix: The QNetworkReply or QNetworkProxy object.
@param authenticator: The QAuthenticator object.
"""
if self._auth_attempt == 0:
username, password = self._auth
authenticator.setUser(username)
authenticator.setPassword(password)
self._auth_attempt += 1
def _on_manager_ssl_errors(self, reply, errors):
"""Ingore all the ssl error
@param reply:
@param errors:
"""
url = str(reply.url().toString())
if self.ignore_ssl_errors:
reply.ignoreSslErrors()
else:
logger.warning('SSL certificate error: %s' % url)
def exit(self):
"""Destroy the Qt main event loop.
"""
GRobot.exit_lock.acquire()
if not self._deleted:
if self.inspector:
self.inspector.close()
sip.delete(self.inspector)
if self.display:
self.webview.close()
sip.delete(self.webview)
if self.page and not sip.isdeleted(self.page):
sip.delete(self.page)
GRobot._liveRobot -= 1
if GRobot._liveRobot == 0 and GRobot._loop is not None:
GRobot._kill_loop=gevent.spawn_later(20,self.kill_loop)
self._deleted = True
GRobot.exit_lock.release()
def kill_loop(self):
GRobot._loop.stop()
GRobot._loop = None
GRobot._app = None
if hasattr(self, 'xvfb'):
GRobot.xvfb.terminate()
def __init__(self):
self.pool = {}
self.lock = RLock()
self.config_time = 0
super(TimerPool, self).__init__()
def __init__(self, xml_rpc):
self.event_channels = set()
self.lock = RLock()
self.rpc = xml_rpc
for name in self.rpc.system.listMethods():
setattr(self, *build_method(self, name))
from cache import CacheStore
from lib.geom import confxml
from lib.system import system, SubprocessException
from task import Provider, Task, TaskStatus, TaskException, VerifyException, query
from dispatcher.rpc import RpcException, accepts, returns, description, private
from dispatcher.rpc import SchemaHelper as h
# Note the following monkey patch is required for pySMART to work correctly
gevent.monkey.patch_subprocess()
from pySMART import Device
EXPIRE_TIMEOUT = timedelta(hours=24)
multipaths = -1
diskinfo_cache = CacheStore()
logger = logging.getLogger('DiskPlugin')
diskinfo_cache_lock = RLock()
class AcousticLevel(enum.IntEnum):
DISABLED = 0
MINIMUM = 1
MEDIUM = 64
MAXIMUM = 127
class DiskProvider(Provider):
@query('disk')
def query(self, filter=None, params=None):
def extend(disk):
disk['online'] = self.is_online(disk['path'])
disk['status'] = diskinfo_cache.get(disk['id'])
def __init__(self):
self.logger = logging.getLogger(self.__class__.__name__)
self.proxies = {}
self.rlock = RLock()
def __init__(self):
self._buffer = BytesIO()
self._read_pos = 0
self._write_pos = 0
self.eof = Event()
self._lock = RLock()
def __init__(self, sql_conn_func):
self.sql_conn_func = sql_conn_func
self.lock = RLock()
def __init__(self):
self._conn = None
self._initialized = None
self._refresh_lock = RLock()
def __init__(self):
self.synchronized_mutex = RLock()
def __init__(self, pipe):
self.pipe = pipe
self.buffer = {}
self.buffer_lock = RLock()
self.log = False
def start():
con = connect()
con.debug = conf['DEBUG_IMAP'] if debug is None else debug
con.lock = RLock()
con.new = new
return con
def __init__(self, with_lock):
if with_lock:
self._lock = RLock()
from six import PY3, iterkeys, iteritems
import io
import json
import logging
import os
import re
from gevent.lock import RLock
from cmsranking.Entity import Entity, InvalidKey, InvalidData
logger = logging.getLogger(__name__)
# Global shared lock for all Store instances.
LOCK = RLock()
class Store(object):
"""A store for entities.
Provide methods to perform the CRUD operations (create, retrieve,
update, delete) on a set of entities, accessed by their unique key.
It's very similar to a dict, except that keys are strings, values
are of a single type (defined at init-time) and it's possible to
get notified when something changes by providing appropriate
callbacks.
"""
def __init__(self, entity, path, all_stores, depends=None):
"""Initialize an empty EntityStore.
#from threading import Lock
from gevent.lock import RLock
from . import *
from app.irsystem.models.helpers import *
from app.irsystem.models.search import *
mutex = RLock()
@irsystem.route('/', methods=['GET'])
def initializeTemplate():
with mutex:
return render_template('search.html')
@irsystem.route('/postquery', methods=['POST'])
def handleQuery():
'''
The EXPECTED incoming request is a json of the form:
{
max_res: integer representing the number of maximum results to be returned per category,
ml_mode: integer identifier in the range [0, 1, 2], with 0 being no relevance feedback,
1 representing logistic regression, and 2 representing Rocchio,
query: Query string with alphanumeric characters only,
reddit_range_utc: array of length 2 specifying the beginning and end UTC
time in seconds of reddit posts that should be returned, ie. [1000000000, 2000000000],
relevance_feedbacks: a json object (dictionary) of the form:
{
'codes_info':
{
'query':
def __init__(self,
hosts,
user=None,
password=None,
port=22,
pkey=None,
num_retries=DEFAULT_RETRIES,
timeout=None,
pool_size=10,
allow_agent=True,
host_config=None,
retry_delay=RETRY_DELAY,
proxy_host=None,
proxy_port=22,
proxy_user=None,
proxy_password=None,
proxy_pkey=None,
forward_ssh_agent=False,
tunnel_timeout=None,
keepalive_seconds=60):
"""
:param hosts: Hosts to connect to
:type hosts: list(str)
:param user: (Optional) User to login as. Defaults to logged in user
:type user: str
:param password: (Optional) Password to use for login. Defaults to
no password
:type password: str
:param port: (Optional) Port number to use for SSH connection. Defaults
to 22.
:type port: int
:param pkey: Private key file path to use. Path must be either absolute
path or relative to user home directory like ``~/<path>``.
:type pkey: str
:param num_retries: (Optional) Number of connection and authentication
attempts before the client gives up. Defaults to 3.
:type num_retries: int
:param retry_delay: Number of seconds to wait between retries. Defaults
to :py:class:`pssh.constants.RETRY_DELAY`
:type retry_delay: int
:param timeout: (Optional) SSH session timeout setting in seconds.
This controls timeout setting of socket operations used for SSH
sessions. Defaults to OS default - usually 60 seconds.
:type timeout: float
:param pool_size: (Optional) Greenlet pool size. Controls
concurrency, on how many hosts to execute tasks in parallel.
Defaults to 10. Overhead in event
loop will determine how high this can be set to, see scaling guide
lines in project's readme.
:type pool_size: int
:param host_config: (Optional) Per-host configuration for cases where
not all hosts use the same configuration.
:type host_config: dict
:param allow_agent: (Optional) set to False to disable connecting to
the system's SSH agent.
:type allow_agent: bool
:param proxy_host: (Optional) SSH host to tunnel connection through
so that SSH clients connect to host via client -> proxy_host -> host
:type proxy_host: str
:param proxy_port: (Optional) SSH port to use to login to proxy host if
set. Defaults to 22.
:type proxy_port: int
:param proxy_user: (Optional) User to login to ``proxy_host`` as.
Defaults to logged in user.
:type proxy_user: str
:param proxy_password: (Optional) Password to login to ``proxy_host``
with. Defaults to no password.
:type proxy_password: str
:param proxy_pkey: (Optional) Private key file to be used for
authentication with ``proxy_host``. Defaults to available keys from
SSHAgent and user's SSH identities.
:type proxy_pkey: Private key file path to use.
:param forward_ssh_agent: (Optional) Turn on SSH agent forwarding -
equivalent to `ssh -A` from the `ssh` command line utility.
Defaults to False if not set.
Requires agent forwarding implementation in libssh2 version used.
:type forward_ssh_agent: bool
:param tunnel_timeout: (Optional) Timeout setting for proxy tunnel
connections.
:type tunnel_timeout: float
:raises: :py:class:`pssh.exceptions.PKeyFileError` on errors finding
provided private key.
"""
BaseParallelSSHClient.__init__(self,
hosts,
user=user,
password=password,
port=port,
pkey=pkey,
allow_agent=allow_agent,
num_retries=num_retries,
timeout=timeout,
pool_size=pool_size,
host_config=host_config,
retry_delay=retry_delay)
self.pkey = _validate_pkey_path(pkey)
self.proxy_host = proxy_host
self.proxy_port = proxy_port
self.proxy_pkey = _validate_pkey_path(proxy_pkey)
self.proxy_user = proxy_user
self.proxy_password = proxy_password
self.forward_ssh_agent = forward_ssh_agent
self._tunnel = None
self._tunnel_in_q = None
self._tunnel_out_q = None
self._tunnel_lock = None
self._tunnel_timeout = tunnel_timeout
self._clients_lock = RLock()
self.keepalive_seconds = keepalive_seconds
def __init__(self, count, freq=0.1):
super(CountDownLatch, self).__init__()
assert count >= 0
self._lock = RLock()
self._count = count
self._freq = freq
def rlock_object(self):
"""Create an appropriate RLock object"""
return RLock()
# @author:九世
# @time:2019/7/2
# @file:mian.py
import gevent
from gevent import monkey
monkey.patch_all()
import config.config
from multiprocessing import Process
from gevent.lock import RLock
import socket
lock=RLock()
def portscan(host,port):
global recvs
lock.acquire()
s=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
s.settimeout(2)
try:
s.connect((host,port))
s.sendall(b'banner')
recvs=s.recv(1024)
print('\033[1;32m[+]\033[0m {}/Open banner:{}'.format(port,recvs.decode('utf-8')))
except Exception as r:
if "'utf-8' codec can't decode" in str(r):
print('\033[1;32m[+]\033[0m {}/Open banner:{}'.format(port, recvs))
pass
lock.release()
def xc(url,rw):
rg=[]
def __init__(self, key=None):
self.lock = RLock()
self.store = SortedDict(key)
def __init__(self, hosts, user=None, password=None, port=22, pkey=None,
num_retries=DEFAULT_RETRIES, timeout=None, pool_size=100,
allow_agent=True, host_config=None, retry_delay=RETRY_DELAY,
forward_ssh_agent=False,
gssapi_auth=False,
gssapi_server_identity=None,
gssapi_client_identity=None,
gssapi_delegate_credentials=False,
identity_auth=True):
"""
:param hosts: Hosts to connect to
:type hosts: list(str)
:param user: (Optional) User to login as. Defaults to logged in user
:type user: str
:param password: (Optional) Password to use for login. Defaults to
no password
:type password: str
:param port: (Optional) Port number to use for SSH connection. Defaults
to 22.
:type port: int
:param pkey: Private key file path to use. Path must be either absolute
path or relative to user home directory like ``~/<path>``.
:type pkey: str
:param num_retries: (Optional) Number of connection and authentication
attempts before the client gives up. Defaults to 3.
:type num_retries: int
:param retry_delay: Number of seconds to wait between retries. Defaults
to :py:class:`pssh.constants.RETRY_DELAY`
:type retry_delay: int
:param timeout: (Optional) Individual SSH client timeout setting in
seconds passed on to each SSH client spawned by `ParallelSSHClient`.
This controls timeout setting of socket operations used for SSH
sessions *on a per session basis* meaning for each individual
SSH session.
Defaults to OS default - usually 60 seconds.
Parallel functions like `run_command` and `join` have a cummulative
timeout setting that is separate to and
not affected by `self.timeout`.
:type timeout: float
:param pool_size: (Optional) Greenlet pool size. Controls
concurrency, on how many hosts to execute tasks in parallel.
Defaults to 100. Overhead in event
loop will determine how high this can be set to, see scaling guide
lines in project's readme.
:type pool_size: int
:param host_config: (Optional) Per-host configuration for cases where
not all hosts use the same configuration.
:type host_config: dict
:param allow_agent: (Optional) set to False to disable connecting to
the system's SSH agent. Currently unused - always off.
:type allow_agent: bool
:param identity_auth: (Optional) set to False to disable attempting to
authenticate with default identity files from
`pssh.clients.base_ssh_client.BaseSSHClient.IDENTITIES`
:type identity_auth: bool
:param proxy_host: (Optional) SSH host to tunnel connection through
so that SSH clients connect to host via client -> proxy_host -> host
:type proxy_host: str
:param proxy_port: (Optional) SSH port to use to login to proxy host if
set. Defaults to 22.
:type proxy_port: int
:param proxy_user: (Optional) User to login to ``proxy_host`` as.
Defaults to logged in user.
:type proxy_user: str
:param proxy_password: (Optional) Password to login to ``proxy_host``
with. Defaults to no password.
:type proxy_password: str
:param proxy_pkey: (Optional) Private key file to be used for
authentication with ``proxy_host``. Defaults to available keys from
SSHAgent and user's SSH identities.
:type proxy_pkey: Private key file path to use.
:param forward_ssh_agent: (Optional) Turn on SSH agent forwarding -
equivalent to `ssh -A` from the `ssh` command line utility.
Defaults to False if not set.
Currently unused meaning always off.
:type forward_ssh_agent: bool
:param gssapi_server_identity: Set GSSAPI server identity.
:type gssapi_server_identity: str
:param gssapi_server_identity: Set GSSAPI client identity.
:type gssapi_server_identity: str
:param gssapi_delegate_credentials: Enable/disable server credentials
delegation.
:type gssapi_delegate_credentials: bool
:raises: :py:class:`pssh.exceptions.PKeyFileError` on errors finding
provided private key.
"""
BaseParallelSSHClient.__init__(
self, hosts, user=user, password=password, port=port, pkey=pkey,
allow_agent=allow_agent, num_retries=num_retries,
timeout=timeout, pool_size=pool_size,
host_config=host_config, retry_delay=retry_delay,
identity_auth=identity_auth)
self.pkey = _validate_pkey_path(pkey)
self.forward_ssh_agent = forward_ssh_agent
self._clients_lock = RLock()
self.gssapi_auth = gssapi_auth
self.gssapi_server_identity = gssapi_server_identity
self.gssapi_client_identity = gssapi_client_identity
self.gssapi_delegate_credentials = gssapi_delegate_credentials
def __init__(self):
self._sessions = {}
self._read_lock = RLock()
gevent.spawn(self.orphan_cleaning)
