def _load_flatened_toml(self, flattened_toml: str, dest: OrderedDict = None) -> OrderedDict: """[summary] Args: flattened_toml (str): [description] dest (OrderedDict, optional): [description]. Defaults to None. Returns: OrderedDict: [description] """ # initialize our return value if dest is None: dest = OrderedDict() # raw configparse parser = ConfigParser( dict_type=OrderedDict, allow_no_value=False, delimiters=("="), comment_prefixes=(self.commment_prefixes), strict=False, empty_lines_in_values=True, interpolation=ExtendedInterpolation(), default_section=self.default_str, ) # read as a simple INI file parser.read_string(flattened_toml) # we are gonna build a json string out of it since it # is a relatively simple file to translate Toml onto and our values are # always json strings json_entries = [] for _section in parser.sections(): # make it a json string section = '"' + strip_str(_section) + '"' key_entries = [] for _option in parser.options(_section): # make it a json string option = '"' + strip_str(_option) + '"' value = parser.get(_section, _option, fallback=None) if value is None: key_entries.append(option + ": null") else: key_entries.append(option + ": " + value) json_entries.append(section + ": {\n\t\t" + ",\n\t\t".join(key_entries) + "\n\t}") json_entry = "{\n" + ", ".join(json_entries) + "\n}" # escape the escape character for json json_entry = json_entry.replace("\\", "\\\\") new_dict = jsonLoads(json_entry, object_pairs_hook=OrderedDict) new_dict = sanitize_value(new_dict) for section in new_dict: # we override sections by section, overloading is done differently dest[section] = new_dict[section] return dest
def permissions_retrieved(self, response): appInfo = response.meta['appInfo'] package = jsonLoads(response.text[response.text.index('\n') + 1:]) permissionData = jsonLoads(package[0][2]) permissions = [] # permissionData may have # nothing, or # grouped permissions, or # grouped permissions + other permissions, or # grouped permissions + other permissions + miscellaneous permissions # permissionData[0] is grouped permissions. if len(permissionData) > 0: if (permissionData[0]): for g in permissionData[0]: # permission group may be empty. if (not g): continue for p in g[2]: permissions.append(p[1]) # permissionData[1] is other permissions. # permissionData[1] may be None if len(permissionData) > 1 and permissionData[1]: for p in permissionData[1][0][2]: permissions.append(p[1]) # permissionData[2] is miscellaneous permissions. if len(permissionData) > 2: for p in permissionData[2]: permissions.append(p[1]) if len(permissionData) > 3: print( 'Unknown data in permission block.\npermissionData={}'.format( permissionData), file=sys.stderr) self.logger.info( f'name={appInfo["name"]}, rating={appInfo["rating"]}, inAppPurchases={appInfo["inAppPurchases"]}, containsAds={appInfo["containsAds"]}, ' f'categories={appInfo["categories"]}, number of reviews={appInfo["num_reviews"]}, install_fee={appInfo["install_fee"]},' f'app_icon={appInfo["app_icon"]}') self.logger.info(f'permissions={permissions}') appInfo['permissions'] = permissions yield appInfo
def jsonBody(self) -> dict: """ It parsed the body as a json if it matches or it will throw an error Returns: dict """ requestBody = self.__getBody() return jsonLoads(requestBody)
def __getToken(self): # 获取token认证 url_token = 'https://openapi.baidu.com/oauth/2.0/token' api_key = 'R26ZZakxixaQbIDGrPkUwOTc' secret_key = '10d76d90116385e126d95e1c277c538c' get_token_url = url_token+'?'+'grant_type=client_credentials&client_id=' + \ api_key+'&client_secret='+secret_key token = requests_get(get_token_url) r = jsonLoads(token.text) return r['access_token']
def checkForUpdate(self): """Notifies user if their copy of OneLauncher is out of date""" current_version = parse_version(Information.Version) repository_url = Information.repoUrl if "github.com" not in repository_url.lower(): self.logger.warning( "Repository URL set in Information.py is not " "at github.com. The system for update notifications" " only supports this site." ) return latest_release_template = ( "https://api.github.com/repos/{user_and_repo}/releases/latest" ) latest_release_url = latest_release_template.format( user_and_repo=repository_url.lower().split("github.com")[1].strip("/") ) try: with urllib.request.urlopen(latest_release_url, timeout=2) as response: release_dictionary = jsonLoads(response.read()) except (urllib.error.URLError, urllib.error.HTTPError) as error: self.AddLog("[E18] Error checking for OneLauncher updates.") self.logger.error(error.reason, exc_info=True) return release_version = parse_version(release_dictionary["tag_name"]) if release_version > current_version: url = release_dictionary["html_url"] name = release_dictionary["name"] description = release_dictionary["body"] messageBox = QtWidgets.QMessageBox(self.winMain) messageBox.setWindowFlag(QtCore.Qt.FramelessWindowHint) messageBox.setIcon(QtWidgets.QMessageBox.Information) messageBox.setStandardButtons(messageBox.Ok) centered_href = ( '<html><head/><body><p align="center"><a href="{url}">' "<span>{name}</span></a></p></body></html>" ).format(url=url, name=name) messageBox.setInformativeText( "There is a new version of OneLauncher available: {href}".format( href=centered_href ) ) messageBox.setDetailedText(description) messageBox.show() else: self.AddLog("OneLauncher is up to date.")
def get_service_data(self, url): """Retrieves data from service, GET method Args: url (str): URL Returns: JSON with service response """ req = Request(url) req.add_header('X-TrackerToken', self.__api_token) response = urlopen(req) data = response.read() return jsonLoads(data.decode('utf-8'))
def loadCache(self): if self.cacheData or self.cacheAuth: if isfile(self.cacheFileName): print "Loading cache for project ID %d..." % self.projectId with open(self.cacheFileName) as f: json = f.read() if json: for (field, value) in jsonLoads(json).iteritems(): setattr(self, field, value) self._processFields() else: print "The cache is empty" else: print "No cache found for project ID %d" % self.projectId if not self.cacheAuth: self._resetAuth() if not self.cacheData: self.updatedAt = self.metadata = self.fieldNames = self.characterData = self.characters = None
def post_service_data(self, url, data): """Retrieves data from service, POST Args: url (str): URL data (str): string for send to server Returns: JSON with service response """ data_to_send = parse.urlencode(data) data_received = None req = Request(url, data=data_to_send.encode('ascii')) req.add_header('X-TrackerToken', self.__api_token) req.add_header('Content-Type', 'application/json') try: with urlopen(req) as response: data_received = response.read() except HTTPError: return None return jsonLoads(data_received.decode('utf-8'))
def loads(*args, **kwds): if 'cls' not in kwds: kwds['cls'] = QtJSONDecoder return jsonLoads(*args, **kwds)
def searchGooglePlay(keyword) -> List[AppItem]: url = 'https://play.google.com/store/search?q=%s&c=apps' % urllib.parse.quote_plus(keyword) page = requests.get(url, verify=True) # "key: 'ds:3'" is not reliable. matches = re.findall(r'<script.*?>AF_initDataCallback\((.+?)\)\s*;\s*</script>', page.text, flags=re.DOTALL) data = None # Typically the target segment is the last match. for i in range(len(matches) - 1, 0, -1): m = matches[i] if 'googleusercontent.com' in m: try: startIndex = m.find('data:') endIndex = m.rfind(']', startIndex + 1) m = m[startIndex + len('data:'):endIndex + 1] data = jsonLoads(m) data = data[0][1] break except: pass if not data: print("We couldn't find anything for your search.") return [] appInfos = [] appSaver = GooglePlayAdvancedSearch.DBUtils.AppAccessor() appsData = None try: while True: appsData = data[0][0][0] print(f'Load {len(appsData)} apps.') for app in appsData: appId = app[12][0] if any(a['id'] == appId for a in appInfos): print(f'Duplicate app id {appId}.') continue appInfo = AppItem() appInfo['name'] = app[2] appInfo['id'] = appId appInfo['rating'] = app[6][0][2][1][1] if app[6] is not None else 0 appInfo['app_icon'] = app[1][1][0][3][2] if app[7]: appInfo['install_fee'] = float(re.search(r'\d+\.\d*', app[7][0][3][2][1][0][2])[0]) else: appInfo['install_fee'] = 0 print(appInfo['id']) appSaver.insertOrUpdateApp(appInfo) appInfos.append(appInfo) if data[0][0][-2]: pageToken = data[0][0][-2][1] else: break print('continue searching') response = requests.post(r'https://play.google.com/_/PlayStoreUi/data/batchexecute?rpcids=qnKhOb&hl=en', headers={"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"}, data={'f.req': r'[[["qnKhOb","[[null,[[10,[10,50]],true,null,[96,27,4,8,57,30,110,79,11,16,49,1,3,9,12,104,55,56,51,10,34,31,77],[null,null,null,[[[[7,31],[[1,52,43,112,92,58,69,31,19,96]]]]]]],null,\"' + pageToken + r'\"]]",null,"generic"]]]'}, verify=True) package = jsonLoads(response.text[response.text.index('\n') + 1:]) data = jsonLoads(package[0][2]) except Exception as e: print(str(e)) if appsData is None: print(f'data:\n{data}') else: print(f'appsData:\n{appsData}') return appInfos
def _jsonLoads(content): try: result = jsonLoads(content) except Exception as err: return False return result
def registerEPs(self, ce_proxy=None): """ Register EPs to Central Engines """ if ce_proxy: print('Starting Client Service register on `{}`...'.format(ce_proxy)) else: print('Starting Client Service register...') # List of Central Engine connections proxyEpsList = {} for currentEP in self.eps: _proxy = '{}:{}'.format(self.eps[currentEP]['ce_ip'], self.eps[currentEP]['ce_port']) # If Central Engine proxy filter is specified, use it if ce_proxy and ce_proxy != _proxy: continue if not proxyEpsList.has_key(_proxy): proxyEpsList[_proxy] = [ ep for ep in self.eps if self.eps[ep]['ce_ip'] == self.eps[currentEP]['ce_ip'] and self.eps[ep]['ce_port'] == self.eps[currentEP]['ce_port'] ] unregistered = True # Try to register to Central Engine, forever while unregistered: for currentCE in proxyEpsList: try: proxy = self.eps[proxyEpsList[currentCE][0]]['proxy'] __proxy = proxy._ServerProxy__host.split('@')[1].split(':') except Exception, e: print('CE proxy error: `{}` on `{}`.'.format(e, __proxy)) continue clientKey = ':{port}'.format(port=self.clientPort) try: userCeClientInfo = proxy.getUserVariable(self.username, 'clients') if not userCeClientInfo: userCeClientInfo = {} else: userCeClientInfo = jsonLoads(userCeClientInfo) while True: ceStatus = proxy.getExecStatusAll(self.username) if ceStatus.startswith('invalid'): break elif ceStatus.startswith('stopped'): # Reset user project proxy.resetProject(self.username) print('User project reset.') break else: print('CE on `{}` is running with status `{}`.'.format( proxy._ServerProxy__host.split('@')[1], ceStatus)) print('Waiting to stop ...') sleep(2) for (prxy, eps) in userCeClientInfo.items(): for ep in eps: if ep in proxyEpsList[currentCE]: print('Warning: epname {} already registered. Trying to stop..'.format(ep)) try: p = xmlrpclib.ServerProxy('http://{}:{}/twisterclient/'.format( prxy.split(':')[0], prxy.split(':')[1])) try: last_seen_alive = self.eps[ep]['proxy'].getEpVariable( self.username, ep, 'last_seen_alive') now_dtime = datetime.today() if last_seen_alive: diff = now_dtime - datetime.strptime(last_seen_alive, '%Y-%m-%d %H:%M:%S') if diff.seconds <= 2.4: proxyEpsList[currentCE].pop(proxyEpsList[currentCE].index(ep)) print('Warning: epname {} is running. Will not register.'.format(ep)) else: p.stopEP(ep) userCeClientInfo[prxy].pop(userCeClientInfo[prxy].index(ep)) if not userCeClientInfo[prxy]: userCeClientInfo.pop(prxy) print('Warning: epname {} stoped. Will register.'.format(ep)) except Exception as e: pass except Exception as e: pass if not proxyEpsList[currentCE]: continue userCeClientInfo.update([(clientKey, proxyEpsList[currentCE]), ]) userCeClientInfo = jsonDumps(userCeClientInfo) proxy.registerClient(self.username, userCeClientInfo) unregistered = False except Exception as e: self.proxyList.pop(currentCE) print('Error: {er}'.format(er=e)) if unregistered: print('Error: Central Engine is down... will retry...') sleep(2)
def searchGooglePlay(keyword) -> List[AppItem]: url = 'https://play.google.com/store/search?q=%s&c=apps' % keyword page = requests.get(url, verify=True) # "key: 'ds:3'" is not reliable. matches = re.findall( r'<script.*?>AF_initDataCallback\(\s*{.*?data:function\(\){return\s+(\[.+?\])\s*}\s*}\s*\)\s*;\s*</script>', page.text, flags=re.DOTALL) data = jsonLoads(matches[-1]) data = data[0][1] if not data: print("We couldn't find anything for your search.") return [] appInfos = [] appSaver = GooglePlayAdvancedSearch.DBUtils.AppAccessor(1) while True: appsData = data[0][0][0] print(f'Load {len(appsData)} apps.') for app in appsData: appId = app[12][0] if any(a['id'] == appId for a in appInfos): print(f'Duplicate app id {appId}.') continue appInfo = AppItem() appInfo['appName'] = app[2] appInfo['id'] = appId appInfo['rating'] = app[6][0][2][1][1] appInfo['app_icon'] = app[1][1][0][3][2] if app[7]: appInfo['install_fee'] = float( re.search(r'\d+\.\d*', app[7][0][3][2][1][0][2])[0]) else: appInfo['install_fee'] = 0 print(appInfo['id']) appSaver.insertOrUpdateApp(appInfo) appInfos.append(appInfo) if data[0][0][-2]: pageToken = data[0][0][-2][1] else: break print('continue searching') response = requests.post( r'https://play.google.com/_/PlayStoreUi/data/batchexecute?rpcids=qnKhOb&hl=en', headers={ "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8" }, data={ 'f.req': r'[[["qnKhOb","[[null,[[10,[10,50]],true,null,[96,27,4,8,57,30,110,79,11,16,49,1,3,9,12,104,55,56,51,10,34,31,77],[null,null,null,[[[[7,31],[[1,52,43,112,92,58,69,31,19,96]]]]]]],null,\"' + pageToken + r'\"]]",null,"generic"]]]' }, verify=True) package = jsonLoads(response.text[response.text.index('\n') + 1:]) data = jsonLoads(package[0][2]) return appInfos
def registerEPs(self, ce_proxy=None): """ Register EPs to Central Engines """ if ce_proxy: print( 'Starting Client Service register on `{}`...'.format(ce_proxy)) else: print('Starting Client Service register...') # List of Central Engine connections proxyEpsList = {} for currentEP in self.eps: _proxy = '{}:{}'.format(self.eps[currentEP]['ce_ip'], self.eps[currentEP]['ce_port']) # If Central Engine proxy filter is specified, use it if ce_proxy and ce_proxy != _proxy: continue if not proxyEpsList.has_key(_proxy): proxyEpsList[_proxy] = [ ep for ep in self.eps if self.eps[ep]['ce_ip'] == self.eps[currentEP]['ce_ip'] and self.eps[ep]['ce_port'] == self.eps[currentEP]['ce_port'] ] unregistered = True # Try to register to Central Engine, forever while unregistered: for currentCE in proxyEpsList: try: proxy = self.eps[proxyEpsList[currentCE][0]]['proxy'] __proxy = proxy._ServerProxy__host.split('@')[1].split(':') create_connection((__proxy[0], __proxy[1]), 2) except Exception, e: print('CE proxy error: `{}` on `{}`.'.format(e, __proxy)) continue clientKey = ':{port}'.format(port=self.clientPort) try: userCeClientInfo = proxy.getUserVariable( self.username, 'clients') if not userCeClientInfo: userCeClientInfo = {} else: userCeClientInfo = jsonLoads(userCeClientInfo) while True: ceStatus = proxy.getExecStatusAll(self.username) if ceStatus.startswith('invalid'): break elif ceStatus.startswith('stopped'): # Reset user project proxy.resetProject(self.username) print('User project reset.') break else: print('CE on `{}` is running with status `{}`.'. format( proxy._ServerProxy__host.split('@')[1], ceStatus)) print('Waiting to stop ...') sleep(2) for (prxy, eps) in userCeClientInfo.items(): for ep in eps: if ep in proxyEpsList[currentCE]: print( 'Warning: epname {} already registered. Trying to stop..' .format(ep)) try: p = xmlrpclib.ServerProxy( 'http://{}:{}/twisterclient/'.format( prxy.split(':')[0], prxy.split(':')[1])) try: last_seen_alive = self.eps[ep][ 'proxy'].getEpVariable( self.username, ep, 'last_seen_alive') now_dtime = datetime.today() if last_seen_alive: diff = now_dtime - datetime.strptime( last_seen_alive, '%Y-%m-%d %H:%M:%S') if diff.seconds <= 2.4: proxyEpsList[currentCE].pop( proxyEpsList[currentCE]. index(ep)) print( 'Warning: epname {} is running. Will not register.' .format(ep)) else: p.stopEP(ep) userCeClientInfo[prxy].pop( userCeClientInfo[prxy].index( ep)) if not userCeClientInfo[prxy]: userCeClientInfo.pop(prxy) print( 'Warning: epname {} stoped. Will register.' .format(ep)) except Exception as e: pass except Exception as e: pass if not proxyEpsList[currentCE]: continue userCeClientInfo.update([ (clientKey, proxyEpsList[currentCE]), ]) userCeClientInfo = jsonDumps(userCeClientInfo) proxy.registerClient(self.username, userCeClientInfo) unregistered = False except Exception as e: self.proxyList.pop(currentCE) print('Error: {er}'.format(er=e)) if unregistered: print('Error: Central Engine is down... will retry...') sleep(2)