def _load_flatened_toml(self,
flattened_toml: str,
dest: OrderedDict = None) -> OrderedDict:
"""[summary]
Args:
flattened_toml (str): [description]
dest (OrderedDict, optional): [description]. Defaults to None.
Returns:
OrderedDict: [description]
"""
# initialize our return value
if dest is None:
dest = OrderedDict()
# raw configparse
parser = ConfigParser(
dict_type=OrderedDict,
allow_no_value=False,
delimiters=("="),
comment_prefixes=(self.commment_prefixes),
strict=False,
empty_lines_in_values=True,
interpolation=ExtendedInterpolation(),
default_section=self.default_str,
)
# read as a simple INI file
parser.read_string(flattened_toml)
# we are gonna build a json string out of it since it
# is a relatively simple file to translate Toml onto and our values are
# always json strings
json_entries = []
for _section in parser.sections():
# make it a json string
section = '"' + strip_str(_section) + '"'
key_entries = []
for _option in parser.options(_section):
# make it a json string
option = '"' + strip_str(_option) + '"'
value = parser.get(_section, _option, fallback=None)
if value is None:
key_entries.append(option + ": null")
else:
key_entries.append(option + ": " + value)
json_entries.append(section + ": {\n\t\t" +
",\n\t\t".join(key_entries) + "\n\t}")
json_entry = "{\n" + ", ".join(json_entries) + "\n}"
# escape the escape character for json
json_entry = json_entry.replace("\\", "\\\\")
new_dict = jsonLoads(json_entry, object_pairs_hook=OrderedDict)
new_dict = sanitize_value(new_dict)
for section in new_dict:
# we override sections by section, overloading is done differently
dest[section] = new_dict[section]
return dest
def permissions_retrieved(self, response):
appInfo = response.meta['appInfo']
package = jsonLoads(response.text[response.text.index('\n') + 1:])
permissionData = jsonLoads(package[0][2])
permissions = []
# permissionData may have
# nothing, or
# grouped permissions, or
# grouped permissions + other permissions, or
# grouped permissions + other permissions + miscellaneous permissions
# permissionData[0] is grouped permissions.
if len(permissionData) > 0:
if (permissionData[0]):
for g in permissionData[0]:
# permission group may be empty.
if (not g):
continue
for p in g[2]:
permissions.append(p[1])
# permissionData[1] is other permissions.
# permissionData[1] may be None
if len(permissionData) > 1 and permissionData[1]:
for p in permissionData[1][0][2]:
permissions.append(p[1])
# permissionData[2] is miscellaneous permissions.
if len(permissionData) > 2:
for p in permissionData[2]:
permissions.append(p[1])
if len(permissionData) > 3:
print(
'Unknown data in permission block.\npermissionData={}'.format(
permissionData),
file=sys.stderr)
self.logger.info(
f'name={appInfo["name"]}, rating={appInfo["rating"]}, inAppPurchases={appInfo["inAppPurchases"]}, containsAds={appInfo["containsAds"]}, '
f'categories={appInfo["categories"]}, number of reviews={appInfo["num_reviews"]}, install_fee={appInfo["install_fee"]},'
f'app_icon={appInfo["app_icon"]}')
self.logger.info(f'permissions={permissions}')
appInfo['permissions'] = permissions
yield appInfo
def jsonBody(self) -> dict:
"""
It parsed the body as a json if it matches or it will throw an error
Returns:
dict
"""
requestBody = self.__getBody()
return jsonLoads(requestBody)
def __getToken(self):
# 获取token认证
url_token = 'https://openapi.baidu.com/oauth/2.0/token'
api_key = 'R26ZZakxixaQbIDGrPkUwOTc'
secret_key = '10d76d90116385e126d95e1c277c538c'
get_token_url = url_token+'?'+'grant_type=client_credentials&client_id=' + \
api_key+'&client_secret='+secret_key
token = requests_get(get_token_url)
r = jsonLoads(token.text)
return r['access_token']
def checkForUpdate(self):
"""Notifies user if their copy of OneLauncher is out of date"""
current_version = parse_version(Information.Version)
repository_url = Information.repoUrl
if "github.com" not in repository_url.lower():
self.logger.warning(
"Repository URL set in Information.py is not "
"at github.com. The system for update notifications"
" only supports this site."
)
return
latest_release_template = (
"https://api.github.com/repos/{user_and_repo}/releases/latest"
)
latest_release_url = latest_release_template.format(
user_and_repo=repository_url.lower().split("github.com")[1].strip("/")
)
try:
with urllib.request.urlopen(latest_release_url, timeout=2) as response:
release_dictionary = jsonLoads(response.read())
except (urllib.error.URLError, urllib.error.HTTPError) as error:
self.AddLog("[E18] Error checking for OneLauncher updates.")
self.logger.error(error.reason, exc_info=True)
return
release_version = parse_version(release_dictionary["tag_name"])
if release_version > current_version:
url = release_dictionary["html_url"]
name = release_dictionary["name"]
description = release_dictionary["body"]
messageBox = QtWidgets.QMessageBox(self.winMain)
messageBox.setWindowFlag(QtCore.Qt.FramelessWindowHint)
messageBox.setIcon(QtWidgets.QMessageBox.Information)
messageBox.setStandardButtons(messageBox.Ok)
centered_href = (
'<html><head/><body><p align="center"><a href="{url}">'
"<span>{name}</span></a></p></body></html>"
).format(url=url, name=name)
messageBox.setInformativeText(
"There is a new version of OneLauncher available: {href}".format(
href=centered_href
)
)
messageBox.setDetailedText(description)
messageBox.show()
else:
self.AddLog("OneLauncher is up to date.")
def get_service_data(self, url):
"""Retrieves data from service, GET method
Args:
url (str): URL
Returns:
JSON with service response
"""
req = Request(url)
req.add_header('X-TrackerToken', self.__api_token)
response = urlopen(req)
data = response.read()
return jsonLoads(data.decode('utf-8'))
def loadCache(self):
if self.cacheData or self.cacheAuth:
if isfile(self.cacheFileName):
print "Loading cache for project ID %d..." % self.projectId
with open(self.cacheFileName) as f:
json = f.read()
if json:
for (field, value) in jsonLoads(json).iteritems():
setattr(self, field, value)
self._processFields()
else:
print "The cache is empty"
else:
print "No cache found for project ID %d" % self.projectId
if not self.cacheAuth:
self._resetAuth()
if not self.cacheData:
self.updatedAt = self.metadata = self.fieldNames = self.characterData = self.characters = None
def post_service_data(self, url, data):
"""Retrieves data from service, POST
Args:
url (str): URL
data (str): string for send to server
Returns:
JSON with service response
"""
data_to_send = parse.urlencode(data)
data_received = None
req = Request(url, data=data_to_send.encode('ascii'))
req.add_header('X-TrackerToken', self.__api_token)
req.add_header('Content-Type', 'application/json')
try:
with urlopen(req) as response:
data_received = response.read()
except HTTPError:
return None
return jsonLoads(data_received.decode('utf-8'))
def loads(*args, **kwds):
if 'cls' not in kwds:
kwds['cls'] = QtJSONDecoder
return jsonLoads(*args, **kwds)
def searchGooglePlay(keyword) -> List[AppItem]:
url = 'https://play.google.com/store/search?q=%s&c=apps' % urllib.parse.quote_plus(keyword)
page = requests.get(url, verify=True)
# "key: 'ds:3'" is not reliable.
matches = re.findall(r'<script.*?>AF_initDataCallback\((.+?)\)\s*;\s*</script>', page.text, flags=re.DOTALL)
data = None
# Typically the target segment is the last match.
for i in range(len(matches) - 1, 0, -1):
m = matches[i]
if 'googleusercontent.com' in m:
try:
startIndex = m.find('data:')
endIndex = m.rfind(']', startIndex + 1)
m = m[startIndex + len('data:'):endIndex + 1]
data = jsonLoads(m)
data = data[0][1]
break
except:
pass
if not data:
print("We couldn't find anything for your search.")
return []
appInfos = []
appSaver = GooglePlayAdvancedSearch.DBUtils.AppAccessor()
appsData = None
try:
while True:
appsData = data[0][0][0]
print(f'Load {len(appsData)} apps.')
for app in appsData:
appId = app[12][0]
if any(a['id'] == appId for a in appInfos):
print(f'Duplicate app id {appId}.')
continue
appInfo = AppItem()
appInfo['name'] = app[2]
appInfo['id'] = appId
appInfo['rating'] = app[6][0][2][1][1] if app[6] is not None else 0
appInfo['app_icon'] = app[1][1][0][3][2]
if app[7]:
appInfo['install_fee'] = float(re.search(r'\d+\.\d*', app[7][0][3][2][1][0][2])[0])
else:
appInfo['install_fee'] = 0
print(appInfo['id'])
appSaver.insertOrUpdateApp(appInfo)
appInfos.append(appInfo)
if data[0][0][-2]:
pageToken = data[0][0][-2][1]
else:
break
print('continue searching')
response = requests.post(r'https://play.google.com/_/PlayStoreUi/data/batchexecute?rpcids=qnKhOb&hl=en',
headers={"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"},
data={'f.req': r'[[["qnKhOb","[[null,[[10,[10,50]],true,null,[96,27,4,8,57,30,110,79,11,16,49,1,3,9,12,104,55,56,51,10,34,31,77],[null,null,null,[[[[7,31],[[1,52,43,112,92,58,69,31,19,96]]]]]]],null,\"'
+ pageToken
+ r'\"]]",null,"generic"]]]'},
verify=True)
package = jsonLoads(response.text[response.text.index('\n') + 1:])
data = jsonLoads(package[0][2])
except Exception as e:
print(str(e))
if appsData is None:
print(f'data:\n{data}')
else:
print(f'appsData:\n{appsData}')
return appInfos
def _jsonLoads(content):
try:
result = jsonLoads(content)
except Exception as err:
return False
return result
def registerEPs(self, ce_proxy=None):
""" Register EPs to Central Engines """
if ce_proxy:
print('Starting Client Service register on `{}`...'.format(ce_proxy))
else:
print('Starting Client Service register...')
# List of Central Engine connections
proxyEpsList = {}
for currentEP in self.eps:
_proxy = '{}:{}'.format(self.eps[currentEP]['ce_ip'], self.eps[currentEP]['ce_port'])
# If Central Engine proxy filter is specified, use it
if ce_proxy and ce_proxy != _proxy:
continue
if not proxyEpsList.has_key(_proxy):
proxyEpsList[_proxy] = [
ep for ep in self.eps if self.eps[ep]['ce_ip'] == self.eps[currentEP]['ce_ip'] and
self.eps[ep]['ce_port'] == self.eps[currentEP]['ce_port']
]
unregistered = True
# Try to register to Central Engine, forever
while unregistered:
for currentCE in proxyEpsList:
try:
proxy = self.eps[proxyEpsList[currentCE][0]]['proxy']
__proxy = proxy._ServerProxy__host.split('@')[1].split(':')
except Exception, e:
print('CE proxy error: `{}` on `{}`.'.format(e, __proxy))
continue
clientKey = ':{port}'.format(port=self.clientPort)
try:
userCeClientInfo = proxy.getUserVariable(self.username, 'clients')
if not userCeClientInfo:
userCeClientInfo = {}
else:
userCeClientInfo = jsonLoads(userCeClientInfo)
while True:
ceStatus = proxy.getExecStatusAll(self.username)
if ceStatus.startswith('invalid'):
break
elif ceStatus.startswith('stopped'):
# Reset user project
proxy.resetProject(self.username)
print('User project reset.')
break
else:
print('CE on `{}` is running with status `{}`.'.format(
proxy._ServerProxy__host.split('@')[1], ceStatus))
print('Waiting to stop ...')
sleep(2)
for (prxy, eps) in userCeClientInfo.items():
for ep in eps:
if ep in proxyEpsList[currentCE]:
print('Warning: epname {} already registered. Trying to stop..'.format(ep))
try:
p = xmlrpclib.ServerProxy('http://{}:{}/twisterclient/'.format(
prxy.split(':')[0], prxy.split(':')[1]))
try:
last_seen_alive = self.eps[ep]['proxy'].getEpVariable(
self.username, ep, 'last_seen_alive')
now_dtime = datetime.today()
if last_seen_alive:
diff = now_dtime - datetime.strptime(last_seen_alive,
'%Y-%m-%d %H:%M:%S')
if diff.seconds <= 2.4:
proxyEpsList[currentCE].pop(proxyEpsList[currentCE].index(ep))
print('Warning: epname {} is running. Will not register.'.format(ep))
else:
p.stopEP(ep)
userCeClientInfo[prxy].pop(userCeClientInfo[prxy].index(ep))
if not userCeClientInfo[prxy]:
userCeClientInfo.pop(prxy)
print('Warning: epname {} stoped. Will register.'.format(ep))
except Exception as e:
pass
except Exception as e:
pass
if not proxyEpsList[currentCE]:
continue
userCeClientInfo.update([(clientKey, proxyEpsList[currentCE]), ])
userCeClientInfo = jsonDumps(userCeClientInfo)
proxy.registerClient(self.username, userCeClientInfo)
unregistered = False
except Exception as e:
self.proxyList.pop(currentCE)
print('Error: {er}'.format(er=e))
if unregistered:
print('Error: Central Engine is down... will retry...')
sleep(2)
def searchGooglePlay(keyword) -> List[AppItem]:
url = 'https://play.google.com/store/search?q=%s&c=apps' % keyword
page = requests.get(url, verify=True)
# "key: 'ds:3'" is not reliable.
matches = re.findall(
r'<script.*?>AF_initDataCallback\(\s*{.*?data:function\(\){return\s+(\[.+?\])\s*}\s*}\s*\)\s*;\s*</script>',
page.text,
flags=re.DOTALL)
data = jsonLoads(matches[-1])
data = data[0][1]
if not data:
print("We couldn't find anything for your search.")
return []
appInfos = []
appSaver = GooglePlayAdvancedSearch.DBUtils.AppAccessor(1)
while True:
appsData = data[0][0][0]
print(f'Load {len(appsData)} apps.')
for app in appsData:
appId = app[12][0]
if any(a['id'] == appId for a in appInfos):
print(f'Duplicate app id {appId}.')
continue
appInfo = AppItem()
appInfo['appName'] = app[2]
appInfo['id'] = appId
appInfo['rating'] = app[6][0][2][1][1]
appInfo['app_icon'] = app[1][1][0][3][2]
if app[7]:
appInfo['install_fee'] = float(
re.search(r'\d+\.\d*', app[7][0][3][2][1][0][2])[0])
else:
appInfo['install_fee'] = 0
print(appInfo['id'])
appSaver.insertOrUpdateApp(appInfo)
appInfos.append(appInfo)
if data[0][0][-2]:
pageToken = data[0][0][-2][1]
else:
break
print('continue searching')
response = requests.post(
r'https://play.google.com/_/PlayStoreUi/data/batchexecute?rpcids=qnKhOb&hl=en',
headers={
"Content-Type":
"application/x-www-form-urlencoded;charset=UTF-8"
},
data={
'f.req':
r'[[["qnKhOb","[[null,[[10,[10,50]],true,null,[96,27,4,8,57,30,110,79,11,16,49,1,3,9,12,104,55,56,51,10,34,31,77],[null,null,null,[[[[7,31],[[1,52,43,112,92,58,69,31,19,96]]]]]]],null,\"'
+ pageToken + r'\"]]",null,"generic"]]]'
},
verify=True)
package = jsonLoads(response.text[response.text.index('\n') + 1:])
data = jsonLoads(package[0][2])
return appInfos
def registerEPs(self, ce_proxy=None):
""" Register EPs to Central Engines """
if ce_proxy:
print(
'Starting Client Service register on `{}`...'.format(ce_proxy))
else:
print('Starting Client Service register...')
# List of Central Engine connections
proxyEpsList = {}
for currentEP in self.eps:
_proxy = '{}:{}'.format(self.eps[currentEP]['ce_ip'],
self.eps[currentEP]['ce_port'])
# If Central Engine proxy filter is specified, use it
if ce_proxy and ce_proxy != _proxy:
continue
if not proxyEpsList.has_key(_proxy):
proxyEpsList[_proxy] = [
ep for ep in self.eps if
self.eps[ep]['ce_ip'] == self.eps[currentEP]['ce_ip'] and
self.eps[ep]['ce_port'] == self.eps[currentEP]['ce_port']
]
unregistered = True
# Try to register to Central Engine, forever
while unregistered:
for currentCE in proxyEpsList:
try:
proxy = self.eps[proxyEpsList[currentCE][0]]['proxy']
__proxy = proxy._ServerProxy__host.split('@')[1].split(':')
create_connection((__proxy[0], __proxy[1]), 2)
except Exception, e:
print('CE proxy error: `{}` on `{}`.'.format(e, __proxy))
continue
clientKey = ':{port}'.format(port=self.clientPort)
try:
userCeClientInfo = proxy.getUserVariable(
self.username, 'clients')
if not userCeClientInfo:
userCeClientInfo = {}
else:
userCeClientInfo = jsonLoads(userCeClientInfo)
while True:
ceStatus = proxy.getExecStatusAll(self.username)
if ceStatus.startswith('invalid'):
break
elif ceStatus.startswith('stopped'):
# Reset user project
proxy.resetProject(self.username)
print('User project reset.')
break
else:
print('CE on `{}` is running with status `{}`.'.
format(
proxy._ServerProxy__host.split('@')[1],
ceStatus))
print('Waiting to stop ...')
sleep(2)
for (prxy, eps) in userCeClientInfo.items():
for ep in eps:
if ep in proxyEpsList[currentCE]:
print(
'Warning: epname {} already registered. Trying to stop..'
.format(ep))
try:
p = xmlrpclib.ServerProxy(
'http://{}:{}/twisterclient/'.format(
prxy.split(':')[0],
prxy.split(':')[1]))
try:
last_seen_alive = self.eps[ep][
'proxy'].getEpVariable(
self.username, ep,
'last_seen_alive')
now_dtime = datetime.today()
if last_seen_alive:
diff = now_dtime - datetime.strptime(
last_seen_alive,
'%Y-%m-%d %H:%M:%S')
if diff.seconds <= 2.4:
proxyEpsList[currentCE].pop(
proxyEpsList[currentCE].
index(ep))
print(
'Warning: epname {} is running. Will not register.'
.format(ep))
else:
p.stopEP(ep)
userCeClientInfo[prxy].pop(
userCeClientInfo[prxy].index(
ep))
if not userCeClientInfo[prxy]:
userCeClientInfo.pop(prxy)
print(
'Warning: epname {} stoped. Will register.'
.format(ep))
except Exception as e:
pass
except Exception as e:
pass
if not proxyEpsList[currentCE]:
continue
userCeClientInfo.update([
(clientKey, proxyEpsList[currentCE]),
])
userCeClientInfo = jsonDumps(userCeClientInfo)
proxy.registerClient(self.username, userCeClientInfo)
unregistered = False
except Exception as e:
self.proxyList.pop(currentCE)
print('Error: {er}'.format(er=e))
if unregistered:
print('Error: Central Engine is down... will retry...')
sleep(2)
