def export_single(filename, link):
jsonrpc_path = global_config.jsonrpc
jsonrpc_user = global_config.jsonrpc_user
jsonrpc_pass = global_config.jsonrpc_pass
if not jsonrpc_path:
print("请设置config.ini中的jsonrpc选项")
exit(1)
jsonreq = json.dumps(
[{
"jsonrpc": "2.0",
"method": "aria2.addUri",
"id": "qwer",
"params": [
[link],
{
"out": filename,
"header": "User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0"
"\r\nReferer:http://pan.baidu.com/disk/home"
}]
}]
)
try:
if jsonrpc_user and jsonrpc_pass:
response = requests.post(url=jsonrpc_path, data=jsonreq, auth=(jsonrpc_user, jsonrpc_pass))
else:
response = requests.post(url=jsonrpc_path, data=jsonreq)
logger.debug(response.text, extra={"type": "jsonreq", "method": "POST"})
except requests.ConnectionError as urle:
print(urle)
raise JsonrpcError("jsonrpc无法连接，请检查jsonrpc地址是否有误！")
if response.ok:
print("已成功添加到jsonrpc\n")
def handle_part(data, match, client, channels):
"""
Someone is leaving a channel. Let's tell everyone about it and
remove them from the channel's user listing (and the channel from the client's).
'^PART (?P<channel>[^\s]+)( :(?P<message>.*))'
:type data: str
:type match: dict
:type client: Client
:type channels: list
"""
channame = match['channel']
logger.debug("{nick} leaves channel {channame}", nick=client.nick, channame=channame)
if not channame in channels:
logger.warn("no channel named {channame}", channame=channame)
return
channel = channels[channame]
client.channels.remove(channel)
channels[channame].clients.remove(client)
announce = Protocol.part(client, channel, match['message'] or 'leaving')
channel.send(announce)
def verify_passwd(self, url, secret=None):
"""
Verify password if url is a private sharing.
:param url: link of private sharing. ('init' must in url)
:type url: str
:param secret: password of the private sharing
:type secret: str
:return: None
"""
if secret:
pwd = secret
else:
# FIXME: Improve translation
pwd = raw_input("Please input this sharing password\n")
data = {'pwd': pwd, 'vcode': ''}
url = "{0}&t={1}&".format(url.replace('init', 'verify'), int(time()))
logger.debug(url, extra={'type': 'url', 'method': 'POST'})
r = self.session.post(url=url, data=data, headers=self.headers)
mesg = r.json()
logger.debug(mesg, extra={'type': 'JSON', 'method': 'POST'})
errno = mesg.get('errno')
if errno == -63:
raise UnknownError
elif errno == -9:
raise VerificationError("提取密码错误\n")
def get_file_infos(self, link, secret=None):
shareinfo = ShareInfo()
js = None
try:
js = self._get_js(link, secret)
except IndexError:
# Retry with new cookies
js = self._get_js(link, secret)
# Fix #15
self.session.get(
'http://d.pcs.baidu.com/rest/2.0/pcs/file?method=plantcookie&type=ett')
self.pcsett = self.session.cookies.get('pcsett')
logger.debug(self.pcsett, extra={
'type': 'cookies', 'method': 'SetCookies'})
if not shareinfo.match(js):
pass
for fi in shareinfo.fileinfo:
if fi['isdir'] == 0:
self.all_files.append(fi)
else:
# recursively get files under the specific path
self.bd_get_files(shareinfo, fi['path'])
# get file details include dlink, path, filename ...
return [self.get_file_info(shareinfo, fsid=f['fs_id'], secret=secret) for f in self.all_files]
def __init__(self, url, method="GET", parameters=None, cookie=None, headers={}):
try:
# 解析url
if type(url) == bytes:
self.__url = url.decode("utf-8")
if type(url) == str:
self.__url = url
logger.debug(self.__url)
scheme, rest = urllib.parse.splittype(self.__url)
# 拆分域名和路径
logger.debug(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
self.__host_absolutely, self.__path = urllib.parse.splithost(rest)
host_list = self.__host_absolutely.split(":")
if len(host_list) == 1:
self.__host = host_list[0]
self.__port = 80
elif len(host_list) == 2:
self.__host = host_list[0]
self.__port = host_list[1]
# 对所传参数进行处理
self.__method = method
self.__data = parameters
self.__cookie = cookie
if parameters != None:
self.__parameters_urlencode_deal = urllib.parse.urlencode(parameters)
else:
self.__parameters_urlencode_deal = ""
self.__jdata = simplejson.dumps(parameters, ensure_ascii=False)
self.__headers = headers
except Exception as e:
logger.error(e)
logger.exception(u"捕获到错误如下:")
def get_dlink(self, link, secret=None):
info = FileInfo()
js = self._get_js(link, secret)
if info.match(js):
extra_params = dict(bdstoken=info.bdstoken, sign=info.sign, timestamp=str(int(time())))
post_form = {
'encrypt': '0',
'product': 'share',
'uk': info.uk,
'primaryid': info.share_id,
'fid_list': '[{0}]'.format(info.fid_list)
}
url = BAIDUPAN_SERVER + 'sharedownload'
response = self._request('POST', url, extra_params=extra_params, post_data=post_form)
if response.ok:
_json = response.json()
errno = _json['errno']
while errno == -20:
verify_params = self._handle_captcha(info.bdstoken)
post_form.update(verify_params)
response = self._request('POST', url, extra_params=extra_params, post_data=post_form)
_json = response.json()
errno = _json['errno']
logger.debug(_json, extra={'type': 'json', 'method': 'POST'})
if errno == 0:
# FIXME: only support single file for now
dlink = _json['list'][0]['dlink']
setattr(info, 'dlink', dlink)
else:
raise UnknownError
return info
def start(self):
while True:
try:
# 1. Capture images from all cameras
logger.debug("Capturing Images")
images = self.get_images()
# 2. Send them to the remote server
logger.debug("Submitting Images")
self.post_images(images)
except:
logger.warning("Unable to retrieve and send images")
# Wait
time.sleep(PERIOD)
def get_images(self):
images = []
for cam in self.cameras:
# Get Image from camera
img = cam.getImage()
images.append(img)
return images
def post_images(self, images):
#Todo: Saving the images to disk until webserver is up and running
for i in xrange(self.n_cameras):
img = images[i]
img.show()
img.save("images/{}-{}.jpg".format(i, time.time()))
def match(self, js):
_filename = re.search(self.filename_pattern, js)
_fileinfo = re.search(self.fileinfo_pattern, js)
if _filename:
self.filename = _filename.group(1).decode('unicode_escape')
if _fileinfo:
self.fileinfo = json.loads(
_fileinfo.group(1).decode('unicode_escape'))
data = re.findall(self.pattern, js)
if not data:
return False
yun_data = dict([i.split(' = ', 1) for i in data])
logger.debug(yun_data, extra={'method': 'GET', 'type': 'javascript'})
# if 'single' not in yun_data.get('SHAREPAGETYPE') or '0' in yun_data.get('LOGINSTATUS'):
# return False
self.uk = yun_data.get('SHARE_UK').strip('"')
# self.bduss = yun_data.get('MYBDUSS').strip('"')
self.share_id = yun_data.get('SHARE_ID').strip('"')
self.fid_list = json.dumps([i['fs_id'] for i in self.fileinfo])
self.sign = yun_data.get('SIGN').strip('"')
if yun_data.get('MYBDSTOKEN'):
self.bdstoken = yun_data.get('MYBDSTOKEN').strip('"')
self.timestamp = yun_data.get('TIMESTAMP').strip('"')
self.sharepagetype = yun_data.get('SHAREPAGETYPE').strip('"')
if self.sharepagetype == DLTYPE_MULTIPLE:
self.filename = os.path.splitext(self.filename)[0] + '-batch.zip'
# if self.bdstoken:
# return True
return True
def download(args):
limit = global_config.limit
output_dir = global_config.dir
parser = argparse.ArgumentParser(description="download command arg parser")
parser.add_argument('-L', '--limit', action="store", dest='limit', help="Max download speed limit.")
parser.add_argument('-D', '--dir', action="store", dest='output_dir', help="Download task to dir.")
parser.add_argument('-S', '--secret', action="store", dest='secret', help="Retrieval password.", default="")
if not args:
parser.print_help()
exit(1)
namespace, links = parser.parse_known_args(args)
secret = namespace.secret
if namespace.limit:
limit = namespace.limit
if namespace.output_dir:
output_dir = namespace.output_dir
# if is wap
links = [link.replace("wap/link", "share/link") for link in links]
# add 'http://'
links = map(add_http, links)
for url in links:
res = parse_url(url)
# normal
if res.get('type') == 1:
pan = Pan()
info = pan.get_dlink(url, secret)
cookies = 'BDUSS={0}'.format(pan.bduss) if pan.bduss else ''
if cookies and pan.pcsett:
cookies += ';pcsett={0}'.format(pan.pcsett)
if cookies:
cookies += '"'
download_command(info.filename, info.dlink, cookies=cookies, limit=limit, output_dir=output_dir)
elif res.get('type') == 4:
pan = Pan()
fsid = res.get('fsid')
newUrl = res.get('url')
info = pan.get_dlink(newUrl, secret, fsid)
cookies = 'BDUSS={0}'.format(pan.bduss) if pan.bduss else ''
if cookies and pan.pcsett:
cookies += ';pcsett={0}'.format(pan.pcsett)
if cookies:
cookies += '"'
download_command(info.filename, info.dlink, cookies=cookies, limit=limit, output_dir=output_dir)
# album
elif res.get('type') == 2:
raise NotImplementedError('This function has not implemented.')
# home
elif res.get('type') == 3:
raise NotImplementedError('This function has not implemented.')
elif res.get('type') == 0:
logger.debug(url, extra={"type": "wrong link", "method": "None"})
continue
else:
continue
sys.exit(0)
def attribute(object, scope):
value = object.raw
if value in scope.items:
output = _new_with_self(_expression_with_scope(scope.items[value], lambda: _item_resolution_scope(scope))(), scope)
logger.debug('attribute {} in {}, returned {}'.format(object.raw, scope._id, output._id))
return output
return create_failed(WRException('Attribute error: `%s`' % value))
def _native_equals(a, b):
try:
if a.raw == b.raw:
return a # or b
return create_failed(False)
except Exception as e:
logger.debug(e) # TODO narrow down exceptions
return create_failed(e)
def _check_verify_code(self):
"""Check if login need to input verify code."""
r = self.session.get(self._check_url)
s = r.text
data = json.loads(s[s.index('{'):-1])
log_message = {'type': 'check loging verify code', 'method': 'GET'}
logger.debug(data, extra=log_message)
if data.get('codestring'):
self.codestring = data.get('codestring')
def addhook(regex, func):
"""
Associates a function with a regex.
:type regex: str
:type func: function
:rtype: None
"""
logger.debug("Adding hook for '{regex}'", regex=regex)
MessageHandler.HOOKS.append((re.compile(regex), func))
def _native_add(a, b):
try:
native = a.raw + b.raw
return {
int: create_number,
list: create_list,
str: create_string,
}[type(native)](native)
except Exception as e:
logger.debug(e) # TODO narrow down exception list
return create_failed(WRException('Cannot add types: %s, %s' % (type(a.raw), type(b.raw))))
def _resolve(ident, scope):
logger.debug("resolve {} (in {})".format(ident, scope._id))
d = _get_scope_dict(scope)
if ident in d:
return d[ident]()
if scope.parent:
return _resolve(ident, scope.parent)
return create_failed(WRException('Key error: `%s`' % ident))
def _get_token(self):
"""Get bdstoken."""
r = self.session.get(self._token_url)
s = r.text
try:
self.token = re.search("login_token='(\w+)';", s).group(1)
# FIXME: if couldn't get the token, we can not get the log message.
log_message = {'type': 'bdstoken', 'method': 'GET'}
logger.debug(self.token, extra=log_message)
except:
raise GetTokenError("Can't get the token")
def start(self):
# Start the thread for each sensor
logger.debug("Starting all DataPoster threads")
for thread in self.threads:
thread.start()
# Wait fo rall threads to finish
# In theory this should run forever
for thread in self.threads:
thread.join()
logger.debug("All DataPoster Threads joined")
def get_all_data(self, start=0, end=-1):
""" prepare data for feeding DNN """
ret = []
temp_queue_set = self.queue_set[start:end]
logger.debug("the shape of queue set: " +
str(np.array(temp_queue_set).shape))
for i in range(len(temp_queue_set[0])):
trs = list(item.transformation_para
for item in np.array(temp_queue_set)[:, i])
attr = self.generate_attr(start, trs)
ret.append(attr)
return ret # num_mutate * num_test
def attribute(object, scope):
value = object.raw
if value in scope.items:
output = _new_with_self(
_expression_with_scope(scope.items[value],
lambda: _item_resolution_scope(scope))(),
scope)
logger.debug('attribute {} in {}, returned {}'.format(
object.raw, scope._id, output._id))
return output
return create_failed(WRException('Attribute error: `%s`' % value))
def start(self):
# Start the thread for each sensor
logger.debug("Starting all DataPoster threads")
for thread in self.threads:
thread.start()
# Wait fo rall threads to finish
# In theory this should run forever
for thread in self.threads:
thread.join()
logger.debug("All DataPoster Threads joined");
def get_file_info(self, shareinfo, fsid, secret=None):
fi = FileInfo()
extra_params = dict(bdstoken=shareinfo.bdstoken,
sign=shareinfo.sign,
timestamp=shareinfo.timestamp)
post_form = {
'encrypt': '0',
'product': 'share',
'uk': shareinfo.uk,
'primaryid': shareinfo.share_id,
'fid_list': json.dumps([fsid]),
}
if self.session.cookies.get('BDCLND'):
post_form['extra'] = '{"sekey":"%s"}' % (url_unquote(
self.session.cookies['BDCLND'])),
logger.debug(post_form, extra={'type': 'form', 'method': 'POST'})
url = BAIDUPAN_SERVER + 'sharedownload'
while True:
response = self._request('POST',
url,
extra_params=extra_params,
post_data=post_form)
if not response.ok:
raise UnknownError
_json = response.json()
errno = _json['errno']
logger.debug(_json, extra={'type': 'json', 'method': 'POST'})
if errno == 0:
fi.filename = _json['list'][0]['server_filename']
fi.path = os.path.dirname(_json['list'][0]['path'])
fi.dlink = _json['list'][0]['dlink']
fi.parent_path = url_unquote(
shareinfo.fileinfo[0]['parent_path'].encode('utf8'))
break
elif errno == -20:
verify_params = self._handle_captcha(shareinfo.bdstoken)
post_form.update(verify_params)
response = self._request('POST',
url,
extra_params=extra_params,
post_data=post_form)
_json = response.json()
errno = _json['errno']
continue
elif errno == 116:
raise DownloadError("The share file does not exist")
else:
raise UnknownError
return fi
def get_trending_repository(self, since: Since = Since.daily):
"""热门仓库
{'href': href, 'url': url, 'description': desc, 'language': language,
'stars': stars, 'folks': folks, 'recent_stars': recent_stars}
"""
items = []
resp = None
try:
with request_session() as s:
params = {'since': since.value}
resp = s.get(TRENDING_REPOSITORY_URL, params=params)
soup = BeautifulSoup(resp.text, features='html.parser')
articles = soup.select('main article.Box-row')
for article in articles:
href = article.select_one('h1.h3 > a')['href']
url = 'https://github.com' + href
logger.info('%s %s', since, href)
# 项目描述
description = '无'
desc = article.select_one('article > p')
if desc:
description = desc.text.strip()
div = article.select_one('div.f6')
spans = div.select('div > span')
# 并非每个仓库都有语言标签
language = '无'
if len(spans) == 3:
language = spans[0].text.strip()
# 最近关注数
recent_stars = spans[-1].text.strip()
star_folk = div.select('a')
stars = star_folk[0].text.strip()
folks = star_folk[1].text.strip()
item = {
'href': href,
'url': url,
'description': description,
'language': language,
'stars': stars,
'folks': folks,
'recent_stars': recent_stars
}
logger.debug('repo:%s', item)
items.append(item)
except:
logger.exception('get trending repository failed')
return (items, resp)
def stop():
output = None
try:
output = runCmdAndGetOutput('ps -ef|grep kubesds-rpc-service')
except Exception:
logger.debug(traceback.format_exc())
if output:
lines = output.splitlines()
if len(lines) <= 1:
return
else:
pid = lines[0].split()[1]
runCmd('kill -9 %s' % pid)
def _parse_command(self, command: str):
cmd_split = shlex.split(command)
invoke = cmd_split[0]
args = cmd_split[1:]
if invoke in self.registered_commands.keys():
logger.debug('Trying executing command "%s"' % invoke)
try:
self.registered_commands[invoke](self).execute(args)
except Exception as e:
logger.error('Failed executing command "' + invoke + '": ', e)
exit(1)
else:
logger.error('Command "%s" not found' % invoke)
def _native_add(a, b):
try:
native = a.raw + b.raw
return {
int: create_number,
list: create_list,
str: create_string,
}[type(native)](native)
except Exception as e:
logger.debug(e) # TODO narrow down exception list
return create_failed(
WRException('Cannot add types: %s, %s' %
(type(a.raw), type(b.raw))))
def repost_replies(account_name):
bf = open('.blacklist_%s'%account_name,'a+')
blacklist = bf.read().splitlines()
bf.close()
rp = open('.reposted_%s'%account_name,'a+')
reposted = rp.read().splitlines()
account = settings.ACCOUNTS.get(account_name)
try:
logging.info('[%s] Getting last mentions offset'%account_name)
bot = TwitterBot(settings.CONSUMER_KEY,settings.CONSUMER_SECRET,
account['key'],account['secret'])
mentions = []
try:
mentions = bot.api.mentions()
logging.info('[%s] Got %d mentions'%(account_name,len(mentions)))
except Exception,e:
logging.error('[%s] Failed to get mentions. %s'%(account_name,e))
for mess in reversed(mentions):
try:
author = mess.author.screen_name
if str(author) in blacklist:
logging.debug('[%s] Author %s blacklisted. Skipping.'%(account_name,str(author)))
continue
if str(mess.id) in reposted:
logging.debug('[%s] Message #%s already reposted. Skipping.'%(account_name,str(mess.id)))
continue
message = mess.text.split(' ')
if message[0] != '@%s'%account_name:
continue #not a "@reply"
trigger = message[1]
triggers = dict(account['triggers'])
if trigger not in triggers:
logging.warning('[%s] Bad message format, sending DM to author'%account_name)
bot.dm(author,account['not_triggered'])
else:
len_params = {'message':'','user':author}
mess_len = len(triggers[trigger]%len_params)
params = {'message':bot.trim_message(' '.join(message[2:]),mess_len),'user':author}
message = triggers[trigger]%params
logging.info('[%s] Tweeting message %s'%(account_name,message))
bot.tweet(message)
rp.write('%s\n'%mess.id)
except Exception,e:
logging.error('%s'%e)
continue
def handle_mode(data, match, client, channels):
"""
Someone is setting their mode. I don't know enough about this to talk about it yet.
'^MODE (?P<nick>[^\s]+) (?P<mode>.*)'
:type data: str
:type match: dict
:type client: Client
:type channels: list
"""
logger.debug("setting {nick}'s mode to {mode}, as per their request", **match)
client.mode = match['mode'].replace('+', '')
def call(function, arguments):
logger.debug("call {} {}".format(function._id, arguments))
scope_with_arguments = _new_with_arguments(function, arguments, scope_stack[-1])
if scope_with_arguments.expression:
scope_stack.append(scope_with_arguments)
output = scope_with_arguments.expression()
scope_stack.pop()
else:
output = scope_with_arguments
output.stack_trace.append(scope_with_arguments)
logger.debug("return {} (from {})".format(output._id, function._id))
return output
def run_server():
# 多线程服务器
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
# 实例化 计算len的类
servicer = AutoControlServicer()
# 注册本地服务,方法CmdCallServicer只有这个是变的
auto_pb2_grpc.add_AutoControlServicer_to_server(servicer, server)
# 监听端口
logger.debug("%s:%s" % (get_docker0_IP(), DEFAULT_PORT))
server.add_insecure_port("%s:%s" % (get_docker0_IP(), DEFAULT_PORT))
# 开始接收请求进行服务
server.start()
return server
def _wait_for_event(self, thread_id, timeout):
'''等待事件
'''
if self._get_lock(thread_id).is_set(): return
time0 = time.time()
ret = self._get_lock(thread_id).wait(timeout)
if ret:
cost_time = time.time() - time0
logger.debug('thread %s wait %s S' % (thread_id, cost_time))
else:
# 超时
logger.warn('thread %s wait for sync event timeout' % thread_id)
self.resume_thread(thread_id) # 防止出错后一直等待
return ret
def exec_cmd(cmd, comment, env=None, cwd=None, backgroud=False):
logger.info('-' * 15 + ' ' + comment + ' ' + '-' * 15)
logger.info('执行命令行命令：')
logger.info(' '.join(cmd))
if env:
logger.info('env: ' + str(env))
if cwd:
logger.info('cwd: ' + str(cwd))
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
cwd=cwd)
if backgroud:
out_value, err_value = None, None
returncode = p.poll()
else:
out_value, err_value = p.communicate()
out_value = out_value.decode()
err_value = err_value.decode()
returncode = p.returncode
logger.debug('*' * 40)
logger.debug('标准输出：\n' + str(out_value))
logger.debug('错误输出：\n' + str(err_value))
logger.debug('*' * 40)
return returncode, out_value, err_value
def get(object, gettable):
try:
index = object.raw
native = gettable.raw
output = native[index]
# hack handle native string get
if type(output) is str:
output = create_string(output)
logger.debug('get {} in {}, returned {}'.format(object.raw, gettable._id, output._id))
return output
except IndexError as e:
return create_failed(WRException('Index out of range: %s' % index))
except KeyError as e:
return create_failed(WRException('Key error: %s' % index))
def server_opened(port, addr='127.0.0.1'):
'''判断服务是否已经打开
'''
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((addr, port))
sock.send('Hello') # 有时connect会成功，但是send才会发现失败
time.sleep(0.4)
sock.send('Hello')
return True
except socket.error, e:
logger.debug('server_opened (%s, %s) %s' % (addr, port, e))
if e.args[0] == 10035: return True
return False
def get_view_id(self, package_name, view_str_id):
'''获取控件整型ID
'''
id_list = [] # 可能会存在多个整型ID（应用和android）
for package_name in [package_name, 'android']:
time0 = time.time()
result = self._send_command('GetViewId', PkgName=package_name, StrId=view_str_id)
if result > 0:
logger.debug('get_view_id %s in %s use %s S, result=%d' % (view_str_id, package_name, time.time() - time0, result))
id_list.append(result)
else:
if package_name == 'android' and len(id_list) == 0:
raise RuntimeError('View id: %s not exist' % view_str_id)
return id_list
def lineReceived(self, data):
"""
:type data: bytes
"""
data = data.decode()
logger.debug('Recieved "{data}" from {nick}',
nick=self._client.nick,
data=data)
response = MessageHandler.handleline(data, self._client, self.factory.channels)
if response:
for data in response:
self.send(data)
def send(self, data):
"""
:type data: str
"""
logger.debug('Sending "{data}" to {nick}',
nick=self._client.nick,
data=data)
if type(data) in (list, tuple):
for line in data:
self.transport.write((line + '\r\n').encode())
else:
self.transport.write((data + '\r\n').encode())
def get_dlink(self, link, secret=None):
info = FileInfo()
js = self._get_js(link, secret)
if info.match(js):
extra_params = dict(bdstoken=info.bdstoken,
sign=info.sign,
timestamp=info.timestamp)
post_form = {
'encrypt': '0',
'product': 'share',
'uk': info.uk,
'primaryid': info.share_id,
'fid_list': '[{0}]'.format(info.fid_list),
}
if self.session.cookies.get('BDCLND'):
post_form['extra'] = '{"sekey":"%s"}' % (url_unquote(
self.session.cookies['BDCLND'])),
logger.debug(post_form, extra={'type': 'form', 'method': 'POST'})
url = BAIDUPAN_SERVER + 'sharedownload'
while True:
response = self._request('POST',
url,
extra_params=extra_params,
post_data=post_form)
if not response.ok:
raise UnknownError
_json = response.json()
errno = _json['errno']
logger.debug(_json, extra={'type': 'json', 'method': 'POST'})
if errno == 0:
# FIXME: only support single file for now
dlink = _json['list'][0]['dlink']
setattr(info, 'dlink', dlink)
break
elif errno == -20:
verify_params = self._handle_captcha(info.bdstoken)
post_form.update(verify_params)
response = self._request('POST',
url,
extra_params=extra_params,
post_data=post_form)
_json = response.json()
errno = _json['errno']
continue
elif errno == 116:
raise DownloadError("The share file does not exist")
else:
raise UnknownError
return info
def send(self, data):
"""
:type data: str
"""
logger.debug('Sending "{data}" to {nick}',
nick=self._client.nick,
data=data)
if type(data) in (list, tuple):
for line in data:
self.transport.write((line + '\r\n').encode())
else:
self.transport.write((data + '\r\n').encode())
def get_data_dir(user,
password,
host,
port=3306,
unix_socket='/tmp/mysql.sock',
charset='utf8',
cursor=None):
logger.debug('user: '******'password: '******'host: ' + str(host) + 'port: ' + str(port) +
'unix_socket: ' + str(unix_socket) + 'charset: ' +
str(charset))
cursor.execute("SHOW VARIABLES LIKE 'datadir'")
value = cursor.fetchall()
datadir = value[0][1]
return datadir
def start(self):
while True:
start_time = time.time()
try:
#self.get_and_send_api()
self.get_and_send_udp()
except Exception, e:
logger.warning("Error forwarding data")
elapsed_time = time.time() - start_time
logger.debug("get_and_send processing time {}".format(elapsed_time))
# Wait before we execute the next round
time.sleep(max(0, PERIOD - elapsed_time))
def lineReceived(self, data):
"""
:type data: bytes
"""
data = data.decode()
logger.debug('Recieved "{data}" from {nick}',
nick=self._client.nick,
data=data)
response = MessageHandler.handleline(data, self._client,
self.factory.channels)
if response:
for data in response:
self.send(data)
def handle_mode(data, match, client, channels):
"""
Someone is setting their mode. I don't know enough about this to talk about it yet.
'^MODE (?P<nick>[^\s]+) (?P<mode>.*)'
:type data: str
:type match: dict
:type client: Client
:type channels: list
"""
logger.debug("setting {nick}'s mode to {mode}, as per their request",
**match)
client.mode = match['mode'].replace('+', '')
def get_view_id(self, package_name, view_str_id):
'''获取控件整型ID
'''
# ret = self.adb.run_shell_cmd('sh %s/SpyHelper.sh getViewId %s %s' % (qt4a_path, package_name, view_str_id))
id_list = [] # 可能会存在多个整型ID（应用和android）
for package_name in [package_name, 'android']:
time0 = time.time()
result = self._send_command('GetViewId', PkgName=package_name, StrId=view_str_id)
if result > 0:
logger.debug('get_view_id %s in %s use %s S, result=%d' % (view_str_id, package_name, time.time() - time0, result))
id_list.append(result)
else:
if package_name == 'android' and len(id_list) == 0:
raise RuntimeError('控件ID：%s 不存在' % view_str_id)
return id_list
def getfunc(data):
"""
Tries hooks until it finds one that matches. The function and matching data is then returned.
:type data: str
:rtype: (regex, function)
"""
for regex, func in MessageHandler.HOOKS:
match = regex.match(data)
if match:
return func, match
logger.debug('Failed to handle line: "{data}"', data=data)
return None, None
def get(object, gettable):
try:
index = object.raw
native = gettable.raw
output = native[index]
# hack handle native string get
if type(output) is str:
output = create_string(output)
logger.debug('get {} in {}, returned {}'.format(
object.raw, gettable._id, output._id))
return output
except IndexError as e:
return create_failed(WRException('Index out of range: %s' % index))
except KeyError as e:
return create_failed(WRException('Key error: %s' % index))
def call(function, arguments):
logger.debug("call {} {}".format(function._id, arguments))
scope_with_arguments = _new_with_arguments(function, arguments,
scope_stack[-1])
if scope_with_arguments.expression:
scope_stack.append(scope_with_arguments)
output = scope_with_arguments.expression()
scope_stack.pop()
else:
output = scope_with_arguments
output.stack_trace.append(scope_with_arguments)
logger.debug("return {} (from {})".format(output._id, function._id))
return output
def format_segment(
record,
indir='',
force_input_dir='',
):
"""Search filesystem for complete file name matching FILE_LOC glob."""
regex = re.compile(indir)
file_loc = regex.sub(force_input_dir or indir, record['FILE_LOC'])
logger.debug('Search for file location: %s', file_loc)
filenames = glob.glob(file_loc)
if filenames:
return {
k: (v if k != 'FILE_LOC' else filenames[0])
for k, v in record.items()
}
return None
def update(connection, sql, **opts):
"""Execute SQL statement and commit changes.
Args:
connection (cx_Oracle.Connection): open database connection object
sql (str): oracle sql select statement with optional args (e.g. :key1)
opts (dict): key/value for optional args in select statement
Returns:
None
"""
cursor = connection.cursor()
cursor.execute(sql, opts)
logger.debug('Commit SQL Statement: %s', cursor.statement)
connection.commit()
def lock_keyguard(self):
'''锁屏
'''
if not self.is_screen_lock_enabled():
# 先设置为有屏幕锁
self.set_screen_lock_enable(True)
self._last_activity_before_lock = self._get_current_activity(True) # 保存锁屏前的Activity
logger.debug('锁屏前Activity为：%r' % self._last_activity_before_lock)
ret = self._lock_keyguard()
if not ret:
logger.warn('lock keyguard failed')
self.wake_screen(False)
time.sleep(1)
self.wake_screen(True)
return self.is_keyguard_locked()
return True
def _wait_for_cpu_low(self, max_p_cpu, max_t_cpu, timeout=20, interval=0.5):
'''等待待注入进程CPU使用率降低到max_p_cpu，线程CPU使用率降低到max_t_cpu
:param max_p_cpu: 进程占用的CPU
:type max_p_cpu: int
:param max_t_cpu: 线程占用的CPU
:type max_t_cpu: int
'''
time0 = time.time()
while time.time() - time0 < timeout:
ret = self._device.adb.get_process_cpu(self._process_name)
if ret != None:
p_cpu, t_cpu = ret
logger.debug('current cpu: %d, %d' % (p_cpu, t_cpu))
if p_cpu < max_p_cpu and t_cpu < max_t_cpu: return
time.sleep(interval)
def start(self):
while True:
start_time = time.time()
try:
#self.get_and_send_api()
self.get_and_send_udp()
except Exception, e:
logger.warning("Error forwarding data")
elapsed_time = time.time() - start_time
logger.debug(
"get_and_send processing time {}".format(elapsed_time))
# Wait before we execute the next round
time.sleep(max(0, PERIOD - elapsed_time))
def get_file_info(self, shareinfo, fsid, secret=None):
fi = FileInfo()
extra_params = dict(bdstoken=shareinfo.bdstoken,
sign=shareinfo.sign, timestamp=shareinfo.timestamp)
post_form = {
'encrypt': '0',
'product': 'share',
'uk': shareinfo.uk,
'primaryid': shareinfo.share_id,
'fid_list': json.dumps([fsid]),
}
if self.session.cookies.get('BDCLND'):
post_form['extra'] = '{"sekey":"%s"}' % (
url_unquote(self.session.cookies['BDCLND'])),
logger.debug(post_form, extra={'type': 'form', 'method': 'POST'})
url = BAIDUPAN_SERVER + 'sharedownload'
while True:
response = self._request(
'POST', url, extra_params=extra_params, post_data=post_form)
if not response.ok:
raise UnknownError
_json = response.json()
errno = _json['errno']
logger.debug(_json, extra={'type': 'json', 'method': 'POST'})
if errno == 0:
fi.filename = _json['list'][0]['server_filename']
fi.path = os.path.dirname(_json['list'][0]['path'])
fi.dlink = _json['list'][0]['dlink']
fi.parent_path = url_unquote(shareinfo.fileinfo[0]['parent_path'].encode('utf8'))
break
elif errno == -20:
verify_params = self._handle_captcha(shareinfo.bdstoken)
post_form.update(verify_params)
response = self._request(
'POST', url, extra_params=extra_params, post_data=post_form)
_json = response.json()
errno = _json['errno']
continue
elif errno == 116:
raise DownloadError("The share file does not exist")
else:
raise UnknownError
return fi
def handle_join(data, match, client, channels):
"""
Someone joined a channel! A few things can happen now.
Either they joined a channel that doesn't exist, and in that case, we create it and
set them as the owner.
If it already exists, let's announce their arrival to the participants.
'^JOIN (?P<channels>.*)'
:type data: str
:type match: dict
:type client: Client
:type channels: list
"""
templates = [
":{identity} JOIN {channel}", # join
":{server} 332 {nick} {channel} :{topic}", # topic
":{server} 353 {nick} = {channel} :{nicks}", # names in channel
":{server} 366 {nick} {channel} :End of /NAMES list."
] # end of names
for channame in match['channels'].split(','):
channame = channame.strip()
logger.debug("{nick} joins channel {channame}",
nick=client.nick,
channame=channame)
if channame not in channels:
channels[channame] = Channel(name=channame, owner=client)
channel = channels[channame]
channel.clients.append(client)
client.channels.append(channel)
nicks = ' '.join(client.nick for client in channel.clients)
client.send([
template.format(channel=channel.name,
identity=client.identity,
nick=client.nick,
nicks=nicks,
server="localhost",
topic=channel.topic) for template in templates
])
announce = Protocol.join(client, channel)
channel.send(announce)
def get_databases_list(user,
password,
host,
port=3306,
unix_socket='/tmp/mysql.sock',
charset='utf8',
cursor=None):
logger.debug('user: '******'password: '******'host: ' + str(host) + 'port: ' + str(port) +
'unix_socket: ' + str(unix_socket) + 'charset: ' +
str(charset))
sql = "SHOW DATABASES"
# sql = "SHOW processlist"
cursor.execute(sql)
values = cursor.fetchall()
dbs = [value[0] for value in values]
return dbs
def get_table_list(table_list,
user,
password,
host,
port=3306,
unix_socket='/tmp/mysql.sock',
charset='utf8',
cursor=None):
logger.debug('table_list: ' + str(table_list))
logger.debug('user: '******'password: '******'host: ' + str(host) + 'port: ' + str(port) +
'unix_socket: ' + str(unix_socket) + 'charset: ' +
str(charset))
# sql_template = "SELECT CONCAT(table_schema,'.',table_name) AS full_name FROM information_schema.tables WHERE table_schema = '{db}' AND table_name in ('{tbs}')"
sql_template = "SELECT {select_content} FROM information_schema.tables WHERE table_schema = '{db}' AND table_name in ('{tbs}') AND ENGINE='InnoDB'"
sql_cache = '\nUNION\n'.join([
sql_template.format(select_content='{select_content}',
db=db,
tbs="', '".join(tbs))
for db, tbs in table_list.items()
])
select_contents = [
"CONCAT(table_schema, '.', table_name, '.', row_format) AS table_list"
]
sqls = [
sql_cache.format(select_content=select_content)
for select_content in select_contents
]
logger.debug('SQL: \n' + sql_cache)
values = {}
for sql in sqls:
cursor.execute(sql)
value = cursor.fetchall()
keys = zip(*cursor.description).__next__()
values.update({keys[0]: [v[0] for v in value]})
create_table = []
logger.debug('')
for tb in values['table_list']:
create_sql = 'SHOW CREATE TABLE {table}'.format(
table='.'.join(tb.split('.')[:2]))
logger.debug(create_sql)
cursor.execute(create_sql)
value = cursor.fetchall()
create_table.append('-- ' + value[0][0] + '\n' + value[0][1] + ';')
values['create_table'] = create_table
return values
def _create_native(value):
output = Object()
output.items = {
'or': LV(lambda: create_scope(
defaults=[('other', None)],
expression=LV(lambda: resolve('self'))
)),
'then': LV(lambda: create_scope(
defaults=[('callable', None)],
expression=LV(lambda: call(resolve('callable'), [(None, LV(lambda: resolve('self')))]))
)),
}
output.parent = scope_stack[0]
output.arguments_scope = output.parent
output.raw = value
logger.debug('create native {} {}'.format(output._id, value))
return output
def run(self):
logger.debug("Start DataPoster: " + str(self.sensor))
while True:
# Wait for the next data point
data = self.socket.recv(512)
# Validate by castings to a float
try:
data = float(data)
except:
logger.warning("Unable to cast value:"+str(data))
continue
# Post the value to the webserver
try:
self.post_data(data)
except:
logger.warning("Unable to POST data")
def handle_join(data, match, client, channels):
"""
Someone joined a channel! A few things can happen now.
Either they joined a channel that doesn't exist, and in that case, we create it and
set them as the owner.
If it already exists, let's announce their arrival to the participants.
'^JOIN (?P<channels>.*)'
:type data: str
:type match: dict
:type client: Client
:type channels: list
"""
templates = [":{identity} JOIN {channel}", # join
":{server} 332 {nick} {channel} :{topic}", # topic
":{server} 353 {nick} = {channel} :{nicks}", # names in channel
":{server} 366 {nick} {channel} :End of /NAMES list."] # end of names
for channame in match['channels'].split(','):
channame = channame.strip()
logger.debug("{nick} joins channel {channame}", nick=client.nick, channame=channame)
if channame not in channels:
channels[channame] = Channel(name=channame, owner=client)
channel = channels[channame]
channel.clients.append(client)
client.channels.append(channel)
nicks = ' '.join(client.nick for client in channel.clients)
client.send([template.format(channel=channel.name,
identity=client.identity,
nick=client.nick,
nicks=nicks,
server="localhost",
topic=channel.topic)
for template in templates])
announce = Protocol.join(client, channel)
channel.send(announce)
def match(self, js):
_filename = re.search(self.filename_pattern, js)
if _filename:
self.filename = _filename.group(1)
data = re.findall(self.pattern, js)
if not data:
return False
yun_data = dict([i.split(' = ', 1) for i in data])
logger.debug(yun_data, extra={'method': 'GET', 'type': 'javascript'})
if 'single' not in yun_data.get('SHAREPAGETYPE') or '0' in yun_data.get('LOGINSTATUS'):
return False
self.uk = yun_data.get('MYUK').strip('"')
# self.bduss = yun_data.get('MYBDUSS').strip('"')
self.share_id = yun_data.get('SHARE_ID').strip('"')
self.fid_list = yun_data.get('FS_ID').strip('"')
self.sign = yun_data.get('SIGN').strip('"')
self.bdstoken = yun_data.get('MYBDSTOKEN').strip('"')
if self.bdstoken:
return True
return False