def show_device(self):
# os.chdir(deviceFilePath)
logger.debug("List all device")
for i in os.listdir(deviceFilePath):
namePath = deviceFilePath + i + '/device/name'
if os.path.isfile(namePath):
logger.debug("Name: %s Device: %s" % (i, file(namePath).read()))
def remove_link(self,one,two):
""" Remove a link between cells one and two """
del self.data[self.ord(one,two)]
self.neighbors[one].remove(two)
self.neighbors[two].remove(one)
logger.debug('Removed a link between {one} and {two}'.format(one=one,two=two))
def stopped_playing(self, segment):
logger.debug("stopped segment %s" % segment)
if self.gui:
self.gui.unhighlight_segment(segment)
if len(self.visualizers) > 0:
if self.ssr and segment["sound_source_id"]:
self.ssr.free_source(segment["sound_source_id"])
def update_status(self, message):
twitter = TwitterWrapper()
logger.debug('called post message')
logger.debug(message)
twitter.update_status(
status=message
)
def add_link(self,one,two,*args,**kwargs):
""" Add a link between cells one and two """
self.data[self.ord(one,two)] = Link(one,two,*args,**kwargs)
self.neighbors[one].add(two)
self.neighbors[two].add(one)
logger.debug('Created a link between {one} and {two}'.format(one=one,two=two))
def regenerate_cluster_certificate(self, host_port):
api = 'http://{}/controller/regenerateCertificate'.format(host_port)
response = self.post(url=api)
if not response.ok:
logger.error("Unable to regenerateCertificate", response.content)
else:
logger.debug("Regenerated Certificate", response.content)
def on_step_options_changed(self, plugin, step_number):
app = get_app()
if not app.running and (app.realtime_mode and plugin == self.name and
app.protocol.current_step_number ==
step_number):
logger.debug('[PSTraceLauncher] on_step_options_changed(): %s step'
'%d' % (plugin, step_number))
def __init__(self, info): logger.debug(info) self._info = info # for test self._info['curr_exp'] = 0 self._info['next_exp'] = 0 self._info['playing_time'] = 0 self._info['epic_progress'] = 0 self._info['skill_rate'] = 0 self._info['hp'] = 0 self._info['mp'] = 0 self._info['atk'] = 0 self._info['atr'] = 0 self._info['def'] = 0 self._info['dfr'] = 0 self._info['ctr'] = 0 self._info['ctd'] = 0 self._info['ccm'] = 0 self._info['dex'] = 0 self._info['hpr'] = 0 self._info['hpt'] = 0 self._info['mpr'] = 0 self._info['mpt'] = 0 self._info['hb'] = 0 self._info['ctm'] = 0 self._info['mov'] = 0
def __call__(self, environ, start_response):
"""
If the wsgi PATH_INFO starts with the static contents location, it will be returned.
Otherwise the wrapped application will be called.
"""
if environ['REQUEST_METHOD'] == 'GET' and environ['PATH_INFO'].startswith('/%s/' % self._location):
logger.info('GET from %s: %s' % (environ.get('REMOTE_ADDR', 'unknown'), environ['PATH_INFO']))
prefix = "/usr/share/skdrepo/"
path = prefix + environ['PATH_INFO'][1:]
try:
f = open(path, 'r')
data = f.read()
f.close()
(mime, encoding) = guess_type(path)
status = '200 OK'
response_headers = [('Content-Type', mime)]
response_body = [data]
except IOError, e:
logger.warning('failed to open file: %s' % path)
status = '404 Not Found'
response_headers = [('Content-Type', 'text/plain')]
response_body = ['404 Not Found - \'%s\'' % path]
start_response(status, response_headers)
logger.debug('response to %s: %s, %s' % (environ['REMOTE_ADDR'], status, str(response_headers)))
return response_body
def send_friend_msg(self, reply_content, uin, msg_id, fail_times=0):
fix_content = str(reply_content.replace("\\", "\\\\\\\\").replace("\n", "\\\\n").replace("\t", "\\\\t"))
rsp = ""
try:
req_url = "http://d1.web2.qq.com/channel/send_buddy_msg2"
data = (
('r',
'{{"to":{0}, "face":594, "content":"[\\"{4}\\", [\\"font\\", {{\\"name\\":\\"Arial\\", \\"size\\":\\"10\\", \\"style\\":[0, 0, 0], \\"color\\":\\"000000\\"}}]]", "clientid":{1}, "msg_id":{2}, "psessionid":"{3}"}}'.format(
uin, self.client_id, msg_id, self.psessionid, fix_content)),
('clientid', self.client_id),
('psessionid', self.psessionid)
)
rsp = self.client.post(req_url, data, self.smart_qq_refer)
rsp_json = json.loads(rsp)
if 'errCode' in rsp_json and rsp_json['errCode'] != 0:
raise ValueError("reply pmchat error" + str(rsp_json['retcode']))
logger.info("RUNTIMELOG Reply successfully.")
logger.debug("RESPONSE Reply response: " + str(rsp))
return rsp_json
except:
if fail_times < 5:
logger.warning("RUNTIMELOG Response Error.Wait for 2s and Retrying." + str(fail_times))
logger.debug("RESPONSE " + str(rsp))
time.sleep(2)
self.send_friend_msg(reply_content, uin, msg_id, fail_times + 1)
else:
logger.warning("RUNTIMELOG Response Error over 5 times.Exit.reply content:" + str(reply_content))
return False
def process_message(self, ircmsg):
msg = Message(ircmsg)
# ignore anything from #speedrunslive to avoid flooding it accidentally
if msg.channel == "#speedrunslive":
pass
# kill command to force disconnect the bot from the server
# WARNING: the bot will not reconnect until manually reset
elif msg.command in {"!kill", ".kill"}:
logger.debug("Kill request detected from " + msg.sender.lower())
if self.has_op(msg.sender):
# actually kills the bot if the sender is privelaged
self.send("QUIT Kill requested by " + msg.sender + "\n")
raise KillException
else:
for command in self.commands:
try:
command(self, msg)
except NameException as e:
logger.error(traceback.format_exc())
message = "There was a problem looking up data for " + str(e) + ". "
message += "Do they have an SRL profile?"
self.sendmsg(msg.channel, message)
except Exception as e:
logger.error(traceback.format_exc())
self.sendmsg(msg.channel, "Something weird happened...")
def get_group_member_info_list(self, group_code):
"""
获取指定群的成员信息
:group_code: int, can be "ture" of "fake" group_code
{"retcode":0,"result":{"stats":[],"minfo":[{"nick":" 信","province":"山东","gender":"male","uin":3964575484,"country":"中国","city":""},{"nick":"崔震","province":"","gender":"unknown","uin":2081397472,"country":"","city":""},{"nick":"云端的猫","province":"山东","gender":"male","uin":3123065696,"country":"中国","city":"青岛"},{"nick":"要有光","province":"山东","gender":"male","uin":2609717081,"country":"中国","city":"青岛"},{"nick":"小莎机器人","province":"广东","gender":"female","uin":495456232,"country":"中国","city":"深圳"}],"ginfo":{"face":0,"memo":"http://hujj009.ys168.com\r\n0086+区(没0)+电话\r\n0086+手机\r\nhttp://john123951.xinwen365.net/qq/index.htm","class":395,"fingermemo":"","code":3943922314,"createtime":1079268574,"flag":16778241,"level":0,"name":"ぁQQぁ","gid":3931577475,"owner":3964575484,"members":[{"muin":3964575484,"mflag":192},{"muin":2081397472,"mflag":65},{"muin":3123065696,"mflag":128},{"muin":2609717081,"mflag":0},{"muin":495456232,"mflag":0}],"option":2},"cards":[{"muin":3964575484,"card":"●s.Εx2(22222)□"},{"muin":495456232,"card":"小莎机器人"}],"vipinfo":[{"vip_level":0,"u":3964575484,"is_vip":0},{"vip_level":0,"u":2081397472,"is_vip":0},{"vip_level":0,"u":3123065696,"is_vip":0},{"vip_level":0,"u":2609717081,"is_vip":0},{"vip_level":0,"u":495456232,"is_vip":0}]}}
:return:dict
"""
if group_code == 0:
return
try:
url = "http://s.web2.qq.com/api/get_group_member_info_ext2?gcode=%s&vfwebqq=%s&t=%s" % (
group_code, self.vfwebqq, int(time.time() * 100))
response = self.client.get(url)
rsp_json = json.loads(response)
logger.debug("get_group_member_info_ext2 info response: {}".format(rsp_json))
retcode = rsp_json["retcode"]
if retcode == 0:
result = rsp_json["result"]
elif retcode == 6:
logger.debug("get_group_member_info_ext2 retcode is 6, trying to get true code.")
result = self.get_group_member_info_list(self.get_true_group_code(group_code))
else:
logger.warning("group_code error.")
return
self.group_member_info[str(group_code)] = result # 缓存群成员信息, 此处会把真假group_code都加入cache
return result
except Exception as ex:
logger.warning("RUNTIMELOG get_group_member_info_ext2. Error: " + str(ex))
return
def send_group_msg(self, reply_content, group_code, msg_id, fail_times=0):
fix_content = str(reply_content.replace("\\", "\\\\\\\\").replace("\n", "\\\\n").replace("\t", "\\\\t"))
rsp = ""
try:
logger.info("Starting send group message: %s" % reply_content)
req_url = "http://d1.web2.qq.com/channel/send_qun_msg2"
data = (
('r',
'{{"group_uin":{0}, "face":564,"content":"[\\"{4}\\",[\\"font\\",{{\\"name\\":\\"Arial\\",\\"size\\":\\"10\\",\\"style\\":[0,0,0],\\"color\\":\\"000000\\"}}]]","clientid":{1},"msg_id":{2},"psessionid":"{3}"}}'.format(
group_code, self.client_id, msg_id, self.psessionid, fix_content)),
('clientid', self.client_id),
('psessionid', self.psessionid)
)
rsp = self.client.post(req_url, data, self.smart_qq_refer)
rsp_json = json.loads(rsp)
if 'retcode' in rsp_json and rsp_json['retcode'] not in MESSAGE_SENT:
raise ValueError("RUNTIMELOG reply group chat error" + str(rsp_json['retcode']))
logger.info("RUNTIMELOG send_qun_msg: Reply '{}' successfully.".format(reply_content))
logger.debug("RESPONSE send_qun_msg: Reply response: " + str(rsp))
return rsp_json
except:
logger.warning("RUNTIMELOG send_qun_msg fail")
if fail_times < 5:
logger.warning("RUNTIMELOG send_qun_msg: Response Error.Wait for 2s and Retrying." + str(fail_times))
logger.debug("RESPONSE send_qun_msg rsp:" + str(rsp))
time.sleep(2)
self.send_group_msg(reply_content, group_code, msg_id, fail_times + 1)
else:
logger.warning("RUNTIMELOG send_qun_msg: Response Error over 5 times.Exit.reply content:" + str(reply_content))
return False
def get_online_friends_list(self):
"""
获取在线好友列表
get_online_buddies2
:return:list
"""
logger.info("RUNTIMELOG Requesting the online buddies.")
response = self.client.get(
'http://d1.web2.qq.com/channel/get_online_buddies2?vfwebqq={0}&clientid={1}&psessionid={2}&t={3}'.format(
self.vfwebqq,
self.client_id,
self.psessionid,
self.client.get_timestamp(),
)
) # {"result":[],"retcode":0}
logger.debug("RESPONSE get_online_buddies2 html:{}".format(response))
try:
online_buddies = json.loads(response)
except ValueError:
logger.warning("get_online_buddies2 response decode as json fail.")
return None
if online_buddies['retcode'] != 0:
logger.warning('get_online_buddies2 retcode is not 0. returning.')
return None
online_buddies = online_buddies['result']
return online_buddies
def get_friend_info(self, tuin):
"""
获取好友详情信息
get_friend_info
{"retcode":0,"result":{"face":0,"birthday":{"month":1,"year":1989,"day":30},"occupation":"","phone":"","allow":1,"college":"","uin":3964575484,"constel":1,"blood":3,"homepage":"http://blog.lovewinne.com","stat":20,"vip_info":0,"country":"中国","city":"","personal":"","nick":" 信","shengxiao":5,"email":"*****@*****.**","province":"山东","gender":"male","mobile":"158********"}}
:return:dict
"""
uin = str(tuin)
if uin not in self.friend_uin_list:
logger.info("RUNTIMELOG Requesting the account info by uin: {}".format(uin))
info = json.loads(self.client.get(
'http://s.web2.qq.com/api/get_friend_info2?tuin={0}&vfwebqq={1}&clientid={2}&psessionid={3}&t={4}'.format(
uin,
self.vfwebqq,
self.client_id,
self.psessionid,
self.client.get_timestamp()
)
))
logger.debug("get_friend_info2 html: {}".format(str(info)))
if info['retcode'] != 0:
logger.warning('get_friend_info2 retcode unknown: {}'.format(info))
return None
info = info['result']
info['account'] = self.uin_to_account(uin)
info['longnick'] = self.get_friend_longnick(uin)
self.friend_uin_list[uin] = info
try:
return self.friend_uin_list[uin]
except:
logger.warning("RUNTIMELOG get_friend_info return fail.")
logger.debug("RUNTIMELOG now uin list: " + str(self.friend_uin_list[uin]))
def uin_to_account(self, tuin):
"""
将uin转换成用户QQ号
:param tuin:
:return:str 用户QQ号
"""
uin_str = str(tuin)
try:
logger.info("RUNTIMELOG Requesting the account by uin: " + str(tuin))
info = json.loads(
self.client.get(
'http://s.web2.qq.com/api/get_friend_uin2?tuin={0}&type=1&vfwebqq={1}&t={2}'.format(
uin_str,
self.vfwebqq,
self.client.get_timestamp()
),
self.smart_qq_refer
)
)
logger.debug("RESPONSE uin_to_account html: " + str(info))
if info['retcode'] != 0:
raise TypeError('uin_to_account retcode error')
info = info['result']['account']
return info
except Exception:
logger.exception("RUNTIMELOG uin_to_account fail")
return None
def get_self_info(self):
"""
获取自己的信息, 并存入self._self_info
get_self_info2
{"retcode":0,"result":{"birthday":{"month":1,"year":1989,"day":30},"face":555,"phone":"","occupation":"","allow":1,"college":"","uin":2609717081,"blood":0,"constel":1,"lnick":"","vfwebqq":"68b5ff5e862ac589de4fc69ee58f3a5a9709180367cba3122a7d5194cfd43781ada3ac814868b474","homepage":"","vip_info":0,"city":"青岛","country":"中国","personal":"","shengxiao":5,"nick":"要有光","email":"","province":"山东","account":2609717081,"gender":"male","mobile":""}}
:return:dict
"""
try_times = 0
while len(self._self_info) is 0:
url = "http://s.web2.qq.com/api/get_self_info2?t={}".format(time.time())
response = self.client.get(url)
logger.debug("get_self_info2 response:{}".format(response))
rsp_json = json.loads(response)
if rsp_json["retcode"] != 0:
try_times += 1
logger.warning("get_self_info2 fail. {}".format(try_times))
if try_times >= 5:
return {}
try:
self._self_info = rsp_json["result"]
except KeyError:
logger.warning("get_self_info2 failed. Retrying.")
continue
return self._self_info
def send_heartbeat(self):
ping_json = json_util.dumps({"src_id":self.node_id})
while(True):
last_ping_sent = time.time()
for node_id in self.connections.keys():
to_destroy = []
node_connections = self.connections.get(node_id , [])
for conn in node_connections:
if(conn.is_external_node):
try:
conn.send(ping_json)
except:
to_destroy.append(conn)
else:
if(time.time()*1000 - conn.last_msg_recv_timestamp > 30*60*1000 and time.time()*1000 - conn.last_msg_sent_timestamp > 30*60*1000):
#30 min no msg received or sent, basically very idle connection , they should probably be closed instead of monitoring
to_destroy.append(conn)
for conn in to_destroy:
self.destroy_connection(conn.ws, conn_obj=conn)
gc.collect()
time_elapsed = time.time()-last_ping_sent
logger.debug("sent a heartbeat")
gevent.sleep(max(0 , 10*60 - (time_elapsed))) # 10 minutes send a heart beat
def async_reader(conn, loop):
data = conn.recv()
logger.debug("RECV DATA: %s", data)
if data is None:
return
if data == b'':
if conn in FD_MAP:
forward_server = FD_MAP[conn]
loop.remove_reader(forward_server.fd)
forward_server.shutdown()
del FD_MAP[forward_server]
del FD_MAP[conn]
loop.remove_reader(conn.fd)
conn.shutdown()
return
if conn not in FD_MAP:
target_host = find_host(data)
if target_host is None:
return
forward_server = ProxySocket.get_client(host=target_host, port=80)
forward_server.connect()
if forward_server and forward_server.fd != -1:
FD_MAP[conn] = forward_server
FD_MAP[forward_server] = conn
loop.add_reader(forward_server.fd, async_reader, forward_server, loop)
logger.info("Create a connection to %s", target_host)
else:
logger.error("FAIL to connect to target host {0}".format(target_host))
FD_MAP[conn].sendall(data)
def resolve_dependencies(self):
# update the definition from a reference, if there is one
if self.content_reference is not None:
if '#' != self.content_reference[:1]:
raise Exception("Only relative 'contentReference' element definitions are supported right now")
elem = self.element.profile.element_with_id(self.content_reference[1:])
if elem is None:
raise Exception("There is no element definiton with id \"{}\", as referenced by {} in {}"
.format(self.content_reference, self.path, self.profile.url))
self._content_referenced = elem.definition
# resolve bindings
if self.binding is not None and self.binding.is_required and (self.binding.valueSet is not None or self.binding.legacy_uri is not None or self.binding.legacy_canonical is not None):
uri = self.binding.valueSet or self.binding.legacy_canonical or self.binding.legacy_uri
if 'http://hl7.org/fhir' != uri[:19]:
logger.debug("Ignoring foreign ValueSet \"{}\"".format(uri))
return
# remove version from canonical URI, if present, e.g. "http://hl7.org/fhir/ValueSet/name-use|4.0.0"
if '|' in uri:
uri = uri.split('|')[0]
valueset = self.element.profile.spec.valueset_with_uri(uri)
if valueset is None:
logger.error("There is no ValueSet for required binding \"{}\" on {} in {}"
.format(uri, self.name or self.prop_name, self.element.profile.name))
else:
self.element.valueset = valueset
self.element.enum = valueset.enum
def update_status( self, event ):
while not self.in_queue.empty():
message = gui_queue.get()
logger.debug('GUI got message %s'%str(message) )
if message[0] == REQUEST_USER_CREDS and not self.logging_in:
self.login_prompt( event=None )
elif message[0] == LOGGED_IN:
self.logged_in = True
self.set_icon()
elif message[0] == STARTED_SESSION:
self.active = True
self.set_icon()
elif message[0] == ENDED_SESSION:
self.active=False
self.paused = False
self.set_icon()
elif message[0] == PAUSED_SESSION :
self.paused = True
self.set_icon()
elif message[0] == RESUMED_SESSION :
self.paused = False
self.set_icon()
elif message[0] == BLURRED_SESSION :
self.blur = True
self.set_icon()
elif message[0] == UNBLURRED_SESSION :
self.blur = False
self.set_icon()
def downloadAllTo(this,root):
for item in this.soup.find_all("item"):
if item.find("enclosure"):
episodeTitle = item.find("title").string + ".mp3.part"
cRoot = root
cPath = cleanPath(this.title)
cFileName = cleanPath(episodeTitle)
pubDate = parseDate(item.find("pubdate").string)
filePath = os.path.join(cRoot,cPath,pubDate.strftime("%Y-%m-%d") + "-" + cFileName)
isOldie = ( datetime.now() - pubDate) > this.dateLimit
if os.path.exists(filePath[:-5]) or isOldie:
logger.debug("Skipping:\t" + filePath[:-5] + "\nReason:\t" + ("age" if isOldie else "already have it.") + "\n")
continue
if not os.path.exists(os.path.join(root,cleanPath(this.title))):
os.makedirs(os.path.join(root,cleanPath(this.title)))
itemUrl = item.find("enclosure").get("url")
audioFile = open(filePath,'wb')
remoteFile = urllib2.urlopen(itemUrl)
logger.info("Fetching:\t" + itemUrl)
audioFile.write(remoteFile.read())
audioFile.close()
remoteFile.close()
os.rename(filePath,filePath[:-5])
def check_root():
logger.info("Root checking...")
if os.getegid() != 0:
logger.info("This program must be run as root..")
sys.exit(0)
logger.info("Root checking success. You have the root permission")
logger.debug("File path:" + os.path.dirname(__file__))
def attack(self, target, num_ships):
logger.debug("Attacking %s" % target)
self.num_ships -= num_ships
attack = "%d %d %d\n" % (self.id, target.id, num_ships)
stdout.write(attack)
stdout.flush()
logger.debug("Attack message: %s" % attack)
def graph():
query = request.args.get('q', None)
_render = request.args.get('render', 'True')
_render = False if _render == 'False' else True
logger.debug('QUERY: ' + query)
start_timer = datetime.now()
if query is not None and _render:
start, end, expressions, data_frame = parse_query(query)
if expressions[0].name == 'metric':
chart = generate_graph(query, start, end, data_frame)
table = ''
elif expressions[0].name == 'log':
chart = '{}'
table = generate_table(start, end, data_frame)
else:
logger.error("Unknown expression type: %s" % str(type(expressions[0])))
chart = '{}'
table = ''
else:
chart = "{}"
table = ''
# query = ''
end_timer = datetime.now()
logger.debug('Took %0.02fs to process this request' % (end_timer-start_timer).total_seconds())
return render('graph.html', 'Graph', chart=chart, table=table, q=query)
def __init__(self, version, updater):
# set version
self.version = version
# set updater
self.updater = updater
# keep state of 'toggle key'
self.toggle_key_active = False
# read user defaults (preferences)
self.prefs = NSUserDefaults.standardUserDefaults()
# register some default values
self.prefs.registerDefaults_(dict(
QuoteFixFixReply = True,
QuoteFixFixReplyAll = True,
QuoteFixFixForward = True,
QuoteFixFixDraft = False,
QuoteFixFixNewMessage = False,
))
# set log level
logger.setLevel(self.is_debugging and logger.DEBUG or logger.WARNING)
logger.debug('debug logging active')
# add menu item for quick enable/disable
Menu.alloc().initWithApp_(self).inject()
# check update interval
self.check_update_interval = self.prefs.int["QuoteFixCheckUpdateInterval"] or 0
# check if we're running in a different Mail version as before
self.check_version()
def configure_interface_ip(intf=None, ip=None):
"""configures ip address for non-local interface"""
cmd = "/sbin/ifconfig" + " " + intf + " " + ip + " up"
try:
os.system(cmd)
except Exception, e:
logger.debug(str(e))
def api_forever():
while True:
command = api_queue.get()
if command == messages.QUIT:
logger.debug("API thread exiting")
sys.exit(0)
command()
def login(self):
logger.info('login...')
if os.path.isfile(COOKIE):
self.cookieJar.load(ignore_discard=True, ignore_expires=True)
else:
self._verifycode = self._getverifycode()
self.pswd = self._preprocess(self._pw, self._verifycode)
self._headers.update({"Referer":"http://ui.ptlogin2.qq.com/cgi-bin/login?target=self&style=5&mibao_css=m_webqq&appid={0}&enable_qlogin=0&no_verifyimg=1&s_url=http%3A%2F%2Fweb.qq.com%2Floginproxy.html&f_url=loginerroralert&strong_login=1&login_state=10&t=20121029001".format(self.appid)})
url = "http://ptlogin2.qq.com/login?u={0}&p={1}&verifycode={2}&aid={3}&u1=http%3A%2F%2Fweb.qq.com%2Floginproxy.html%3Flogin2qq%3D1%26webqq_type%3D10&h=1&ptredirect=0&ptlang=2052&from_ui=1&pttype=1&dumy=&fp=loginerroralert&action=3-25-30079&mibao_css=m_webqq&t=1&g=1".format(self.qq, self.pswd, self._verifycode[1], self.appid)
res = self._request(url=url, cookie=True)
if res.find("成功") != -1:
pass
elif res.find("验证码") != -1:
print("验证码错误")
self._getverifycode()
self.login()
else:
logger.error(res)
raise Exception("登陆错误")
self.cookies.update(dict([(x.name,x.value) for x in self.cookieJar]))
tmp = self.get_login_info()
logger.debug(tmp)
if os.path.isfile(COOKIE) and tmp: #cookie timeout
self._login_info.update(tmp)
self.name_info()
self.__poll()
else:
self.login()
def __init__(self, spec, resource):
assert 'content' in resource
self.spec = spec
self.definition = resource
self.url = resource.get('url')
if self.url in self.spec.settings.enum_namemap:
self.name = self.spec.settings.enum_namemap[self.url]
else:
self.name = self.spec.safe_enum_name(resource.get('name'), ucfirst=True)
self.codes = None
self.generate_enum = False
concepts = self.definition.get('concept', [])
if resource.get('experimental'):
return
self.generate_enum = 'complete' == resource['content']
if not self.generate_enum:
logger.debug("Will not generate enum for CodeSystem \"{}\" whose content is {}"
.format(self.url, resource['content']))
return
assert concepts, "Expecting at least one code for \"complete\" CodeSystem"
if len(concepts) > 200:
self.generate_enum = False
logger.info("Will not generate enum for CodeSystem \"{}\" because it has > 200 ({}) concepts"
.format(self.url, len(concepts)))
return
self.codes = self.parsed_codes(concepts)
def get_next_table(self):
'''
Read from the currently cached line and up to the start of the next
table (or EOF), and parse current table's content.
'''
# The document should now have the table name line cached.
# If it doesn't, there are no more tables to read.
if self.line == '':
return False
# Table name line looks something like this: "## 1.1 PERSON".
# Parse out the actual table name.
table_name = _get_table_name(self.line)
self.add_table(table_name)
# Everything after the table name line and up to the table header
# is the content (documentation) of the table.
content = []
parser = None
for line in self.fileobj:
for parser in table_parsers:
if parser.matches_header(line):
logger.debug('[parser] matched header for %s', table_name)
break
else:
parser = None
if parser:
break
content.append(line)
else:
line = None
# If a line is present, a table header matched. Skip the
# line '--- | --- | --- | --- | ---' that follows.
if line is not None:
logger.debug('[parser] parsing fields for %s', table_name)
next(self.fileobj)
# Parse the rows in the table.
for line in self.fileobj:
field = parser.parse_row(line)
# If a valid field is parsed, add and continue. Otherwise
# set the line for the next iteration.
if not field:
self.line = line
break
self.add_field(table_name, *field)
# Read the rest of the table contents, whatever comes after
# the fields specification table, up to the next table.
if not table_name_re.match(self.line):
content.append(line)
self.line = ''
for line in self.fileobj:
if table_name_re.match(line):
self.line = line
break
content.append(line)
self.add_table_content(table_name, ''.join(content).strip())
return True
async def parse_response(self, incoming):
data = incoming
msg_type = None
first = str(data[:40])
# Detect type of incoming data
if (data != ''):
# search for id_catalog in all data to be sure to get configuration
# detected
if ("id_catalog" in data):
logger.debug(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
logger.debug('Incoming message type : config detected')
msg_type = 'msg_config'
logger.debug(data)
elif ("id" in first):
logger.debug(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
logger.debug('Incoming message type : data detected')
msg_type = 'msg_data'
logger.debug(data)
elif ("doctype" in first):
logger.debug(
'Incoming message type : html detected (probable 404)')
msg_type = 'msg_html'
logger.debug(data)
elif ("productName" in first):
logger.debug(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
logger.debug('Incoming message type : Info detected')
msg_type = 'msg_info'
logger.debug(data)
else:
logger.debug('Incoming message type : no type detected')
logger.debug(data)
if not (msg_type is None):
try:
if (msg_type == 'msg_config'):
parsed = json.loads(data)
# logger.debug(parsed)
await self.parse_config_data(parsed=parsed)
elif (msg_type == 'msg_data'):
parsed = json.loads(data)
# logger.debug(parsed)
await self.parse_devices_data(parsed=parsed)
elif (msg_type == 'msg_html'):
logger.debug("HTML Response ?")
elif (msg_type == 'msg_info'):
pass
else:
# Default json dump
logger.debug()
logger.debug(
json.dumps(parsed,
sort_keys=True,
indent=4,
separators=(',', ': ')))
except Exception as e:
logger.error('Cannot parse response !')
# logger.error('Response :')
# logger.error(data)
if (e != 'Expecting value: line 1 column 1 (char 0)'):
logger.error("Error : ", e)
logger.error(parsed)
logger.info('Incoming data parsed successfully !')
return (0)
async def incomingTriage(self):
bytes_str = self.incoming_bytes
# If not MQTT client, return incoming message to use it with anything.
if self.mqtt_client is None:
return bytes_str
else:
incoming = None
first = str(bytes_str[:40]) # Scanning 1st characters
try:
if ("Uri-Origin: /refresh/all" in first in first):
pass
elif ("PUT /devices/data" in first) or ("/devices/cdata"
in first):
logger.debug('PUT /devices/data message detected !')
try:
incoming = self.parse_put_response(bytes_str)
await self.parse_response(incoming)
except BaseException:
logger.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
logger.error('RAW INCOMING :')
logger.error(bytes_str)
logger.error('END RAW')
logger.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
elif ("scn" in first):
try:
incoming = get(bytes_str)
await self.parse_response(incoming)
logger.info('Scenarii message processed !')
logger.info("##################################")
except BaseException:
logger.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
logger.error('RAW INCOMING :')
logger.error(bytes_str)
logger.error('END RAW')
logger.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
elif ("POST" in first):
try:
incoming = self.parse_put_response(bytes_str)
await self.parse_response(incoming)
logger.info('POST message processed !')
except BaseException:
logger.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
logger.error('RAW INCOMING :')
logger.error(bytes_str)
logger.error('END RAW')
logger.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
elif ("HTTP/1.1" in first): # (bytes_str != 0) and
response = self.response_from_bytes(
bytes_str[len(self.cmd_prefix):])
incoming = response.data.decode("utf-8")
try:
await self.parse_response(incoming)
except BaseException:
logger.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
logger.error('RAW INCOMING :')
logger.error(bytes_str)
logger.error('END RAW')
logger.error(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
else:
logger.warn("Didn't detect incoming type, here it is :")
logger.warn(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
logger.warn('RAW INCOMING :')
logger.warn(bytes_str)
logger.warn('END RAW')
logger.warn(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>")
except Exception as e:
logger.error("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
logger.error('receiveMessage error')
logger.error('RAW :')
logger.error(bytes_str)
logger.error("Incoming payload :")
logger.error(incoming)
logger.error("Error :")
logger.error(e)
logger.error('Exiting to ensure systemd restart....')
sys.exit() # Exit all to ensure systemd restart
def _scan_attempt(self, path_keywords, result_path=None, last_matched=None, result_type='attribute', retry=0):
"""One scan attempt for exploration by level-based keywords scan
:param path_keywords: 2-D keyword list, do recursive fuzzy match
:param result_path: list storing deterministic result path
:param last_matched: record last matched keywords, if explore failed, remove it from path_keywords
:param result_type: attribute - result keywords are matched attribute of elements.
pattern - result keywords are matched patterns defined in conf file.
:param retry: retry flag for scrolling search
:return: matched keyword list, or error message
"""
# fix "default parameter is mutable" warning
if result_path is None:
result_path = []
# Return condition: Every level of path_keywords is exhausted - level 0 DOES match
# If dest_keywords is defined, double check for destination match
if not path_keywords:
if self._dest_keywords or self._dest_activities:
if self._dest_keywords and self.wait_for_destination(self._dest_keywords, timeout=10):
logger.info(u'Destination keywords matched. Path: ->%s' % u'->'.join(result_path))
return {'status': 'success', 'path': result_path, 'package': self.package}
elif self._dest_activities and self.wait_for_activities(self._dest_activities, timeout=10):
logger.info(u'Destination activities matched. Path: ->%s' % u'->'.join(result_path))
return {'status': 'success', 'path': result_path, 'package': self.package}
else:
logger.debug('Level 0 matched, but wrong destination')
return {'status': 'deadend', 'last_turn': last_matched, 'path': result_path,
'package': self.package}
else:
logger.info('Level 0 keywords matched')
return {'status': 'success', 'path': result_path, 'package': self.package}
# 2D list: fuzzy keyword path
level = 0
for kwline in path_keywords:
# self.skip_irrelevant()
logger.debug(u'Try level {0} keywords'.format(level))
for kw in kwline:
elements = self.find_elements_by_keyword(kw, clickable_only=False, exact=False,
text_max_len=32, sort_elements=True)
if elements:
logger.debug(u'\tMatched {0} elements'.format(len(elements)))
for e in elements:
e_info = layout.ElementInfo(e)
e_location = e.location
logger.debug(u'\t\tTap element {0}'.format(e_info))
if not self.tap_test(e):
logger.warning(u'\t\tTap failed, try next element')
continue
else:
last_matched = {'lv': level, 'kw': kw}
if result_type == 'attribute':
# Use matched element attribute as keyword
for s in [e_info.text, e_info.res_id, e_info.desc]:
if re.search(kw, s, re.I):
kw = s
logger.info(
u'Level {0} matched\n\tKeyword: {1}, Position: {2}'.format(level, kw, e_location))
result_path.append(kw)
# Recursion: with path_keywords level 0 - current level
return self._scan_attempt(path_keywords[:level], result_path=result_path,
last_matched=last_matched, result_type=result_type)
level += 1
# Tried all remaining levels, none matching
# scroll to end and retry
if retry == 0:
# very naive scrolling
logger.debug(u'\tNothing found, try scrolling to end ...')
self.scroll_down(swipe=3)
return self._scan_attempt(path_keywords, result_path=result_path,
last_matched=last_matched, result_type=result_type, retry=1)
# scrolled and retried, return result
else:
if last_matched is None:
return {'status': 'failed', 'path': result_path, 'package': self.package}
else:
return {'status': 'deadend', 'last_turn': last_matched, 'path': result_path, 'package': self.package}
def tap_keyword(self, keyword, siblings_on=True, retry=3):
try:
tap_suc = False
logger.debug(u'Try to click by keyword {0}'.format(keyword))
# Quick try
elements = self.find_elements_by_keyword(keyword, clickable_only=False, exact=True)
logger.debug(u'\tQuick try, find {0} elements'.format(len(elements)))
for e in elements:
if self.tap_test(e):
return True
# Comprehensive Try
logger.debug(u'\tComprehensive try')
elements = self.find_elements_by_keyword(keyword, clickable_only=False,
sort_elements=True, text_max_len=32)
if not elements:
logger.debug(u'\tTry scroll')
elements = self.find_elements_by_keyword(keyword, clickable_only=False, scroll=True,
sort_elements=True, text_max_len=32)
if not elements:
logger.error(u'\tCannot find keyword "{0}"'.format(keyword))
return False
for e in elements:
logger.debug(u'\tTry tapping: {0}'.format(layout.ElementInfo(e)))
tap_suc = self.tap_test(e)
if tap_suc:
break
else:
if siblings_on and not e.get_attribute('clickable') == 'true':
logger.debug(u'\t\tTap failed, try siblings')
tap_suc = self.tap_siblings(e)
if tap_suc:
break
logger.debug(u'\tTap failed, try other matches')
if not tap_suc:
logger.error(u'Click by keyword "{0}" unsuccessful by all means'.format(keyword))
return tap_suc
except NoSuchElementException:
if retry > 0:
retry = retry - 1
self.skip_irrelevant()
return self.tap_keyword(keyword, siblings_on=siblings_on, retry=retry)
def skip_irrelevant(self, initial=True, limit=None, detect_login=None):
""" Skip irrelevant activities and dialogs like update and permission notifications
Will call itself recursively until no irrelevant things found
This function may be invoked frequently to prevent stuck
"""
if initial:
self.rec_count = 0
if detect_login is None:
detect_login = self.app_style == 'international'
if limit is None:
if self.SKIP_IRR_LIMIT:
limit = self.SKIP_IRR_LIMIT
else:
limit = 20
elif self.rec_count >= self.SKIP_IRR_LIMIT:
raise myexceptions.SkipIrrelevantExceedLimit
else:
self.rec_count += 1
try:
cur_act = self.driver.current_activity
if not self.is_in_app():
logger.error('!!! APP not running, raise exception ...')
raise myexceptions.AppNotRunningException
logger.debug(u"Try to check and skip irrelevant activities {1}, current activity: {0}"
.format(cur_act, "(%d)" % self.rec_count if self.rec_count else ""))
clickable = self.driver.find_elements_by_android_uiautomator('new UiSelector().clickable(true)')
textedit = self.driver.find_elements_by_class_name('android.widget.EditText')
logger.verbose(u'''Found:
{0} clickable elements
{1} TextEdit elements'''.format(len(clickable), len(textedit)))
# Nothing can be clicked, wait or swipe
if len(clickable) == 0 \
and len(textedit) == 0:
if not self.waited:
logger.debug(u'Seems to be in loading page, wait then try again ...')
timeout = 6
source_before = self.page_source
while timeout:
sleep(1)
source_after = self.page_source
if difflib.SequenceMatcher(None, source_before, source_after).ratio() < 0.8:
break
source_before = source_after
timeout -= 1
self.waited = 1
elif self.waited == 1:
logger.debug(u'\ttap then wait ...')
self.driver.tap([(self.window_size['width'] / 2, self.window_size['height'] / 2)])
sleep(1)
self.waited = 2
elif self.waited > 1:
logger.debug(u'\tswipe then wait ...')
self.swipe_left()
sleep(1)
self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login)
return
# Welcome page
skippable = self.contain_skip_text()
get_sinks = lambda: self.driver.find_elements_by_xpath('//*[not(*)]')
if not self.is_in_dialog() \
and ((skippable or (len(clickable) in range(1, 5) and len(textedit) == 0))
and len(get_sinks()) < 20):
logger.debug(u'Seems to be in welcome page, try bypassing it')
if detect_login and self.contain_login_text():
logger.debug(u'Find login keywords in welcome page, break skip_irrelevant')
return
if skippable:
for kw in self.skipkw:
for e in self.find_elements_by_keyword(kw):
if self.tap_test(e, diff=0.98):
self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login)
return
safe_clickable = self.find_safe_clickable_elements()
if safe_clickable:
to_tap = safe_clickable[-1]
ele_info = layout.ElementInfo(to_tap)
logger.debug(u'Tapped {0}'.format(ele_info))
if not self.tap_test(to_tap, diff=0.98):
logger.warning(u'Tap failed: {0}, try swiping'.format(ele_info))
self.swipe_left()
self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login)
return
# Dialog
# TODO: decide cancel/ok by context
if self.is_in_dialog() \
and len(clickable) in range(1, 5) \
and len(textedit) == 0:
logger.debug(u'Seems to be a dialog, try bypassing it')
if detect_login and self.contain_login_text():
logger.debug(u'Find login keywords in welcome page, break skip_irrelevant')
return
safe_clickable = self.find_safe_clickable_elements()
if not safe_clickable:
raise myexceptions.NoSafeClickableElement("Seems like a dialog that requiring update")
source_before_tap = self.page_source
to_tap = safe_clickable[-1]
ele_info = layout.ElementInfo(to_tap)
logger.debug(u'Tapped {0}'.format(ele_info))
self.tap(to_tap)
if self.driver.current_activity == cur_act \
and difflib.SequenceMatcher(None, self.page_source.lower(),
source_before_tap).ratio() > 0.95:
logger.warning(u'Tap failed: {0}'.format(ele_info))
self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login)
return
# City list
cities_pattern = u'(?:鞍山|安庆|安阳|安顺|北京|天津|上海|深圳|广州|成都|南京|重庆|杭州)市?'
filtered = self.find_elements_by_keyword(cities_pattern, clickable_only=False, exact=True, scroll=False)
if len(filtered) > 3:
logger.debug(u'Seems to be a city select page, try bypassing it')
to_tap = filtered[0]
ele_info = layout.ElementInfo(to_tap)
logger.debug(u'Tapped {0}'.format(ele_info))
self.tap(to_tap)
if self.driver.current_activity == cur_act:
logger.warning(u'Tap failed: {0}'.format(ele_info))
self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login)
return
except NoSuchElementException:
self.skip_irrelevant(initial=False, limit=limit, detect_login=detect_login)
def add_content(self, content):
logger.debug('[parser] added top-level content')
self.model['content'] = content
def run(self, case_email_path):
"""
执行测试任务
:param case_email_path: 列表，第一个元素是测试用例文件路径，第二个元素是收件人的txt文件路径
:return:
"""
try:
if int(cfg.getConfig('is_git')):
logger.info('准备从git上拉取最新版本')
repo = Repo(cfg.getConfig('git_path'))
remote = repo.remote()
remote.pull()
logger.info('从git上拉取版本成功')
except Exception as err:
logger.error(err)
file_name = None
error_msg = None
case_path = case_email_path[0]
email_path = case_email_path[1]
build_path = os.path.join(case_path, 'build.xml')
logger.info(f'开始执行测试任务{build_path}')
try:
start_time = time.strftime('%Y-%m-%d %H:%M:%S')
res = os.popen(
'ant -f {}'.format(build_path)).readlines() # 执行测试，并等待测试完成
logger.debug(res)
length = len(res)
for i in range(length, -1, -1):
if 'Failed' in res[i]: # 如果有失败日志，打印出
error_msg = '{}\n{}'.format(res[i], res[i - 1])
logger.error(error_msg)
break
if 'xslt' in res[i] and 'Processing' in res[i] and 'to' in res[
i]: # 获取测试报告文件名
line = res[i].strip()
logger.debug(line)
if '/' in line:
file_name = line.split('/')[-1]
else:
file_name = line.split('\\')[-1]
logger.info(file_name)
break
del res
if file_name:
logger.info('测试任务执行完成')
time.sleep(2)
msg = self.parse_html(file_name, case_path) # 重组html
sendMsg(msg['fail_case'],
email_path,
failure_num=msg['failure_num']) # 发送邮件
string = f"{start_time},{build_path},{msg['total_num']},{msg['failure_num']}\n"
self.lock.acquire()
logger.info(f'写测试记录到本地, {string}')
with open(os.path.join(case_path,
cfg.getConfig('record_name')),
'a',
encoding='utf-8') as f:
f.write(string)
self.lock.release()
logger.info('测试完成')
else:
error_msg = 'html格式的测试报告未找到'
except Exception as err:
error_msg = err
logger.error(err)
if error_msg:
logger.error(f'测试任务执行失败，失败信息：{error_msg}')
html = f'<html><body>' \
f'<h3>异常提醒：{build_path} 测试任务执行失败，请重新执行！ 失败信息：{error_msg}</h3>' \
f'<p style="color:blue;">此邮件自动发出，请勿回复。</p></body></html>'
try:
sendMsg(html, email_path, is_path=False)
except Exception as err:
logger.error(err)
def dealHomeGoods(self):
result_datas = []
try:
page_source = self.getHtml(self.home_url, self.header)
soup = BeautifulSoup(page_source, 'lxml')
pre_load_data = soup.findAll('li',
{'class': 'content_banner_item'})
for load_data in pre_load_data:
logger.debug('load_data: {}'.format(load_data))
resultData = {}
resultData['CHANNEL'.lower()] = self.Channel
resultData['STATUS'.lower()] = '01'
try:
resultData['LINK'.lower()] = load_data.find(
'a').attrs['href']
resultData['TITLE'.lower()] = load_data.find(
'img').attrs['alt']
resultData['MAIN_IMAGE'.lower()] = urljoin(
self.home_url,
load_data.find('img').attrs['src'])
resultData['CREATE_TIME'.lower()] = time.strftime(
'%Y%m%d%H%M%S', time.localtime(time.time()))
except Exception:
map_name = load_data.find('img').attrs['usemap'][1:]
maps = soup.find('map', {'name': map_name}).findAll('area')
for map in maps:
href = map.attrs['href']
if href.find('women') != -1:
resultData['MIN_IMAGE'.lower()] = 'WOMEN'
else:
resultData['MIN_IMAGE'.lower()] = 'MEN'
resultData['LINK'.lower()] = map.attrs['href']
resultData['TITLE'.lower()] = load_data.find(
'img').attrs['alt']
resultData['MAIN_IMAGE'.lower()] = urljoin(
self.home_url,
load_data.find('img').attrs['src'])
resultData['CREATE_TIME'.lower()] = time.strftime(
'%Y%m%d%H%M%S', time.localtime(time.time()))
t_resultData = copy.copy(resultData)
result_datas.append(t_resultData)
continue
else:
continue
result_datas.append(resultData)
result_datas = self._rm_duplicate(result_datas, 'LINK'.lower())
if len(result_datas) == 0:
logger.error('page_source: {}'.format(page_source))
raise ValueError('not get valid data')
format_select = r'SELECT ID FROM {} WHERE CHANNEL="{{channel}}" and LINK="{{link}}" ORDER BY CREATE_TIME DESC'
good_datas = result_datas
select_sql = format_select.format(self.TABLE_NAME_BANNER)
table = self.TABLE_NAME_BANNER
replace_insert_columns = [
'CHANNEL', 'LINK', 'MAIN_IMAGE', 'CREATE_TIME', 'MIN_IMAGE',
'STATUS', 'TITLE'
]
select_columns = ['ID']
return self._saveDatas(good_datas, table, select_sql,
replace_insert_columns, select_columns)
except Exception, e:
logger.error('Get home goods infos error:{},retry it'.format(e))
raise
def add_table_content(self, table, content):
logger.debug('[parser] added table %s content', table)
self.model['tables'][table]['content'] = content
def crawl(dest_filename, multipart_chunksize, url):
t0 = time.time()
# 如果没有指定本地保存时的文件名，则默认使用 URL 中最后一部分作为文件名
official_filename = dest_filename if dest_filename else url.split('/')[-1] # 正式文件名
temp_filename = official_filename + '.swp' # 没下载完成时，临时文件名
config_filename = official_filename + '.swp.cfg' # 没下载完成时，存储 ETag 等信息的配置文件名
# 获取文件的大小和 ETag
r = custom_request('HEAD', url, info='header message')
if not r: # 请求失败时，r 为 None
logger.error('Failed to get header message on URL [{}]'.format(url))
return
file_size = int(r.headers['Content-Length'])
ETag = r.headers['ETag']
logger.info('File size: {} bytes, ETag: {}'.format(file_size, ETag))
# 如果正式文件存在
if os.path.exists(official_filename):
if os.path.getsize(official_filename) == file_size: # 且大小与待下载的目标文件大小一致时
logger.warning('The file [{}] has already been downloaded'.format(official_filename))
return
else: # 大小不一致时，提醒用户要保存的文件名已存在，需要手动处理，不能随便覆盖
logger.warning('The filename [{}] has already exist, but it does not match the remote file'.format(official_filename))
return
# 首先需要判断此文件支不支持 Range 下载，请求第 1 个字节即可
headers = {'Range': 'bytes=0-0'}
r = custom_request('HEAD', url, info='Range: bytes=0-0', headers=headers)
if not r: # 请求失败时，r 为 None
logger.error('Failed to get [Range: bytes=0-0] on URL [{}]'.format(url))
return
if r.status_code != 206: # 不支持 Range 下载时
logger.warning('The file [{}] does not support breakpoint retransmission'.format(official_filename))
# 需要重新从头开始下载 (wb 模式)
with tqdm(total=file_size, unit='B', unit_scale=True, unit_divisor=1024, desc=official_filename) as bar: # 打印下载时的进度条，并动态显示下载速度
r = custom_request('GET', url, info='all content', stream=True)
if not r: # 请求失败时，r 为 None
logger.error('Failed to get all content on URL [{}]'.format(url))
return
with open(temp_filename, 'wb') as fp:
for chunk in r.iter_content(chunk_size=multipart_chunksize):
if chunk:
fp.write(chunk)
bar.update(len(chunk))
# 整个文件内容被成功下载后，将临时文件名修改回正式文件名、删除配置文件
os.rename(temp_filename, official_filename)
if os.path.exists(config_filename):
os.remove(config_filename)
logger.info('{} downloaded'.format(official_filename))
logger.info('Cost {:.2f} seconds'.format(time.time() - t0))
else: # 支持 Range 下载时
# 获取文件的总块数
div, mod = divmod(file_size, multipart_chunksize)
parts_count = div if mod == 0 else div + 1 # 计算出多少个分块
logger.info('Chunk size: {} bytes, total parts: {}'.format(multipart_chunksize, parts_count))
# 如果临时文件存在
if os.path.exists(temp_filename):
if os.path.getsize(temp_filename) != file_size: # 说明此临时文件有问题，需要先删除它
os.remove(temp_filename)
else: # 临时文件有效时
if not os.path.exists(config_filename): # 如果不存在配置文件时
os.remove(temp_filename)
else: # 如果配置文件也在，则继续判断 ETag 是否一致
with open(config_filename, 'r') as fp:
cfg = json.load(fp)
if cfg['ETag'] != ETag: # 如果不一致
os.remove(temp_filename)
else: # 从配置文件中读取已下载的分块号集合，从而得出未下载的分块号集合
succeed_parts = {part['PartNumber'] for part in cfg['parts']} # 之前已下载好的分块号集合
succeed_parts_size = sum([part['Size'] for part in cfg['parts']]) # 已下载的块的总大小，注意是列表推导式不是集合推导式
parts = set(range(parts_count)) - succeed_parts # 本次需要下载的分块号集合
# 再次判断临时文件在不在，如果不存在时，表示要下载所有分块号
if not os.path.exists(temp_filename):
succeed_parts_size = 0
parts = range(parts_count)
# 由于 _fetchByRange() 中使用 rb+ 模式，必须先保证文件存在，所以要先创建指定大小的临时文件 (用0填充)
f = open(temp_filename, 'wb')
f.seek(file_size - 1)
f.write(b'\0')
f.close()
with open(config_filename, 'w') as fp: # 创建配置文件，写入 ETag
cfg = {
'ETag': ETag,
'parts': []
}
json.dump(cfg, fp)
logger.debug('The remaining parts that need to be downloaded: {}'.format(set(parts)))
# 多线程并发下载
workers = min(8, len(parts))
failed_parts = 0 # 下载失败的分块数目
# 创建互斥锁
lock = threading.Lock()
# 固定住 lock、url、temp_filename、config_filename，不用每次都传入相同的参数
_fetchByRange_partial = partial(_fetchByRange, lock, url, temp_filename, config_filename)
with futures.ThreadPoolExecutor(workers) as executor:
to_do = []
# 创建并排定Future
for part_number in parts:
# 重要: 通过块号计算出块的起始与结束位置，最后一块(编号从0开始，所以最后一块编号为 parts_count - 1)需要特殊处理
if part_number != parts_count-1:
start = part_number * multipart_chunksize
stop = (part_number + 1) * multipart_chunksize - 1
else:
start = part_number * multipart_chunksize
stop = file_size - 1
future = executor.submit(_fetchByRange_partial, part_number, start, stop)
to_do.append(future)
# 获取Future的结果，futures.as_completed(to_do)的参数是Future列表，返回迭代器，
# 只有当有Future运行结束后，才产出future
done_iter = futures.as_completed(to_do)
with tqdm(total=file_size, initial=succeed_parts_size, unit='B', unit_scale=True, unit_divisor=1024, desc=official_filename) as bar: # 打印下载时的进度条，并动态显示下载速度
for future in done_iter: # future变量表示已完成的Future对象，所以后续future.result()绝不会阻塞
result = future.result()
if result.get('failed'):
failed_parts += 1
else:
bar.update(result.get('part')['Size'])
if failed_parts > 0:
logger.error('Failed to download {}, failed parts: {}, successful parts: {}'.format(official_filename, failed_parts, parts_count-failed_parts))
else:
# 整个文件内容被成功下载后，将临时文件名修改回正式文件名、删除配置文件
os.rename(temp_filename, official_filename)
if os.path.exists(config_filename):
os.remove(config_filename)
logger.info('{} downloaded'.format(official_filename))
logger.info('Cost {:.2f} seconds'.format(time.time() - t0))
# Standard-Lib imports
# Third-Party imports
# BITSON Imports
from utils import *
from logger import logger
logger.debug("Loading " + __name__ + " plugin...")
@bot.message_handler(commands=['flip'])
def flip(message):
if in_time(message):
user_id = str(message.from_user.id)
username = str(message.from_user.username)
chat_id = str(message.chat.id)
args = get_content(message)
coin = ['cara', 'seca']
if not args:
bot.send_message(chat_id, 'Ingrese su elección... \n '
'¿cara o seca? \n'
'/flip <eleccion>',
parse_mode='Markdown')
return
if args != '-?' and args in coin:
coin = random.choice(coin)
if args == coin:
bot.send_message(chat_id,
'Salio %s. Ganaste.' % coin,
parse_mode="Markdown")
def get_albums(url):
'''请求入口页面，BeautifulSoup解析返回的HTTP响应，从中获取所有图集的URL
:return:
find_albums: 本次请求入口页面后，一共找到多个图集
new_albums: 有多少个是新增的（即数据库之前没有记录的）
'''
find_albums = 0 # 一共找到多个图集
new_albums = 0 # 有多少个是新增的（即数据库之前没有记录的）
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'
}
# 返回入口页面的HTTP响应
# TODO: 捕获request.get方法的异常，比如连接超时、被拒绝等
resp = get_response(url, info='start url', headers=headers)
if not resp: # 请求失败时，resp为None，不能往下执行
logger.error('Exception {} on URL [{}]'.format(e.__class__, url))
return
# 使用lxml解析器，解析返回的响应（HTML文档）
soup = BeautifulSoup(resp.text, 'lxml')
# 每个图集按年份/月份被放在 <div class='all'></div> 下面的每个<a href="图集URL">图集标题<a> 中
try:
a_tags = soup.find('div', {
'class': 'all'
}).find_all('a') # <class 'bs4.element.ResultSet'>
except Exception as e:
logger.error('Exception {} on URL [{}]'.format(e.__class__, url))
return
logger.critical('URL [{}] has [{}] <a></a> tags'.format(url, len(a_tags)))
for a in a_tags:
# 判断每个<a></a>标签中的URL是不是符合图集URL格式，如果不是，则递归调用它看看它下面有没有相同URL
# 因为有一个 https://www.mzitu.com/old/
if re.match(r'https://www.mzitu.com/\d+', a['href']):
data = {
'album_title': a.get_text(), # 每个图集的标题
'album_url': a['href'], # 每个图集的URL
'created_at': datetime.utcnow(), # 何时添加到MongoDB的
'visited': 0 # 表明此图集URL没被访问过
}
if TEST_NUM and TEST_NUM == collection_albums.count_documents(
{}): # 如果是测试，MongoDB中永远只能存在 TEST_NUM 个集合
find_albums += 1 # 增加找到的图集计数
continue
else:
if not collection_albums.find_one(
{'album_url': data['album_url']}): # 如果是新图集，保存到MongoDB
new_albums += 1 # 增加新图集计数
try:
collection_albums.insert_one(data)
logger.debug(
'Successfully saved album {} [{}] to MongoDB'.
format(data['album_title'], data['album_url']))
except pymongo.errors.DuplicateKeyError as e: # 多线程可能会报异常
logger.error(
'Album {} [{}] already exsits in MongoDB'.format(
data['album_title'], data['album_url']))
find_albums += 1 # 增加找到的图集计数
else: # 此<a></a>标签包含的URL不是一个合法的图集地址，即将递归获取它下面的图集
logger.critical(
'Tag [<a href="{}">{}</a>] is invalid, recursive call this function'
.format(a['href'], a.get_text()))
recursive_find_albums, recursive_new_albums = get_albums(a['href'])
find_albums += recursive_find_albums
new_albums += recursive_new_albums
return find_albums, new_albums
def load():
"""load model from path bound to local from the container."""
l_time = time.time()
logger.debug("## >> load-time {:.2f} s".format(time.time() - l_time))
def get_image_pages(album):
'''请求图集页面，BeautifulSoup解析返回的HTTP响应，从中获取此图集下所有图片页面（非真正图片资源）'''
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'
}
# 返回一个图集页面的HTTP响应
# TODO: 捕获request.get方法的异常，比如连接超时、被拒绝等
resp = get_response(album['album_url'], info='album url', headers=headers)
if not resp: # 请求失败时，resp为None，不能往下执行
return {
'failed': True # 用于告知get_image_pages()的调用方，请求此图集URL时失败了
}
# 使用lxml解析器，解析返回的响应（HTML文档）
soup = BeautifulSoup(resp.text, 'lxml')
# 图集发布日期，后续保存时要按年份/月份创建目录
try:
date_span = soup.find('div', {
'class': 'main-meta'
}).find_all('span')[1].get_text() # 类似于'发布于 2014-06-20 13:09'
except Exception as e:
logger.error('Exception {} on URL [{}]'.format(e.__class__,
album['album_url']))
return {
'failed': True # 用于告知get_image_pages()的调用方，请求此图集URL时失败了
}
published_at = re.search(r'\d+-\d+', date_span).group() # 类似于2014-06
# 图集有多少张图片
try:
images_num = int(
soup.find('div', {
'class': 'pagenavi'
}).find_all('span')[-2].get_text())
except Exception as e:
logger.error('Exception {} on URL [{}]'.format(e.__class__,
album['album_url']))
return {
'failed': True # 用于告知get_image_pages()的调用方，请求此图集URL时失败了
}
logger.debug('Album {} [{}] has [{}] images'.format(
album['album_title'], album['album_url'], images_num))
# 按日期创建目录
date_path = os.path.join(download_path, published_at)
if not os.path.exists(date_path):
try:
os.mkdir(date_path)
logger.debug('Create date directory [{}]'.format(date_path))
except FileExistsError as e: # 多线程时，可能会存在一个线程刚创建了日期目录，另一个线程也刚好想创建，结果慢的会报异常
pass
# 为每个图集创建一个目录
album_name = '[{}P] '.format(images_num) + re.sub(
'[\/:*?"<>|]', '_', album['album_title']) # 注意要去除标题的非法字符
album_path = os.path.join(date_path, album_name)
if not os.path.exists(album_path):
try:
os.mkdir(album_path)
logger.debug('Create album directory [{}]'.format(album_path))
except FileExistsError as e: # 多线程时，可能会存在一个线程刚创建了该图集目录，另一个线程也刚好想创建，结果慢的会报异常
pass
# 构建图集下所有包含真实图片资源的页面URL
for i in range(1, images_num + 1):
data = {
'image_page_url':
album['album_url'] + '/' + str(i), # 每张图片的所在页面的URL，还不是图片的真实地址
'image_idx': i, # 每张图片的所在页面在图集中的序号
'album_url': album['album_url'], # 所属图集的URL
'album_title': album['album_title'], # 所属图集的标题
'album_path':
os.path.join(published_at,
album_name), # 所属图集的相对路径，不使用绝对路径是因为可能会移动此脚本
'created_at': datetime.utcnow(), # 何时添加到MongoDB的
'visited': 0 # 表明此图片页面URL没被访问过
}
# 保存到MongoDB
if not collection_image_pages.find_one(
{'image_page_url': data['image_page_url']}):
try:
collection_image_pages.insert_one(data)
logger.debug(
'Successfully saved image page No.{} [{}] of album {} [{}] to MongoDB'
.format(data['image_idx'], data['image_page_url'],
data['album_title'], data['album_url']))
except pymongo.errors.DuplicateKeyError as e: # 多线程可能会报异常
logger.error(
'Image page No.{} [{}] of album {} [{}] already exsits in MongoDB'
.format(data['image_idx'], data['image_page_url'],
data['album_title'], data['album_url']))
# 更新图集的collection，增加字段
data = {
'album_published_at': re.search('\s(.*)',
date_span).group(1), # 图集的发布时间
'album_images_num': images_num, # 图集的图片数
'album_path': os.path.join(published_at,
album_name), # 图集的保存路径，不使用绝对路径是因为可能会移动此脚本
'visited': 1, # 表明此图集URL被访问过，下次不用再访问它来获取此图集下的图片页面
'visited_at': datetime.utcnow() # 何时被访问过
}
collection_albums.update_one({'album_url': album['album_url']},
{'$set': data})
return {
'failed': False # 用于告知get_image_pages()的调用方，此图集被正常访问
}
async def on_greeting(event):
'''Greets someone'''
# telethon.events.newmessage.NewMessage.Event
# telethon.events.messageedited.MessageEdited.Event
if not event.chat:
logger.error(f'event.chat empty. event: { event }')
raise events.StopPropagation
if not hasattr(event.chat, 'username'):
logger.error(f'event.chat not found username:{event.chat}')
raise events.StopPropagation
if event.chat.username == account['bot_name']: # 不监听当前机器人消息
logger.debug(
f'不监听当前机器人消息, event.chat.username: { event.chat.username }')
raise events.StopPropagation
# 是否拒绝来自其它机器人发在群里的消息
if 'block_bot_msg' in config and config['block_bot_msg']:
if hasattr(event.message.sender, 'bot') and event.message.sender.bot:
logger.debug(
f'不监听所有机器人消息, event.chat.username: { event.chat.username }')
raise events.StopPropagation
# if not event.is_group:# channel 类型
if True: # 所有消息类型，支持群组
message = event.message
text = message.text
if message.file and message.file.name:
# text += ' file:{}'.format(message.file.name)# 追加上文件名
text += ' {}'.format(message.file.name) # 追加上文件名
# 打印消息
_title = ''
if not hasattr(event.chat, 'title'):
logger.warning(f'event.chat not found title:{event.chat}')
else:
_title = f'event.chat.title:{event.chat.title},'
logger.debug(
f'event.chat.username: {event.chat.username},event.chat.id:{event.chat.id},{_title} event.message.id:{event.message.id},text:{text}'
)
# 1.方法(失败)：转发消息
# chat = 'keyword_alert_bot' #能转发 但是不能真对特定用户。只能转发给当前允许账户的bot
# from_chat = 'tianfutong'
# chat = 349506543# 无法使用chat_id直接转发 没有任何反应
# chat = 1354871670
# await message.forward_to('keyword_alert_bot')
# await client.forward_messages(chat, message)
# await bot.forward_messages(chat, message)
# await client.forward_messages(chat, message.id, from_chat)
# 2.方法：直接发送新消息,非转发.但是可以url预览达到效果
# 查找当前频道的所有订阅
sql = """
select u.chat_id,l.keywords,l.id,l.chat_id
from user_subscribe_list as l
INNER JOIN user as u on u.id = l.user_id
where (l.channel_name = ? or l.chat_id = ?) and l.status = 0 order by l.create_time desc
"""
find = utils.db.connect.execute_sql(
sql, (event.chat.username, str(event.chat_id))).fetchall()
if find:
logger.info(
f'channel: {event.chat.username}; all chat_id & keywords:{find}'
) # 打印当前频道，订阅的用户以及关键字
for receiver, keywords, l_id, l_chat_id in find:
try:
# 消息发送去重规则
MSG_UNIQUE_RULE_MAP = {
'SUBSCRIBE_ID': f'{receiver}_{l_id}',
'MESSAGE_ID': f'{receiver}_{message.id}',
}
if 'msg_unique_rule' not in config:
config['msg_unique_rule'] = 'SUBSCRIBE_ID'
assert config[
'msg_unique_rule'] in MSG_UNIQUE_RULE_MAP, 'config "msg_unique_rule" error!!!'
CACHE_KEY_UNIQUE_SEND = MSG_UNIQUE_RULE_MAP[
config['msg_unique_rule']]
logger.debug(
f'msg_unique_rule:{config["msg_unique_rule"]} --> {CACHE_KEY_UNIQUE_SEND}'
)
# 优先返回可预览url
channel_url = f'https://t.me/{event.chat.username}/' if event.chat.username else get_channel_url(
event.chat.username, event.chat_id)
channel_msg_url = f'{channel_url}{message.id}'
send_cache_key = f'_LAST_{l_id}_{message.id}_send'
if isinstance(event, events.MessageEdited.Event): # 编辑事件
# 24小时内新建2秒后的编辑不提醒
if cache.get(send_cache_key) and (
event.message.edit_date - event.message.date
) > datetime.timedelta(seconds=2):
logger.error(
f'{channel_msg_url} repeat send. deny!')
continue
if not l_chat_id: # 未记录频道id
logger.info(
f'update user_subscribe_list.chat_id:{event.chat_id} where id = {l_id} '
)
re_update = utils.db.user_subscribe_list.update(
chat_id=str(event.chat_id)).where(
utils.User_subscribe_list.id == l_id)
re_update.execute()
chat_title = event.chat.username or event.chat.title
if is_regex_str(keywords): # 输入为正则字符串
regex_match = js_to_py_re(keywords)(
text) # 进行正则匹配 只支持ig两个flag
if isinstance(regex_match, regex.Match): #search()结果
regex_match = [regex_match.group()]
regex_match_str = [] # 显示内容
for _ in regex_match:
item = ''.join(_) if isinstance(_, tuple) else _
if item:
regex_match_str.append(item) # 合并处理掉空格
regex_match_str = list(set(regex_match_str)) # 处理重复元素
if regex_match_str: # 默认 findall()结果
# # {chat_title} \n\n
channel_title = f"\n\nCHANNEL: {chat_title}" if not event.chat.username else ""
message_str = f'[#FOUND]({channel_msg_url}) **{regex_match_str}**{channel_title}'
if cache.add(CACHE_KEY_UNIQUE_SEND, 1, expire=5):
logger.info(
f'REGEX: receiver chat_id:{receiver}, l_id:{l_id}, message_str:{message_str}'
)
if isinstance(event,
events.NewMessage.Event): # 新建事件
cache.set(send_cache_key, 1,
expire=86400) # 发送标记缓存一天
await bot.send_message(receiver,
message_str,
link_preview=True,
parse_mode='markdown')
else:
# 已发送该消息
logger.debug(
f'REGEX send repeat. rule_name:{config["msg_unique_rule"]} {CACHE_KEY_UNIQUE_SEND}:{channel_msg_url}'
)
continue
else:
logger.debug(
f'regex_match empty. regex:{keywords} ,message: t.me/{event.chat.username}/{event.message.id}'
)
else: #普通模式
if keywords in text:
# # {chat_title} \n\n
channel_title = f"\n\nCHANNEL: {chat_title}" if not event.chat.username else ""
message_str = f'[#FOUND]({channel_msg_url}) **{keywords}**{channel_title}'
if cache.add(CACHE_KEY_UNIQUE_SEND, 1, expire=5):
logger.info(
f'TEXT: receiver chat_id:{receiver}, l_id:{l_id}, message_str:{message_str}'
)
if isinstance(event,
events.NewMessage.Event): # 新建事件
cache.set(send_cache_key, 1,
expire=86400) # 发送标记缓存一天
await bot.send_message(receiver,
message_str,
link_preview=True,
parse_mode='markdown')
else:
# 已发送该消息
logger.debug(
f'TEXT send repeat. rule_name:{config["msg_unique_rule"]} {CACHE_KEY_UNIQUE_SEND}:{channel_msg_url}'
)
continue
except errors.rpcerrorlist.UserIsBlockedError as _e:
# User is blocked (caused by SendMessageRequest) 用户已手动停止bot
logger.error(f'{_e}')
pass # 关闭全部订阅
except ValueError as _e:
# 用户从未使用bot
logger.error(f'{_e}')
# 删除用户订阅和id
isdel = utils.db.user.delete().where(
utils.User.chat_id == receiver).execute()
user_id = utils.db.user.get_or_none(chat_id=receiver)
if user_id:
isdel2 = utils.db.user_subscribe_list.delete().where(
utils.User_subscribe_list.user_id ==
user_id.id).execute()
except AssertionError as _e:
raise _e
except Exception as _e:
logger.error(f'{_e}')
else:
logger.debug(
f'sql find empty. event.chat.username:{event.chat.username}, find:{find}, sql:{sql}'
)
if 'auto_leave_channel' in config and config['auto_leave_channel']:
if event.chat.username: # 公开频道/组
logger.info(f'Leave Channel/group: {event.chat.username}')
await leave_channel(event.chat.username)
def WebScrapper(URL, sourceLink, Id=0):
global count
global linkDicts
if URL[-1] == "/":
URL = URL[:-1]
cursor = collection.find()
try:
for record in cursor:
if record.get("link") == URL:
currentDate = datetime.now()
crawledDate = record["crawlDate"]
timeDifference = currentDate - crawledDate
if record["isCrawled"] and timeDifference < Config[
"Crawling_Time"]:
logger.debug(URL + " Already Crawled")
return
elif record["isCrawled"] and timeDifference >= Config[
"Crawling_Time"]:
linkDicts[Id] = record
del linkDicts[Id]["_id"]
if linkDicts[Id]["sourceLink"] != "Manual Input":
sourceLink = linkDicts[Id]["sourceLink"]
collection.delete_one(record)
count -= 1
break
if not record["isCrawled"]:
linkDicts[Id] = record
del linkDicts[Id]["_id"]
sourceLink = linkDicts[Id]["sourceLink"]
collection.delete_one(record)
count -= 1
break
except KeyboardInterrupt:
exit()
except:
if Id == 0:
logger.warning("Can not connect to database")
logger.info("Retrying...")
WebScrapper(URL, sourceLink, Id)
try:
r = requests.get(URL, headers=headers)
logger.debug("Crawling " + URL)
except KeyboardInterrupt:
exit()
except:
connected = isConnected(Id)
if connected:
return
elif not connected:
WebScrapper(URL, sourceLink, Id)
statusCode = r.status_code
contentType = r.headers['content-type']
contentLength = len(r.content)
rText = r.text
soup = BeautifulSoup(rText, 'html.parser')
aTag = soup.find_all("a")
Links = []
for a in aTag:
try:
link = a['href']
except:
continue
if len(link) > 1 and link[0] == "/":
link = requests.compat.urljoin(URL, link)
if link[-1] == "/":
link = link[:-1]
elif link[:4] == "http":
link = link
if link[-1] == "/":
link = link[:-1]
else:
continue
Links.append(link)
AddData(Id, URL, sourceLink, True, statusCode, contentType, contentLength,
rText, Links)
linksList.append(Links)
sourceList.append(URL)
def get_image(image_page):
'''请求包含图片资源链接的图片页面，BeautifulSoup解析返回的HTTP响应，从中获取真正的图片资源URL'''
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36'
}
# 返回一个图片所在页面的HTTP响应
# TODO: 捕获request.get方法的异常，比如连接超时、被拒绝等
resp = get_response(image_page['image_page_url'],
info='image page url',
headers=headers)
if not resp: # 请求失败时，resp为None，不能往下执行
return {
'failed': True # 用于告知get_image()的调用方，请求此图片页面URL时失败了
}
# 使用lxml解析器，解析返回的响应（HTML文档）
soup = BeautifulSoup(resp.text, 'lxml')
# 获取图片的真实地址
try:
image_url = soup.find('div', {
'class': 'main-image'
}).find('img')['src']
except Exception as e: # 偶尔有一些图集中间某几个图片页面会返回响应，但响应中没有main-image，即找不到有效的图片资源URL，比如https://www.mzitu.com/57773/41
logger.error(
'Image page No.{} [{}] of album {} [{}] has no valid image URL'.
format(image_page['image_idx'], image_page['image_page_url'],
image_page['album_title'], image_page['album_url']))
# 更新图片页面的collection，增加字段
data = {
'visited': 1, # 表明此图片页面URL被访问过，下次不用再访问它来获取它对应的真实图片资源的URL
'visited_at': datetime.utcnow() # 何时被访问过
}
collection_image_pages.update_one(
{'image_page_url': image_page['image_page_url']}, {'$set': data})
return {
'failed': True # 用于告知get_image()的调用方，请求此图片页面URL时失败了
}
# 还有一些image_url的值不是有效的图片链接，比如https://www.mzitu.com/57773/40获得的image_url就是'https:\n</p>\n</div>\r\n <div class='
if not re.match(
r'https://.*?\.(jpe|jpg|jpeg|png|gif)', image_url
): # https://www.mzitu.com/23077/18图片页面返回的是https://i.meizitu.net/2014/03/20140315qc18.jpe
logger.error(
'Image page No.{} [{}] of album {} [{}] has no valid image URL'.
format(image_page['image_idx'], image_page['image_page_url'],
image_page['album_title'], image_page['album_url']))
# 更新图片页面的collection，增加字段
data = {
'visited': 1, # 表明此图片页面URL被访问过，下次不用再访问它来获取它对应的真实图片资源的URL
'visited_at': datetime.utcnow() # 何时被访问过
}
collection_image_pages.update_one(
{'image_page_url': image_page['image_page_url']}, {'$set': data})
return {
'failed': True # 用于告知get_image()的调用方，请求此图片页面URL时失败了
}
# 准备用这个名称来保存图片
image_name = image_url.split('/')[-1]
# 图片保存的相对路径，不使用绝对路径是因为可能会移动此脚本
image_path = os.path.join(image_page['album_path'], image_name)
data = {
'image_url': image_url,
'image_path': image_path,
'image_idx': image_page['image_idx'], # 每张图片在图集中的序号
'album_url': image_page['album_url'], # 所属图集的URL
'album_title': image_page['album_title'], # 所属图集的标题
'created_at': datetime.utcnow(), # 何时添加到MongoDB的
'visited': 0 # 表明此图片资源URL没被访问过
}
# 保存到MongoDB
if not collection_images.find_one({'image_url': data['image_url']}):
try:
collection_images.insert_one(data)
logger.debug(
'Successfully saved image No.{} [{}] of album {} [{}] to MongoDB'
.format(data['image_idx'], data['image_url'],
data['album_title'], data['album_url']))
except pymongo.errors.DuplicateKeyError as e: # 多线程可能会报异常
logger.error(
'Image No.{} [{}] of album {} [{}] already exsits in MongoDB'.
format(data['image_idx'], data['image_url'],
data['album_title'], data['album_url']))
# 更新图片页面的collection，增加字段
data = {
'visited': 1, # 表明此图片页面URL被访问过，下次不用再访问它来获取它对应的真实图片资源的URL
'visited_at': datetime.utcnow() # 何时被访问过
}
collection_image_pages.update_one(
{'image_page_url': image_page['image_page_url']}, {'$set': data})
return {
'failed': False # 用于告知get_image()的调用方，此图片页面被正常访问
}
def fetch(self, begin, end, first_response):
headers = dict((k.title(), v) for k, v in self.headers.items())
retry_num = 0
while self.keep_running:
retry_num += 1
if retry_num > 20:
logger.warn("RangeFetch try max times, exit. %s", self.url)
self.close()
break
expect_len = end - begin + 1
headers['Range'] = 'bytes=%d-%d' % (begin, end)
if first_response:
response = first_response
else:
try:
response = request_gae_proxy(self.method, self.url,
headers, self.body)
except GAE_Exception as e:
logger.warning('RangeFetch %s request fail:%r',
headers['Range'], e)
continue
if response.app_msg:
response.worker.close("range get gae status:%d" %
response.app_status)
continue
response.status = response.app_status
if response.headers.get('Location', None):
self.url = urlparse.urljoin(self.url,
response.headers.get('Location'))
logger.warn('RangeFetch Redirect(%r) status:%s', self.url,
response.status)
continue
if response.status >= 300:
#logger.error('RangeFetch %r return %s :%s', self.url, response.status, cgi.escape(response.body))
response.worker.close("range status:%s" % response.status)
continue
content_range = response.headers.get('Content-Range', "")
if not content_range:
logger.warning(
'RangeFetch "%s %s" return headers=%r, retry %s-%s',
self.method, self.url, response.headers, begin, end)
# if len(response.body) < 2048:
#logger.warn('body:%s', cgi.escape(response.body))
# response.worker.close("no range")
continue
content_length = int(response.headers.get('Content-Length', 0))
data_readed = 0
while True:
if data_readed >= content_length:
percent = begin * 100 / self.req_end
logger.debug(
'RangeFetch [%s] %d%% length:%s range:%s %s %s',
response.ssl_sock.ip, percent, content_length,
content_range, self.url, response.task.get_trace())
break
data = response.task.read()
if not data:
logger.warn("RangeFetch [%s] get body fail, begin:%d %s",
response.ssl_sock.ip, begin, self.url)
break
data_len = len(data)
data_readed += data_len
if data_len > expect_len:
logger.warn("RangeFetch expect:%d, get:%d", expect_len,
data_len)
data = data[:expect_len]
data_len = expect_len
self.put_data(begin, data)
expect_len -= data_len
begin += data_len
if begin >= end + 1:
break
logger.warn("RangeFetch get left, begin:%d end:%d", begin, end)
# Save trained model: separate network structure (stored as JSON) and trained weights (stored as HDF5)
save_keras_sequential(network, RELATIVE_PATH, filename_to_save)
def create_model() -> Sequential:
network = Sequential()
network.add(Dense(500, activation='relu', input_dim=100))
network.add(Dense(500, activation='relu'))
network.add(Dense(1, activation='linear'))
return network
if __name__ == "__main__":
# Load the training data; here: complete data about stock A
logger.debug("Data loading...")
full_stock_market_data = read_stock_market_data(
[CompanyEnum.COMPANY_A, CompanyEnum.COMPANY_B],
['1962-2011', '2012-2017'])
company_a_stock_data = full_stock_market_data[CompanyEnum.COMPANY_A]
logger.debug(
f"Data for Stock A loaded: {company_a_stock_data.get_row_count()} prices and dates read."
)
learn_nn_and_save(company_a_stock_data, MODEL_FILE_NAME_STOCK_A)
company_b_stock_data = full_stock_market_data[CompanyEnum.COMPANY_B]
logger.debug(
f"Data for Stock B loaded: {company_b_stock_data.get_row_count()} prices and dates read."
def run(self):
req_range_begin = None
req_range_end = None
for k, v in self.headers.items():
# logger.debug("range req head:%s => %s", k, v)
if k.lower() == "range":
req_range_begin, req_range_end = tuple(
x for x in re.search(r'bytes=(\d*)-(\d*)', v).group(1, 2))
# break
response_headers = dict(
(k.title(), v) for k, v in self.response.headers.items())
content_range = response_headers['Content-Range']
res_begin, res_end, res_length = tuple(
int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)',
content_range).group(1, 2, 3))
self.req_begin = res_end + 1
if req_range_begin and req_range_end:
self.req_end = int(req_range_end)
else:
self.req_end = res_length - 1
self.wait_begin = res_begin
if self.wait_begin == 0 and self.req_end == res_length - 1:
response_headers['Content-Length'] = str(res_length)
del response_headers['Content-Range']
state_code = 200
else:
response_headers['Content-Range'] = 'bytes %s-%s/%s' % (
res_begin, self.req_end, res_length)
response_headers['Content-Length'] = str(self.req_end - res_begin +
1)
state_code = 206
response_headers["Persist"] = ""
response_headers["Connection"] = "Persist"
logger.info('RangeFetch %d-%d started(%r) ', res_begin, self.req_end,
self.url)
try:
self.wfile.write("HTTP/1.1 %d OK\r\n" % state_code)
for key in response_headers:
if key in skip_response_headers:
continue
value = response_headers[key]
#logger.debug("Head %s: %s", key.title(), value)
send_header(self.wfile, key, value)
self.wfile.write("\r\n")
except Exception as e:
self.keep_running = False
logger.info("RangeFetch send response fail:%r %s", e, self.url)
return
data_left_to_fetch = self.req_end - self.req_begin + 1
fetch_times = int(
(data_left_to_fetch + front.config.AUTORANGE_MAXSIZE - 1) /
front.config.AUTORANGE_MAXSIZE)
thread_num = min(front.config.AUTORANGE_THREADS, fetch_times)
for i in xrange(0, thread_num):
threading.Thread(target=self.fetch_worker).start()
threading.Thread(target=self.fetch,
args=(res_begin, res_end, self.response)).start()
ok = "ok"
while self.keep_running and \
(front.config.use_ipv6 == "force_ipv6" and \
check_local_network.IPv6.is_ok() or \
front.config.use_ipv6 != "force_ipv6" and \
check_local_network.is_ok()) and \
self.wait_begin < self.req_end + 1:
with self.lock:
if self.wait_begin not in self.data_list:
self.waiter.wait()
if self.wait_begin not in self.data_list:
logger.error("get notify but no data")
continue
else:
data = self.data_list[self.wait_begin]
del self.data_list[self.wait_begin]
self.wait_begin += len(data)
self.data_size -= len(data)
self.all_data_size[self] = self.data_size
try:
ret = self.wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
logger.debug("send to browser wfile.write ret:%d, retry",
ret)
ret = self.wfile.write(data)
logger.debug("send to browser wfile.write ret:%d", ret)
del data
except Exception as e:
logger.info('RangeFetch client closed(%s). %s', e, self.url)
ok = None
break
self.keep_running = False
self.all_data_size.pop(self, None)
return ok
from cfg import config
from logger import logger
from getActorsInMovie import get_actors_in_movie
from getTopMovies import get_top_movies
if __name__ == "__main__":
logger.debug('Starting process')
# Get the top movies list
top_movies_list = get_top_movies()
# We will store actor names and their number of movies in this dict
actor_movie_counts = {}
logger.debug("Extracted " + str(len(top_movies_list)) + " top movies info")
# For testing purposes use only top 15 movies
# Remove this line below to run the algorithm on all 250 movies
top_movies_list = top_movies_list[:15]
for movie in top_movies_list:
movie_name = movie['name']
movie_link = movie['link']
# domain name is missing from the movie link, so append manually
full_movie_link = config['host_name'] + movie_link
actor_names = get_actors_in_movie(full_movie_link)
logger.debug("Extracted " + str(len(actor_names)) +
" actor names in movie " + movie_name)
plt.figure()
plt.plot(dates[INPUT_SIZE:], awaited_results,
color="black") # current prices in reality
plt.plot(dates[INPUT_SIZE:], prediction_results,
color="green") # predicted prices by neural network
plt.title('current prices / predicted prices by date')
plt.ylabel('price')
plt.xlabel('date')
plt.legend(['current', 'predicted'], loc='best')
plt.show()
if __name__ == "__main__":
logger.debug("Data loading...")
training_stock_market_data = read_stock_market_data(
[CompanyEnum.COMPANY_A, CompanyEnum.COMPANY_B], [PERIOD_1])
test_stock_market_data = read_stock_market_data(
[CompanyEnum.COMPANY_A, CompanyEnum.COMPANY_B], [PERIOD_2])
company_a_training_stock_data: StockData = training_stock_market_data[
CompanyEnum.COMPANY_A]
company_a_test_stock_data: StockData = test_stock_market_data[
CompanyEnum.COMPANY_A]
logger.debug(f"Data for Stock A loaded")
learn_nn_and_save(company_a_training_stock_data, company_a_test_stock_data,
MODEL_FILE_NAME_STOCK_A)
company_b_training_stock_data: StockData = training_stock_market_data[
def __init__(self, error_code, message):
logger.debug("GAE_Exception %r %r", error_code, message)
self.error_code = error_code
self.message = "%r:%s" % (error_code, message)
def create_status_index():
"""
记录启动铁笼日志，数据铁笼的运行监控信息，状态信息，由后台写入ES中，其文档id就是ids_id
"""
res = es.indices.exists(index=es_status_index)
logger.debug("%s es.indices.exists is %s", es_status_index, res)
if not res:
# 创建索引,建立mapping索引
mappings = {
"mappings": {
"doc": {
"properties": {
"company_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"created_time": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"destroy_time": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"id": {
"type": "keyword"
},
"live_time": {
"type": "long"
},
"request_id": {
"type": "keyword"
},
"service_id": {
"type": "keyword"
},
"user_id": {
"type": "keyword"
},
"blockchain": {
"type": "keyword"
},
"status": {
"type": "keyword"
},
"service_name": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
logger.debug(mappings)
res = es.indices.create(index=es_status_index, body=mappings)
logger.debug("es.indices.create result is %s", res)
def handler(method, host, url, headers, body, wfile, fallback=None):
if not url.startswith("http") and not url.startswith("HTTP"):
logger.error("gae:%s", url)
return
request_time = time.time()
org_headers = dict(headers)
remove_list = []
req_range_begin = ""
req_range_end = ""
req_range = ""
for k, v in headers.items():
if v == "":
remove_list.append(k)
continue
if k.lower() == "range":
req_range = v
req_range_begin, req_range_end = tuple(
x for x in re.search(r'bytes=(\d*)-(\d*)', v).group(1, 2))
# fix bug for android market app: Mobogenie
# GAE url_fetch refuse empty value in header.
for key in remove_list:
del headers[key]
# force to get content range
# reduce wait time
if method == "GET":
if req_range_begin and not req_range_end:
# don't known how many bytes to get, but get from begin position
req_range_begin = int(req_range_begin)
headers["Range"] = "bytes=%d-%d" % (
req_range_begin,
req_range_begin + front.config.AUTORANGE_MAXSIZE - 1)
logger.debug("change Range %s => %s %s", req_range,
headers["Range"], url)
elif req_range_begin and req_range_end:
req_range_begin = int(req_range_begin)
req_range_end = int(req_range_end)
if req_range_end - req_range_begin + 1 > front.config.AUTORANGE_MAXSIZE:
headers["Range"] = "bytes=%d-%d" % (
req_range_begin,
req_range_begin + front.config.AUTORANGE_MAXSIZE - 1)
# remove wait time for GAE server to get knowledge that content
# size exceed the max size per fetch
logger.debug("change Range %s => %s %s", req_range,
headers["Range"], url)
elif not req_range_begin and req_range_end:
# get the last n bytes of content
pass
else:
# no begin and no end
# don't add range, some host like github don't support Range.
# headers["Range"] = "bytes=0-%d" % config.AUTORANGE_MAXSIZE
pass
try:
response = request_gae_proxy(method, url, headers, body)
# http://en.wikipedia.org/wiki/Chunked_transfer_encoding
response.headers.pop("Transfer-Encoding", None)
# gae代理请求
except GAE_Exception as e:
logger.warn("GAE %s %s request fail:%r", method, url, e)
if fallback and host.endswith(front.config.GOOGLE_ENDSWITH):
return fallback()
send_response(wfile, e.error_code, body=e.message)
return_fail_message(wfile)
return "ok"
if response.app_msg:
# XX-net 自己数据包
send_response(wfile, response.app_status, body=response.app_msg)
return "ok"
else:
response.status = response.app_status
if response.status == 206:
# use org_headers
# RangeFetch need to known the real range end
# 需要分片
return RangeFetch2(method, url, org_headers, body, response,
wfile).run()
response_headers = {}
# 初始化给客户端的headers
for key, value in response.headers.items():
key = key.title()
if key in skip_response_headers:
continue
response_headers[key] = value
response_headers["Persist"] = ""
response_headers["Connection"] = "Persist"
if 'X-Head-Content-Length' in response_headers:
if method == "HEAD":
response_headers['Content-Length'] = response_headers[
'X-Head-Content-Length']
del response_headers['X-Head-Content-Length']
# 只是获取头
content_length = int(response.headers.get('Content-Length', 0))
content_range = response.headers.get('Content-Range', '')
# content_range 分片时合并用到
if content_range and 'bytes */' not in content_range:
start, end, length = tuple(
int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)',
content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length - 1, content_length
# 未分片
if method == "HEAD":
body_length = 0
else:
body_length = end - start + 1
def send_response_headers():
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
for key, value in response_headers.items():
send_header(wfile, key, value)
# logger.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
# 写入除body外内容
def is_text_content_type(content_type):
mct, _, sct = content_type.partition('/')
if mct == 'text':
return True
if mct == 'application':
sct = sct.split(';', 1)[0]
if (sct in ('json', 'javascript', 'x-www-form-urlencoded')
or sct.endswith(('xml', 'script')) or sct.startswith(
('xml', 'rss', 'atom'))):
return True
return False
data0 = ""
content_type = response_headers.get("Content-Type", "")
content_encoding = response_headers.get("Content-Encoding", "")
if body_length and \
content_encoding == "gzip" and \
response.gps < "GPS 3.3.2" and \
is_text_content_type(content_type):
url_guess_type = guess_type(url)[0]
if url_guess_type is None or is_text_content_type(url_guess_type):
# try decode and detect type
min_block = min(1024, body_length)
data0 = response.task.read(min_block)
if not data0 or len(data0) == 0:
logger.warn("recv body fail:%s", url)
return
gzip_decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
decoded_data0 = gzip_decompressor.decompress(data0)
deflate_decompressor = zlib.decompressobj(-zlib.MAX_WBITS)
decoded_data1 = None
if len(decoded_data0) > 1:
CMF, FLG = bytearray(decoded_data0[:2])
if CMF & 0x0F == 8 and CMF & 0x80 == 0 and (
(CMF << 8) + FLG) % 31 == 0:
decoded_data1 = deflate_decompressor.decompress(
decoded_data0[2:])
if decoded_data1 is None and len(decoded_data0) > 0:
try:
decoded_data1 = deflate_decompressor.decompress(
decoded_data0)
if deflate_decompressor.unused_data != '':
decoded_data1 = None
except:
pass
if decoded_data1:
try:
response_headers.pop("Content-Length", None)
if "deflate" in headers.get("Accept-Encoding", ""):
# return deflate data if accept deflate
response_headers["Content-Encoding"] = "deflate"
send_response_headers()
while True:
wfile.write(decoded_data0)
if response.task.body_readed >= body_length:
break
data = response.task.read()
decoded_data0 = gzip_decompressor.decompress(data)
logger.info(
"GAE send ungziped deflate data to browser t:%d s:%d %s %s %s",
(time.time() - request_time) * 1000,
content_length, method, url,
response.task.get_trace())
else:
# inflate data and send
del response_headers["Content-Encoding"]
send_response_headers()
while True:
wfile.write(decoded_data1)
if response.task.body_readed >= body_length:
break
data = response.task.read()
decoded_data0 = gzip_decompressor.decompress(data)
decoded_data1 = deflate_decompressor.decompress(
decoded_data0)
logger.info(
"GAE send ungziped data to browser t:%d s:%d %s %s %s",
(time.time() - request_time) * 1000,
content_length, method, url,
response.task.get_trace())
return
except Exception as e:
logger.info(
"gae_handler.handler try decode and send response fail. e:%r %s",
e, url)
return
try:
send_response_headers()
if data0:
wfile.write(data0)
body_sended = len(data0)
else:
body_sended = 0
except Exception as e:
logger.info("gae_handler.handler send response fail. e:%r %s", e, url)
return
while True:
# 可能分片发给客户端
if body_sended >= body_length:
break
data = response.task.read()
if not data:
logger.warn("get body fail, until:%d %s",
body_length - body_sended, url)
break
body_sended += len(data)
try:
# https 包装
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
#logger.debug("send to browser wfile.write ret:%d", ret)
#ret = wfile.write(data)
wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE,
errno.ECONNRESET) or 'bad write retry' in repr(e_b):
logger.info(
'gae_handler send to browser return %r %r, len:%d, sended:%d',
e_b, url, body_length, body_sended)
else:
logger.info('gae_handler send to browser return %r %r', e_b,
url)
return
# 完整一次https请求
logger.info("GAE t:%d s:%d %s %s %s", (time.time() - request_time) * 1000,
content_length, method, url, response.task.get_trace())
return "ok"
def fix(self):
try:
# if toggle key is active, temporarily switch the active state
is_active = self.app.toggle_key_active ^ self.app.is_active
# check if we can proceed
if not is_active:
logger.debug("QuoteFix is not active, so no QuoteFixing for you!")
return
# Grab some variables we need to perform our business
view = self.composeWebView() # contains the message editor
backend = self.backEnd()
htmldom = view.mainFrame().DOMDocument()
htmlroot = htmldom.documentElement()
messageType = self.messageType()
# XXX: hack alert! if message type is DRAFT, but we can determine this
# is actually a Send Again action, adjust the message type.
origmsg = backend.originalMessage()
if origmsg and messageType == DRAFT and origmsg.type() == 0:
messageType = SENDAGAIN
# send original HTML to menu for debugging
self.app.html = htmlroot.innerHTML()
# provide custom attribution?
attributor = None
if self.app.use_custom_reply_attribution and messageType in [
REPLY, REPLY_ALL, REPLY_AS
]:
logger.debug("calling customize_attribution() for reply{-all,-as}")
attributor = CustomizedAttribution.customize_reply
elif self.app.use_custom_sendagain_attribution and messageType in [
SENDAGAIN
]:
logger.debug("calling customize_attribution() for Send Again")
attributor = CustomizedAttribution.customize_sendagain
elif self.app.use_custom_forwarding_attribution and messageType == FORWARD:
logger.debug("calling customize_attribution() for forwarding")
attributor = CustomizedAttribution.customize_forward
if attributor:
# play nice with Attachment Tamer
try:
message = backend.draftMessage()
except:
try:
copy = backend.copyOfContentsForDraft_shouldBePlainText_isOkayToForceRichText_(
True, False, True)
except:
# Yosemite
copy = backend.copyOfContentsForDraft_shouldBePlainText_isOkayToForceRichText_isMailDropPlaceholderMessage_(
True, False, True, False)
message = backend._makeMessageWithContents_isDraft_shouldSign_shouldEncrypt_shouldSkipSignature_shouldBePlainText_(
copy, True, False, False, False, False)
try:
for original in objc.getInstanceVariable(
backend, '_originalMessages'):
attributor(
app=self.app,
editor=self,
dom=htmldom,
reply=message,
inreplyto=original,
)
backend.setHasChanges_(False)
except:
# ignore when not debugging
if self.app.is_debugging:
raise
# should we be quotefixing?
if not self.app.is_quotefixing:
logger.debug(
'quotefixing turned off in preferences, skipping that part')
elif messageType not in self.app.message_types_to_quotefix:
logger.debug('message type "%s" not in %s, not quotefixing' %
(messageType, self.app.message_types_to_quotefix))
else:
# remove attachment placeholders?
if self.app.remove_attachment_placeholders:
self.remove_attachment_placeholders(backend, htmlroot)
# move cursor to end of document
view.moveToEndOfDocument_(self)
# remove quotes?
if self.app.remove_quotes:
logger.debug('calling remove_quotes()')
self.remove_quotes(htmldom, self.app.remove_quotes_level)
# make quotes selectable?
if self.app.selectable_quotes:
logger.debug('calling make_selectable_quotes()')
self.make_selectable_quotes(view, htmldom)
# remove signature from sender
if not self.app.keep_sender_signature:
logger.debug('calling remove_old_signature()')
self.remove_old_signature(htmldom, view)
# place cursor above own signature (if any)
logger.debug('calling move_above_new_signature()')
if self.move_above_new_signature(htmldom, view):
# insert a paragraph break?
if not self.app.no_whitespace_below_quote:
view.insertParagraphSeparator_(self)
else:
view.insertNewline_(self)
# perform some general cleanups
logger.debug('calling cleanup_layout()')
self.cleanup_layout(htmlroot, backend)
# move cursor to top of document
if self.app.move_cursor_to_top:
view.moveToBeginningOfDocument_(self)
# move to beginning of line
logger.debug('calling view.moveToBeginningOfLine()')
view.moveToBeginningOfLine_(self)
# done
logger.debug('QuoteFixing done')
except Exception:
logger.critical(traceback.format_exc())
if self.app.is_debugging:
NSRunAlertPanel(
'QuoteFix caught an exception',
'The QuoteFix plug-in caught an exception:\n\n' +
traceback.format_exc() +
'\nPlease contact the developer quoting the contents of this alert.',
None, None, None)
def record_status(record_id=None):
"""
记录启动铁笼日志，数据铁笼的运行监控信息，状态信息，由后台写入ES中，其文档id就是ids_id
"""
# 铁笼启动是否已经记录
if record_id is None:
record_id = worker.get_id()
logger.debug("id is %s", record_id)
isRecorded = os.environ["IS_RECORD"]
service_id = os.environ["HOSTNAME"]
if "service_id" in os.environ:
service_id = os.environ['service_id']
handler = os.environ["Handler"]
runtime = os.environ["Runtime"]
serverName = os.environ["SERVER_NAME"]
if "service_name_cn" in os.environ:
serverName = os.environ["service_name_cn"]
content = "启动铁笼" + service_id
now = datetime.now()
logger.debug(now.strftime("%Y-%m-%d %H:%M:%S"))
dt = time.time()
nowFormat = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(dt)))
user_id = os.environ["creator"]
if isRecorded == "0":
# 插入数据
action = {
"id": record_id,
"service_id": service_id,
"service_name": serverName,
"request_id": "",
"user_id": user_id,
"company_name": "",
"live_time": 0,
"destroy_time": None,
"created_time": nowFormat,
"status": "运行中",
"blockchain": "数安链"
}
res = es.index(index=es_status_index, doc_type="doc", body=action)
logger.debug("insert status result: %s", res)
# {'_index': 'fcs', '_type': 'type_doc', '_id': 'z4dbDW8BtULaBa6qlsZO', '_version': 1, 'result': 'created', '_shards': {'total': 2, 'successful': 2, 'failed': 0}, '_seq_no': 0, '_primary_term': 1}
if res["result"] == "created":
os.environ["IS_RECORD"] = res["_id"]
logger.debug("IS_RECORD is %s", os.environ["IS_RECORD"])
else:
# # 根据关键词查找
# doc = {
# "query": {
# "match": {
# # "_id": "jId_RW8BtULaBa6q8cmB"
# "_id": isRecorded
# }
# }
# }
# res = es.search(index=es_status_index, body=doc)
# # {'took': 97, 'timed_out': False, '_shards': {'total': 5, 'successful': 5, 'skipped': 0, 'failed': 0}, 'hits': {'total': 1, 'max_score': 1.0, 'hits': [{'_index': 'fcs_status', '_type': 'doc', '_id': 'jId_RW8BtULaBa6q8cmB', '_score': 1.0, '_source': {'id': 1210409175465730048, 'service_id': 'aa527f36-da1c-4406-baa6-d62f1943c9c3', 'service_name': '铁笼调用联通测试', 'request_id': '', 'user_id': 'a6837cf7-704a-41bd-a38f-fbfb8c4841bb', 'company_name': '', 'live_time': 0, 'destroy_time': '2019-12-27 11:56:52', 'created_time': '2019-12-27 11:56:50', 'status': '已销毁', 'blockchain': '数安链'}}]}}
# # {'took': 14, 'timed_out': False, '_shards': {'total': 5, 'successful': 5, 'skipped': 0, 'failed': 0}, 'hits': {'total': 0, 'max_score': None, 'hits': []}}
# created_time = nowFormat
# if res["hits"]["total"] > 0:
# created_time = res["hits"]["hits"][0]["_source"]["created_time"]
# logger.debug("created_time is %s", created_time)
# created_time_ms = int(round(time.mktime(time.strptime(created_time, "%Y-%m-%d %H:%M:%S"))) * 1000)
# logger.debug("created_time_ms is %s", created_time_ms)
# destroy_time_ms = int(round(dt*1000))
# logger.debug("destroy_time_ms is %s", destroy_time_ms)
# live_time_ms = destroy_time_ms - created_time_ms
# logger.debug("live_time_ms is %s", live_time_ms)
# # 更新数据
# action = {
# "doc": {
# # "id": isRecorded,
# # "service_id": hostName,
# # "service_name": serverName,
# # "request_id": "",
# # "user_id": user_id,
# # "company_name": "",
# "live_time": live_time_ms,
# "destroy_time": nowFormat,
# # "created_time": createTime,
# "status": "已销毁",
# "blockchain": "数安链"
# }
# }
# logger.debug("update status %s", os.environ["IS_RECORD"])
# res = es.update(index=es_status_index, id=isRecorded, doc_type="doc", body=action)
# logger.debug("es.update result is %s", res)
# 根据id查询
try:
res = es.get(index=es_status_index, doc_type="doc", id=isRecorded)
# {'_index': 'fcs_status', '_type': 'doc', '_id': 'jId_RW8BtULaBa6q8cmB', '_version': 2, 'found': True, '_source': {'id': 1210409175465730048, 'service_id': 'aa527f36-da1c-4406-baa6-d62f1943c9c3', 'service_name': '铁笼调用联通测试', 'request_id': '', 'user_id': 'a6837cf7-704a-41bd-a38f-fbfb8c4841bb', 'company_name': '', 'live_time': 0, 'destroy_time': '2019-12-27 11:56:52', 'created_time': '2019-12-27 11:56:50', 'status': '已销毁', 'blockchain': '数安链'}}
created_time = res["_source"]["created_time"]
logger.debug("created_time is %s", created_time)
created_time_ms = int(
round(
time.mktime(
time.strptime(created_time, "%Y-%m-%d %H:%M:%S"))) *
1000)
logger.debug("created_time_ms is %s", created_time_ms)
destroy_time_ms = int(round(dt * 1000))
logger.debug("destroy_time_ms is %s", destroy_time_ms)
live_time_ms = destroy_time_ms - created_time_ms
logger.debug("live_time_ms is %s", live_time_ms)
# 更新数据
action = {
"doc": {
# "id": isRecorded,
# "service_id": hostName,
# "service_name": serverName,
# "request_id": "",
# "user_id": user_id,
# "company_name": "",
"live_time": live_time_ms,
"destroy_time": nowFormat,
# "created_time": createTime,
"status": "已销毁",
"blockchain": "数安链"
}
}
logger.debug("update status %s", os.environ["IS_RECORD"])
res = es.update(index=es_status_index,
id=isRecorded,
doc_type="doc",
body=action)
# logger.debug("es.update result is", res)
logger.debug("status_index res is %s", res)
except elasticsearch.exceptions.NotFoundError as e:
logger.debug('错误类型是%s', e.__class__.__name__)
logger.debug('错误明细是%s', e)
result = traceback.format_exc()
logger.debug(result)
except Exception as e:
logger.debug('错误类型是%s', e.__class__.__name__)
logger.debug('错误明细是%s', e)
result = traceback.format_exc()
logger.debug(result)
def finishLoadingEditor(self, original):
logger.debug('DocumentEditor finishLoadingEditor')
original(self)
self.fix()
def __init__(self,
filepath: str = None,
adjacency_list: AdjacencyList = None,
adjacency_matrix: AdjacencyMatrix = None,
version=0) -> None:
"""
Note: All the variables are exlusive, that means if on is supplied the others should not be used
:param filepath: The path from which the graph should be loaded
:param adjacency_list: The ajd. list from which the graph should be load
:param adjacency_matrix:
"""
# Filepath to the grad which should be loaded
self.filepath = filepath
# 2d list. which holds all adjacent nodes in the form of and adjacency list
self.adjacency_list = adjacency_list
# 2d list. which holds all adjacent nodes in the form of and adjacency matrix
self.adjacency_matrix = adjacency_matrix
# Total number of vertices
self.vertice_count = None
self.j = None
self.version = 0
self.values = list()
# Load from a file
# https://stackoverflow.com/questions/1369526/what-is-the-python-keyword-with-used-for
if filepath:
logger.info("Loading from" + filepath)
# print("Loading from " + filepath)
# Get file descriptor
try:
f = open(self.filepath, "r")
try:
# Load data into the adjacency_list
self.j = json.load(f)
self.version = self.j["version"]
if self.version == 0:
self.adjacency_list = self.j["adj_list"]
logger.info("adjacency list" +
str(self.adjacency_list))
# This is the cycle free adj. list representation.
# Normal: [ This version: [
# [1], [1],
# [0] []
# ] ]
if self.version == 0.1:
self.adjacency_list = self.j["adj_list"]
logger.info("Adjacency list" +
str(self.adjacency_list))
if self.version == 1:
self.adjacency_list = self.convert_from_adjacency_matrix(
self.j["adj_matrix"])
# Load values for the nodes(Optional)
if self.version == 2:
self.adjacency_list = self.j["adj_list"]
self.node_attribs = self.j["nodes"]
logger.info("adjacency list" +
str(self.adjacency_list))
logger.info("nodes" + str(self.node_attribs))
try:
self.values = self.j["values"]
logger.debug("Values are %s", self.values)
except KeyError:
logger.debug("Values list is not specified")
except json.JSONDecodeError:
logger.error("Colud not decode " + filepath + "to Json")
raise DecodingError
except FileNotFoundError:
logger.error("Could not load" + filepath)
raise LoadingError
finally:
# Close file
f.close()
else:
if self.adjacency_matrix is not None:
self.adjacency_list = self.convert_from_adjacency_matrix(
self.adjacency_matrix)
self.version = version
# Get the vertice count
self.vertice_count = len(self.adjacency_list)
self.is_binary_tree = False
self.node_id = 0
