def spider_closed(self, spider):
stats = self.crawler.stats.get_stats()
stats['siteid'] = self.siteid
stats['siteinfo'] = self.siteinfo[self.siteid]
import common
com = common.Common()
com.add_crawl_log(spider, self.siteid, stats)
def getContent(self):
cn = common.Common()
ret = []
try:
# html = cn.getHtmlAsBrowser(self._url,self._refer,self._host)
html = cn.getHtml(self._url)
soup = BeautifulSoup(html, 'html5lib')
trs = soup.select('table .tab tbody tr .xian')
# print len(trs)
isodd = True
#因为有循环，故先初始化变量
title = None
url = None
time = None
for tr in trs:
if isodd: #奇数行
href = tr.a['href'] #链接
url = self._baseurl + href
title = tr.a.string.strip()
else: #偶数行
time = tr.string.replace('[',
'').replace(']',
'') #时间[2017-03-10]
if not isodd: #偶数行之后存储记录
ret.append({'time': time, 'title': title, 'url': url})
isodd = not isodd
except Exception as e:
print str(e)
return ret
class Loop(Thread):
com: common.Common = common.Common()
mqtt: MQTTClient = MQTTClient(MessageReceiver(com), 'HTTPLooper')
def __init__(self, config):
self.mqtt.start_loop()
for topic in config['topics']['subscribe']:
self.mqtt.subscribe(topic)
super(Loop, self).__init__()
def run(self):
jobs = self.com.jobs()
while jobs > 0:
for task in self.com.tasks():
if task[0] == common.PUBLISH:
self.mqtt.publish(task[1], task[2])
elif task[0] == common.SUBSCRIBE:
self.mqtt.subscribe(task[1])
jobs = self.com.jobs()
self.mqtt.stop_loop()
def stop(self):
self.com.stop()
def getContent(self):
cn = common.Common()
ret = []
try:
html = cn.getHtml(self._url)
soup = BeautifulSoup(html, 'html5lib')
lis = soup.select("#tigtag2_right li")
for li in lis:
for child in li.children:
# print child.name
name = child.name
if name == "span":
time = child.text#[截止:2017-03-30]
endtime = re.search(r'\d{4}-\d{1,2}-\d{1,2}',time).group(0)
continue
if name == "a":
if child.has_attr('title'):
title = child['title']
href = child['href']
url = self._baseurl + str(href).replace("..","")
# print title,url,endtime
# break
ret.append({'endtime': endtime, 'title': title, 'url': url})
except Exception as e:
print str(e)
return ret
def __init__(self, swname, portInfo):
print portInfo
self.sw_serial = swname
# portInfo.pop('name')
self.portInfo = portInfo
self.cfg = self.getCfg()
self.cmdList = self.getCmdList()
print portInfo.values()
self.ip = portInfo['ip']
self.online_port_list = []
self.name = swname
self.rdb = fleta_dbms.FletaDb()
# self.portInfo.pop('name')
self.password = portInfo['password']
self.username = portInfo['username']
self.fileName = os.path.join('data', '%s_%s.tmp' % (
self.name,
self.ip,
))
self.com = common.Common()
self.net_connect = ConnectHandler(**self.portInfo)
self.portinfolist = []
self.portlist = []
self.r = self.redis()
self.now = datetime.datetime.now()
self.store_time = self.now.strftime('%Y-%m-%d %H:%M:%S')
self.lhour = ''
def __init__(self, swname, portInfo, num):
self.portInfo = portInfo
self.cfg = self.getCfg()
self.cmdList = self.getCmdList()
self.ip = portInfo['ip']
self.crc = num
self.name = swname
# self.portInfo.pop('name')
self.password = portInfo['password']
self.username = portInfo['username']
self.fileName = os.path.join('data', '%s_%s.tmp' % (
self.name,
self.ip,
))
self.com = common.Common()
self.net_connect = ConnectHandler(**self.portInfo)
self.portinfolist = []
self.portlist = []
self.r = self.redis()
self.now = datetime.datetime.now()
self.store_time = self.now.strftime('%Y-%m-%d %H:%M:%S')
self.lhour = ''
def __init__(self):
com = common.Common()
str = "http://www.51voa.com%s"
for x in com.get_crawl_urls(1, True):
if x.get('url', ''):
self.start_urls.append(str % x.get('url', ''))
self._urls[str % x.get('url', '')] = x.get('link_id', 0)
class Deal:
CONST = const
CONST.BUFFSIZE = 1024
comm = common.Common()
def __init__(self):
obj = socket.socket()
obj.connect((self.comm.getClientConf("server", "host"),
int(self.comm.getClientConf("server", "port"))))
ret_bytes = obj.recv(self.CONST.BUFFSIZE)
ret_str = self.comm.tanslate(str(ret_bytes))
print(ret_str)
while True:
inp = raw_input(self.comm.tanslate("你好请问您有什么问题？ \n >>>"))
if inp == "q":
inp = cryptofactory.CryptoFactory().encrypto(
inp, self.comm.getClientConf("transmission", "crypto_key"))
data = self.comm.createPacket(inp, 0)
obj.sendall(bytes(data))
break
elif inp == "upload":
upload.Upload(obj)
else:
inp = cryptofactory.CryptoFactory().encrypto(
inp, self.comm.getClientConf("transmission", "crypto_key"))
data = self.comm.createPacket(inp, 99999, "text")
obj.sendall(bytes(data))
ret_bytes = obj.recv(self.CONST.BUFFSIZE)
ret_str = self.comm.tanslate(str(ret_bytes))
print(ret_str + "112233")
def __init__(self, fullpath):
self.dbg = debug.Debug()
self.comm = common.Common()
self.recordfile = fullpath
self.user_id_list = []
self.user_id_list_commit_idx = 0
self.setpath(fullpath)
def getContent(self):
cn = common.Common()
ret = []
try:
data = cn.postUseRawHeader(self._url, self._header, self._data)
soup = BeautifulSoup(data, 'html5lib')
trs = soup.select("#bulletininfotable_table_body tr")
for tr in trs:
contents = tr.contents
title = contents[3].text.strip()
id = contents[3].a['value']
url = self._baseurl.format(id)
times = contents[5].text
found = re.findall(r"\d{4}-\d{1,2}-\d{1,2}", times)
time = found[0]
endtime = found[1]
ret.append({
'time': time,
'title': title,
'url': url,
'endtime': endtime
})
except Exception as e:
print str(e)
return ret
def sell_times(self, BuyTime):
sell_times = (common.Common().get_time() -
self.time_change(BuyTime)).total_seconds() / 86400
if sell_times > 10:
return 0.1
else:
return int(sell_times) * 0.01
def getContent(self):
cn = common.Common()
ret = []
try:
self._url = self._url.format(cn.makeUTCTime())
data = cn.getHTTPS(self._url)
dics = eval(data)
notices = dics['notices']
for notice in notices:
title = notice['title']
issueTime = notice['issueTime']
time = re.search(r"\d{4}-\d{1,2}-\d{1,2}", issueTime).group(0)
begintime = notice['bidBeginTime']
endtime = notice['bidEndTime']
id = notice['id']
url = self._baseurl + id
# begintime = re.search(r"\d{4}-\d{1,2}-\d{1,2}",bidBeginTime).group(0)
# endtime = re.search(r"\d{4}-\d{1,2}-\d{1,2}",bidEndTime).group(0)
ret.append({
'time': time,
'title': title,
'url': url,
'begintime': begintime,
'endtime': endtime
})
except Exception as e:
print str(e)
return ret
def getContent(self):
cn = common.Common()
ret = []
try:
html = cn.getHtml(self._url)
soup = BeautifulSoup(html, 'html5lib')
tds = soup.select(".liebiaobg td")
i = 0
href = None
title = None
time = None
for td in tds:
i += 1
if i == 1:#img标签，没用
continue
if i == 2:#a标签，title和href
href = td.a['href']
url = self._baseurl + href
title = td.text.strip()
continue
if i == 3:#时间，[2017-03-11]
time = td.text.replace('[','').replace(']','')
# print time,title,url
ret.append({'time': time, 'title': title, 'url': url})
i = 0
except Exception as e:
print str(e)
return ret
def getContent(self):
cn = common.Common()
ret = []
try:
data = cn.post(self._url, self._posturl, self._para, self._header)
# print data
soup = BeautifulSoup(data, 'html5lib')
trs = soup.select("#moredingannctable tbody tr")
# print len(trs)
for tr in trs:
if tr.has_attr('onclick'):
# print tr.attrs['onclick']
watchContent = tr.attrs['onclick']
match = re.findall(r"\d+", watchContent)
# [u'80244', u'1']
flag = match[1]
fid = match[0]
href = "/zfcg/" + flag + "/bidingAnncDetail_" + fid + ".html"
url = self._baseurl + href
a = tr.a
span = tr.span
if a is not None:
title = a.text
if span is not None:
time = span.text
# print time,title,url
ret.append({'time': time, 'title': title, 'url': url})
# break
except Exception as e:
print str(e)
return ret
def spider_closed(self, spider):
stats = self.crawler.stats.get_stats()
# print spider.state.values()
import common
com = common.Common()
com.add_crawl_log(spider, 1, stats)
log.msg("spider closed", level=log.INFO)
print "spider closed!"
def __init__(self):
self.mdb = mongodb.MongoDB()
self.collection = 'bugContent'
self.cm = common.Common()
self.titles = [
u'缺陷编号', u'缺陷标题', u'缺陷描述', u'重现步骤', u'缺陷状态', u'所属终端', u'所属项目',
u'开发人员', u'测试人员', u'创建日期'
]
def __init__(self, L):
# print(a)
# cm = request.GET.get('name', '')
self.mk = monkey.Monkey()
self.cm = common.Common()
self.cf = configparser.ConfigParser()
self.L = L
print(L)
def __init__(self):
self.com = common.Common()
self.dec = common.Decode()
# self.logger = self.com.flog()
# self.conn_string = "host='localhost' dbname='fleta' user='******' password='******'"
self.conn_string = self.getConnStr()
# print self.conn_string
self.cfg = self.getCfg()
def __init__(self):
self._com = common.Common()
host = "http://www.jd5.com%s"
for x in self._com.get_crawl_urls(self.siteid, True):
if x.get('url', ''):
self.start_urls.append(host % x.get('url', ''))
self._urls[host % x.get('url', '')] = x.get('link_id', 0)
print self._urls
dispatcher.connect(self.spider_closed, signals.spider_closed)
def main():
b1 = b.Demo()
b1.abc()
print("multi is ", a1.multi(2, 3))
# object2 = a2.Ob2()
# print(object2.add(2,3))
c1 = c.Common()
# del c1
c1.json()
c.Common().json_dump()
#list
l = c.List()
l.make_alist()
l.remove_duplicate()
#tuple
t = c.Tuple()
t.make()
#set
s = c.Set()
s.make()
#dict
d = c.Dict()
d.make()
anyobject = c.AnyObject("A", "B")
d.of(anyobject)
#Operation
o = c.Ops()
s1 = o.make()
print(s1)
r1 = "abc"[::-1]
print(r1)
#Lambda
lamb = c.Lambda()
print(lamb.make(5))
class UmiChestServer(socketserver.BaseRequestHandler):
comm=common.Common()
CONST=const
CONST.METHOD=constTable.constTable().METHOD
def handle(self):
conn = self.request
conn.sendall(bytes("你好，我是机器人"))
up=uploadDeal.UploadDeal()
cf=cryptofactory.CryptoFactory()
while True:
flag=False
totol_data=""
ret_bytes = str(conn.recv(1024))
totol_data+=ret_bytes
if ret_bytes[-1:] != "\0":
while True:
ret_bytes = str(conn.recv(1024))
totol_data+=ret_bytes
if ret_bytes[-1:] == "\0" or not ret_bytes:
break
ret_str_packet = str(totol_data)
ret_str_packet = ret_str_packet.split("\0")
# 去除\0的空元素
ret_str_packet=ret_str_packet[:-1]
for ret_str in ret_str_packet:
ret_str_list=ret_str[:-1].split("\n")
# print(ret_str_list)
ret_dict={}
for r in ret_str_list:
r=r.split(":")
# print(r)
if len(r)>1:
ret_dict[r[0]]=r[1]
# print(ret_dict['md5'])
ret_msg=""
data=cf.decrypto(ret_dict['data'],self.comm.getClientConf("transmission","crypto_key"))
if data == "q":
flag=True
break
elif ret_dict['method'].upper() == self.CONST.METHOD["upload"]:
ret_msg = up.savePackage(data,ret_dict)
# print(ret_msg)
if up.isFinish(ret_dict,len(data)):
up.syntheticFile(ret_dict)
else:
ret_msg = data+"11ss22"
conn.sendall(bytes(ret_msg))
if flag:
break
def __init__(self, ident):
self.dbg = debug.Debug()
self.com = common.Common()
self.gbl = common_var
self.ident = ident
new_name = re.sub(r'[^0-9A-Za-z]', '_', ident)
self.uixml = "ui_" + str(new_name) + ".xml" #ui控件文件名称
self.adbpath = self.gbl.path_adb #adb软件路径
self.uixml_tmp = os.path.join(self.gbl.path_tmp, self.uixml) #ui控件全路径
self.uixml_tmp2 = "/sdcard/" + self.uixml #终端里的 ui控件全路径
self.dy_packet = "com.ss.android.ugc.aweme" #抖音包名称
self.dy_activity = ".splash.SplashActivity" #抖音active名称(软件启动时使用)
def getContent(self):
cn = common.Common()
ret = []
try:
html = cn.getHtml(self._url)
soup = BeautifulSoup(html, 'html5lib')
#TODO
# lis = soup.select("")
# ret.append({'time': time, 'title': title, 'url': url})
except Exception as e:
print str(e)
return ret
def run(self, dirpath, file):
self.common = common.Common()
root_path = self.common.createDirByKey(file, self.key)
filelist = os.listdir(dirpath)
for zipfile in filelist:
cmd = "unzip -jxo " + dirpath + '/' + zipfile + ' -d ' + root_path
print 'cmd :', cmd
rtn = os.system(cmd)
if rtn == 0:
filename = dirpath + '/' + zipfile
os.remove(filename)
return True
def __init__(self, portInfo):
self.cfg = self.getCfg()
self.cmdList = self.getCmdList()
self.ip = portInfo['ip']
self.name = portInfo['name']
self.password = portInfo['password']
self.username = portInfo['username']
self.port = portInfo['port']
self.fileName = os.path.join('data', '%s_%s.tmp' % (
self.name,
self.ip,
))
self.com = common.Common()
def getContent(self):
cn = common.Common()
ret = []
try:
html = cn.getHtml(self._url)
soup = BeautifulSoup(html, 'html5lib')
trs = soup.select(".list_table tr")
for tr in trs:
time = re.search(r"\d{4}-\d{1,2}-\d{1,2}",tr.text).group(0)
href = tr.a['href']
title = tr.a.text.strip()
url = self._baseurl + href
ret.append({'time': time, 'title': title, 'url': url})
except Exception as e:
print str(e)
return ret
def __init__(self, portInfo):
self.portInfo = portInfo
self.cfg = self.getCfg()
self.cmdList = self.getCmdList()
self.ip = portInfo['ip']
print '03 :', portInfo.keys()
self.name = portInfo['name']
self.portInfo.pop('name')
self.password = portInfo['password']
self.username = portInfo['username']
self.fileName = os.path.join('data', '%s_%s.tmp' % (
self.name,
self.ip,
))
self.com = common.Common()
self.net_connect = ConnectHandler(**self.portInfo)
def getContent(self):
cn = common.Common()
ret = []
try:
html = cn.getHtml(self._url + str(self._pageSize))
soup = BeautifulSoup(html, 'html5lib')
lis = soup.select(".List2 ul li")
for li in lis:
time = li.span.text
href = li.a['href']
url = self._baseurl + href
title = li.a.text
ret.append({'time': time, 'title': title, 'url': url})
except Exception as e:
print str(e)
return ret
def __init__(self):
self.cf = ConfigParser.ConfigParser()
self.cf.read('monkey.conf')
self.cm = common.Common()
# add by jianan
self.crashCount = 0
self.anrCount = 0
self.tasksCrashCount = 0
self.tasksAnrCount = 0
self.tasks = {}
self.tasks["pkgName"] = self.cf.get('monkey_conf', 'package_name')
# self.tasks["duration"] = self.hours
self.tasks["tasks"] = []
def getContent(self):
cn = common.Common()
ret = []
try:
html = cn.getHtml(self._url)
soup = BeautifulSoup(html, 'html5lib')
lis = soup.select(".nei02_04_01 li")
for li in lis:
title = li.em.text.strip()
href = li.em.a['href']
time = li.i.text
url = self._baseurl + href
ret.append({'time': time, 'title': title, 'url': url})
except Exception as e:
print str(e)
return ret