def is_valid_submission(json_data):
try:
#jsonschema.validate(json_data, config.SUBMIT_SCHEMA)
return True
except jsonschema.ValidationError as e:
log("Invalid client response:", e, json_data)
return False
def get_game_for_client(self, client_id):
# Check if client is already in active game
game_state = self.db.get_client_game(client_id)
if game_state:
#log('Client in game', game_state.id)
if game_state and game_state.end_time > lib.now():
if game_state.has_client(client_id):
client = game_state.client(client_id)
else:
client = ClientState(client_id)
self.db.add_client_to_game(game_state, client)
return game_state, client
# Try to add to the client to the most recently started game
minimum_end_time = lib.now() + config.MIN_TIME_LEFT_MS
game_state = self.db.get_latest_game(minimum_end_time)
if not game_state:
# Start a new game for the client
game_state = self._new_game()
else:
log('Add client to existing game', game_state.id)
log('Adding client to game ', game_state.id)
client = ClientState(client_id)
game_state = self.db.add_client_to_game(game_state, client)
return game_state, client
def trace_ids_iterator(self,
start: datetime,
end: datetime,
filter_expr: str = 'ok or !ok'):
if not self.__less_than_24_hours(start, end):
end = start + timedelta(seconds=23 * 60 * 60)
response = self.get_trace_summaries(start=start,
end=end,
filter_expression=filter_expr)
next_token = response.get('NextToken', None)
while True:
summaries = response["TraceSummaries"]
log(f"Fetching {len(summaries)} trace summaries IDs")
for summary in summaries:
yield summary['Id']
if next_token is None:
break
summaries = self.get_trace_summaries(start=start,
end=end,
filter_expression=filter_expr,
next_token=next_token)
next_token = summaries.get('NextToken', None)
def rows2txt():
db = DdwDb()
count = db.get_book_count()["count"]
pageSize = 100
page = count / pageSize + 1
for i in range(0, page + 1):
step = i * pageSize + 94
rows = db.list(step)
if rows is not None:
for item in rows:
print "book -------------------------------" + str(item["id"])
path = mkdir(item["id"])
chs = db.get_chapters(item["id"])
if chs is not None:
for ch in chs:
# db.update_book_last_one(item['id'], one['id'], one['title'], one['create_at'])
chtext = db.get_chapter_text(ch["id"])
if chtext is not None:
try:
file_path = os.path.join(path, str(ch["id"]))
text = htmlstrip(chtext["text"])
# write to file
write2file(file_path, text)
# update chapter size and text
db.update_chapter_size(ch["id"], len(text))
db.update_chapter_text(ch["id"], text)
print ch["id"]
except Exception, ex:
log("Error: ch " + str(ch["id"]) + " " + str(Exception) + ":" + str(ex))
def save_book(id):
dw = DdwDb()
book = dw.get_book(id)
if book is None:
print '-------------begin ' + str(id) + '-------------'
book = get_book_detail(id)
if book is not None:
print '-------------saving-------------'
#author
author_id = 0
author = dw.get_author(book['author'])
if author is None:
author_id = dw.insert_author(book['author'])
else:
author_id = int(author['id'])
#category
cate_id = 0
cate = dw.get_category(book['cate'])
if cate is None:
cate_id = dw.insert_category(book['cate'])
else:
cate_id = int(cate['id'])
#book
desc = book['desc']
try:
desc = htmlstrip(desc).strip()
except Exception, ex:
log("Error: desc.error")
dw.insert_book(id, book['name'], desc, author_id, cate_id,
book['is_finish'], book['recommend_count'])
print '-------------end of ' + str(id) + '-----------'
def get_chapter(book_id, id):
try:
_url = __url__ + "%d/%d/%d.html" % (book_id / 1000, book_id, id)
print _url
html = urllib2.urlopen(_url).read()
"""
soup = BeautifulSoup(html,fromEncoding="gbk")
info = soup.findAll(attrs={'id' : "booktext"})[0]
info.strong.extract()
s = info.prettify()
s = s.replace('<div id="booktext">\n<!--go-->','')
s = s.replace('<!--over-->\n</div>','')
s = s.strip()
print s
return unicode(s, 'utf-8', "ignore")
"""
html = unicode(html, __encode__, "ignore")
m = re.compile(r'<!--go-->(?P<T>.*?)<!--over-->', re.S).search(html)
text = ''
if m:
text = m.group('T')
text = re.sub(r"(<STRONG>.*</STRONG><br><br>)", "", text)
return text.strip()
except urllib2.HTTPError:
log("Error: get_chapter book_id " + str(book_id) + " not exists.")
except Exception, ex:
log("Error: get_chapter book_id " + str(book_id) + " " +
str(Exception) + ":" + str(ex))
def read_file(fobj_read, delimiter=';', encoding='utf-8-sig', skip=0):
def get_real_header(header):
""" Get real header cut at the first empty column """
new_header = []
for head in header:
if head:
new_header.append(head)
else:
break
return new_header
def check_id_column(header):
try:
header.index('id')
except ValueError as ve:
log_error("No External Id (id) column defined, please add one")
raise ve
def skip_line(reader):
log_info("Skipping until line %s excluded" % skip)
for _ in xrange(1, skip):
reader.next()
log('open %s' % fobj_read.name)
reader = UnicodeReader(fobj_read,
delimiter=delimiter,
encoding='utf-8-sig')
header = reader.next()
header = get_real_header(header)
check_id_column(header)
skip_line(reader)
data = [l for l in reader]
return header, data
def rows2txt ():
db = DdwDb()
count = db.get_book_count()['count']
pageSize = 100
page = count / pageSize + 1
for i in range(0,page+1):
step = i * pageSize+94
rows = db.list(step)
if rows is not None:
for item in rows:
print "book -------------------------------" + str(item["id"])
path = mkdir(item["id"])
chs = db.get_chapters(item['id'])
if chs is not None:
for ch in chs:
#db.update_book_last_one(item['id'], one['id'], one['title'], one['create_at'])
chtext = db.get_chapter_text(ch['id'])
if chtext is not None:
try:
file_path = os.path.join(path, str(ch['id']))
text = htmlstrip(chtext["text"])
#write to file
write2file(file_path, text)
#update chapter size and text
db.update_chapter_size(ch['id'], len(text))
db.update_chapter_text(ch['id'], text)
print ch['id']
except Exception,ex:
log("Error: ch "+ str(ch['id']) + " " + str(Exception) + ":" + str(ex) )
def do_page(page, temp):
print page
params = dictify(temp)
if not 'sesso' in params:
return
sex = params['sesso'].lower()
if sex == 'm':
target = male
elif sex == 'f':
target = female
else:
lib.log('sesso', sex, page)
return
item = pywikibot.ItemPage.fromPage(page)
if not item.exists():
item = wdapi.createItem(page)
prop.setTarget(target)
add, reason = wdapi.canClaimBeAdded(item, prop, checkDupe=True)
if add:
print 'Adding a claim...'
item.addClaim(prop)
prop.addSource(lib.claim)
else:
print 'Not adding because: "{0}"'.format(reason)
def iterator(self):
log('Parsing inputs')
sqs_event = AwsSqsEvent(event=self.input_data)
for record in sqs_event.each_record():
sqs_record = AwsSqsEventRecord(record)
body = json.loads(sqs_record.get_body())
yield body
def get_chapter(book_id, id):
try:
_url = __url__ + "%d/%d/%d.html" % (book_id/1000, book_id, id)
print _url
html = urllib2.urlopen(_url).read()
"""
soup = BeautifulSoup(html,fromEncoding="gbk")
info = soup.findAll(attrs={'id' : "booktext"})[0]
info.strong.extract()
s = info.prettify()
s = s.replace('<div id="booktext">\n<!--go-->','')
s = s.replace('<!--over-->\n</div>','')
s = s.strip()
print s
return unicode(s, 'utf-8', "ignore")
"""
html = unicode(html, __encode__, "ignore")
m = re.compile(r'<!--go-->(?P<T>.*?)<!--over-->', re.S).search(html)
text = ''
if m:
text = m.group('T')
text = re.sub(r"(<STRONG>.*</STRONG><br><br>)", "", text)
return text.strip()
except urllib2.HTTPError:
log("Error: get_chapter book_id "+ str(book_id) + " not exists.")
except Exception,ex:
log("Error: get_chapter book_id "+ str(book_id) + " " + str(Exception) + ":" + str(ex) )
def save_book (id):
dw = DdwDb()
book = dw.get_book(id)
if book is None:
print '-------------begin '+ str(id)+'-------------'
book = get_book_detail(id)
if book is not None:
print '-------------saving-------------'
#author
author_id = 0
author = dw.get_author(book['author'])
if author is None:
author_id = dw.insert_author(book['author'])
else:
author_id = int(author['id'])
#category
cate_id = 0
cate = dw.get_category(book['cate'])
if cate is None:
cate_id = dw.insert_category(book['cate'])
else:
cate_id = int(cate['id'])
#book
desc = book['desc']
try:
desc = htmlstrip(desc).strip()
except Exception,ex:
log("Error: desc.error")
dw.insert_book(id, book['name'], desc, author_id, cate_id, book['is_finish'],book['recommend_count'])
print '-------------end of '+ str(id)+'-----------'
def do_page(page, temp):
print page
params = dictify(temp)
if not 'sesso' in params:
return
sex = params['sesso'].lower()
if sex == 'm':
target = male
elif sex == 'f':
target = female
else:
lib.log('sesso', sex, page)
return
item = pywikibot.ItemPage.fromPage(page)
if not item.exists():
item = wdapi.createItem(page)
prop.setTarget(target)
add, reason = wdapi.canClaimBeAdded(item, prop, checkDupe=True)
if add:
print 'Adding a claim...'
item.addClaim(prop)
prop.addSource(lib.claim)
else:
print 'Not adding because: "{0}"'.format(reason)
def __init__(self, parent=None):
QWidget.__init__(self, parent)
self.ui = MainWindow.Ui_MainWindow()
self.ui.setupUi(self)
# Translations
try:
if os.path.exists(os.getcwd() + "/translations/"):
dirr = os.getcwd() + "/translations/"
else:
dirr = '/usr/share/urtdsc/'
translator = QTranslator(app)
translator.load("urtdsc-" + QLocale.system().name() + ".qm", dirr)
app.installTranslator(translator)
lib.log(
'1', "Loaded translation for: " + QLocale.system().name() +
" from " + dirr)
except:
lib.log('1', "Cant load translations!")
# Settings
self.settings = QSettings('urtdsc', 'settings')
self.restoreGeometry(self.settings.value('geometry').toByteArray())
self.restoreState(self.settings.value('state').toByteArray())
if os.path.exists(os.path.expanduser('~/.q3a/q3ut4/demos')):
pass
else:
nodemos = '1'
self.fillDemosList(1)
self.ui.label.setText("")
# We need this for translations >.<
self.ui.copyDemoToDesktop.setText(self.tr("Copy demo to desktop"))
self.ui.viewAllScreenshots.setText(self.tr("View all screenshots"))
self.ui.createArchive.setText(self.tr("Create archive..."))
self.ui.menu_File.setTitle(self.tr("&File"))
self.ui.action_Options.setText(self.tr("&Options"))
self.ui.action_Exit.setText(self.tr("&Exit"))
self.ui.menu_Demos.setTitle(self.tr("&Demos"))
self.ui.action_Update_list.setText(self.tr("&Update list"))
self.ui.menu_Help.setTitle(self.tr("&Help"))
self.ui.action_About_urtdsc.setText(self.tr("About &UrTDSC..."))
self.ui.actionAbout_Qt.setText(self.tr("About &Qt..."))
QMetaObject.connectSlotsByName(self)
self.ui.demosList.itemClicked.connect(self.showDemoInfo)
self.ui.demosList.itemSelectionChanged.connect(self.showDemoInfo)
self.ui.demosList.itemActivated.connect(self.showDemoInfo)
self.ui.demosList.customContextMenuRequested.connect(
self.demoContextMenu)
self.ui.copyDemoToDesktop.clicked.connect(self.copydemos)
self.ui.viewAllScreenshots.clicked.connect(self.allScreenshotsDialog)
self.ui.action_Update_list.triggered.connect(self.fillDemosList)
self.ui.action_About_urtdsc.triggered.connect(self.aboutWindow)
self.ui.actionAbout_Qt.triggered.connect(self.aboutQt)
self.ui.action_Exit.triggered.connect(self.close)
def execute(self, sql, params=()):
try:
cur = self.connection.cursor()
return cur.execute(sql, params)
except Exception as e:
lib.log("Error at SQL")
lib.log(sql)
raise e
def alert(msg, level):
"""
alert user
@param msg string the message to send
@param grade int error level 1: fatal 2: warning
@return null
"""
lib.log(msg, '/data0/logs/mysqlmond.pid')
def test_post(self):
splunk.access_token = ''
splunk.project_id = ''
splunk.log({
'event': 'event',
'duration': '5ms'
})
def restartSeq(self, seqname):
try:
cur = self.connection.cursor()
sql = "alter sequence %s restart with 1" % seqname
return cur.execute(sql)
except Exception as e:
lib.log("Error at SQL")
lib.log(sql)
raise e
def truncateTable(self, tablename):
try:
cur = self.connection.cursor()
sql = "truncate table %s;" % tablename
return cur.execute(sql)
except Exception as e:
lib.log("Error at SQL")
lib.log(sql)
raise e
def __init__(self, parent=None):
QWidget.__init__(self, parent)
self.ui = MainWindow.Ui_MainWindow()
self.ui.setupUi(self)
# Translations
try:
if os.path.exists(os.getcwd() + "/translations/"):
dirr = os.getcwd() + "/translations/"
else:
dirr = '/usr/share/urtdsc/'
translator = QTranslator(app)
translator.load("urtdsc-" + QLocale.system().name() + ".qm", dirr)
app.installTranslator(translator)
lib.log('1', "Loaded translation for: " + QLocale.system().name() + " from " + dirr)
except:
lib.log('1', "Cant load translations!")
# Settings
self.settings = QSettings('urtdsc', 'settings')
self.restoreGeometry(self.settings.value('geometry').toByteArray())
self.restoreState(self.settings.value('state').toByteArray())
if os.path.exists(os.path.expanduser('~/.q3a/q3ut4/demos')):
pass
else:
nodemos = '1'
self.fillDemosList(1)
self.ui.label.setText("")
# We need this for translations >.<
self.ui.copyDemoToDesktop.setText(self.tr("Copy demo to desktop"))
self.ui.viewAllScreenshots.setText(self.tr("View all screenshots"))
self.ui.createArchive.setText(self.tr("Create archive..."))
self.ui.menu_File.setTitle(self.tr("&File"))
self.ui.action_Options.setText(self.tr("&Options"))
self.ui.action_Exit.setText(self.tr("&Exit"))
self.ui.menu_Demos.setTitle(self.tr("&Demos"))
self.ui.action_Update_list.setText(self.tr("&Update list"))
self.ui.menu_Help.setTitle(self.tr("&Help"))
self.ui.action_About_urtdsc.setText(self.tr("About &UrTDSC..."))
self.ui.actionAbout_Qt.setText(self.tr("About &Qt..."))
QMetaObject.connectSlotsByName(self)
self.ui.demosList.itemClicked.connect(self.showDemoInfo)
self.ui.demosList.itemSelectionChanged.connect(self.showDemoInfo)
self.ui.demosList.itemActivated.connect(self.showDemoInfo)
self.ui.demosList.customContextMenuRequested.connect(self.demoContextMenu)
self.ui.copyDemoToDesktop.clicked.connect(self.copydemos)
self.ui.viewAllScreenshots.clicked.connect(self.allScreenshotsDialog)
self.ui.action_Update_list.triggered.connect(self.fillDemosList)
self.ui.action_About_urtdsc.triggered.connect(self.aboutWindow)
self.ui.actionAbout_Qt.triggered.connect(self.aboutQt)
self.ui.action_Exit.triggered.connect(self.close)
def collect(hosts):
if ping_method == 'fping':
responses = fping(hosts)
else:
responses = ping(hosts)
for (host, delay) in responses:
add_value(host, delay)
log('collecting host ' + host + ': delay ' + delay, 1)
def parse_ping_response(string):
if string.find('avg') == -1:
log('ping: no answer, ping output was' + string, 2)
return 'U'
stats_start = string.find('=')
avg_start = string.find('/', stats_start)+1
avg_end = string.find('/', avg_start)
return string[avg_start:avg_end]
def text ():
_count = 22962
for i in range(2900, _count+1):
#if i%10==1:
#print "sleeping..."
#time.sleep(2)
try:
save_text(i)
except Exception,ex:
log("Error: chapters book_id "+ str(i) + " " + str(Exception)+":"+str(ex) )
def ClientRead(data):
opcode_and_fin = data[0]
msg_len = data[1] - 128
mask = data[2:6]
encrypted_msg = data[6: 6+msg_len]
msg = bytearray([encrypted_msg[i] ^ mask[i % 4]
for i in range(msg_len)])
clientMsg = str(msg, "utf-8")
log('Incoming message: ' + str(clientMsg))
return clientMsg
def books ():
_count = 24221
for i in range(24154, _count+1):
if i%50==1:
print "sleeping..."
time.sleep(2)
try:
save_book(i)
except Exception,ex:
log("Error: book_id "+ str(i) + " " + str(Exception)+":"+str(ex) )
def chapters ():
_count = 24221
for i in range(24154, _count+1):
#if i%10==1:
#print "sleeping..."
#time.sleep(2)
try:
save_chapters(i)
except Exception,ex:
log("Error: chapters book_id "+ str(i) + " " + str(Exception)+":"+str(ex) )
def split_sort(split, header, data):
split_index = 0
if split:
try:
split_index = header.index(split)
except ValueError as ve:
log("column %s not defined" % split)
raise ve
data = sorted(data, key=lambda d: d[split_index])
return data, split_index
def chapters():
_count = 24221
for i in range(24154, _count + 1):
#if i%10==1:
#print "sleeping..."
#time.sleep(2)
try:
save_chapters(i)
except Exception, ex:
log("Error: chapters book_id " + str(i) + " " + str(Exception) +
":" + str(ex))
def captureAndUpload(self, imgName):
localtime = str(time.strftime("%H%M%S", time.localtime()))
homeImgName = imgName + '_' + localtime + '.png'
self.driver.save_screenshot(homeImgName)
localImg = os.path.abspath(homeImgName)
self.localImgList.append(localImg)
ossPath = self.getOssPath(homeImgName)
fullOssPath = self.cdnHost + '/' + ossPath
self.ossImgList.append(fullOssPath)
upload(ossPath, localImg)
log('异常截图已上传: ' + fullOssPath)
def text():
_count = 22962
for i in range(2900, _count + 1):
#if i%10==1:
#print "sleeping..."
#time.sleep(2)
try:
save_text(i)
except Exception, ex:
log("Error: chapters book_id " + str(i) + " " + str(Exception) +
":" + str(ex))
def books():
_count = 24221
for i in range(24154, _count + 1):
if i % 50 == 1:
print "sleeping..."
time.sleep(2)
try:
save_book(i)
except Exception, ex:
log("Error: book_id " + str(i) + " " + str(Exception) + ":" +
str(ex))
def parse_fping_response(string):
end_host = string.find(' ')
start_delay = string.find(':')+2
host = string[0:end_host]
if string.find('avg') == -1:
log('fping: no answer, fping output was' + string, 2)
return (host, 'U')
stats_start = string.find('max')+5
avg_start = string.find('/', stats_start)+1
avg_end = string.find('/', avg_start)
return (host, string[avg_start:avg_end])
def on_event(self, event, extension):
try:
vrs = extension.variables
data = event.get_data()
if "reset" in data:
log("LIST DOCSETS RESET", LogType.DEBUG)
# Unselect docset
extension.unselect_docset()
# List docsets
return RenderResultListAction(
extension.get_sorted_docsets(vrs.query))
log("QUERY IN DOCSET", LogType.DEBUG)
# Select a docset
extension.select_docset(data)
# Reset query to allow keyword search in the docset
vrs.query = ""
# Update UI query string
extension.update_ui_query_string(event, vrs.query)
# you may want to return another list of results
return RenderResultListAction([
ResultItems.docset_searchforin_result(vrs.sel_docset,
vrs.query),
ResultItems.docset_change_result(),
])
except Exception as error: # pylint: disable=broad-except
log(error, LogType.ERROR)
log(traceback.format_exc(), LogType.DEBUG)
def get_chapters(id):
try:
_url = __url__ + "%d/%d/index.html" % ((id / 1000), id)
print _url
html = urllib2.urlopen(_url).read()
soup = BeautifulSoup(html, fromEncoding="gbk")
a = dict()
volumes = soup.findAll(attrs={'class': re.compile("^book_article_")})
flag = False
i = 0
j = 0
sort = 0
count = len(volumes)
_re = re.compile(r'(?P<T>\d+?)\.html')
for v in volumes:
if v.attrs[0][1] == "book_article_texttitle":
a[i] = {}
a[i]['title'] = v.contents[0]
a[i]['chs'] = []
flag = True
if v.attrs[0][1] == "book_article_listtext" and flag == True:
chs = v.findAll({"a": True})
for x in chs:
id = int(_re.search(x.attrs[0][1]).group("T"))
title = x.contents[0]
var = id, title, sort
a[i]['chs'].append(var)
sort += 1
if j < count - 1:
if volumes[j + 1].attrs[0][1] == "book_article_texttitle":
flag = False
i += 1
j += 1
return a
except urllib2.HTTPError:
log("Error: get_chapters book_id " + str(id) + " not exists.")
except Exception, ex:
log("Error: get_chapters book_id " + str(id) + " " + str(Exception) +
":" + str(ex))
def get_chapters (id):
try:
_url = __url__ + "%d/%d/index.html" % ((id/1000), id)
print _url
html = urllib2.urlopen(_url).read()
soup = BeautifulSoup(html,fromEncoding="gbk")
a = dict()
volumes = soup.findAll(attrs={'class' : re.compile("^book_article_")})
flag = False
i = 0
j = 0
sort = 0
count = len(volumes)
_re = re.compile(r'(?P<T>\d+?)\.html')
for v in volumes:
if v.attrs[0][1] == "book_article_texttitle":
a[i] = {}
a[i]['title'] = v.contents[0]
a[i]['chs'] = []
flag = True
if v.attrs[0][1] == "book_article_listtext" and flag == True:
chs = v.findAll({"a":True})
for x in chs:
id = int(_re.search(x.attrs[0][1]).group("T"))
title = x.contents[0]
var = id, title, sort
a[i]['chs'].append(var)
sort += 1
if j < count-1 :
if volumes[j+1].attrs[0][1] == "book_article_texttitle":
flag = False
i += 1
j += 1
return a
except urllib2.HTTPError:
log("Error: get_chapters book_id "+ str(id) + " not exists.")
except Exception,ex:
log("Error: get_chapters book_id "+ str(id) + " " + str(Exception) + ":" + str(ex) )
def get_book_detail(id):
try:
_url = __url__ + "info/%d/%d.htm" % ((id / 1000), id)
html = urllib2.urlopen(_url).read()
#html = unicode(html, __encode__, "ignore")
soup = BeautifulSoup(html)
info = soup.findAll(id=['In'])[0]
#print type(info)
#print info.__dict__
#print info.prettify()
a = dict()
m = re.compile(r'作 者：(?P<T>.*?)<', re.S).search(info.prettify())
if m: a['author'] = m.group('T').strip()
m = re.compile(r'类 别：(?P<T>.*?)<', re.S).search(info.prettify())
if m: a['cate'] = m.group('T').strip()
m = re.compile(r'状 态：(?P<T>.*?)<', re.S).search(info.prettify())
if m:
tmp = m.group('T').strip()
a['is_finish'] = False
if tmp == "已完成":
a['is_finish'] = True
m = re.compile(r'票 票：(?P<T>.*?)<', re.S).search(info.prettify())
if m: a['recommend_count'] = int(m.group('T'))
m = re.compile(r'内容简介：(?P<T>.*/>)(?P<Name>.*?)最新章节',
re.S).search(info.prettify())
if m:
a['desc'] = m.group('T').strip() #md.convert
a['name'] = m.group('Name').strip()
#print a
return a
except urllib2.HTTPError:
log("Error: book_id " + str(id) + " not exists.")
except Exception, ex:
log("Error: book_id " + str(id) + " " + str(Exception) + ":" + str(ex))
def msgHandler(msg):
msgTag = ''
msgAtt = ''
msgCmd = ''
result = ''
message = msg.split('/')
log(message)
if len(message) == 1:
msgTag = message[0]
if len(message) == 2:
msgTag = message[0]
msgAtt = message[1]
if len(message) == 3:
msgTag = message[0]
msgAtt = message[1]
msgCmd = message[2]
log(msgTag + '/' + msgAtt + '/' + msgCmd)
for tag in tags:
if msgTag != '':
if tags[tag]['name'] == msgTag:
if msgCmd == '':
# Request received
if msgAtt == 'io':
result = str(tags[tag][msgAtt].value())
else:
if msgAtt != '':
try:
result = str(tags[tag][msgAtt])
except:
result = 'Invalid attribute...'
else:
result = 'attribute missing...'
else:
# Command received
if type(tags[tag][msgAtt]) is str:
tags[tag][msgAtt] = msgCmd
else:
if msgAtt == 'io':
tags[tag][msgAtt].value(int(msgCmd))
else:
tags[tag][msgAtt] = int(msgCmd)
result = 'Command: ' + msgTag + '/' + msgAtt + '/' + msgCmd + ' received!'
if len(msg) == 0:
result = 'connected!'
if result != '':
log(result)
clientWrite(result)
if result == '':
log('cannot process request or command...')
clientWrite('cannot process request or command...')
def mkdesc ():
db = DdwDb()
count = db.get_book_count()['count']
pageSize = 100
page = count / pageSize + 1
for i in range(0,page+1):
step = i * pageSize
rows = db.list(step)
if rows is not None:
for item in rows:
print "book -------------------------------" + str(item["id"])
b = db.get_book(item['id'])
try:
desc = htmlstrip(b["description"]).strip()
db.update_book_desc(item['id'], desc)
except Exception,ex:
log("Error: mkdesc "+ str(item['id']) + " " + str(Exception) + ":" + str(ex),"desc.error")
def get_book_detail (id):
try:
_url = __url__ + "info/%d/%d.htm" % ((id/1000), id)
html = urllib2.urlopen(_url).read()
#html = unicode(html, __encode__, "ignore")
soup = BeautifulSoup(html)
info = soup.findAll(id=['In'])[0]
#print type(info)
#print info.__dict__
#print info.prettify()
a = dict()
m = re.compile(r'作 者：(?P<T>.*?)<', re.S).search(info.prettify())
if m: a['author'] = m.group('T').strip()
m = re.compile(r'类 别：(?P<T>.*?)<', re.S).search(info.prettify())
if m: a['cate'] = m.group('T').strip()
m = re.compile(r'状 态：(?P<T>.*?)<', re.S).search(info.prettify())
if m:
tmp = m.group('T').strip()
a['is_finish'] = False
if tmp == "已完成":
a['is_finish'] = True
m = re.compile(r'票 票：(?P<T>.*?)<', re.S).search(info.prettify())
if m: a['recommend_count'] = int(m.group('T'))
m = re.compile(r'内容简介：(?P<T>.*/>)(?P<Name>.*?)最新章节', re.S).search(info.prettify())
if m:
a['desc'] = m.group('T').strip()#md.convert
a['name'] = m.group('Name').strip()
#print a
return a
except urllib2.HTTPError:
log("Error: book_id "+ str(id) + " not exists.")
except Exception,ex:
log("Error: book_id "+ str(id) + " " + str(Exception) + ":" + str(ex) )
def mkdesc():
db = DdwDb()
count = db.get_book_count()["count"]
pageSize = 100
page = count / pageSize + 1
for i in range(0, page + 1):
step = i * pageSize
rows = db.list(step)
if rows is not None:
for item in rows:
print "book -------------------------------" + str(item["id"])
b = db.get_book(item["id"])
try:
desc = htmlstrip(b["description"]).strip()
db.update_book_desc(item["id"], desc)
except Exception, ex:
log("Error: mkdesc " + str(item["id"]) + " " + str(Exception) + ":" + str(ex), "desc.error")
def on_event(self, event, extension):
try:
vrs = extension.variables
vrs.keyword = event.query.split(" ")[
0] # store docset used keyword
query = vrs.query = event.query.replace(vrs.keyword + " ", "", 1)
# Is a docset selected ?
if vrs.is_sel:
log("EXEC SEARCH", LogType.DEBUG)
vrs.query = query.strip()
return RenderResultListAction([
ResultItems.docset_searchforin_result(
vrs.sel_docset, vrs.query),
ResultItems.docset_change_result(),
])
query = event.query.replace(vrs.keyword + " ", "", 1)
# If User types a docset ID withinz keyword
if Docsets.docsetDict.get(query):
log("SELECTED A DOCSET", LogType.DEBUG)
# Select a docset
extension.select_docset(query)
# Update UI query string
extension.update_ui_query_string(event, "")
return RenderResultListAction([
ResultItems.docset_searchforin_result(vrs.sel_docset, ""),
ResultItems.docset_change_result(),
])
# ELSE User would select a docset
log("LIST DOCSETS", LogType.DEBUG)
# Unselect docset
extension.unselect_docset()
# List docsets
return RenderResultListAction(extension.get_sorted_docsets(query))
except Exception as error: # pylint: disable=broad-except
log(error, LogType.ERROR)
log(traceback.format_exc(), LogType.DEBUG)
def timed(*args, **kw):
global is_start
global nesting_level
if not is_start:
print()
is_start = True
log("{}.".format(method.__name__))
nesting_level += 1
start_time = time.time()
result = method(*args, **kw)
end_time = time.time()
nesting_level -= 1
log("/{} ({:0.2f} sec)".format(method.__name__, end_time - start_time))
is_start = False
return result
def __init__(self):
"""Initiate and include some class"""
dir = os.path.dirname(__file__)
includedir = os.path.join(dir, "../")
sys.path.append(includedir)
from lib.log import *
from conf.Config import *
self.Conf = Config()
self.Log = log()
def save_text (id):
dw = DdwDb()
book = dw.get_book(id)
if book is not None:
print '-------------begin '+ str(id)+'-------------'
chs = dw.get_chapters(id)
if chs is not None or len(chs) > 0:
for var in chs:
print var['id']
text = ''
t = dw.get_chapter_text(var['id'])
if t is None:
try:
text = get_chapter(id, var['id'])
if text is not None and text != '':
dw.insert_chapter_text(var['id'], text)
print '-------------end of '+ str(id)+'-----------'
except Exception,ex:
log("Error: get_chapter "+ str(id) + " " + str(Exception)+":"+str(ex) )
print "Info: chapter "+str(var['id'])+" exists"
else:
print 'Info: have no chapters'
def test_login(self):
try:
#输入账号密码
accountInput = self.driver.find_element_by_id(
"j-phone-normal-input")
accountInput.send_keys(self.account)
pswInput = self.driver.find_element_by_id(
"j-password-normal-input")
pswInput.send_keys(self.password)
#输入回车
pswInput.send_keys(Keys.RETURN)
try:
#输入账号密码登录后，看下错误弹窗有无出来
WebDriverWait(self.driver, 2).until(
EC.visibility_of_element_located(
(By.CLASS_NAME, "xh-message")))
#如果有出来，截图，并且发钉钉消息
log('发现异常')
self.captureAndUpload('error')
sendDingMsg(
self.createDingMessageJson(title='天幕管理后台前端监控',
subtitle='登录测试未通过',
imgList=self.ossImgList))
except:
#如果没有出来，检查进入主页后有无class为org-list的元素（页面是否正常加载）
WebDriverWait(self.driver, 2).until(
EC.presence_of_element_located(
(By.CLASS_NAME, "org-list")))
except:
#若报错，截图，发钉钉消息
log('发现异常')
self.captureAndUpload('error')
sendDingMsg(
self.createDingMessageJson(title='天幕管理后台前端监控',
subtitle='登录测试未通过',
imgList=self.ossImgList))
def setUp(self):
driverPath = os.path.abspath('chromedriver')
webaddr = webInfo['addr']
option = webdriver.ChromeOptions()
option.add_argument("headless")
self.driver = Chrome(driverPath, options=option)
self.driver.get(webaddr)
#模拟登录账号
self.account = webInfo['account']
#模拟登录密码
self.password = webInfo['password']
#图片上传oss的地址
self.ossPath = webInfo['ossPath']
#图片的cdn域名
self.cdnHost = webInfo['cdnHost']
#若有错误，截图后图片的本地的地址
self.localImgList = []
#若有错误，截图并上传到oss后，oss的地址
self.ossImgList = []
log('网址已打开: ' + webaddr + '。现在登录, 账号: ' + self.account + ', 密码: ' +
self.password)
def save_text(id):
dw = DdwDb()
book = dw.get_book(id)
if book is not None:
print '-------------begin ' + str(id) + '-------------'
chs = dw.get_chapters(id)
if chs is not None or len(chs) > 0:
for var in chs:
print var['id']
text = ''
t = dw.get_chapter_text(var['id'])
if t is None:
try:
text = get_chapter(id, var['id'])
if text is not None and text != '':
dw.insert_chapter_text(var['id'], text)
print '-------------end of ' + str(
id) + '-----------'
except Exception, ex:
log("Error: get_chapter " + str(id) + " " +
str(Exception) + ":" + str(ex))
print "Info: chapter " + str(var['id']) + " exists"
else:
print 'Info: have no chapters'
def scheduler():
min = (utime.localtime()[4])
hour = int((utime.localtime()[3]) + 7)
# Adjust hours between 24-30
if hour > 23 and hour < 31:
hour = hour - 24
currTime = str(hour) + ':' + str(min)
print(currTime)
clientWrite('time-' + currTime)
if hour == 23 and min == 45:
try:
os.remove('logs.txt')
log('logs cleared...\n')
except:
log('logs file purge error!')
if min == 58:
lib.syncClock()
for tag in tags:
schedule = tags[tag]['schedule']
sched = schedule.split('/')
for hm in sched:
start = hm.split(' ')[0]
startH = int(start.split(':')[0])
startM = int(start.split(':')[1])
stop = hm.split(' ')[1]
stopH = int(stop.split(':')[0])
stopM = int(stop.split(':')[1])
if hour == startH and min >= startM and min < stopM:
if tags[tag]['io'].value() == 1:
log('start tag: ' + tags[tag]['name'])
tags[tag]['io'].value(0)
clientWrite(tags[tag]['name'] + '/0')
if ((hour == startH) and (min >= stopM)) or ((hour == startH + 1) and (min - stopM < 0)):
# if hour == startH:
if tags[tag]['io'].value() == 0:
log('stop tag: ' + tags[tag]['name'])
# print(type(tag))
tags[tag]['io'].value(1)
clientWrite(tags[tag]['name'] + '/1')
def graph(name, functions, settings, preview): # x_pos, y_pos, scale):
x_pos, y_pos, scale = settings['x_pos'], settings['y_pos'], settings['scale']
log("pl.graph")
grapher.main(name, functions, x_pos, y_pos, scale / 10 * const.PIXELS_PER_MM, preview)
# ping a list of hosts
def ping(hosts):
for host in hosts:
pipe = subprocess.Popen([ping_executable, '-c', str(packets), host],
stdout=subprocess.PIPE)
response = pipe.communicate()[0].splitlines()[-1] # last line
yield (host, parse_ping_response(response))
# collect data
def collect(hosts):
if ping_method == 'fping':
responses = fping(hosts)
else:
responses = ping(hosts)
for (host, delay) in responses:
add_value(host, delay)
log('collecting host ' + host + ': delay ' + delay, 1)
if __name__ == "__main__":
from lib.get_hosts import get_hosts
log('Collect started', 1)
tstart = time.time()
collect(get_hosts())
tend = time.time() - tstart
log('Collect finished and ran for ' + str(tend), 1)
def main():
log("started")
log(sys.path)
log("wait while unlocking...")
while is_locked():
time.sleep(1)
lock()
try:
log("generating cnc")
gen_cnc()
log("sending cnc")
send_cnc()
log("done")
except Exception as e:
log("ERROR")
tb = traceback.format_exc()
print(tb, file=sys.stderr)
unlock()
log("stopped")
def unlock():
os.remove('.lock')
log("unlocked")
def lock():
open('.lock', 'a').close()
log("locked")
