async def check(url, cms_tuple): # 检测主函数
global SESSION, SUCCESS, REQUEST_DICT
if (len(SUCCESS) > 0):
return (False, tuple())
cms_name = cms_tuple[0]
cms_path = cms_tuple[1]
cms_match_pattern = cms_tuple[2]
cms_id = cms_tuple[3]
message_list = []
try:
if (cms_path in REQUEST_DICT):
data = REQUEST_DICT[cms_path]
if (rule(RULE_NAME, cms_match_pattern, data)):
message_list.append(cms_name)
try:
CONN, CURSOR = connect()
cms_hit = select_cms_hit(CURSOR, cms_id)
update_cms(CONN, CURSOR, cms_hit, cms_id) # hit++
CONN.close()
except Exception:
pass
return (True, tuple(message_list))
else:
return (False, tuple(message_list))
else:
return (False, tuple(message_list))
except Exception:
return (False, tuple(message_list))
def importTags():
(connection, cursor) = general.connect()
print 'Parsing the tags file'
tags = parse_tags(config.websiteProject + "/tags/tags")
labels = get_labels_from_source(config.websiteProject + "/tags/tmp/")
print 'Inserting (or updating) the tags'
for tag, label in tags.iteritems():
if label not in labels:
print 'ERROR, label', label, 'not found in auxiliary files. Have you ran `make tags`?'
sys.exit()
if tag_exists(tag, cursor) and get_label(tag, cursor) != label:
print 'The label for tag %s has changed from \'%s\'to \'%s\'' % (
tag, get_label(tag, cursor), label)
if not tag_exists(tag, cursor):
print 'New tag %s with label \'%s\'' % (tag, label)
info = labels[label]
insert_tag(tag, (label, info[0], info[2][1], info[1][1], info[1][0],
info[1][2], info[2][3], info[1][4]), cursor)
general.close(connection)
async def check(url, cms_tuple, timeout): # MD5检测
global SESSION, NUM
if (len(SUCCESS) > 0):
return (False, tuple())
cms_name = cms_tuple[0]
cms_path = cms_tuple[1]
cms_match_pattern = cms_tuple[2]
cms_id = cms_tuple[3]
cms_hit = cms_tuple[4]
message_list = []
target_url = url + cms_path
if (VERBOSE is True):
message('#', 'target:%s' % target_url)
try:
req = SESSION.get(
target_url, headers=requests_headers(), timeout=timeout)
if (req.status_code == 200 and cms_match_pattern.lower() in req.text.lower()):
message_list.append(cms_name)
try:
CONN, CURSOR = connect()
update(CONN, CURSOR, cms_hit, cms_id) # hit++
close(CONN)
except Exception:
pass
return (True, tuple(message_list))
else:
return (False, tuple(message_list))
except Exception:
return (False, tuple(message_list))
finally:
NUM += 1
def importTitles():
global connection
(connection, cursor) = general.connect()
print 'Creating a database version of the table of contents'
print 'Parsing the files, linking chapters to file names'
titles = get_titles(config.websiteProject + "/tags/tmp/")
print 'Parsing the big table of contents'
sections = parse_book_toc(config.websiteProject + "/tags/tmp/book.toc")
# print out new or changed titles before updating the database
for section in sections:
if title_exists(section[2]) == 0 and not section[2] == 'Bibliography':
print 'New or changed section \'%s\'' % (section[2])
print 'Inserting the information into the database'
for section in sections:
# the bibliography doesn't correspond to a file, we can safely ignore it
if section[2] == 'Bibliography':
continue
insert_title(section[1], section[2],
find_file_for_section(titles, sections, section[1]))
general.close(connection)
def main(dataset):
# Database Connector
dbc = general.connect()
# if dataset in 'SFBS':
# df = distance.distance(dbc, dataset, 'src', 'dest')
# else:
# df = distance.distance(dbc, dataset, 'start', 'location')
# print(df)
# if dataset in 'SFBS':
# df = duration.duration(dbc, dataset)
# else:
# df = duration.duration(dbc, dataset)
# print(df)
# if dataset in 'SFBS':
# df = saturation.saturation(dbc, dataset, 'with', 'src', 'dest', 1, 3, 'SFBS1')
# df = saturation.saturation(dbc, dataset, 'with', 'src', 'dest', 2, 3, 'SFBS2')
# else:
# df = saturation(dbc, dataset, 'act', 'start', None, 1, 0, 'SFFD')
if dataset in 'SFBS':
forecast.count_model('data/station70_forecast')
elif dataset in 'SFFD':
forecast.count_model('data/satur')
elif dataset in 'ISSUE':
# FORECAST
forecast.model_cross_version(['data/hadoop-version_2_prior-2.csv', 'data/hbase-version_0_prior-2.csv', 'data/hive-version_2_prior-2.csv', 'data/hadoop-version_1_prior-2.csv'], 'data/hadoop-version_3_prior-2.csv')
forecast.model_cross_version(['data/hadoop-version_2_prior-4.csv', 'data/hbase-version_0_prior-4.csv', 'data/hive-version_2_prior-4.csv', 'data/hadoop-version_1_prior-4.csv'], 'data/hadoop-version_3_prior-4.csv')
forecast.model_cross_version(['data/hadoop-version_2_prior-8.csv', 'data/hbase-version_0_prior-8.csv', 'data/hive-version_2_prior-8.csv', 'data/hadoop-version_1_prior-8.csv'], 'data/hadoop-version_3_prior-8.csv')
forecast.model_cross_version(['data/hadoop-version_2_prior-10.csv', 'data/hbase-version_0_prior-10.csv', 'data/hive-version_2_prior-10.csv', 'data/hadoop-version_1_prior-10.csv'], 'data/hadoop-version_3_prior-10.csv')
def handle(name, key, cms_id, header, body, title): # 对数据库取出的信息进行拆分并开始指纹识别
# 满足一个条件即可的情况
success_list = []
if ('||' in key and '&&' not in key and '(' not in key):
for rule in key.split('||'):
if check_rule(rule, header, body, title):
success_list.append(name)
break
# 只有一个条件的情况
elif ('||' not in key and '&&' not in key and '(' not in key):
if check_rule(key, header, body, title):
success_list.append(name)
# 需要同时满足条件的情况
elif ('&&' in key and '||' not in key and '(' not in key):
num = 0
for rule in key.split('&&'):
if check_rule(rule.strip(), header, body, title):
num += 1
if (num == len(key.split('&&'))):
success_list.append(name)
else:
# 与条件下存在并条件: 1||2||(3&&4)
if ('&&' in re.findall(rbracket, key)[0]):
for rule in key.split('||'):
if ('&&' in rule):
num = 0
for _rule in rule.split('&&'):
if (check_rule(_rule, header, body, title)):
num += 1
if (num == len(rule.split('&&'))):
success_list.append(name)
break
else:
if (check_rule(rule, header, body, title)):
success_list.append(name)
break
else:
# 并条件下存在与条件： 1&&2&&(3||4)
for rule in key.split('&&'):
num = 0
if '||' in rule:
for _rule in rule.split('||'):
if (check_rule(_rule, title, body, header)):
num += 1
break
else:
if (check_rule(rule, title, body, header)):
num += 1
if (num == len(key.split('&&'))):
success_list.append(name)
for each in success_list:
try:
CONN, CURSOR = connect()
hit = select_fofa_hit(CURSOR, cms_id)
update_fofa(CONN, CURSOR, hit, cms_id) # hit++
CONN.close()
except Exception:
pass
return success_list
def __init__(self, usernames: List[str]):
self.conn = general.connect()
self.usernames = usernames
for name in usernames:
if general.get_user(self.conn, name) is None:
general.register(self.conn, name)
self.users = [User(self.conn, username=name) for name in usernames]
def updateCounts():
(connection, cursor) = general.connect()
for tag, label in tags:
print "updating " + tag
update(tag, tag_node_count[tag], tag_edge_count[tag], tag_total_edge_count[tag], tag_chapter_count[tag], tag_section_count[tag], tag_use_count[tag], tag_indirect_use_count[tag], cursor)
general.close(connection)
def insertDependencies():
(connection, cursor) = general.connect()
for tag, label in tags:
for child in tags_refs[tag]:
addDependency(tag, child, cursor)
general.close(connection)
def updateCounts():
(connection, cursor) = general.connect()
for tag, label in tags:
print "updating " + tag
update(tag, tag_node_count[tag], tag_edge_count[tag], tag_total_edge_count[tag], tag_chapter_count[tag], tag_section_count[tag], tag_use_count[tag], tag_indirect_use_count[tag], cursor)
general.close(connection)
def insertDependencies():
(connection, cursor) = general.connect()
for tag, label in tags:
for child in tags_refs[tag]:
addDependency(tag, child, cursor)
general.close(connection)
def clearCounts():
(connection, cursor) = general.connect()
try:
query = 'DELETE FROM graphs'
cursor.execute(query)
except sqlite3.Error, e:
print "An error occurred:", e.args[0]
def clearSearchTable():
(connection, cursor) = general.connect()
try:
query = 'DELETE FROM tags_search'
connection.execute(query)
except sqlite3.Error, e:
print "An error occurred:", e.args[0]
def clearCounts():
(connection, cursor) = general.connect()
try:
query = 'DELETE FROM graphs'
cursor.execute(query)
except sqlite3.Error, e:
print "An error occurred:", e.args[0]
def importLaTeX():
global connection
(connection, cursor) = general.connect()
n = 0
while n < len(tags):
tag = tags[n][0]
label = tags[n][1]
text = ''
if label in label_linenumbers.keys():
update_linenumbers(tag, label_linenumbers[label])
if label in label_texts:
text = text + label_texts[label]
if label in proof_texts:
text = text + '\n' + proof_texts[label]
# if text has changed and current text isn't empty (i.e. not a new tag)
if get_text(tag) != text and get_text(tag) != '':
print "The text of tag", tag, "has changed",
if label in proof_texts and extract_proofs(get_text(tag)) != extract_proofs(text):
print "as well as its proof",
if label in reference_texts and get_reference(tag) != reference_texts[label]:
print "as well as its reference",
if label in slogan_texts and get_slogan(tag) != slogan_texts[label]:
print "as well as its slogan",
if label in history_texts and get_history(tag) != history_texts[label]:
print "as well as its historical remark",
print ""
# update anyway to fill tags_search which is emptied every time
update_text(tag, text)
# if there is a reference, update it
if label in reference_texts:
update_reference(tag, reference_texts[label])
else:
update_reference(tag, "")
# if there is a slogan, update it
if label in slogan_texts:
update_slogan(tag, slogan_texts[label])
else:
update_slogan(tag, "")
# if there is a historical remark, update it
if label in history_texts:
update_history(tag, history_texts[label])
else:
update_history(tag, "")
n = n + 1
general.close(connection)
def __init__(self,
conn: Optional[redis.Redis] = None,
username: Optional[str] = None):
self.conn = conn or general.connect()
self.token = None
self.username = None
if username is not None:
self.username = username
self.token = general.login(self.conn, username)
def updateLineCounts():
(connection, cursor) = general.connect()
command = "wc -l *.tex"
(output, error) = general.execute(command)
for line in filter(None, output.split("\n")):
(linecount, filename) = line.split()
update("linecount " + filename, linecount, cursor)
general.close(connection)
def updateLineCounts():
(connection, cursor) = general.connect()
command = "wc -l *.tex"
(output, error) = general.execute(command)
for line in filter(None, output.split("\n")):
(linecount, filename) = line.split()
update("linecount " + filename, linecount, cursor)
general.close(connection)
def updatePageCounts():
(connection, cursor) = general.connect()
command = "ls tags/tmp/*.log"
(output, error) = general.execute(command)
for filename in filter(None, output.split("\n")):
f = open(config.websiteProject + "/" + filename)
for line in f:
if "Output written on " in line:
update("pagecount " + filename.split("/")[-1].split(".")[0], line.split(" ")[4][1:], cursor)
general.close(connection)
def clearHistory():
(connection, cursor) = general.connect()
try:
query = "DELETE FROM commits"
cursor.execute(query)
query = "DELETE FROM changes"
cursor.execute(query)
except sqlite3.Error, e:
print "An error occurred:", e.args[0]
def clearHistory():
(connection, cursor) = general.connect()
try:
query = 'DELETE FROM commits'
cursor.execute(query)
query = 'DELETE FROM changes'
cursor.execute(query)
except sqlite3.Error, e:
print "An error occurred:", e.args[0]
def clearBibliography():
(connection, cursor) = general.connect()
try:
query = 'DELETE FROM bibliography_items'
cursor.execute(query)
query = 'DELETE FROM bibliography_values'
cursor.execute(query)
except sqlite3.Error, e:
print "An error occurred:", e.args[0]
def updatePageCounts():
(connection, cursor) = general.connect()
command = "ls tags/tmp/*.log"
(output, error) = general.execute(command)
for filename in filter(None, output.split("\n")):
f = open(config.websiteProject + "/" + filename)
for line in f:
if "Output written on " in line:
update("pagecount " + filename.split("/")[-1].split(".")[0],
line.split(" ")[4][1:], cursor)
general.close(connection)
def importHistory(commit):
history = load_back(commit)
(connection, cursor) = general.connect()
# process commits
print " Processing commits"
for commit in history.commits:
insertCommitInfo(commit, getCommitInfo(commit), cursor)
print " Processing histories"
for tag in history.env_histories:
# there must be a tag, otherwise it is not accessible on the website (intermediate commits can have "tag-less tags")
if tag.env.tag != "":
process_tag(tag, cursor)
general.close(connection)
def importHistory(commit):
history = load_back(commit)
(connection, cursor) = general.connect()
# process commits
print " Processing commits"
for commit in history.commits:
insertCommitInfo(commit, getCommitInfo(commit), cursor)
print " Processing histories"
for tag in history.env_histories:
# there must be a tag, otherwise it is not accessible on the website (intermediate commits can have "tag-less tags")
if tag.env.tag != "":
process_tag(tag, cursor)
general.close(connection)
def importBootstrap():
(connection, cursor) = general.connect()
bootstrap_file = open("tags/bootstrap.txt", 'r')
for line in bootstrap_file:
tag, label = line.strip().split(',')
if not tag_exists(tag, cursor):
try:
query = 'INSERT INTO tags (tag, label, active) VALUES (?, ?, "FALSE")'
cursor.execute(query, (tag, label))
except sqlite3.Error, e:
print tag, label
print "An error occurred:", e.args[0]
else:
set_inactive(tag, cursor)
def checkTags():
(connection, cursor) = general.connect()
print 'Parsing the tags file'
active_tags = parse_tags(config.websiteProject + "/tags/tags").keys()
tags = get_tags(cursor)
for tag in tags:
# check whether the tag is no longer used in the project
if tag[0] not in active_tags and is_active(tag[0], cursor):
print ' ', tag[0], 'has become inactive'
set_inactive(tag[0], cursor)
# probably not necessary, but check whether a tag is again used in the project
if tag[1] == 'FALSE' and tag[0] in active_tags:
print ' ', tag[0], 'has become active again'
set_active(tag[0], cursor)
general.close(connection)
def importTitles():
global connection
(connection, cursor) = general.connect()
print 'Creating a database version of the table of contents'
print 'Parsing the files, linking chapters to file names'
titles = get_titles(config.websiteProject + "/tags/tmp/")
print 'Parsing the big table of contents'
sections = parse_book_toc(config.websiteProject + "/tags/tmp/book.toc")
# print out new or changed titles before updating the database
for section in sections:
if title_exists(section[2]) == 0 and not section[2] == 'Bibliography':
print 'New or changed section \'%s\'' % (section[2])
print 'Inserting the information into the database'
for section in sections:
insert_title(section[1], section[2], find_file_for_section(titles, sections, section[1]))
general.close(connection)
def importTags():
(connection, cursor) = general.connect()
print 'Parsing the tags file'
tags = parse_tags(config.websiteProject + "/tags/tags")
labels = get_labels_from_source(config.websiteProject + "/tags/tmp/")
print 'Inserting (or updating) the tags'
for tag, label in tags.iteritems():
if label not in labels:
print 'ERROR, label', label, 'not found in auxiliary files. Have you ran `make tags`?'
sys.exit()
if tag_exists(tag, cursor) and get_label(tag, cursor) != label:
print 'The label for tag %s has changed from \'%s\'to \'%s\'' % (tag, get_label(tag, cursor), label)
if not tag_exists(tag, cursor):
print 'New tag %s with label \'%s\'' % (tag, label)
info = labels[label]
insert_tag(tag, (label, info[0], info[2][1], info[1][1], info[1][0], info[1][2], info[2][3], info[1][4]), cursor)
general.close(connection)
def updateCommits():
(connection, cursor) = general.connect()
(what, error) = general.execute("git whatchanged --reverse -p tags/tags")
creation = {}
modified = {}
for line in what.split("\n"):
if line == "":
continue
if line.find('commit') == 0:
commit = line[7:]
new = 1
continue
if line.find('Date') == 0:
date = line[12:]
continue
if line.find('#') >= 0:
continue
if line.find('@@') == 0:
new = 0
continue
if new == 0:
c = line[0]
if not c == '+':
continue
line = line.lstrip(c)
tag = line.split(',')[0]
label = line.split(',')[1]
if tag not in creation:
creation[tag] = [date, commit]
modified[tag] = [date, commit]
tags = getTags(cursor)
for tag in tags:
updateCommitInformation(tag, creation[tag], modified[tag], cursor)
general.close(connection)
import general, macros.parse (connection, cursor) = general.connect() print 'Emptying the table' macros.parse.clearMacros(cursor) print 'Importing macros from tex/preamble.tex' macros.parse.addMacros(cursor) general.close(connection)
CURSOR = None
CONN = None
def select(sql):
global CURSOR
try:
CURSOR.execute(sql)
return CURSOR.fetchall()
except Exception:
return None
if (__name__ == '__main__'):
# --------------------------------------------
CONN, CURSOR = connect()
if (CONN is None and CURSOR is None):
message('-', ' Connect database failed')
exit(1)
# --------------------------------------------
parser = argparse.ArgumentParser(
description='Check CMS for website(s).', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'url', nargs='+', help='The website URL/ The URLs File')
parser.add_argument(
'--version', action='version', version='SoFinger Version: 0.1 Beta', help='Show version and exit')
parser.add_argument('--time', nargs='?', type=int,
default=3, const=1, help=' Timeout of requests')
parser.add_argument('--thread', nargs='?', type=int,
default=20, const=1, help=' Thread number for tasks')
def importBibliography():
(connection, cursor) = general.connect()
f = open(config.websiteProject + "/my.bib")
items = []
in_item = 0
line_nr = 0
for line in f:
line_nr = line_nr + 1
# beginning of a new item
if line[0] == '@':
if in_item == 1:
print "On line:", line_nr, "of my.bib:"
print line
print "Nested items. Exiting."
exit(1)
# clear previous item
item = [[], {}]
bib_type = line.partition('{')[0].strip('@').lower()
name = line.partition('{')[2].strip().strip(',')
item[0] = (bib_type, name)
in_item = 1
continue
# end of an item
if line[0] == '}':
if in_item == 0:
print "On line:", line_nr, "of my.bib:"
print line
print "Nested items. Exiting."
exit(1)
# add a *copy* to the list of items
items.append(list(item))
in_item = 0
continue
# can ignore current line if not in an item
if in_item == 0:
continue
# ignore comments
if line[0] == '%' or line[0:2] == '//':
continue
# Get key value pair from the line if it contains = sign
if '=' in line:
key = line.partition('=')[0].strip().lower()
value = line.partition('=')[2].strip().strip(',')[1:-1]
item[1][key] = value
# Check for correctness
if not (line.strip()[-2:] == '},' or line.strip()[-2:] == '",' or line.strip()[-1] == '}' or line.strip()[-1:] == '"'):
print "Warning: run over on line:", line_nr, "in my.bib:"
print line
print "Should not happen: each key = value pair should be on a line!"
print "Exiting"
exit(1)
for item in items:
insertItem(item, cursor)
general.close(connection)
try:
query = 'DELETE FROM comments WHERE id = ?'
connection.execute(query, [ID])
except sqlite3.Error, e:
print "An error occurred:", e.args[0]
if not len(sys.argv) == 2:
print 'You must supply one argument, namely the id of the comment you wish to remove'
raise Exception('Wrong number of arguments')
ID = int(sys.argv[1])
print 'Trying to remove the comment with id', int(ID)
(connection, cursor) = general.connect()
if not commentExists(ID):
print 'There is no such comment in the database'
else:
comment = getComment(ID)
choice = raw_input('Are you sure you wish to remove this comment by ' +
comment[1] + ' on tag ' + comment[3] + '? (Y/N): ')
if string.upper(choice) == 'Y':
deleteComment(ID)
print 'Comment removed!'
connection.commit()
connection.close()
def __init__(self, conn: Optional[redis.Redis] = None):
self.conn = conn or general.connect()
self.msg = None
