def importTitles():
global connection
(connection, cursor) = general.connect()
print 'Creating a database version of the table of contents'
print 'Parsing the files, linking chapters to file names'
titles = get_titles(config.websiteProject + "/tags/tmp/")
print 'Parsing the big table of contents'
sections = parse_book_toc(config.websiteProject + "/tags/tmp/book.toc")
# print out new or changed titles before updating the database
for section in sections:
if title_exists(section[2]) == 0 and not section[2] == 'Bibliography':
print 'New or changed section \'%s\'' % (section[2])
print 'Inserting the information into the database'
for section in sections:
# the bibliography doesn't correspond to a file, we can safely ignore it
if section[2] == 'Bibliography':
continue
insert_title(section[1], section[2],
find_file_for_section(titles, sections, section[1]))
general.close(connection)
async def check(url, cms_tuple, timeout): # MD5检测
global SESSION, NUM
if (len(SUCCESS) > 0):
return (False, tuple())
cms_name = cms_tuple[0]
cms_path = cms_tuple[1]
cms_match_pattern = cms_tuple[2]
cms_id = cms_tuple[3]
cms_hit = cms_tuple[4]
message_list = []
target_url = url + cms_path
if (VERBOSE is True):
message('#', 'target:%s' % target_url)
try:
req = SESSION.get(
target_url, headers=requests_headers(), timeout=timeout)
if (req.status_code == 200 and cms_match_pattern.lower() in req.text.lower()):
message_list.append(cms_name)
try:
CONN, CURSOR = connect()
update(CONN, CURSOR, cms_hit, cms_id) # hit++
close(CONN)
except Exception:
pass
return (True, tuple(message_list))
else:
return (False, tuple(message_list))
except Exception:
return (False, tuple(message_list))
finally:
NUM += 1
def importTags():
(connection, cursor) = general.connect()
print 'Parsing the tags file'
tags = parse_tags(config.websiteProject + "/tags/tags")
labels = get_labels_from_source(config.websiteProject + "/tags/tmp/")
print 'Inserting (or updating) the tags'
for tag, label in tags.iteritems():
if label not in labels:
print 'ERROR, label', label, 'not found in auxiliary files. Have you ran `make tags`?'
sys.exit()
if tag_exists(tag, cursor) and get_label(tag, cursor) != label:
print 'The label for tag %s has changed from \'%s\'to \'%s\'' % (
tag, get_label(tag, cursor), label)
if not tag_exists(tag, cursor):
print 'New tag %s with label \'%s\'' % (tag, label)
info = labels[label]
insert_tag(tag, (label, info[0], info[2][1], info[1][1], info[1][0],
info[1][2], info[2][3], info[1][4]), cursor)
general.close(connection)
def insertDependencies():
(connection, cursor) = general.connect()
for tag, label in tags:
for child in tags_refs[tag]:
addDependency(tag, child, cursor)
general.close(connection)
def updateCounts():
(connection, cursor) = general.connect()
for tag, label in tags:
print "updating " + tag
update(tag, tag_node_count[tag], tag_edge_count[tag], tag_total_edge_count[tag], tag_chapter_count[tag], tag_section_count[tag], tag_use_count[tag], tag_indirect_use_count[tag], cursor)
general.close(connection)
def insertDependencies():
(connection, cursor) = general.connect()
for tag, label in tags:
for child in tags_refs[tag]:
addDependency(tag, child, cursor)
general.close(connection)
def updateCounts():
(connection, cursor) = general.connect()
for tag, label in tags:
print "updating " + tag
update(tag, tag_node_count[tag], tag_edge_count[tag], tag_total_edge_count[tag], tag_chapter_count[tag], tag_section_count[tag], tag_use_count[tag], tag_indirect_use_count[tag], cursor)
general.close(connection)
def importLaTeX():
global connection
(connection, cursor) = general.connect()
n = 0
while n < len(tags):
tag = tags[n][0]
label = tags[n][1]
text = ''
if label in label_linenumbers.keys():
update_linenumbers(tag, label_linenumbers[label])
if label in label_texts:
text = text + label_texts[label]
if label in proof_texts:
text = text + '\n' + proof_texts[label]
# if text has changed and current text isn't empty (i.e. not a new tag)
if get_text(tag) != text and get_text(tag) != '':
print "The text of tag", tag, "has changed",
if label in proof_texts and extract_proofs(get_text(tag)) != extract_proofs(text):
print "as well as its proof",
if label in reference_texts and get_reference(tag) != reference_texts[label]:
print "as well as its reference",
if label in slogan_texts and get_slogan(tag) != slogan_texts[label]:
print "as well as its slogan",
if label in history_texts and get_history(tag) != history_texts[label]:
print "as well as its historical remark",
print ""
# update anyway to fill tags_search which is emptied every time
update_text(tag, text)
# if there is a reference, update it
if label in reference_texts:
update_reference(tag, reference_texts[label])
else:
update_reference(tag, "")
# if there is a slogan, update it
if label in slogan_texts:
update_slogan(tag, slogan_texts[label])
else:
update_slogan(tag, "")
# if there is a historical remark, update it
if label in history_texts:
update_history(tag, history_texts[label])
else:
update_history(tag, "")
n = n + 1
general.close(connection)
def updateLineCounts():
(connection, cursor) = general.connect()
command = "wc -l *.tex"
(output, error) = general.execute(command)
for line in filter(None, output.split("\n")):
(linecount, filename) = line.split()
update("linecount " + filename, linecount, cursor)
general.close(connection)
def updateLineCounts():
(connection, cursor) = general.connect()
command = "wc -l *.tex"
(output, error) = general.execute(command)
for line in filter(None, output.split("\n")):
(linecount, filename) = line.split()
update("linecount " + filename, linecount, cursor)
general.close(connection)
def updatePageCounts():
(connection, cursor) = general.connect()
command = "ls tags/tmp/*.log"
(output, error) = general.execute(command)
for filename in filter(None, output.split("\n")):
f = open(config.websiteProject + "/" + filename)
for line in f:
if "Output written on " in line:
update("pagecount " + filename.split("/")[-1].split(".")[0], line.split(" ")[4][1:], cursor)
general.close(connection)
def updatePageCounts():
(connection, cursor) = general.connect()
command = "ls tags/tmp/*.log"
(output, error) = general.execute(command)
for filename in filter(None, output.split("\n")):
f = open(config.websiteProject + "/" + filename)
for line in f:
if "Output written on " in line:
update("pagecount " + filename.split("/")[-1].split(".")[0],
line.split(" ")[4][1:], cursor)
general.close(connection)
def importHistory(commit):
history = load_back(commit)
(connection, cursor) = general.connect()
# process commits
print " Processing commits"
for commit in history.commits:
insertCommitInfo(commit, getCommitInfo(commit), cursor)
print " Processing histories"
for tag in history.env_histories:
# there must be a tag, otherwise it is not accessible on the website (intermediate commits can have "tag-less tags")
if tag.env.tag != "":
process_tag(tag, cursor)
general.close(connection)
def importHistory(commit):
history = load_back(commit)
(connection, cursor) = general.connect()
# process commits
print " Processing commits"
for commit in history.commits:
insertCommitInfo(commit, getCommitInfo(commit), cursor)
print " Processing histories"
for tag in history.env_histories:
# there must be a tag, otherwise it is not accessible on the website (intermediate commits can have "tag-less tags")
if tag.env.tag != "":
process_tag(tag, cursor)
general.close(connection)
def checkTags():
(connection, cursor) = general.connect()
print 'Parsing the tags file'
active_tags = parse_tags(config.websiteProject + "/tags/tags").keys()
tags = get_tags(cursor)
for tag in tags:
# check whether the tag is no longer used in the project
if tag[0] not in active_tags and is_active(tag[0], cursor):
print ' ', tag[0], 'has become inactive'
set_inactive(tag[0], cursor)
# probably not necessary, but check whether a tag is again used in the project
if tag[1] == 'FALSE' and tag[0] in active_tags:
print ' ', tag[0], 'has become active again'
set_active(tag[0], cursor)
general.close(connection)
def importTitles():
global connection
(connection, cursor) = general.connect()
print 'Creating a database version of the table of contents'
print 'Parsing the files, linking chapters to file names'
titles = get_titles(config.websiteProject + "/tags/tmp/")
print 'Parsing the big table of contents'
sections = parse_book_toc(config.websiteProject + "/tags/tmp/book.toc")
# print out new or changed titles before updating the database
for section in sections:
if title_exists(section[2]) == 0 and not section[2] == 'Bibliography':
print 'New or changed section \'%s\'' % (section[2])
print 'Inserting the information into the database'
for section in sections:
insert_title(section[1], section[2], find_file_for_section(titles, sections, section[1]))
general.close(connection)
def importTags():
(connection, cursor) = general.connect()
print 'Parsing the tags file'
tags = parse_tags(config.websiteProject + "/tags/tags")
labels = get_labels_from_source(config.websiteProject + "/tags/tmp/")
print 'Inserting (or updating) the tags'
for tag, label in tags.iteritems():
if label not in labels:
print 'ERROR, label', label, 'not found in auxiliary files. Have you ran `make tags`?'
sys.exit()
if tag_exists(tag, cursor) and get_label(tag, cursor) != label:
print 'The label for tag %s has changed from \'%s\'to \'%s\'' % (tag, get_label(tag, cursor), label)
if not tag_exists(tag, cursor):
print 'New tag %s with label \'%s\'' % (tag, label)
info = labels[label]
insert_tag(tag, (label, info[0], info[2][1], info[1][1], info[1][0], info[1][2], info[2][3], info[1][4]), cursor)
general.close(connection)
def updateCommits():
(connection, cursor) = general.connect()
(what, error) = general.execute("git whatchanged --reverse -p tags/tags")
creation = {}
modified = {}
for line in what.split("\n"):
if line == "":
continue
if line.find('commit') == 0:
commit = line[7:]
new = 1
continue
if line.find('Date') == 0:
date = line[12:]
continue
if line.find('#') >= 0:
continue
if line.find('@@') == 0:
new = 0
continue
if new == 0:
c = line[0]
if not c == '+':
continue
line = line.lstrip(c)
tag = line.split(',')[0]
label = line.split(',')[1]
if tag not in creation:
creation[tag] = [date, commit]
modified[tag] = [date, commit]
tags = getTags(cursor)
for tag in tags:
updateCommitInformation(tag, creation[tag], modified[tag], cursor)
general.close(connection)
if tag in getChildren(candidate): result = result + 1
return result
# TODO move to file and call somewhere else
def clearDependencies():
(connection, cursor) = general.connect()
try:
query = 'DELETE FROM dependencies'
cursor.execute(query)
except sqlite3.Error, e:
print "An error occurred:", e.args[0]
general.close(connection)
def addDependency(source, target, cursor):
try:
query = 'INSERT INTO dependencies (source, target) VALUES (?, ?)'
cursor.execute(query, [source, target])
except sqlite3.Error, e:
print "An error occurred:", e.args[0]
def insertDependencies():
(connection, cursor) = general.connect()
for tag, label in tags:
for child in tags_refs[tag]:
addDependency(tag, child, cursor)
if tag in getChildren(candidate): result = result + 1
return result
# TODO move to file and call somewhere else
def clearDependencies():
(connection, cursor) = general.connect()
try:
query = 'DELETE FROM dependencies'
cursor.execute(query)
except sqlite3.Error, e:
print "An error occurred:", e.args[0]
general.close(connection)
def addDependency(source, target, cursor):
try:
query = 'INSERT INTO dependencies (source, target) VALUES (?, ?)'
cursor.execute(query, [source, target])
except sqlite3.Error, e:
print "An error occurred:", e.args[0]
def insertDependencies():
(connection, cursor) = general.connect()
for tag, label in tags:
for child in tags_refs[tag]:
addDependency(tag, child, cursor)
def importBibliography():
(connection, cursor) = general.connect()
f = open(config.websiteProject + "/my.bib")
items = []
in_item = 0
line_nr = 0
for line in f:
line_nr = line_nr + 1
# beginning of a new item
if line[0] == '@':
if in_item == 1:
print "On line:", line_nr, "of my.bib:"
print line
print "Nested items. Exiting."
exit(1)
# clear previous item
item = [[], {}]
bib_type = line.partition('{')[0].strip('@').lower()
name = line.partition('{')[2].strip().strip(',')
item[0] = (bib_type, name)
in_item = 1
continue
# end of an item
if line[0] == '}':
if in_item == 0:
print "On line:", line_nr, "of my.bib:"
print line
print "Nested items. Exiting."
exit(1)
# add a *copy* to the list of items
items.append(list(item))
in_item = 0
continue
# can ignore current line if not in an item
if in_item == 0:
continue
# ignore comments
if line[0] == '%' or line[0:2] == '//':
continue
# Get key value pair from the line if it contains = sign
if '=' in line:
key = line.partition('=')[0].strip().lower()
value = line.partition('=')[2].strip().strip(',')[1:-1]
item[1][key] = value
# Check for correctness
if not (line.strip()[-2:] == '},' or line.strip()[-2:] == '",' or line.strip()[-1] == '}' or line.strip()[-1:] == '"'):
print "Warning: run over on line:", line_nr, "in my.bib:"
print line
print "Should not happen: each key = value pair should be on a line!"
print "Exiting"
exit(1)
for item in items:
insertItem(item, cursor)
general.close(connection)