Exemple #1
0
def importTitles():
    global connection
    (connection, cursor) = general.connect()

    print 'Creating a database version of the table of contents'
    print 'Parsing the files, linking chapters to file names'
    titles = get_titles(config.websiteProject + "/tags/tmp/")
    print 'Parsing the big table of contents'
    sections = parse_book_toc(config.websiteProject + "/tags/tmp/book.toc")

    # print out new or changed titles before updating the database
    for section in sections:
        if title_exists(section[2]) == 0 and not section[2] == 'Bibliography':
            print 'New or changed section \'%s\'' % (section[2])

    print 'Inserting the information into the database'
    for section in sections:
        # the bibliography doesn't correspond to a file, we can safely ignore it
        if section[2] == 'Bibliography':
            continue

        insert_title(section[1], section[2],
                     find_file_for_section(titles, sections, section[1]))

    general.close(connection)
Exemple #2
0
async def check(url, cms_tuple, timeout):  # MD5检测
    global SESSION, NUM
    if (len(SUCCESS) > 0):
        return (False, tuple())
    cms_name = cms_tuple[0]
    cms_path = cms_tuple[1]
    cms_match_pattern = cms_tuple[2]
    cms_id = cms_tuple[3]
    cms_hit = cms_tuple[4]
    message_list = []
    target_url = url + cms_path
    if (VERBOSE is True):
        message('#', 'target:%s' % target_url)
    try:
        req = SESSION.get(
            target_url, headers=requests_headers(),  timeout=timeout)
        if (req.status_code == 200 and cms_match_pattern.lower() in req.text.lower()):
            message_list.append(cms_name)
            try:
                CONN, CURSOR = connect()
                update(CONN, CURSOR, cms_hit, cms_id)  # hit++
                close(CONN)
            except Exception:
                pass
            return (True, tuple(message_list))
        else:
            return (False, tuple(message_list))
    except Exception:
        return (False, tuple(message_list))
    finally:
        NUM += 1
Exemple #3
0
def importTags():
    (connection, cursor) = general.connect()

    print 'Parsing the tags file'
    tags = parse_tags(config.websiteProject + "/tags/tags")

    labels = get_labels_from_source(config.websiteProject + "/tags/tmp/")

    print 'Inserting (or updating) the tags'
    for tag, label in tags.iteritems():
        if label not in labels:
            print 'ERROR, label', label, 'not found in auxiliary files. Have you ran `make tags`?'
            sys.exit()

        if tag_exists(tag, cursor) and get_label(tag, cursor) != label:
            print 'The label for tag %s has changed from \'%s\'to \'%s\'' % (
                tag, get_label(tag, cursor), label)
        if not tag_exists(tag, cursor):
            print 'New tag %s with label \'%s\'' % (tag, label)

        info = labels[label]
        insert_tag(tag, (label, info[0], info[2][1], info[1][1], info[1][0],
                         info[1][2], info[2][3], info[1][4]), cursor)

    general.close(connection)
def insertDependencies():
  (connection, cursor) = general.connect()

  for tag, label in tags:
    for child in tags_refs[tag]:
      addDependency(tag, child, cursor)

  general.close(connection)
def updateCounts():
  (connection, cursor) = general.connect()

  for tag, label in tags:
    print "updating " + tag
    update(tag, tag_node_count[tag], tag_edge_count[tag], tag_total_edge_count[tag], tag_chapter_count[tag], tag_section_count[tag], tag_use_count[tag], tag_indirect_use_count[tag], cursor)

  general.close(connection)
Exemple #6
0
def insertDependencies():
  (connection, cursor) = general.connect()

  for tag, label in tags:
    for child in tags_refs[tag]:
      addDependency(tag, child, cursor)

  general.close(connection)
Exemple #7
0
def updateCounts():
  (connection, cursor) = general.connect()

  for tag, label in tags:
    print "updating " + tag
    update(tag, tag_node_count[tag], tag_edge_count[tag], tag_total_edge_count[tag], tag_chapter_count[tag], tag_section_count[tag], tag_use_count[tag], tag_indirect_use_count[tag], cursor)

  general.close(connection)
Exemple #8
0
def importLaTeX():
  global connection
  (connection, cursor) = general.connect()

  n = 0
  while n < len(tags):
    tag = tags[n][0]
    label = tags[n][1]
  
    text = ''
  
    if label in label_linenumbers.keys():
      update_linenumbers(tag, label_linenumbers[label])
  
    if label in label_texts:
      text = text + label_texts[label]
  
      if label in proof_texts:
        text = text + '\n' + proof_texts[label]

    # if text has changed and current text isn't empty (i.e. not a new tag)
    if get_text(tag) != text and get_text(tag) != '':
      print "The text of tag", tag, "has changed",
      if label in proof_texts and extract_proofs(get_text(tag)) != extract_proofs(text):
        print "as well as its proof",
      if label in reference_texts and get_reference(tag) != reference_texts[label]:
        print "as well as its reference",
      if label in slogan_texts and get_slogan(tag) != slogan_texts[label]:
        print "as well as its slogan",
      if label in history_texts and get_history(tag) != history_texts[label]:
        print "as well as its historical remark",
      print ""
        
    # update anyway to fill tags_search which is emptied every time
    update_text(tag, text)

    # if there is a reference, update it
    if label in reference_texts:
      update_reference(tag, reference_texts[label])
    else:
      update_reference(tag, "")
  
    # if there is a slogan, update it
    if label in slogan_texts:
      update_slogan(tag, slogan_texts[label])
    else:
      update_slogan(tag, "")
  
    # if there is a historical remark, update it
    if label in history_texts:
      update_history(tag, history_texts[label])
    else:
      update_history(tag, "")
  
    n = n + 1

  general.close(connection)
Exemple #9
0
def updateLineCounts():
  (connection, cursor) = general.connect()

  command = "wc -l *.tex"
  (output, error) = general.execute(command)

  for line in filter(None, output.split("\n")):
    (linecount, filename) = line.split()
    update("linecount " + filename, linecount, cursor)

  general.close(connection)
Exemple #10
0
def updateLineCounts():
    (connection, cursor) = general.connect()

    command = "wc -l *.tex"
    (output, error) = general.execute(command)

    for line in filter(None, output.split("\n")):
        (linecount, filename) = line.split()
        update("linecount " + filename, linecount, cursor)

    general.close(connection)
Exemple #11
0
def updatePageCounts():
  (connection, cursor) = general.connect()

  command = "ls tags/tmp/*.log"
  (output, error) = general.execute(command)
  for filename in filter(None, output.split("\n")):
    f = open(config.websiteProject + "/" + filename)
    for line in f:
      if "Output written on " in line:
        update("pagecount " + filename.split("/")[-1].split(".")[0], line.split(" ")[4][1:], cursor)

  general.close(connection)
Exemple #12
0
def updatePageCounts():
    (connection, cursor) = general.connect()

    command = "ls tags/tmp/*.log"
    (output, error) = general.execute(command)
    for filename in filter(None, output.split("\n")):
        f = open(config.websiteProject + "/" + filename)
        for line in f:
            if "Output written on " in line:
                update("pagecount " + filename.split("/")[-1].split(".")[0],
                       line.split(" ")[4][1:], cursor)

    general.close(connection)
Exemple #13
0
def importHistory(commit):
    history = load_back(commit)

    (connection, cursor) = general.connect()

    # process commits
    print "  Processing commits"
    for commit in history.commits:
        insertCommitInfo(commit, getCommitInfo(commit), cursor)

    print "  Processing histories"
    for tag in history.env_histories:
        # there must be a tag, otherwise it is not accessible on the website (intermediate commits can have "tag-less tags")
        if tag.env.tag != "":
            process_tag(tag, cursor)

    general.close(connection)
Exemple #14
0
def importHistory(commit):
    history = load_back(commit)

    (connection, cursor) = general.connect()

    # process commits
    print "  Processing commits"
    for commit in history.commits:
        insertCommitInfo(commit, getCommitInfo(commit), cursor)

    print "  Processing histories"
    for tag in history.env_histories:
        # there must be a tag, otherwise it is not accessible on the website (intermediate commits can have "tag-less tags")
        if tag.env.tag != "":
            process_tag(tag, cursor)

    general.close(connection)
Exemple #15
0
def checkTags():
  (connection, cursor) = general.connect()

  print 'Parsing the tags file'
  active_tags = parse_tags(config.websiteProject + "/tags/tags").keys()

  tags = get_tags(cursor)

  for tag in tags:
    # check whether the tag is no longer used in the project
    if tag[0] not in active_tags and is_active(tag[0], cursor):
      print '   ', tag[0], 'has become inactive'
      set_inactive(tag[0], cursor)

    # probably not necessary, but check whether a tag is again used in the project
    if tag[1] == 'FALSE' and tag[0] in active_tags:
      print '   ', tag[0], 'has become active again'
      set_active(tag[0], cursor)

  general.close(connection)
Exemple #16
0
def importTitles():
  global connection
  (connection, cursor) = general.connect()

  print 'Creating a database version of the table of contents'
  print 'Parsing the files, linking chapters to file names'
  titles = get_titles(config.websiteProject + "/tags/tmp/")
  print 'Parsing the big table of contents'
  sections = parse_book_toc(config.websiteProject + "/tags/tmp/book.toc")

  # print out new or changed titles before updating the database
  for section in sections:
    if title_exists(section[2]) == 0 and not section[2] == 'Bibliography':
      print 'New or changed section \'%s\'' % (section[2])

  print 'Inserting the information into the database'
  for section in sections:

    insert_title(section[1], section[2], find_file_for_section(titles, sections, section[1]))

  general.close(connection)
Exemple #17
0
def importTags():
  (connection, cursor) = general.connect()

  print 'Parsing the tags file'
  tags = parse_tags(config.websiteProject + "/tags/tags")

  labels = get_labels_from_source(config.websiteProject + "/tags/tmp/")

  print 'Inserting (or updating) the tags'
  for tag, label in tags.iteritems():
    if label not in labels:
      print 'ERROR, label', label, 'not found in auxiliary files. Have you ran `make tags`?'
      sys.exit()

    if tag_exists(tag, cursor) and get_label(tag, cursor) != label:
      print 'The label for tag %s has changed from \'%s\'to \'%s\'' % (tag, get_label(tag, cursor), label)
    if not tag_exists(tag, cursor):
      print 'New tag %s with label \'%s\'' % (tag, label)

    info = labels[label]
    insert_tag(tag, (label, info[0], info[2][1], info[1][1], info[1][0], info[1][2], info[2][3], info[1][4]), cursor)

  general.close(connection)
def updateCommits():
  (connection, cursor) = general.connect()

  (what, error) = general.execute("git whatchanged --reverse -p tags/tags")
  creation = {}
  modified = {}
  for line in what.split("\n"):
    if line == "":
      continue
    if line.find('commit') == 0:
      commit = line[7:]
      new = 1
      continue
    if line.find('Date') == 0:
      date = line[12:]
      continue
    if line.find('#') >= 0:
      continue
    if line.find('@@') == 0:
      new = 0
      continue
    if new == 0:
      c = line[0]
      if not c == '+':
        continue
      line = line.lstrip(c)
      tag = line.split(',')[0]
      label = line.split(',')[1]
      if tag not in creation:
        creation[tag] = [date, commit]
      modified[tag] = [date, commit]

  tags = getTags(cursor)
  for tag in tags:
    updateCommitInformation(tag, creation[tag], modified[tag], cursor)

  general.close(connection)
Exemple #19
0
    if tag in getChildren(candidate): result = result + 1

  return result

# TODO move to file and call somewhere else
def clearDependencies():
  (connection, cursor) = general.connect()

  try:
    query = 'DELETE FROM dependencies'
    cursor.execute(query)

  except sqlite3.Error, e:
    print "An error occurred:", e.args[0]

  general.close(connection)

def addDependency(source, target, cursor):
  try:
    query = 'INSERT INTO dependencies (source, target) VALUES (?, ?)'
    cursor.execute(query, [source, target])

  except sqlite3.Error, e:
    print "An error occurred:", e.args[0]

def insertDependencies():
  (connection, cursor) = general.connect()

  for tag, label in tags:
    for child in tags_refs[tag]:
      addDependency(tag, child, cursor)
    if tag in getChildren(candidate): result = result + 1

  return result

# TODO move to file and call somewhere else
def clearDependencies():
  (connection, cursor) = general.connect()

  try:
    query = 'DELETE FROM dependencies'
    cursor.execute(query)

  except sqlite3.Error, e:
    print "An error occurred:", e.args[0]

  general.close(connection)

def addDependency(source, target, cursor):
  try:
    query = 'INSERT INTO dependencies (source, target) VALUES (?, ?)'
    cursor.execute(query, [source, target])

  except sqlite3.Error, e:
    print "An error occurred:", e.args[0]

def insertDependencies():
  (connection, cursor) = general.connect()

  for tag, label in tags:
    for child in tags_refs[tag]:
      addDependency(tag, child, cursor)
Exemple #21
0
def importBibliography():
  (connection, cursor) = general.connect()
  f = open(config.websiteProject + "/my.bib")

  items = []
  in_item = 0
  line_nr = 0

  for line in f:
    line_nr = line_nr + 1
    # beginning of a new item
    if line[0] == '@':

      if in_item == 1:
        print "On line:", line_nr, "of my.bib:"
	print line
        print "Nested items. Exiting."
	exit(1)

      # clear previous item
      item = [[], {}]

      bib_type = line.partition('{')[0].strip('@').lower()
      name = line.partition('{')[2].strip().strip(',')

      item[0] = (bib_type, name)

      in_item = 1

      continue
  
    # end of an item
    if line[0] == '}':

      if in_item == 0:
        print "On line:", line_nr, "of my.bib:"
        print line
	print "Nested items. Exiting."
	exit(1)

      # add a *copy* to the list of items
      items.append(list(item))

      in_item = 0

      continue

    # can ignore current line if not in an item
    if in_item == 0:

      continue
 
    # ignore comments
    if line[0] == '%' or line[0:2] == '//':

      continue

    # Get key value pair from the line if it contains = sign
    if '=' in line:
      key = line.partition('=')[0].strip().lower()
      value = line.partition('=')[2].strip().strip(',')[1:-1]
      item[1][key] = value

      # Check for correctness
      if not (line.strip()[-2:] == '},' or line.strip()[-2:] == '",' or line.strip()[-1] == '}' or line.strip()[-1:] == '"'):
        print "Warning: run over on line:", line_nr, "in my.bib:"
        print line
        print "Should not happen: each key = value pair should be on a line!"
	print "Exiting"
	exit(1)

  for item in items:
    insertItem(item, cursor)

  general.close(connection)