async def check(url, cms_tuple): # 检测主函数 global SESSION, SUCCESS, REQUEST_DICT if (len(SUCCESS) > 0): return (False, tuple()) cms_name = cms_tuple[0] cms_path = cms_tuple[1] cms_match_pattern = cms_tuple[2] cms_id = cms_tuple[3] message_list = [] try: if (cms_path in REQUEST_DICT): data = REQUEST_DICT[cms_path] if (rule(RULE_NAME, cms_match_pattern, data)): message_list.append(cms_name) try: CONN, CURSOR = connect() cms_hit = select_cms_hit(CURSOR, cms_id) update_cms(CONN, CURSOR, cms_hit, cms_id) # hit++ CONN.close() except Exception: pass return (True, tuple(message_list)) else: return (False, tuple(message_list)) else: return (False, tuple(message_list)) except Exception: return (False, tuple(message_list))
def importTags(): (connection, cursor) = general.connect() print 'Parsing the tags file' tags = parse_tags(config.websiteProject + "/tags/tags") labels = get_labels_from_source(config.websiteProject + "/tags/tmp/") print 'Inserting (or updating) the tags' for tag, label in tags.iteritems(): if label not in labels: print 'ERROR, label', label, 'not found in auxiliary files. Have you ran `make tags`?' sys.exit() if tag_exists(tag, cursor) and get_label(tag, cursor) != label: print 'The label for tag %s has changed from \'%s\'to \'%s\'' % ( tag, get_label(tag, cursor), label) if not tag_exists(tag, cursor): print 'New tag %s with label \'%s\'' % (tag, label) info = labels[label] insert_tag(tag, (label, info[0], info[2][1], info[1][1], info[1][0], info[1][2], info[2][3], info[1][4]), cursor) general.close(connection)
async def check(url, cms_tuple, timeout): # MD5检测 global SESSION, NUM if (len(SUCCESS) > 0): return (False, tuple()) cms_name = cms_tuple[0] cms_path = cms_tuple[1] cms_match_pattern = cms_tuple[2] cms_id = cms_tuple[3] cms_hit = cms_tuple[4] message_list = [] target_url = url + cms_path if (VERBOSE is True): message('#', 'target:%s' % target_url) try: req = SESSION.get( target_url, headers=requests_headers(), timeout=timeout) if (req.status_code == 200 and cms_match_pattern.lower() in req.text.lower()): message_list.append(cms_name) try: CONN, CURSOR = connect() update(CONN, CURSOR, cms_hit, cms_id) # hit++ close(CONN) except Exception: pass return (True, tuple(message_list)) else: return (False, tuple(message_list)) except Exception: return (False, tuple(message_list)) finally: NUM += 1
def importTitles(): global connection (connection, cursor) = general.connect() print 'Creating a database version of the table of contents' print 'Parsing the files, linking chapters to file names' titles = get_titles(config.websiteProject + "/tags/tmp/") print 'Parsing the big table of contents' sections = parse_book_toc(config.websiteProject + "/tags/tmp/book.toc") # print out new or changed titles before updating the database for section in sections: if title_exists(section[2]) == 0 and not section[2] == 'Bibliography': print 'New or changed section \'%s\'' % (section[2]) print 'Inserting the information into the database' for section in sections: # the bibliography doesn't correspond to a file, we can safely ignore it if section[2] == 'Bibliography': continue insert_title(section[1], section[2], find_file_for_section(titles, sections, section[1])) general.close(connection)
def main(dataset): # Database Connector dbc = general.connect() # if dataset in 'SFBS': # df = distance.distance(dbc, dataset, 'src', 'dest') # else: # df = distance.distance(dbc, dataset, 'start', 'location') # print(df) # if dataset in 'SFBS': # df = duration.duration(dbc, dataset) # else: # df = duration.duration(dbc, dataset) # print(df) # if dataset in 'SFBS': # df = saturation.saturation(dbc, dataset, 'with', 'src', 'dest', 1, 3, 'SFBS1') # df = saturation.saturation(dbc, dataset, 'with', 'src', 'dest', 2, 3, 'SFBS2') # else: # df = saturation(dbc, dataset, 'act', 'start', None, 1, 0, 'SFFD') if dataset in 'SFBS': forecast.count_model('data/station70_forecast') elif dataset in 'SFFD': forecast.count_model('data/satur') elif dataset in 'ISSUE': # FORECAST forecast.model_cross_version(['data/hadoop-version_2_prior-2.csv', 'data/hbase-version_0_prior-2.csv', 'data/hive-version_2_prior-2.csv', 'data/hadoop-version_1_prior-2.csv'], 'data/hadoop-version_3_prior-2.csv') forecast.model_cross_version(['data/hadoop-version_2_prior-4.csv', 'data/hbase-version_0_prior-4.csv', 'data/hive-version_2_prior-4.csv', 'data/hadoop-version_1_prior-4.csv'], 'data/hadoop-version_3_prior-4.csv') forecast.model_cross_version(['data/hadoop-version_2_prior-8.csv', 'data/hbase-version_0_prior-8.csv', 'data/hive-version_2_prior-8.csv', 'data/hadoop-version_1_prior-8.csv'], 'data/hadoop-version_3_prior-8.csv') forecast.model_cross_version(['data/hadoop-version_2_prior-10.csv', 'data/hbase-version_0_prior-10.csv', 'data/hive-version_2_prior-10.csv', 'data/hadoop-version_1_prior-10.csv'], 'data/hadoop-version_3_prior-10.csv')
def handle(name, key, cms_id, header, body, title): # 对数据库取出的信息进行拆分并开始指纹识别 # 满足一个条件即可的情况 success_list = [] if ('||' in key and '&&' not in key and '(' not in key): for rule in key.split('||'): if check_rule(rule, header, body, title): success_list.append(name) break # 只有一个条件的情况 elif ('||' not in key and '&&' not in key and '(' not in key): if check_rule(key, header, body, title): success_list.append(name) # 需要同时满足条件的情况 elif ('&&' in key and '||' not in key and '(' not in key): num = 0 for rule in key.split('&&'): if check_rule(rule.strip(), header, body, title): num += 1 if (num == len(key.split('&&'))): success_list.append(name) else: # 与条件下存在并条件: 1||2||(3&&4) if ('&&' in re.findall(rbracket, key)[0]): for rule in key.split('||'): if ('&&' in rule): num = 0 for _rule in rule.split('&&'): if (check_rule(_rule, header, body, title)): num += 1 if (num == len(rule.split('&&'))): success_list.append(name) break else: if (check_rule(rule, header, body, title)): success_list.append(name) break else: # 并条件下存在与条件: 1&&2&&(3||4) for rule in key.split('&&'): num = 0 if '||' in rule: for _rule in rule.split('||'): if (check_rule(_rule, title, body, header)): num += 1 break else: if (check_rule(rule, title, body, header)): num += 1 if (num == len(key.split('&&'))): success_list.append(name) for each in success_list: try: CONN, CURSOR = connect() hit = select_fofa_hit(CURSOR, cms_id) update_fofa(CONN, CURSOR, hit, cms_id) # hit++ CONN.close() except Exception: pass return success_list
def __init__(self, usernames: List[str]): self.conn = general.connect() self.usernames = usernames for name in usernames: if general.get_user(self.conn, name) is None: general.register(self.conn, name) self.users = [User(self.conn, username=name) for name in usernames]
def updateCounts(): (connection, cursor) = general.connect() for tag, label in tags: print "updating " + tag update(tag, tag_node_count[tag], tag_edge_count[tag], tag_total_edge_count[tag], tag_chapter_count[tag], tag_section_count[tag], tag_use_count[tag], tag_indirect_use_count[tag], cursor) general.close(connection)
def insertDependencies(): (connection, cursor) = general.connect() for tag, label in tags: for child in tags_refs[tag]: addDependency(tag, child, cursor) general.close(connection)
def clearCounts(): (connection, cursor) = general.connect() try: query = 'DELETE FROM graphs' cursor.execute(query) except sqlite3.Error, e: print "An error occurred:", e.args[0]
def clearSearchTable(): (connection, cursor) = general.connect() try: query = 'DELETE FROM tags_search' connection.execute(query) except sqlite3.Error, e: print "An error occurred:", e.args[0]
def importLaTeX(): global connection (connection, cursor) = general.connect() n = 0 while n < len(tags): tag = tags[n][0] label = tags[n][1] text = '' if label in label_linenumbers.keys(): update_linenumbers(tag, label_linenumbers[label]) if label in label_texts: text = text + label_texts[label] if label in proof_texts: text = text + '\n' + proof_texts[label] # if text has changed and current text isn't empty (i.e. not a new tag) if get_text(tag) != text and get_text(tag) != '': print "The text of tag", tag, "has changed", if label in proof_texts and extract_proofs(get_text(tag)) != extract_proofs(text): print "as well as its proof", if label in reference_texts and get_reference(tag) != reference_texts[label]: print "as well as its reference", if label in slogan_texts and get_slogan(tag) != slogan_texts[label]: print "as well as its slogan", if label in history_texts and get_history(tag) != history_texts[label]: print "as well as its historical remark", print "" # update anyway to fill tags_search which is emptied every time update_text(tag, text) # if there is a reference, update it if label in reference_texts: update_reference(tag, reference_texts[label]) else: update_reference(tag, "") # if there is a slogan, update it if label in slogan_texts: update_slogan(tag, slogan_texts[label]) else: update_slogan(tag, "") # if there is a historical remark, update it if label in history_texts: update_history(tag, history_texts[label]) else: update_history(tag, "") n = n + 1 general.close(connection)
def __init__(self, conn: Optional[redis.Redis] = None, username: Optional[str] = None): self.conn = conn or general.connect() self.token = None self.username = None if username is not None: self.username = username self.token = general.login(self.conn, username)
def updateLineCounts(): (connection, cursor) = general.connect() command = "wc -l *.tex" (output, error) = general.execute(command) for line in filter(None, output.split("\n")): (linecount, filename) = line.split() update("linecount " + filename, linecount, cursor) general.close(connection)
def updatePageCounts(): (connection, cursor) = general.connect() command = "ls tags/tmp/*.log" (output, error) = general.execute(command) for filename in filter(None, output.split("\n")): f = open(config.websiteProject + "/" + filename) for line in f: if "Output written on " in line: update("pagecount " + filename.split("/")[-1].split(".")[0], line.split(" ")[4][1:], cursor) general.close(connection)
def clearHistory(): (connection, cursor) = general.connect() try: query = "DELETE FROM commits" cursor.execute(query) query = "DELETE FROM changes" cursor.execute(query) except sqlite3.Error, e: print "An error occurred:", e.args[0]
def clearHistory(): (connection, cursor) = general.connect() try: query = 'DELETE FROM commits' cursor.execute(query) query = 'DELETE FROM changes' cursor.execute(query) except sqlite3.Error, e: print "An error occurred:", e.args[0]
def clearBibliography(): (connection, cursor) = general.connect() try: query = 'DELETE FROM bibliography_items' cursor.execute(query) query = 'DELETE FROM bibliography_values' cursor.execute(query) except sqlite3.Error, e: print "An error occurred:", e.args[0]
def importHistory(commit): history = load_back(commit) (connection, cursor) = general.connect() # process commits print " Processing commits" for commit in history.commits: insertCommitInfo(commit, getCommitInfo(commit), cursor) print " Processing histories" for tag in history.env_histories: # there must be a tag, otherwise it is not accessible on the website (intermediate commits can have "tag-less tags") if tag.env.tag != "": process_tag(tag, cursor) general.close(connection)
def importBootstrap(): (connection, cursor) = general.connect() bootstrap_file = open("tags/bootstrap.txt", 'r') for line in bootstrap_file: tag, label = line.strip().split(',') if not tag_exists(tag, cursor): try: query = 'INSERT INTO tags (tag, label, active) VALUES (?, ?, "FALSE")' cursor.execute(query, (tag, label)) except sqlite3.Error, e: print tag, label print "An error occurred:", e.args[0] else: set_inactive(tag, cursor)
def checkTags(): (connection, cursor) = general.connect() print 'Parsing the tags file' active_tags = parse_tags(config.websiteProject + "/tags/tags").keys() tags = get_tags(cursor) for tag in tags: # check whether the tag is no longer used in the project if tag[0] not in active_tags and is_active(tag[0], cursor): print ' ', tag[0], 'has become inactive' set_inactive(tag[0], cursor) # probably not necessary, but check whether a tag is again used in the project if tag[1] == 'FALSE' and tag[0] in active_tags: print ' ', tag[0], 'has become active again' set_active(tag[0], cursor) general.close(connection)
def importTitles(): global connection (connection, cursor) = general.connect() print 'Creating a database version of the table of contents' print 'Parsing the files, linking chapters to file names' titles = get_titles(config.websiteProject + "/tags/tmp/") print 'Parsing the big table of contents' sections = parse_book_toc(config.websiteProject + "/tags/tmp/book.toc") # print out new or changed titles before updating the database for section in sections: if title_exists(section[2]) == 0 and not section[2] == 'Bibliography': print 'New or changed section \'%s\'' % (section[2]) print 'Inserting the information into the database' for section in sections: insert_title(section[1], section[2], find_file_for_section(titles, sections, section[1])) general.close(connection)
def importTags(): (connection, cursor) = general.connect() print 'Parsing the tags file' tags = parse_tags(config.websiteProject + "/tags/tags") labels = get_labels_from_source(config.websiteProject + "/tags/tmp/") print 'Inserting (or updating) the tags' for tag, label in tags.iteritems(): if label not in labels: print 'ERROR, label', label, 'not found in auxiliary files. Have you ran `make tags`?' sys.exit() if tag_exists(tag, cursor) and get_label(tag, cursor) != label: print 'The label for tag %s has changed from \'%s\'to \'%s\'' % (tag, get_label(tag, cursor), label) if not tag_exists(tag, cursor): print 'New tag %s with label \'%s\'' % (tag, label) info = labels[label] insert_tag(tag, (label, info[0], info[2][1], info[1][1], info[1][0], info[1][2], info[2][3], info[1][4]), cursor) general.close(connection)
def updateCommits(): (connection, cursor) = general.connect() (what, error) = general.execute("git whatchanged --reverse -p tags/tags") creation = {} modified = {} for line in what.split("\n"): if line == "": continue if line.find('commit') == 0: commit = line[7:] new = 1 continue if line.find('Date') == 0: date = line[12:] continue if line.find('#') >= 0: continue if line.find('@@') == 0: new = 0 continue if new == 0: c = line[0] if not c == '+': continue line = line.lstrip(c) tag = line.split(',')[0] label = line.split(',')[1] if tag not in creation: creation[tag] = [date, commit] modified[tag] = [date, commit] tags = getTags(cursor) for tag in tags: updateCommitInformation(tag, creation[tag], modified[tag], cursor) general.close(connection)
import general, macros.parse (connection, cursor) = general.connect() print 'Emptying the table' macros.parse.clearMacros(cursor) print 'Importing macros from tex/preamble.tex' macros.parse.addMacros(cursor) general.close(connection)
CURSOR = None CONN = None def select(sql): global CURSOR try: CURSOR.execute(sql) return CURSOR.fetchall() except Exception: return None if (__name__ == '__main__'): # -------------------------------------------- CONN, CURSOR = connect() if (CONN is None and CURSOR is None): message('-', ' Connect database failed') exit(1) # -------------------------------------------- parser = argparse.ArgumentParser( description='Check CMS for website(s).', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( 'url', nargs='+', help='The website URL/ The URLs File') parser.add_argument( '--version', action='version', version='SoFinger Version: 0.1 Beta', help='Show version and exit') parser.add_argument('--time', nargs='?', type=int, default=3, const=1, help=' Timeout of requests') parser.add_argument('--thread', nargs='?', type=int, default=20, const=1, help=' Thread number for tasks')
def importBibliography(): (connection, cursor) = general.connect() f = open(config.websiteProject + "/my.bib") items = [] in_item = 0 line_nr = 0 for line in f: line_nr = line_nr + 1 # beginning of a new item if line[0] == '@': if in_item == 1: print "On line:", line_nr, "of my.bib:" print line print "Nested items. Exiting." exit(1) # clear previous item item = [[], {}] bib_type = line.partition('{')[0].strip('@').lower() name = line.partition('{')[2].strip().strip(',') item[0] = (bib_type, name) in_item = 1 continue # end of an item if line[0] == '}': if in_item == 0: print "On line:", line_nr, "of my.bib:" print line print "Nested items. Exiting." exit(1) # add a *copy* to the list of items items.append(list(item)) in_item = 0 continue # can ignore current line if not in an item if in_item == 0: continue # ignore comments if line[0] == '%' or line[0:2] == '//': continue # Get key value pair from the line if it contains = sign if '=' in line: key = line.partition('=')[0].strip().lower() value = line.partition('=')[2].strip().strip(',')[1:-1] item[1][key] = value # Check for correctness if not (line.strip()[-2:] == '},' or line.strip()[-2:] == '",' or line.strip()[-1] == '}' or line.strip()[-1:] == '"'): print "Warning: run over on line:", line_nr, "in my.bib:" print line print "Should not happen: each key = value pair should be on a line!" print "Exiting" exit(1) for item in items: insertItem(item, cursor) general.close(connection)
try: query = 'DELETE FROM comments WHERE id = ?' connection.execute(query, [ID]) except sqlite3.Error, e: print "An error occurred:", e.args[0] if not len(sys.argv) == 2: print 'You must supply one argument, namely the id of the comment you wish to remove' raise Exception('Wrong number of arguments') ID = int(sys.argv[1]) print 'Trying to remove the comment with id', int(ID) (connection, cursor) = general.connect() if not commentExists(ID): print 'There is no such comment in the database' else: comment = getComment(ID) choice = raw_input('Are you sure you wish to remove this comment by ' + comment[1] + ' on tag ' + comment[3] + '? (Y/N): ') if string.upper(choice) == 'Y': deleteComment(ID) print 'Comment removed!' connection.commit() connection.close()
def __init__(self, conn: Optional[redis.Redis] = None): self.conn = conn or general.connect() self.msg = None