def analyseArticle(file_name): # Container of relationships. relations = {} # Revisions to compare. revision_curr = Revision() revision_prev = Revision() text_curr = None # Access the file. dumpIterator = dump.Iterator(file_name) # Iterate over the pages. for page in dumpIterator.readPages(): i = 0 # Iterate over revisions of the article. for revision in page.readRevisions(): vandalism = False #print "processing rev", revision.getId() # Update the information about the previous revision. revision_prev = revision_curr if (revision.getSha1() == None): revision.setSha1( Text.calculateHash(revision.getText().encode("utf-8"))) if (revision.getSha1() in spam): vandalism = True #TODO: SPAM detection: DELETION if (revision.getComment() != None and revision.getComment().find(FLAG) > 0): pass else: if (revision_prev.length > PREVIOUS_LENGTH) and (len( revision.getText()) < CURR_LENGTH) and (( (len(revision.getText()) - revision_prev.length) / float(revision_prev.length)) <= CHANGE_PERCENTAGE): vandalism = True revision_curr = revision_prev #if (vandalism): #print "---------------------------- FLAG 1" #print "SPAM", revision.getId() #print revision.getText() #print if (not vandalism): # Information about the current revision. revision_curr = Revision() revision_curr.id = i revision_curr.wikipedia_id = int(revision.getId()) revision_curr.length = len(revision.getText()) revision_curr.timestamp = revision.getTimestamp() revision_curr.comment = revision.getComment() # Relation of the current relation. relation = Relation() relation.revision = int(revision.getId()) relation.length = len(revision.getText()) # Some revisions don't have contributor. if (revision.getContributor() != None): revision_curr.contributor_id = revision.getContributor( ).getId() revision_curr.contributor_name = revision.getContributor( ).getUsername().encode('utf-8') relation.author = revision.getContributor().getUsername( ).encode('utf-8') else: revision_curr.contributor_id = 'Not Available ' + revision.getId( ) revision_curr.contribur_name = 'Not Available ' + revision.getId( ) relation.author = 'Not Available ' + revision.getId() # Content within the revision. text_curr = revision.getText().encode('utf-8') text_curr = text_curr.lower() revision_curr.content = text_curr # Perform comparison. vandalism = determineAuthorship(revision_curr, revision_prev, text_curr, relation) if (not vandalism): #print "NOT SPAM", revision.getId() # Add the current revision with all the information. revisions.update( {revision_curr.wikipedia_id: revision_curr}) relations.update({revision_curr.wikipedia_id: relation}) revision_order.append((revision_curr.wikipedia_id, False)) # Update the fake revision id. i = i + 1 # Calculate the number of tokens in the revision. total = 0 for p in revision_curr.ordered_paragraphs: for paragraph_curr in revision_curr.paragraphs[p]: for hash_sentence_curr in paragraph_curr.sentences.keys( ): for sentence_curr in paragraph_curr.sentences[ hash_sentence_curr]: total = total + len(sentence_curr.words) revision_curr.total_tokens = total relation.total_tokens = total else: #print "---------------------------- FLAG 2" #print "SPAM", revision.getId() #print revision.getText() #print revision_order.append((revision_curr.wikipedia_id, True)) revision_curr = revision_prev spam.append(revision.getSha1()) return (revisions, revision_order, relations)
def analyseArticle(file_name): # Container of revisions. revisions = {} # Revisions to compare. revision_curr = Revision() revision_prev = Revision() text_curr = None # Access the file. dumpIterator = dump.Iterator(file_name) # Iterate over the pages. for page in dumpIterator.readPages(): i = 0 # Iterate over revisions of the article. for revision in page.readRevisions(): vandalism = False # Update the information about the previous revision. revision_prev = revision_curr if (revision.getSha1() == None): revision.setSha1(Text.calculateHash(revision.getText().encode("utf-8"))) if (revision.getSha1() in spam): vandalism = True #TODO: SPAM detection: DELETION if (revision.getComment()!= None and revision.getComment().find(FLAG) > 0): pass else: if (revision_prev.length > PREVIOUS_LENGTH) and (len(revision.getText()) < CURR_LENGTH) and (((len(revision.getText())-revision_prev.length)/float(revision_prev.length)) <= CHANGE_PERCENTAGE): vandalism = True revision_curr = revision_prev #if (vandalism): #print "---------------------------- FLAG 1" #print revision.getId() #print revision.getText() #print if (not vandalism): # Information about the current revision. revision_curr = Revision() revision_curr.id = i revision_curr.wikipedia_id = int(revision.getId()) revision_curr.length = len(revision.getText()) # Some revisions don't have contributor. if (revision.getContributor() != None): revision_curr.contributor_id = revision.getContributor().getId() revision_curr.contributor_name = revision.getContributor().getUsername() else: revision_curr.contributor_id = 'Not Available' revision_curr.contribur_name = 'Not Available' # Content within the revision. text_curr = revision.getText().encode('utf-8') text_curr = text_curr.lower() revision_curr.content = text_curr # Perform comparison. vandalism = determineAuthorship(revision_curr, revision_prev, text_curr) if (not vandalism): # Add the current revision with all the information. revisions.update({revision_curr.wikipedia_id : revision_curr}) # Update the fake revision id. i = i+1 else: #print "---------------------------- FLAG 2" #print revision.getId() #print revision.getText() #print revision_curr = revision_prev spam.append(revision.getSha1()) return revisions
def getTagDatesFromPage(file_name): # Compile regexp reglist = list() reglist.append({ "tagname": "maintained", "regexp": re.compile( '\{\{(articleissues\|((?:(?!\}\}).)*\||)|multiple issues\|((?:(?!\}\}).)*\||)|)maintained((\||=)(?:(?!\}\}).)*|)\}\}', re.IGNORECASE) }) reglist.append({ "tagname": "good article", "regexp": re.compile( '\{\{(articleissues\|((?:(?!\}\}).)*\||)|multiple issues\|((?:(?!\}\}).)*\||)|)good article((\||=)(?:(?!\}\}).)*|)\}\}', re.IGNORECASE) }) #reglist.append({"tagname": "featured article", "regexp": re.compile('\{\{(articleissues\|((?:(?!\}\}).)*\||)|multiple issues\|((?:(?!\}\}).)*\||)|)featured article((\||=)(?:(?!\}\}).)*|)\}\}', re.IGNORECASE)}) #reglist.append({"tagname": "featured article", "regexp": re.compile('\|currentstatus=FA', re.IGNORECASE)}) reglist.append({ "tagname": "npov", "regexp": re.compile( '\{\{(articleissues\|((?:(?!\}\}).)*\||)|multiple issues\|((?:(?!\}\}).)*\||)|)(pov|npov)((\||=)(?:(?!\}\}).)*|)\}\}', re.IGNORECASE) }) reglist.append({ "tagname": "disputed", "regexp": re.compile( '\{\{(articleissues\|((?:(?!\}\}).)*\||)|multiple issues\|((?:(?!\}\}).)*\||)|)disputed((\||=)(?:(?!\}\}).)*|)\}\}', re.IGNORECASE) }) re_user = re.compile('({{|\[\[)user.*?[:|](.*?)[}/\]|]', re.IGNORECASE) #"({{|\[\[)user[\s\S]*?[:|]([\s\S]*?)[}/\]|]" # Access the file. dumpIterator = dump.Iterator(file_name) # Revisions to compare. revision_curr = Revision() revision_prev = Revision() text_curr = None listOfTagChanges = {} all_contributors = { "maintained": {}, "good article": {}, "featured article": {}, "npov": {}, "disputed": {} } # Iterate over the pages. for page in dumpIterator.readPages(): # Iterate over revisions of the article. i = 0 prev_matched = list() for revision in page.readRevisions(): revision.wikipedia_id = int(revision.getId()) revision.timestamp = revision.getTimestamp() # Some revisions don't have contributor. if (revision.getContributor() != None): revision.contributor_id = revision.getContributor().getId() revision.contributor_name = revision.getContributor( ).getUsername() else: revision.contributor_id = 'Not Available' revision.contribur_name = 'Not Available' text_curr = revision.getText() if (text_curr): text_curr = text_curr.encode('utf-8') text_curr = text_curr.lower() else: continue matched = list() aux = list() for regexp in reglist: m = regexp["regexp"].search(text_curr) if m: mc = re_user.split(m.group(0)) i = 2 users = [] users.append(mc[2]) while (i + 3 < len(mc)): #print m[i+3] users.append(mc[i + 3]) i = i + 3 #print regexp["tagname"], contributor #print regexp["tagname"], users matched.append(regexp["tagname"]) aux.append((regexp["tagname"], users)) ## #m_user = re_user.search(m.group(0)) #contributor = m_user.group(2) if "|currentstatus=FA" in text_curr: matched.append("featured article") aux.append((revision.contributor_name, "featured article")) # Calculate additions for (match, contributor) in aux: if not (match in prev_matched): if not (revision.timestamp in listOfTagChanges.keys()): listOfTagChanges[revision.timestamp] = list() listOfTagChanges[revision.timestamp].append({ "rev": revision.wikipedia_id, "type": "addition", "tagname": match, "wikiname": revision.contributor_name, "timestamp": revision.timestamp, "date": datetime.datetime.fromtimestamp(int( revision.timestamp)).strftime('%Y-%m-%d %H:%M:%S') }) all_contributors[match].update({ revision.timestamp: { "rev": revision.wikipedia_id, "user": contributor, "date": datetime.datetime.fromtimestamp(int( revision.timestamp)).strftime('%Y-%m-%d %H:%M:%S') } }) # Calculate removals for match in prev_matched: if not (match in matched): if not (revision.timestamp in listOfTagChanges.keys()): listOfTagChanges[revision.timestamp] = list() listOfTagChanges[revision.timestamp].append({ "rev": revision.wikipedia_id, "type": "removal", "tagname": match, "wikiname": revision.contributor_name, "timestamp": revision.timestamp, "date": datetime.datetime.fromtimestamp(int( revision.timestamp)).strftime('%Y-%m-%d %H:%M:%S') }) prev_matched = matched return listOfTagChanges, all_contributors
def analyseArticle(file_name): # Container of relationships. relations = {} # Revisions to compare. revision_curr = Revision() revision_prev = Revision() text_curr = None # Access the file. dumpIterator = dump.Iterator(file_name) # Iterate over the pages. for page in dumpIterator.readPages(): i = 0 # Iterate over revisions of the article. for revision in page.readRevisions(): vandalism = False #print "processing rev", revision.getId() # Update the information about the previous revision. revision_prev = revision_curr if (revision.getSha1() == None): revision.setSha1(Text.calculateHash(revision.getText().encode("utf-8"))) if (revision.getSha1() in spam): vandalism = True #TODO: SPAM detection: DELETION if (revision.getComment()!= None and revision.getComment().find(FLAG) > 0): pass else: if (revision_prev.length > PREVIOUS_LENGTH) and (len(revision.getText()) < CURR_LENGTH) and (((len(revision.getText())-revision_prev.length)/float(revision_prev.length)) <= CHANGE_PERCENTAGE): vandalism = True revision_curr = revision_prev #if (vandalism): #print "---------------------------- FLAG 1" #print "SPAM", revision.getId() #print revision.getText() #print if (not vandalism): # Information about the current revision. revision_curr = Revision() revision_curr.id = i revision_curr.wikipedia_id = int(revision.getId()) revision_curr.length = len(revision.getText()) revision_curr.timestamp = revision.getTimestamp() revision_curr.comment = revision.getComment() # Relation of the current relation. relation = Relation() relation.revision = int(revision.getId()) relation.length = len(revision.getText()) # Some revisions don't have contributor. if (revision.getContributor() != None): revision_curr.contributor_id = revision.getContributor().getId() revision_curr.contributor_name = revision.getContributor().getUsername().encode('utf-8') relation.author = revision.getContributor().getUsername().encode('utf-8') else: revision_curr.contributor_id = 'Not Available ' + revision.getId() revision_curr.contribur_name = 'Not Available ' + revision.getId() relation.author = 'Not Available ' + revision.getId() # Content within the revision. text_curr = revision.getText().encode('utf-8') text_curr = text_curr.lower() revision_curr.content = text_curr # Perform comparison. vandalism = determineAuthorship(revision_curr, revision_prev, text_curr, relation) if (not vandalism): #print "NOT SPAM", revision.getId() # Add the current revision with all the information. revisions.update({revision_curr.wikipedia_id : revision_curr}) relations.update({revision_curr.wikipedia_id : relation}) revision_order.append((revision_curr.wikipedia_id, False)) # Update the fake revision id. i = i+1 # Calculate the number of tokens in the revision. total = 0 for p in revision_curr.ordered_paragraphs: for paragraph_curr in revision_curr.paragraphs[p]: for hash_sentence_curr in paragraph_curr.sentences.keys(): for sentence_curr in paragraph_curr.sentences[hash_sentence_curr]: total = total + len(sentence_curr.words) revision_curr.total_tokens = total relation.total_tokens = total else: #print "---------------------------- FLAG 2" #print "SPAM", revision.getId() #print revision.getText() #print revision_order.append((revision_curr.wikipedia_id, True)) revision_curr = revision_prev spam.append(revision.getSha1()) return (revisions, revision_order, relations)
def analyseArticle(file_name): # Container of revisions. revisions = {} revision_order = [] # Revisions to compare. revision_curr = Revision() revision_prev = Revision() text_curr = None # Access the file. dumpIterator = dump.Iterator(file_name) # Iterate over the pages. for page in dumpIterator.readPages(): i = 0 # Iterate over revisions of the article. for revision in page.readRevisions(): vandalism = False # Update the information about the previous revision. revision_prev = revision_curr if (revision.getSha1() == None): revision.setSha1( Text.calculateHash(revision.getText().encode("utf-8"))) if (revision.getSha1() in spam): vandalism = True #TODO: SPAM detection: DELETION if (revision.getComment() != None and revision.getComment().find(FLAG) > 0): pass else: if (revision_prev.length > PREVIOUS_LENGTH) and (len( revision.getText()) < CURR_LENGTH) and (( (len(revision.getText()) - revision_prev.length) / float(revision_prev.length)) <= CHANGE_PERCENTAGE): print "VANDALISM: CHANGE PERCETANGE" vandalism = True #if (vandalism): #print "---------------------------- FLAG 1" #print revision.getId() #print revision.getText() #print if (not vandalism): # Information about the current revision. revision_curr = Revision() revision_curr.id = i revision_curr.wikipedia_id = int(revision.getId()) revision_curr.length = len(revision.getText()) # Some revisions don't have contributor. if (revision.getContributor() != None): revision_curr.contributor_id = revision.getContributor( ).getId() revision_curr.contributor_name = revision.getContributor( ).getUsername() else: revision_curr.contributor_id = 'Not Available' revision_curr.contribur_name = 'Not Available' # Content within the revision. text_curr = revision.getText().encode('utf-8') text_curr = text_curr.lower() revision_curr.content = text_curr # Perform comparison. vandalism = determineAuthorship(revision_curr, revision_prev, text_curr) if (not vandalism): # Add the current revision with all the information. revisions.update( {revision_curr.wikipedia_id: revision_curr}) # Update the fake revision id. i = i + 1 # Update the index of processed revisions. revision_order.append((revision_curr.wikipedia_id, False)) else: #print "detected vandalism in here ...................................." #print "---------------------------- FLAG 2" #print revision.getId() #print revision.getText() #print spam.append(revision.getSha1()) revision_order.append((revision_curr.wikipedia_id, True)) revision_curr = revision_prev else: # revision.getText() # #print spam.append(revision.getSha1()) revision_order.append((revision_curr.wikipedia_id, True)) revision_curr = revision_prev return (revisions, revision_order)
def analyseArticle(file_name): # Container of relationships. relations = {} # Revisions to compare. revision_curr = Revision() revision_prev = Revision() text_curr = None # Access the file. dumpIterator = mwIterator.from_file(open_file(file_name)) # Iterate over the pages. for page in dumpIterator: i = 0 # Iterate over revisions of the article. for revision in page: vandalism = False # Update the information about the previous revision. revision_prev = revision_curr if (revision.sha1 == None): revision.sha1 = Text.calculateHash(revision.text) if (revision.sha1 in spam): vandalism = True #TODO: SPAM detection: DELETION if (revision.comment!= None and revision.comment.find(FLAG) > 0): pass else: if (revision_prev.length > PREVIOUS_LENGTH) and (len(revision.text) < CURR_LENGTH) and (((len(revision.text)-revision_prev.length)/float(revision_prev.length)) <= CHANGE_PERCENTAGE): vandalism = True revision_curr = revision_prev if (not vandalism): # Information about the current revision. revision_curr = Revision() revision_curr.id = i revision_curr.wikipedia_id = int(revision.id) revision_curr.length = len(revision.text) revision_curr.timestamp = revision.timestamp # Relation of the current relation. relation = Relation() relation.revision = int(revision.id) relation.length = len(revision.text) # Some revisions don't have contributor. if (revision.contributor != None): revision_curr.contributor_id = revision.contributor.id revision_curr.contributor_name = revision.contributor.user_text relation.author = revision.contributor.user_text else: revision_curr.contributor_id = 'Not Available ' + revision.id revision_curr.contribur_name = 'Not Available ' + revision.id relation.author = 'Not Available ' + revision.id # Content within the revision. text_curr = revision.text.lower() # Perform comparison. vandalism = determineAuthorship(revision_curr, revision_prev, text_curr, relation) if (not vandalism): # Add the current revision with all the information. revisions.update({revision_curr.wikipedia_id : revision_curr}) relations.update({revision_curr.wikipedia_id : relation}) revision_order.append((revision_curr.wikipedia_id, False)) # Update the fake revision id. i = i+1 # Calculate the number of tokens in the revision. total = 0 for p in revision_curr.ordered_paragraphs: for paragraph_curr in revision_curr.paragraphs[p]: for hash_sentence_curr in paragraph_curr.sentences.keys(): for sentence_curr in paragraph_curr.sentences[hash_sentence_curr]: total = total + len(sentence_curr.words) revision_curr.total_tokens = total relation.total_tokens = total else: revision_order.append((revision_curr.wikipedia_id, True)) revision_curr = revision_prev spam.append(revision.sha1) return (revisions, revision_order, relations)