class AsonamHoneypotImporter(AbstractController):
def __init__(self, db):
super(AsonamHoneypotImporter, self).__init__(db)
self.twitter_rest_api = Twitter_Rest_Api(self._db)
self._data_path = self._config_parser.eval(self.__class__.__name__,
"data_path")
def execute(self, window_start=None):
all_sub_files = listdir(self._data_path)
for data_file in all_sub_files:
self.add_tweet_data_to_db(data_file)
def add_tweet_data_to_db(self, data_file):
type_to_users_tweets_dict = self.load_file(self._data_path + data_file)
for author_type in type_to_users_tweets_dict:
tweets_ids = []
for author in type_to_users_tweets_dict[author_type]:
tweets_ids.extend(
type_to_users_tweets_dict[author_type][author])
tweets = self.get_tweets(tweets_ids, str(author_type))
self.twitter_rest_api._save_posts_and_authors(
tweets, str(author_type))
def load_file(self, file_path):
file = open(file_path, 'r', newline='')
csv_file = csv.reader(file, delimiter=' ')
type_to_users_tweets = defaultdict(dict)
for row in csv_file:
author_type = row[0]
author_id = row[1]
tweet_ids = row[2:]
type_to_users_tweets[author_type][author_id] = tweet_ids
file.close()
return type_to_users_tweets
def get_tweets(self, tweets_ids, author_type=""):
return self.twitter_rest_api.get_tweets_by_ids(tweets_ids, author_type)
class Trec2012MicroblogTrackImporter(Method_Executor):
def __init__(self, db):
Method_Executor.__init__(self, db)
self._topics_path = self._config_parser.eval(self.__class__.__name__,
"topics_path")
self._judgment_path = self._config_parser.eval(self.__class__.__name__,
"judgment_path")
self._num_of_relevant_tweets = self._config_parser.eval(
self.__class__.__name__, "num_of_relevant_tweets")
self._num_of_description_words = self._config_parser.eval(
self.__class__.__name__, "num_of_description_words")
self._twitter_api = Twitter_Rest_Api(db)
def load_data(self):
topics = self._read_trec_topics(self._topics_path)
topic_judgments = self._read_judgments(self._judgment_path)
claims = self._extract_claims_from_judgments(topics, topic_judgments)
self._db.addPosts(claims)
self._create_tweet_corpus_from_judgments(self._judgment_path)
def set_description_from_relevant(self):
claims = self._db.get_claims()
topic_judgments = self._read_judgments(self._judgment_path)
posts = self._db.get_posts()
post_dict = {p.post_id: p for p in posts}
for claim in claims:
topic_id = int(claim.claim_id)
# tweets = self._twitter_api.get_tweets_by_ids(topic_judgments[topic_id][:self._num_of_relevant_tweets])
# posts, authors = self._db.convert_tweets_to_posts_and_authors(tweets, self._domain)
posts = list(
map(post_dict.get,
topic_judgments[topic_id][:self._num_of_relevant_tweets]))
claim_content = OrderedSet(claim.keywords.lower().split())
for post in posts:
list(map(claim_content.add, clean_tweet(post.content).split()))
# if len(claim_content) > 25:
# break
claim.description = clean_claim_description(
' '.join(claim_content), True)
self._db.addPosts(claims)
def set_description_from_tf_idf_best(self):
tf_idf_vectorizer = TfidfVectorizer(stop_words='english')
posts = self._db.get_posts()
corpus = [clean_claim_description(p.content, True) for p in posts]
tf_idf_vectorizer.fit_transform(corpus)
word_tf_idf_dict = defaultdict(
float,
list(
zip(tf_idf_vectorizer.get_feature_names(),
tf_idf_vectorizer.idf_)))
post_dict = {p.post_id: p for p in posts}
topic_judgments = self._read_judgments(self._judgment_path)
claims = self._db.get_claims()
for i, claim in enumerate(claims):
init_query_words = set(claim.keywords.lower().split())
claim_content = set()
relevant_posts_ids = topic_judgments[int(claim.claim_id)]
words = set()
for post_id in relevant_posts_ids:
if post_id in post_dict:
words.update(
clean_claim_description(
post_dict.get(post_id).content, True).split())
best_words = sorted(words,
key=lambda k: word_tf_idf_dict[k],
reverse=True)[:self._num_of_description_words +
len(init_query_words)]
claim_content.update(best_words)
pass
claim_content = claim_content - init_query_words
claim_description = clean_claim_description(
' '.join(claim_content), True)
claim.description = ' '.join(claim_description.split())
self._db.addPosts(claims)
def _read_trec_topics(self, topics_path):
trec_topic_fields = ['num', 'query', 'querytime', 'querytweettime']
TrecTopic = namedtuple('TrecTopic', trec_topic_fields)
topic_file = open(topics_path)
trec_topics = []
for topic_xml in topic_file.read().split('\n\n')[:-1]:
trec_topic_dict = xmltodict.parse(topic_xml)
trec_topic = TrecTopic._make(
[trec_topic_dict['top'][field] for field in trec_topic_fields])
trec_topics.append(trec_topic)
return trec_topics
# def _read_judgments(self, judgment_path):
# judgments = pd.read_csv(judgment_path, sep=' ', names=['topic_id', 'Q', 'tweet_id', 'rel'])
# tweets = self._twitter_api.get_tweets_by_ids(judgments['tweet_id'], pre_save=False)
# topic_tweet_id_dict = dict(judgments[['topic_id', 'tweet_id']].to_records(index=False))
# tweet_id_rel_dict = dict(judgments[['tweet_id', 'rel']].to_records(index=False))
# posts, authors = self._db.convert_tweets_to_posts_and_authors(tweets, u'Trec2012')
def _read_judgments(self, judgment_path):
topic_high_relevant_judgment_dict = defaultdict(list)
topic_relevant_judgment_dict = defaultdict(list)
for topic, Q, docid, rel in csv.reader(open(judgment_path, "rb"),
delimiter=' '):
if int(rel) > 1:
topic_high_relevant_judgment_dict[int(topic)].append(docid)
elif int(rel) == 1:
topic_relevant_judgment_dict[int(topic)].append(docid)
for topic_id, tweet_ids in topic_relevant_judgment_dict.items():
# if topic_id not in topic_high_relevant_judgment_dict:
topic_high_relevant_judgment_dict[topic_id].extend(tweet_ids)
return topic_high_relevant_judgment_dict
def _extract_claims_from_judgments(self, topics, topic_judgments):
claims = []
for trec_topic in topics:
# tweet_id = tweet_ids[0]
topic_id = int(parse.parse('Number: MB{}', trec_topic.num)[0])
# tweets = self._twitter_api.get_tweets_by_ids(topic_judgments[topic_id][:10])
# posts, authors = self._db.convert_tweets_to_posts_and_authors(tweets, self._domain)
claim_content = set(trec_topic.query.split())
# for post in []:
# claim_content.update(clean_tweet(post.content).split())
# if len(claim_content) > 25:
# break
claim = self._convet_trec_topic_to_claim(' '.join(claim_content),
topic_id, trec_topic)
claims.append(claim)
return claims
def _convet_trec_topic_to_claim(self, claim_content, topic_id, trec_topic):
claim = Claim()
claim.claim_id = topic_id
claim.verdict_date = parser.parse(trec_topic.querytime).date()
claim.domain = 'Trec2012'
claim.title = trec_topic.query
claim.keywords = trec_topic.query
claim.description = claim_content
return claim
def _create_tweet_corpus_from_judgments(self, judgment_path):
judgment_df = pd.read_csv(
judgment_path,
delimiter=' ',
names=['topic', 'Q', 'docid', 'rel'],
)
tweet_ids = judgment_df['docid'].tolist()
tweets = self._twitter_api.get_tweets_by_ids(tweet_ids, pre_save=False)
posts, authors = self._db.convert_tweets_to_posts_and_authors(
tweets, 'Trec2012')
claim_tweet_connections = []
for post in posts:
post.post_id = str(post.post_osn_id)
post_osn_id_posts_dict = set(p.post_osn_id for p in posts)
for topic_id, post_osn_id in judgment_df[['topic', 'docid'
]].to_records(index=False):
if post_osn_id in post_osn_id_posts_dict:
claim_tweet_connection = Claim_Tweet_Connection()
claim_tweet_connection.claim_id = str(topic_id)
claim_tweet_connection.post_id = str(post_osn_id)
claim_tweet_connections.append(claim_tweet_connection)
self._db.addPosts(claim_tweet_connections)
self._db.addPosts(posts)
judgment_df[judgment_df['docid'].isin(post_osn_id_posts_dict)].to_csv(
judgment_path + '_filtered', sep=' ', header=False, index=False)