def run(self):
year, month = self.findFirstMonth()
#year = 2015
#month = 2
if not year or not month: return
d = datetime(year, month, 1)
while not self.finish_flag and d <= datetime.now():
feed = self.getFeed(self.url + "/%s/%s/%s/feed" % (d.year, d.month, d.day))
for entry in feed.entries:
if self.finish_flag: break
if entry.slash_comments > 0:
comments_feed = self.getFeed(entry.wfw_commentrss)
if comments_feed:
for comment_entry in comments_feed.entries:
fe = FeedEntry.fromFeedParserEntry(comments_feed.feed.link, comment_entry)
fe.account = self.account
fe.campaign = self.campaign
self.queue.put(fe)
d = d + timedelta(days=1)
if d.day == 1: #cambio el mes, me fijo que si posts en el nuevo mes
while not self.finish_flag and datetime.now():
dummy_feed = self.getFeed(self.url + "/%s/%s/feed" % (d.year, d.month))
if dummy_feed.entries:
break
d = (d + timedelta(days=32)).replace(day=1) #agrego 1 mes
if d > datetime.now():
acc = MongoManager.getAccount(id=self.account.getId())
camp = acc.getCampaign(id =self.campaign.getId())
camp.addHistoryFetchedForum(self.url)
MongoManager.saveCampaign(acc, camp)
def run(self):
year, month = self.findFirstMonth()
#year = 2015
#month = 2
if not year or not month: return
d = datetime(year, month, 1)
while not self.finish_flag and d <= datetime.now():
feed = self.getFeed(self.url + "/%s/%s/%s/feed" %
(d.year, d.month, d.day))
for entry in feed.entries:
if self.finish_flag: break
if entry.slash_comments > 0:
comments_feed = self.getFeed(entry.wfw_commentrss)
if comments_feed:
for comment_entry in comments_feed.entries:
fe = FeedEntry.fromFeedParserEntry(
comments_feed.feed.link, comment_entry)
fe.account = self.account
fe.campaign = self.campaign
self.queue.put(fe)
d = d + timedelta(days=1)
if d.day == 1: #cambio el mes, me fijo que si posts en el nuevo mes
while not self.finish_flag and datetime.now():
dummy_feed = self.getFeed(self.url + "/%s/%s/feed" %
(d.year, d.month))
if dummy_feed.entries:
break
d = (d + timedelta(days=32)).replace(day=1) #agrego 1 mes
if d > datetime.now():
acc = MongoManager.getAccount(id=self.account.getId())
camp = acc.getCampaign(id=self.campaign.getId())
camp.addHistoryFetchedForum(self.url)
MongoManager.saveCampaign(acc, camp)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--regenerate', action="store_true", default=False)
parser.add_argument('--account', default=None)
parser.add_argument('--list', action="store_true", default=False)
parser.add_argument('--start', default=None)
parser.add_argument('--end', default=None)
parser.add_argument('--clear', action="store_true", default=False)
args, known = parser.parse_known_args()
campaign = None
if args.account:
account = MongoManager.getAccount(name=args.account)
if not account:
pprint("Account %s not found" % args.account)
exit(1)
campaign = account.getActiveCampaigns()[0]
summarizer = Summarizer()
if not args.list and not args.clear:
summarizer.start(campaign=campaign, regenerate=args.regenerate)
elif args.clear and campaign:
summarizer.clearSummarization(campaign)
elif args.list and campaign and args.start and args.end:
print args
start = datetime.strptime(args.start, "%Y-%m-%dT%H")
end = datetime.strptime(args.end, "%Y-%m-%dT%H")
records = summarizer.getSummarizedData(campaign,start,end)
return res
"""
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--regenerate', action="store_true", default=False)
parser.add_argument('--account', default=None)
parser.add_argument('--list', action="store_true", default=False)
parser.add_argument('--start', default=None)
parser.add_argument('--end', default=None)
parser.add_argument('--clear', action="store_true", default=False)
args, known = parser.parse_known_args()
campaign = None
if args.account:
account = MongoManager.getAccount(name=args.account)
if not account:
pprint("Account %s not found" % args.account)
exit(1)
campaign = account.getActiveCampaigns()[0]
summarizer = Summarizer()
if not args.list and not args.clear:
summarizer.start(campaign=campaign, regenerate=args.regenerate)
elif args.clear and campaign:
summarizer.clearSummarization(campaign)
elif args.list and campaign and args.start and args.end:
print args
start = datetime.strptime(args.start, "%Y-%m-%dT%H")
end = datetime.strptime(args.end, "%Y-%m-%dT%H")
records = summarizer.getSummarizedData(campaign, start, end)
