def run(self): year, month = self.findFirstMonth() #year = 2015 #month = 2 if not year or not month: return d = datetime(year, month, 1) while not self.finish_flag and d <= datetime.now(): feed = self.getFeed(self.url + "/%s/%s/%s/feed" % (d.year, d.month, d.day)) for entry in feed.entries: if self.finish_flag: break if entry.slash_comments > 0: comments_feed = self.getFeed(entry.wfw_commentrss) if comments_feed: for comment_entry in comments_feed.entries: fe = FeedEntry.fromFeedParserEntry(comments_feed.feed.link, comment_entry) fe.account = self.account fe.campaign = self.campaign self.queue.put(fe) d = d + timedelta(days=1) if d.day == 1: #cambio el mes, me fijo que si posts en el nuevo mes while not self.finish_flag and datetime.now(): dummy_feed = self.getFeed(self.url + "/%s/%s/feed" % (d.year, d.month)) if dummy_feed.entries: break d = (d + timedelta(days=32)).replace(day=1) #agrego 1 mes if d > datetime.now(): acc = MongoManager.getAccount(id=self.account.getId()) camp = acc.getCampaign(id =self.campaign.getId()) camp.addHistoryFetchedForum(self.url) MongoManager.saveCampaign(acc, camp)
def run(self): year, month = self.findFirstMonth() #year = 2015 #month = 2 if not year or not month: return d = datetime(year, month, 1) while not self.finish_flag and d <= datetime.now(): feed = self.getFeed(self.url + "/%s/%s/%s/feed" % (d.year, d.month, d.day)) for entry in feed.entries: if self.finish_flag: break if entry.slash_comments > 0: comments_feed = self.getFeed(entry.wfw_commentrss) if comments_feed: for comment_entry in comments_feed.entries: fe = FeedEntry.fromFeedParserEntry( comments_feed.feed.link, comment_entry) fe.account = self.account fe.campaign = self.campaign self.queue.put(fe) d = d + timedelta(days=1) if d.day == 1: #cambio el mes, me fijo que si posts en el nuevo mes while not self.finish_flag and datetime.now(): dummy_feed = self.getFeed(self.url + "/%s/%s/feed" % (d.year, d.month)) if dummy_feed.entries: break d = (d + timedelta(days=32)).replace(day=1) #agrego 1 mes if d > datetime.now(): acc = MongoManager.getAccount(id=self.account.getId()) camp = acc.getCampaign(id=self.campaign.getId()) camp.addHistoryFetchedForum(self.url) MongoManager.saveCampaign(acc, camp)
if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--regenerate', action="store_true", default=False) parser.add_argument('--account', default=None) parser.add_argument('--list', action="store_true", default=False) parser.add_argument('--start', default=None) parser.add_argument('--end', default=None) parser.add_argument('--clear', action="store_true", default=False) args, known = parser.parse_known_args() campaign = None if args.account: account = MongoManager.getAccount(name=args.account) if not account: pprint("Account %s not found" % args.account) exit(1) campaign = account.getActiveCampaigns()[0] summarizer = Summarizer() if not args.list and not args.clear: summarizer.start(campaign=campaign, regenerate=args.regenerate) elif args.clear and campaign: summarizer.clearSummarization(campaign) elif args.list and campaign and args.start and args.end: print args start = datetime.strptime(args.start, "%Y-%m-%dT%H") end = datetime.strptime(args.end, "%Y-%m-%dT%H") records = summarizer.getSummarizedData(campaign,start,end)
return res """ if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--regenerate', action="store_true", default=False) parser.add_argument('--account', default=None) parser.add_argument('--list', action="store_true", default=False) parser.add_argument('--start', default=None) parser.add_argument('--end', default=None) parser.add_argument('--clear', action="store_true", default=False) args, known = parser.parse_known_args() campaign = None if args.account: account = MongoManager.getAccount(name=args.account) if not account: pprint("Account %s not found" % args.account) exit(1) campaign = account.getActiveCampaigns()[0] summarizer = Summarizer() if not args.list and not args.clear: summarizer.start(campaign=campaign, regenerate=args.regenerate) elif args.clear and campaign: summarizer.clearSummarization(campaign) elif args.list and campaign and args.start and args.end: print args start = datetime.strptime(args.start, "%Y-%m-%dT%H") end = datetime.strptime(args.end, "%Y-%m-%dT%H") records = summarizer.getSummarizedData(campaign, start, end)