def twitter2all(): from twitter import get_twitter_status prevtime = load_prev_time(conf.twitter_user) statuses = get_twitter_status(conf.twitter_user, prevtime) if not statuses: return if prevtime is None: log("first time fetch %s's tweets, skip", conf.twitter_user) save_prev_time(conf.twitter_user, statuses[-1][1]) return maxtime = prevtime for status, pubdate in statuses: if not can_pub(status): log("[skipping] %s can not be published because of include and exlucde conf", status) continue log("[publishing] %s : %s", pubdate, status, ) if pub2all(status) and maxtime < pubdate: maxtime = pubdate sleep(10) save_prev_time(conf.twitter_user, maxtime)
def twitter2all(): from twitter import get_twitter_status prevtime = load_prev_time(conf.twitter_user) statuses = get_twitter_status(conf.twitter_user, prevtime) if not statuses: return if prevtime is None: log("first time fetch %s's tweets, skip", conf.twitter_user) save_prev_time(conf.twitter_user, statuses[-1][1]) return maxtime = prevtime for status, pubdate in statuses: if not can_pub(status): log( "[skipping] %s can not be published because of include and exlucde conf", status) continue log( "[publishing] %s : %s", pubdate, status, ) if pub2all(status) and maxtime < pubdate: maxtime = pubdate sleep(10) save_prev_time(conf.twitter_user, maxtime)
def feeds2all(): from rss import get_rss_entries lasttimes = read_rss_lasttimes() if lasttimes is None: lasttimes = {} for format_, url in conf.feeds: lasttime = lasttimes.get(url, None) if lasttime is None: log("first time fetching %s, skip", url) lasttimes[url] = gmtime() save_rss_lasttimes(lasttimes) continue statuses = get_rss_entries(url, lasttime) maxtime = lasttime for status, publishtime in statuses: status = format_ % status log("[publishing] %s : %s", strftime("%Y-%m-%d %H:%M:%S", publishtime), status, ) if pub2all(status) and maxtime < publishtime: maxtime = publishtime sleep(10) lasttimes[url] = maxtime save_rss_lasttimes(lasttimes)
def feeds2all(): from rss import get_rss_entries lasttimes = loadfrom('rss_lasttimes') if lasttimes is None: lasttimes = {} for format_, url in conf.feeds: lasttime = lasttimes.get(url, None) if lasttime is None: lasttimes[url] = gmtime() dumpto('rss_lasttimes', lasttimes) continue statuses = get_rss_entries(url, lasttime) maxtime = lasttime for status, publishtime in statuses: status = format_ % status print publishtime, status if pub2all(status): if maxtime < publishtime: maxtime = publishtime sleep(10) lasttimes[url] = maxtime dumpto('rss_lasttimes', lasttimes)
def twitter2all(): from twitter import get_twitter_status prevtime = load_prev_time(conf.twitter_user) statuses = get_twitter_status(conf.twitter_user, prevtime) if prevtime is None: log("first time fetch %s's tweets, skip", conf.twitter_user) save_prev_time(conf.twitter_user, statuses[-1][1]) return maxtime = prevtime for status, pubdate in statuses: if status[0] in conf.exclude: continue log( "[publishing] %s : %s", pubdate, status, ) if pub2all(status) and maxtime < pubdate: maxtime = pubdate sleep(10) save_prev_time(conf.twitter_user, maxtime)
def feeds2all(): from rss import get_rss_entries lasttimes = read_rss_lasttimes() if lasttimes is None: lasttimes = {} for format_, url in conf.feeds: lasttime = lasttimes.get(url, None) if lasttime is None: log("first time fetching %s, skip", url) lasttimes[url] = gmtime() save_rss_lasttimes(lasttimes) continue statuses = get_rss_entries(url, lasttime) maxtime = lasttime for status, publishtime in statuses: status = format_ % status log( "[publishing] %s : %s", strftime("%Y-%m-%d %H:%M:%S", publishtime), status, ) if pub2all(status) and maxtime < publishtime: maxtime = publishtime sleep(10) lasttimes[url] = maxtime save_rss_lasttimes(lasttimes)
def feeds2all(): from rss import get_rss_entries lasttimes = read_rss_lasttimes() if lasttimes is None: lasttimes = {} for feed in conf.feeds: format_, url = feed[0:2] nhead = feed[2] if feed[2:] else 0 lasttime = lasttimes.get(url, None) if lasttime is None: log("first time fetching %s, skip", url) lasttimes[url] = gmtime() lasttimes['links_' + url] = [] save_rss_lasttimes(lasttimes) continue statuses = get_rss_entries(url, lasttime, nhead=nhead) maxtime = lasttime for entry, publishtime in statuses: status = format_ % entry if not can_pub(status): log( "[skipping] %s can not be published because of include and exlucde conf", status) continue if entry['link'] in lasttimes['links_' + url]: log( "[skipping] %s can not be published because it has already bean published", status) continue log( "[publishing] %s : %s", strftime("%Y-%m-%d %H:%M:%S", publishtime) if publishtime is not None else 'None', status, ) if pub2all(status, entry): if publishtime is not None and maxtime < publishtime: maxtime = publishtime lasttimes['links_' + url].append(entry['link']) sleep(10) lasttimes[url] = maxtime if maxtime != lasttime else gmtime() lasttimes['links_' + url] = lasttimes['links_' + url][-100:] save_rss_lasttimes(lasttimes)
def twitter2all(): from twitter import get_twitter_status prevtime = load_prev_time(conf.twitter_user) statuses = get_twitter_status(conf.twitter_user, prevtime) for status, pubdate in statuses: if status[0] in conf.exclude: continue print pubdate, status if pub2all(status): save_prev_time(conf.twitter_user, pubdate) sleep(10)
def sina2all(): from sina import get_sina_status import shelve from time import sleep db = shelve.open('sina.db') statuses = get_sina_status() statuses.reverse() for status in statuses: print dir(status) bd = {str(status.id):status.text} if status.text in conf.exclude: continue print status if db.has_key(str(status.id)): pass else: pub2all(status.text) sleep(10) print status.text db.update(bd) db.sync()
def feeds2all(): from rss import get_rss_entries lasttimes = read_rss_lasttimes() if lasttimes is None: lasttimes = {} for feed in conf.feeds: format_, url = feed[0:2] nhead = feed[2] if feed[2:] else 0 lasttime = lasttimes.get(url, None) if lasttime is None: log("first time fetching %s, skip", url) lasttimes[url] = gmtime() lasttimes['links_' + url] = [] save_rss_lasttimes(lasttimes) continue statuses = get_rss_entries(url, lasttime, nhead=nhead) maxtime = lasttime for entry, publishtime in statuses: status = format_ % entry if not can_pub(status): log("[skipping] %s can not be published because of include and exlucde conf", status) continue if entry['link'] in lasttimes['links_' + url]: log("[skipping] %s can not be published because it has already bean published", status) continue log("[publishing] %s : %s", strftime("%Y-%m-%d %H:%M:%S", publishtime) if publishtime is not None else 'None', status, ) if pub2all(status, entry): if publishtime is not None and maxtime < publishtime: maxtime = publishtime lasttimes['links_' + url].append(entry['link']) sleep(10) lasttimes[url] = maxtime if maxtime != lasttime else gmtime() lasttimes['links_' + url] = lasttimes['links_' + url][-100:] save_rss_lasttimes(lasttimes)
def twitter2all(): from twitter import get_twitter_status prevtime = load_prev_time(conf.twitter_user) statuses = get_twitter_status(conf.twitter_user, prevtime) if prevtime is None: log("first time fetch %s's tweets, skip", conf.twitter_user) save_prev_time(conf.twitter_user, statuses[-1][1]) return maxtime = prevtime for status, pubdate in statuses: if status[0] in conf.exclude: continue log("[publishing] %s : %s", pubdate, status, ) if pub2all(status) and maxtime < pubdate: maxtime = pubdate sleep(10) save_prev_time(conf.twitter_user, maxtime)