import datetime import urlparse from commonutil import dateutil import configmanager.models from configmanager import cmapi class PageConstantTitle(configmanager.models.ConfigItem): pass cmapi.registerModel(PageConstantTitle) def isConstantTitle(titleConfig, url, title, sideEffect): if not url: return False netloc = urlparse.urlparse(url).netloc key = netloc value = cmapi.getItemValue(key, {}, modelname=PageConstantTitle) record = value.get(title) if not record: record = {} count = record.get('c', 0) isconstant = count >= titleConfig.get('occurrence', 1) if sideEffect: nnow = datetime.datetime.utcnow() record['c'] = count + 1 record['u'] = dateutil.getDateAs14(nnow) if len(value) > 20: for ik, iv in value.items(): if (nnow - dateutil.parseDate14(iv['u'])).days >= titleConfig.get('cache.day', 7):
from commonutil import dateutil import configmanager.models from configmanager import cmapi class DatasourceHistory(configmanager.models.ConfigItem): pass class DatasourceArchive(configmanager.models.ConfigItem): pass class LatestItem(configmanager.models.ConfigItem): pass cmapi.registerModel(DatasourceHistory) cmapi.registerModel(DatasourceArchive) cmapi.registerModel(LatestItem) def _getDatasourceHistoryKey(): return 'list' def getDatasourceHistory(): key = _getDatasourceHistoryKey() items = cmapi.getItemValue(key, [], modelname=DatasourceHistory) return items def saveDatasourceHistory(datasourceHistory): key = _getDatasourceHistoryKey() cmapi.saveItem(key, datasourceHistory, modelname=DatasourceHistory) def _saveHistory(datasource, items): sourceadded = datasource.get('added')
import configmanager.models from configmanager import cmapi class WordsRequest(configmanager.models.ConfigItem): pass cmapi.registerModel(WordsRequest) def saveWordsRequest(keyname, data): cmapi.saveItem(keyname, data, modelname=WordsRequest) def getWordsRequest(keyname): value = cmapi.getItemValue(keyname, modelname=WordsRequest) cmapi.removeItem(keyname, modelname=WordsRequest) return value
from commonutil import dateutil import configmanager.models from configmanager import cmapi import globalconfig from headline import datareceiver class LatestItem(configmanager.models.ConfigItem): pass class ChannelGroup(configmanager.models.ConfigItem): pass cmapi.registerModel(LatestItem) cmapi.registerModel(ChannelGroup) def getChannelGroups(slug): result = cmapi.getItemValue(slug, [], modelname=ChannelGroup) if not result: result = cmapi.getItemValue('default', [], modelname=ChannelGroup) return result def receiveData(datasource, items): if datasource.get('charts'): keyname = 'chartses' else: keyname = 'sites' _saveDatasource(datasource, items, keyname) class DataReceiver(datareceiver.BasicDataReceiver):
import configmanager.models from configmanager import cmapi from headline import datareceiver class DatasourceHistory(configmanager.models.ConfigItem): pass cmapi.registerModel(DatasourceHistory) def receiveData(datasource, items): if not datasource.get('charts'): _saveDatasourceHistory(datasource, items) class DataReceiver(datareceiver.BasicDataReceiver): def __init__(self, name): datareceiver.BasicDataReceiver.__init__(self, name) def onData(self, datasource, items): receiveData(datasource, items) datareceiver.registerReceiver(DataReceiver('Source History Receiver')) def _getDatasourceHistoryKey(slug): return slug def _saveDatasourceHistory(datasource, items): _MAX_COUNT = 20 slug = datasource['slug'] value = getDatasourceHistory(slug)
import configmanager.models from configmanager import cmapi class HotWord(configmanager.models.ConfigItem): pass cmapi.registerModel(HotWord) def saveWords(keyname, value): cmapi.saveItem(keyname, value, modelname=HotWord) def getWords(keyname): return cmapi.getItemValue(keyname, {}, modelname=HotWord)
import configmanager.models from configmanager import cmapi class HotEvents(configmanager.models.ConfigItem): pass class HotEvent(configmanager.models.ConfigItem): pass cmapi.registerModel(HotEvents) cmapi.registerModel(HotEvent) def saveEvents(scope, value): cmapi.saveItem(scope, value, modelname=HotEvents) def getEvents(scope): return cmapi.getItemValue(scope, {}, modelname=HotEvents) def saveEvent(scope, event): cmapi.saveItem(scope + '.' + str(event['id']), event, modelname=HotEvent) def removeEvent(scope, eventId): cmapi.removeItem(scope + '.' + str(eventId), modelname=HotEvent) def getEvent(scope, eventId): return cmapi.getItemValue(scope + '.' + str(eventId), {}, modelname=HotEvent) def saveHistoryEvents(scope, value): cmapi.saveItem(scope + '.history', value, modelname=HotEvents)