def __init__(self, snspocket = None): super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF) self.sp = snspocket # SNSPocket object self.msgMapping = { "Email":1, "FacebookFeed":2, "RSS":3, "RSS2RW":4, "RSSSummary":5, "RenrenBlog":6, "RenrenFeed": 7, "RenrenPhoto": 8, "RenrenShare":9, "RenrenStatus":10, "RenrenStatusDirect": 11, "SQLite":12, "SinaWeiboBase":13, "SinaWeiboStatus":14, "SinaWeiboWapStatus":15, "TencentWeiboStatus":16, "TwitterStatus":17, "InstagramFeed":18, "DoubanFeed":19} self.platform_num = 19 self.inputnum = 10 self.condLast = time.time() try: self.queue_conf = json.loads(open('conf' + delimeter + 'queue.json', 'r').read()) except IOError, e: logger.warning("No conf/queue.json, use defaults") self.queue_conf = {}
def __init__(self, snspocket = None): super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF) self.sp = snspocket # SNSPocket object try: self.queue_conf = json.loads(open('conf/queue.json', 'r').read()) except IOError, e: logger.warning("No conf/queue.json, use defaults") self.queue_conf = {}
class Feature(object): """docstring for Feature""" env = { "dir_conf": "./conf", "dir_kdb": "./kdb", } feature_extractors = [] try: awjson = json.loads(open('conf/autoweight.json').read()) features = awjson['features'] import plugin for f in features: module_name = f[0] class_name = f[1] #print module_name #print class_name mo = __import__("ranking.plugin.%s" % module_name, fromlist=["ranking.plugin"]) cl = getattr(mo, class_name) feature_extractors.append(cl(env)) #TODO: # Make the dynamic import method better (more standard ways). # The current import method is borrowed from: # http://stackoverflow.com/questions/301134/dynamic-module-import-in-python # It just works. #cl = __import__("plugin.%s.%s" % (module_name, class_name), fromlist=["plugin.%s" % module_name]) #cl = getattr(getattr(plugin, module_name), class_name) except IOError: logger.warning('No "conf/autoweight.json"!') except KeyError: logger.warning('No "features" defined"!') def __init__(self): super(Feature, self).__init__() @staticmethod def extract(msg): ''' Feature extraction. It will extract features to a dict and store in "msg.feature". msg: an snstype.Message object ''' if not isinstance(msg, snstype.Message): logger.warning( "Cannot extract feature for non snstype.Message object") return # Add all kinds of features msg.feature = {} for fe in Feature.feature_extractors: fe.add_features(msg)
def __init__(self, snspocket=None): super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF) self.sp = snspocket # SNSPocket object try: self.queue_conf = json.loads(open('conf/queue.json', 'r').read()) except IOError, e: logger.warning("No conf/queue.json, use defaults") self.queue_conf = {}
def load_weight(self, fn = None): if fn is None: fn = 'conf/weights.json' try: self.feature_weight = json.loads(open(fn, 'r').read()) self.feature_name = self.feature_weight.keys() logger.info("Loaded weights: %s", self.feature_weight) except IOError: logger.warning("No '%s' weights config file, use empty setting.", fn) self.feature_weight = {} self.feature_name = self.feature_weight.keys()
def __init__(self, snspocket = None): super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF) self.sp = snspocket # SNSPocket object #self.__mount_default_home_timeline_count() #self.queue_conf = json.load(open('conf/queue.json', 'r')) #self.feature_weight = self.queue_conf['feature_weight'] try: self.queue_conf = json.loads(open('conf/queue.json', 'r').read()) except IOError, e: logger.warning("No conf/queue.json, use defaults") self.queue_conf = {}
def load_weight(self, fn=None): if fn is None: fn = 'conf/weights.json' try: self.feature_weight = json.loads(open(fn, 'r').read()) self.feature_name = self.feature_weight.keys() logger.info("Loaded weights: %s", self.feature_weight) except IOError: logger.warning("No '%s' weights config file, use empty setting.", fn) self.feature_weight = {} self.feature_name = self.feature_weight.keys()
def __init__(self, env): super(FeatureEcho, self).__init__(env) fn_channel = self.env['dir_conf'] + '/channel.json' self.schema = {"echo": "numeric"} self.username = [] fields = ["user_name", "username", "address"] with open(fn_channel) as fp: for ch in json.loads(fp.read()): for f in fields: if f in ch: self.username.append(ch[f])
def export_arff(message_list, ds_name, fn_arff): ''' Export message_list to Weka's arff file ds_name: the name of data set. Shown in first line of arff file. ''' all_tags = json.loads(open('tag_mapping.json').read()) all_tags_r = {} for (k, v) in all_tags.iteritems(): all_tags_r[v] = k all_tags_r[0] = "null" with open(fn_arff, 'w') as fp: fp.write("@relation %s\n\n" % (ds_name)) fn = [] # Write schema fp.write("@attribute id numeric\n") for fe in Feature.feature_extractors: for (f, t) in fe.schema.iteritems(): fp.write("@attribute %s %s\n" % (f, t)) fn.append(f) fp.write("@attribute class {%s}\n" % (",".join(all_tags.keys()))) # Write data fp.write("\n\n@data\n") for m in message_list: # Ignore multi tagged messages for simplicity if len(m.tags) == 1: i = str(m.msg_id) t = all_tags_r[m.tags.keys()[0]] Feature.extract(m) fields = [str(m.feature[f]) for f in fn] fields.insert(0, i) fields.append(t) fp.write(",".join(fields) + "\n")
def load_weights(): try: iweight = json.loads(open('conf/weights.json', 'r').read()) except: iweight = None return iweight
def load_weight(self, fn=None): if fn is None: fn = 'conf/weights.json' self.feature_weight = json.loads(open(fn, 'r').read()) self.feature_name = self.feature_weight.keys() logger.info("Loaded weights: %s", self.feature_weight)
def load_weight(self, fn = None): if fn is None: fn = 'conf/weights.json' self.feature_weight = json.loads(open(fn, 'r').read()) self.feature_name = self.feature_weight.keys() logger.info("Loaded weights: %s", self.feature_weight)