def __init__(self, snspocket = None):
super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF)
self.sp = snspocket # SNSPocket object
self.msgMapping = {
"Email":1,
"FacebookFeed":2,
"RSS":3,
"RSS2RW":4,
"RSSSummary":5,
"RenrenBlog":6,
"RenrenFeed": 7,
"RenrenPhoto": 8,
"RenrenShare":9,
"RenrenStatus":10,
"RenrenStatusDirect": 11,
"SQLite":12,
"SinaWeiboBase":13,
"SinaWeiboStatus":14,
"SinaWeiboWapStatus":15,
"TencentWeiboStatus":16,
"TwitterStatus":17,
"InstagramFeed":18,
"DoubanFeed":19}
self.platform_num = 19
self.inputnum = 10
self.condLast = time.time()
try:
self.queue_conf = json.loads(open('conf' + delimeter + 'queue.json', 'r').read())
except IOError, e:
logger.warning("No conf/queue.json, use defaults")
self.queue_conf = {}
def __init__(self, snspocket = None):
super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF)
self.sp = snspocket # SNSPocket object
try:
self.queue_conf = json.loads(open('conf/queue.json', 'r').read())
except IOError, e:
logger.warning("No conf/queue.json, use defaults")
self.queue_conf = {}
class Feature(object):
"""docstring for Feature"""
env = {
"dir_conf": "./conf",
"dir_kdb": "./kdb",
}
feature_extractors = []
try:
awjson = json.loads(open('conf/autoweight.json').read())
features = awjson['features']
import plugin
for f in features:
module_name = f[0]
class_name = f[1]
#print module_name
#print class_name
mo = __import__("ranking.plugin.%s" % module_name,
fromlist=["ranking.plugin"])
cl = getattr(mo, class_name)
feature_extractors.append(cl(env))
#TODO:
# Make the dynamic import method better (more standard ways).
# The current import method is borrowed from:
# http://stackoverflow.com/questions/301134/dynamic-module-import-in-python
# It just works.
#cl = __import__("plugin.%s.%s" % (module_name, class_name), fromlist=["plugin.%s" % module_name])
#cl = getattr(getattr(plugin, module_name), class_name)
except IOError:
logger.warning('No "conf/autoweight.json"!')
except KeyError:
logger.warning('No "features" defined"!')
def __init__(self):
super(Feature, self).__init__()
@staticmethod
def extract(msg):
'''
Feature extraction.
It will extract features to a dict and store in "msg.feature".
msg: an snstype.Message object
'''
if not isinstance(msg, snstype.Message):
logger.warning(
"Cannot extract feature for non snstype.Message object")
return
# Add all kinds of features
msg.feature = {}
for fe in Feature.feature_extractors:
fe.add_features(msg)
def __init__(self, snspocket=None):
super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF)
self.sp = snspocket # SNSPocket object
try:
self.queue_conf = json.loads(open('conf/queue.json', 'r').read())
except IOError, e:
logger.warning("No conf/queue.json, use defaults")
self.queue_conf = {}
def load_weight(self, fn = None):
if fn is None:
fn = 'conf/weights.json'
try:
self.feature_weight = json.loads(open(fn, 'r').read())
self.feature_name = self.feature_weight.keys()
logger.info("Loaded weights: %s", self.feature_weight)
except IOError:
logger.warning("No '%s' weights config file, use empty setting.", fn)
self.feature_weight = {}
self.feature_name = self.feature_weight.keys()
def __init__(self, snspocket = None):
super(SRFEQueue, self).__init__(self.SQLITE_QUEUE_CONF)
self.sp = snspocket # SNSPocket object
#self.__mount_default_home_timeline_count()
#self.queue_conf = json.load(open('conf/queue.json', 'r'))
#self.feature_weight = self.queue_conf['feature_weight']
try:
self.queue_conf = json.loads(open('conf/queue.json', 'r').read())
except IOError, e:
logger.warning("No conf/queue.json, use defaults")
self.queue_conf = {}
def load_weight(self, fn=None):
if fn is None:
fn = 'conf/weights.json'
try:
self.feature_weight = json.loads(open(fn, 'r').read())
self.feature_name = self.feature_weight.keys()
logger.info("Loaded weights: %s", self.feature_weight)
except IOError:
logger.warning("No '%s' weights config file, use empty setting.",
fn)
self.feature_weight = {}
self.feature_name = self.feature_weight.keys()
def __init__(self, env):
super(FeatureEcho, self).__init__(env)
fn_channel = self.env['dir_conf'] + '/channel.json'
self.schema = {"echo": "numeric"}
self.username = []
fields = ["user_name", "username", "address"]
with open(fn_channel) as fp:
for ch in json.loads(fp.read()):
for f in fields:
if f in ch:
self.username.append(ch[f])
def export_arff(message_list, ds_name, fn_arff):
'''
Export message_list to Weka's arff file
ds_name: the name of data set. Shown in first line
of arff file.
'''
all_tags = json.loads(open('tag_mapping.json').read())
all_tags_r = {}
for (k, v) in all_tags.iteritems():
all_tags_r[v] = k
all_tags_r[0] = "null"
with open(fn_arff, 'w') as fp:
fp.write("@relation %s\n\n" % (ds_name))
fn = []
# Write schema
fp.write("@attribute id numeric\n")
for fe in Feature.feature_extractors:
for (f, t) in fe.schema.iteritems():
fp.write("@attribute %s %s\n" % (f, t))
fn.append(f)
fp.write("@attribute class {%s}\n" % (",".join(all_tags.keys())))
# Write data
fp.write("\n\n@data\n")
for m in message_list:
# Ignore multi tagged messages for simplicity
if len(m.tags) == 1:
i = str(m.msg_id)
t = all_tags_r[m.tags.keys()[0]]
Feature.extract(m)
fields = [str(m.feature[f]) for f in fn]
fields.insert(0, i)
fields.append(t)
fp.write(",".join(fields) + "\n")
def export_arff(message_list, ds_name, fn_arff):
'''
Export message_list to Weka's arff file
ds_name: the name of data set. Shown in first line
of arff file.
'''
all_tags = json.loads(open('tag_mapping.json').read())
all_tags_r = {}
for (k, v) in all_tags.iteritems():
all_tags_r[v] = k
all_tags_r[0] = "null"
with open(fn_arff, 'w') as fp:
fp.write("@relation %s\n\n" % (ds_name))
fn = []
# Write schema
fp.write("@attribute id numeric\n")
for fe in Feature.feature_extractors:
for (f, t) in fe.schema.iteritems():
fp.write("@attribute %s %s\n" % (f, t))
fn.append(f)
fp.write("@attribute class {%s}\n" % (",".join(all_tags.keys())))
# Write data
fp.write("\n\n@data\n")
for m in message_list:
# Ignore multi tagged messages for simplicity
if len(m.tags) == 1:
i = str(m.msg_id)
t = all_tags_r[m.tags.keys()[0]]
Feature.extract(m)
fields = [str(m.feature[f]) for f in fn]
fields.insert(0, i)
fields.append(t)
fp.write(",".join(fields) + "\n")
def load_weights():
try:
iweight = json.loads(open('conf/weights.json', 'r').read())
except:
iweight = None
return iweight
def load_weights():
try:
iweight = json.loads(open('conf/weights.json', 'r').read())
except:
iweight = None
return iweight
def load_weight(self, fn=None):
if fn is None:
fn = 'conf/weights.json'
self.feature_weight = json.loads(open(fn, 'r').read())
self.feature_name = self.feature_weight.keys()
logger.info("Loaded weights: %s", self.feature_weight)
def load_weight(self, fn = None):
if fn is None:
fn = 'conf/weights.json'
self.feature_weight = json.loads(open(fn, 'r').read())
self.feature_name = self.feature_weight.keys()
logger.info("Loaded weights: %s", self.feature_weight)
