def hgetall(self, name, callbacks=None):
response = super(MarshalRedis, self).hgetall(name)
if not callbacks:
callbacks = dict(METADATA_CUSTOM_CALLBACK.items() +
FEATURES_CUSTOM_CALLBACK.items() +
POLICY_CUSTOM_CALLBACK.items())
return self.pairs_to_dict_typed(response, callbacks) or {}
def set_metadata(self, pipe=None):
self.tmetadata.last_update = time.time()
if not pipe:
with self.redis.pipeline() as p:
p.hmset(self.tmetadata.namespace, self.tmetadata.as_dict,
dict(FEATURES_CUSTOM_CALLBACK.items() + METADATA_CUSTOM_CALLBACK.items()))
p.expire(self.tmetadata.namespace, self.tmetadata.global_ttl)
p.execute()
else:
if self.tmetadata.learningBlueMode and not self.tmetadata.blue_creation_date:
self.tmetadata.blue_creation_date = time.time()
elif self.tmetadata.learningRedMode and not self.tmetadata.red_creation_date:
self.tmetadata.red_creation_date = time.time()
pipe.hmset(self.tmetadata.namespace, self.tmetadata.as_dict,
dict(FEATURES_CUSTOM_CALLBACK.items() + METADATA_CUSTOM_CALLBACK.items()))
pipe.expire(self.tmetadata.namespace, self.tmetadata.global_ttl)
pipe.execute()
def get_features(self, namespace):
if namespace == 'list':
callback = {}
[
callback.update({key: str(value)})
for key, value in FEATURES_CUSTOM_CALLBACK.items()
]
return callback
return self.redis.hgetall('config:themis:features:%s' % namespace)
def add_default_metadata(self, target, config_file):
if target in RESERVERD_KEYWORDS:
raise ValueError('Reserved word found: %s. Use another name' % ', '.join(RESERVERD_KEYWORDS))
tmetadata = ThemisMetaData(**ThemisMetaData.METADATA_DEFAULT_VALUES)
if not config_file:
global_config = DEFAULT_FEATURES_VALUES
else:
with open(config_file) as f:
_, global_config, _ = yaml.load_all(f)
tmetadata.update_features(**global_config)
self.redis.hmset(target, tmetadata.as_dict, dict(FEATURES_CUSTOM_CALLBACK.items() + METADATA_CUSTOM_CALLBACK.items()))
def set_metadata(self, pipe=None):
self.tmetadata.last_update = time.time()
if not pipe:
with self.redis.pipeline() as p:
p.hmset(
self.tmetadata.namespace, self.tmetadata.as_dict,
dict(FEATURES_CUSTOM_CALLBACK.items() +
METADATA_CUSTOM_CALLBACK.items()))
p.expire(self.tmetadata.namespace, self.tmetadata.global_ttl)
p.execute()
else:
if self.tmetadata.learningBlueMode and not self.tmetadata.blue_creation_date:
self.tmetadata.blue_creation_date = time.time()
elif self.tmetadata.learningRedMode and not self.tmetadata.red_creation_date:
self.tmetadata.red_creation_date = time.time()
pipe.hmset(
self.tmetadata.namespace, self.tmetadata.as_dict,
dict(FEATURES_CUSTOM_CALLBACK.items() +
METADATA_CUSTOM_CALLBACK.items()))
pipe.expire(self.tmetadata.namespace, self.tmetadata.global_ttl)
pipe.execute()
def hset(self, name, key, value, feat_mapping=False):
custom_callbacks = dict(FEATURES_CUSTOM_CALLBACK.items() +
METADATA_CUSTOM_CALLBACK.items())
if feat_mapping:
try:
if custom_callbacks[key] in MARSHAL_TYPES:
if value.lower().title() in ['True', 'False']:
value = value.lower().title()
value = marshal.dumps(literal_eval(value))
else:
value = custom_callbacks[key](value)
except Exception, e:
raise ValueError('Wrong type found: %s Exception: %s' %
(value, e))
def add_default_metadata(self, target, config_file):
if target in RESERVERD_KEYWORDS:
raise ValueError('Reserved word found: %s. Use another name' %
', '.join(RESERVERD_KEYWORDS))
tmetadata = ThemisMetaData(**ThemisMetaData.METADATA_DEFAULT_VALUES)
if not config_file:
global_config = DEFAULT_FEATURES_VALUES
else:
with open(config_file) as f:
_, global_config, _ = yaml.load_all(f)
tmetadata.update_features(**global_config)
self.redis.hmset(
target, tmetadata.as_dict,
dict(FEATURES_CUSTOM_CALLBACK.items() +
METADATA_CUSTOM_CALLBACK.items()))
def call_feeders(self, redis, pdata, milter_object, milter_from_ipaddress,
milter_subject):
if not self.feederFeaturesEnabled:
return
if not self.tmetadata:
raise ValueError(
'tmetadata attribute is empty, expected ThemisMetaData object')
self.tmetadata.build_namespace(milter_object)
self.build_namespaces(milter_object)
now_in_timestamp, now_date_obj = self.get_time()
count_sent_feature_reset, ip_reputation_feature_reset, subject_feature_match = [
False, False, False
]
### ipReputationFeature feeder
if self.tmetadata.ip_reputation_lastupdate:
past_salt = datetime.fromtimestamp(
self.tmetadata.ip_reputation_lastupdate) + timedelta(
hours=pdata.ipprobation)
if not past_salt > now_date_obj:
ip_reputation_feature_reset = True
# subjectReputationFeature feeder
if self.tmetadata.last_subject == milter_subject and self.tmetadata.subject_lastupdate:
subject_feature_match = True
past_salt = datetime.fromtimestamp(
float(self.tmetadata.subject_lastupdate)) + timedelta(
hours=pdata.subjectprobation)
if not past_salt > now_date_obj:
subject_feature_match = 'first_subject_count'
# countSentMessagesFeature
if self.tmetadata.sentmessages_lastupdate:
past_salt = datetime.fromtimestamp(
self.tmetadata.sentmessages_lastupdate) + timedelta(
hours=pdata.countsentprobation)
if not past_salt > now_date_obj:
count_sent_feature_reset = True
# Update all keys with current timestamp
self.tmetadata.__dict__.update(
dict.fromkeys([
'ip_reputation_lastupdate', 'subject_lastupdate',
'sentmessages_lastupdate'
], now_in_timestamp))
self.tmetadata.last_subject = milter_subject
# update features
with redis.pipeline() as pipe:
# Ip Reputation Feature
if ip_reputation_feature_reset:
pipe.delete(self.ipReputationFeatureNamespace)
pipe.zrem(self.ipReputationFeatureGroupByNamespace,
milter_object)
# Count Sent Feature
elif count_sent_feature_reset:
pipe.zrem(self.countSentMessagesFeatureGroupByNamespace,
milter_object)
# Subect Feature
elif subject_feature_match == 'first_subject_count':
pipe.zrem(self.subjectReputationFeatureGroupByNamespace,
milter_object)
self.tmetadata.subject_repeated_count = 1
elif subject_feature_match:
self.tmetadata.subject_repeated_count += 1
else:
pipe.zrem(self.subjectReputationFeatureGroupByNamespace,
milter_object)
pipe.hmset(
self.tmetadata.namespace, self.tmetadata.as_dict,
dict(FEATURES_CUSTOM_CALLBACK.items() +
METADATA_CUSTOM_CALLBACK.items()))
pipe.expire(self.tmetadata.namespace, self.tmetadata.global_ttl)
pipe.sadd(self.ipReputationFeatureNamespace, milter_from_ipaddress)
pipe.expire(self.ipReputationFeatureNamespace,
self.tmetadata.global_ttl)
# Execute commands and get the penultimate element of the list, it is pipe.sadd(...)
ip_reputation_incr = pipe.execute()[-2]
# Increase for grouping objects by score
with redis.pipeline() as pipe:
# We increase if it is true because a new ip is added in the set
if ip_reputation_incr:
pipe.zincrby(self.ipReputationFeatureGroupByNamespace,
milter_object, 1)
# We increase because the subject matched, the last one is equal to the current
if subject_feature_match:
pipe.zincrby(self.subjectReputationFeatureGroupByNamespace,
milter_object, 1)
# A simple count of objects, increase every time this method is executed
pipe.zincrby(self.countSentMessagesFeatureGroupByNamespace,
milter_object, 1)
pipe.execute()
def strict_check(self):
[
self.__dict__.update({key: type_info(self.__dict__[key])})
for key, type_info in FEATURES_CUSTOM_CALLBACK.items()
]
def _validate(self):
features_keys = FEATURES_CUSTOM_CALLBACK.keys()
for entry in self.as_dict.keys():
if not entry in features_keys:
raise NameError('Wrong key found: %s' % entry)
self.strict_check()
def get_features(self, namespace):
if namespace == 'list':
callback = {}
[callback.update({key : str(value)}) for key, value in FEATURES_CUSTOM_CALLBACK.items()]
return callback
return self.redis.hgetall('config:themis:features:%s' % namespace)
def call_feeders(self, redis, pdata, milter_object, milter_from_ipaddress, milter_subject):
if not self.feederFeaturesEnabled:
return
if not self.tmetadata:
raise ValueError('tmetadata attribute is empty, expected ThemisMetaData object')
self.tmetadata.build_namespace(milter_object)
self.build_namespaces(milter_object)
now_in_timestamp, now_date_obj = self.get_time()
count_sent_feature_reset, ip_reputation_feature_reset, subject_feature_match = [False, False, False]
### ipReputationFeature feeder
if self.tmetadata.ip_reputation_lastupdate:
past_salt = datetime.fromtimestamp(self.tmetadata.ip_reputation_lastupdate) + timedelta(hours=pdata.ipprobation)
if not past_salt > now_date_obj:
ip_reputation_feature_reset = True
# subjectReputationFeature feeder
if self.tmetadata.last_subject == milter_subject and self.tmetadata.subject_lastupdate:
subject_feature_match = True
past_salt = datetime.fromtimestamp(float(self.tmetadata.subject_lastupdate)) + timedelta(hours=pdata.subjectprobation)
if not past_salt > now_date_obj:
subject_feature_match = 'first_subject_count'
# countSentMessagesFeature
if self.tmetadata.sentmessages_lastupdate:
past_salt = datetime.fromtimestamp(self.tmetadata.sentmessages_lastupdate) + timedelta(hours=pdata.countsentprobation)
if not past_salt > now_date_obj:
count_sent_feature_reset = True
# Update all keys with current timestamp
self.tmetadata.__dict__.update(dict.fromkeys(['ip_reputation_lastupdate', 'subject_lastupdate', 'sentmessages_lastupdate'], now_in_timestamp))
self.tmetadata.last_subject = milter_subject
# update features
with redis.pipeline() as pipe:
# Ip Reputation Feature
if ip_reputation_feature_reset:
pipe.delete(self.ipReputationFeatureNamespace)
pipe.zrem(self.ipReputationFeatureGroupByNamespace, milter_object)
# Count Sent Feature
elif count_sent_feature_reset:
pipe.zrem(self.countSentMessagesFeatureGroupByNamespace, milter_object)
# Subect Feature
elif subject_feature_match == 'first_subject_count':
pipe.zrem(self.subjectReputationFeatureGroupByNamespace, milter_object)
self.tmetadata.subject_repeated_count = 1
elif subject_feature_match:
self.tmetadata.subject_repeated_count += 1
else:
pipe.zrem(self.subjectReputationFeatureGroupByNamespace, milter_object)
pipe.hmset(self.tmetadata.namespace, self.tmetadata.as_dict,
dict(FEATURES_CUSTOM_CALLBACK.items() + METADATA_CUSTOM_CALLBACK.items()))
pipe.expire(self.tmetadata.namespace, self.tmetadata.global_ttl)
pipe.sadd(self.ipReputationFeatureNamespace, milter_from_ipaddress)
pipe.expire(self.ipReputationFeatureNamespace, self.tmetadata.global_ttl)
# Execute commands and get the penultimate element of the list, it is pipe.sadd(...)
ip_reputation_incr = pipe.execute()[-2]
# Increase for grouping objects by score
with redis.pipeline() as pipe:
# We increase if it is true because a new ip is added in the set
if ip_reputation_incr:
pipe.zincrby(self.ipReputationFeatureGroupByNamespace, milter_object, 1)
# We increase because the subject matched, the last one is equal to the current
if subject_feature_match:
pipe.zincrby(self.subjectReputationFeatureGroupByNamespace, milter_object, 1)
# A simple count of objects, increase every time this method is executed
pipe.zincrby(self.countSentMessagesFeatureGroupByNamespace, milter_object, 1)
pipe.execute()
def strict_check(self):
[self.__dict__.update({key : type_info(self.__dict__[key])}) for key, type_info in FEATURES_CUSTOM_CALLBACK.items()]
def _validate(self):
features_keys = FEATURES_CUSTOM_CALLBACK.keys()
for entry in self.as_dict.keys():
if not entry in features_keys:
raise NameError('Wrong key found: %s' % entry)
self.strict_check()