def deserialize_as_json_for_cache(element):
if element is None:
LOGERR_IF_ENABLED(SOURCE_MODULE, "[-] Element for deserialize is None!")
try:
return json.loads(element)
except Exception as ex:
LOGERR_IF_ENABLED(SOURCE_MODULE, "[-] Got exception during deserializing: {0}".format(ex))
return None
def job_for_cwe_updater():
LOGINFO_IF_ENABLED(SOURCE_MODULE, '[+] Start CWE updater job')
if check_internet_connection():
parser = make_parser()
cwe_handler = CWEHandler()
parser.setContentHandler(cwe_handler)
source = SETTINGS.get("cwe", {}).get(
"source", "http://cwe.mitre.org/data/xml/cwec_v2.8.xml.zip")
try:
LOGINFO_IF_ENABLED(SOURCE_MODULE, '[+] Start downloading file')
data, response = get_file(getfile=source)
if 'error' not in response:
LOGINFO_IF_ENABLED(SOURCE_MODULE, '[+] Start parsing CWE data')
parser.parse(data)
LOGINFO_IF_ENABLED(SOURCE_MODULE,
'[+] Complete parsing CWE data')
for cwe in cwe_handler.cwe:
cwe['description_summary'] = cwe[
'description_summary'].replace("\t\t\t\t\t", " ")
item = {'tag': 'cwe', 'state': 'parsed', 'data': cwe}
dc.append(item)
LOGINFO_IF_ENABLED(
SOURCE_MODULE,
"[===========================================================================]"
)
LOGINFO_IF_ENABLED(
SOURCE_MODULE,
'[+] CWE update complete at: {}'.format(datetime.utcnow()))
LOGINFO_IF_ENABLED(
SOURCE_MODULE,
"[===========================================================================]"
)
return False
else:
LOGERR_IF_ENABLED(
SOURCE_MODULE,
'[-] There are some errors in server response: {}'.format(
response))
return False
except Exception as ex:
LOGERR_IF_ENABLED(
SOURCE_MODULE,
"Got exception during downloading CWE source: {0}".format(ex))
return False
else:
LOGERR_IF_ENABLED(SOURCE_MODULE, '[-] No internet connection!')
return False
def set_ping_counter(value):
try:
return stats.set('ping_counter', value)
except Exception as ex:
LOGERR_IF_ENABLED(
SOURCE_MODULE,
'[e] Got an exception with set ping counter value: {}'.format(ex))
return False
def drop_plugins_in_cache():
try:
stats.delete("run_plugins")
except Exception as ex:
LOGERR_IF_ENABLED(
SOURCE_MODULE,
'[e] Got an exception when drop collection with plugins in cache {}'
.format(ex))
def get_ping_counter():
try:
return stats.get('ping_counter')
except Exception as ex:
LOGERR_IF_ENABLED(
SOURCE_MODULE,
'[e] Got an exception with get ping counter value: {}'.format(ex))
return 0
def increment_ping_counter():
try:
return stats.incr('ping_counter')
except Exception as ex:
LOGERR_IF_ENABLED(
SOURCE_MODULE,
'[e] Got an exception with incr ping counter value: {}'.format(ex))
return False
def check_internet_connection():
url = 'http://www.google.com/'
timeout = 5
try:
_ = requests.get(url, timeout=timeout)
return True
except requests.ConnectionError:
LOGERR_IF_ENABLED(SOURCE_MODULE, "[I] Internet connection is lost")
return False
def serialize_as_json_for_cache(element):
def dt_converter(o):
if isinstance(o, datetime):
return o.__str__()
try:
return json.dumps(element, default=dt_converter)
except Exception as ex:
LOGERR_IF_ENABLED(SOURCE_MODULE, "[-] Got exception during serializing: {0}".format(ex))
return None
def update(self, state):
self._observer_state = state
try:
return self._cache.set(self._collection_name, decode_state(state))
except redis.ConnectionError as ce:
LOGERR_IF_ENABLED(
SOURCE_MODULE,
'[e] Redis ConnectionError exception: {}'.format(ce))
return 0
def deserialize_json_for_postgres(source):
if isinstance(source, list):
return source
else:
try:
a = ast.literal_eval(source)
except Exception:
LOGERR_IF_ENABLED(SOURCE_MODULE, '[-] Got invalid format for deserialize: {0}'.format(source))
return {}
if isinstance(a, dict):
return a
return json.loads(a)
def update(self, state):
def dt_converter(o):
if isinstance(o, datetime):
return o.__str__()
self._observer_state = state
message = {"state": decode_state(state), "ts": datetime.utcnow()}
try:
self._cache.rpush(self._collection_name,
json.dumps(message, default=dt_converter))
except redis.ConnectionError as ce:
LOGERR_IF_ENABLED(
SOURCE_MODULE,
'[e] Redis ConnectionError exception: {}'.format(ce))
return 0
def check_redis_cache_connection():
try:
ping = cache.ping()
if ping:
return True
else:
LOGINFO_IF_ENABLED(
SOURCE_MODULE,
'[-] Redis is not available now for unknown reason')
return False
except Exception as ex:
LOGERR_IF_ENABLED(
SOURCE_MODULE,
'[e] Got an exception with check redis (cache) connection with ping: {}'
.format(ex))
return False