def update_hash_prefix_cache(self):
"""Update locally cached threat lists.
"""
self.api_client.fair_use_delay()
self.storage.cleanup_full_hashes()
threat_lists_to_remove = dict()
for ts, cs in self.storage.get_threat_lists():
threat_lists_to_remove[repr(ts)] = ts
threat_lists = self.api_client.get_threats_lists()
for entry in threat_lists:
threat_list = ThreatList.from_api_entry(entry)
if self.platforms is None or threat_list.platform_type in self.platforms:
self.storage.add_threat_list(threat_list)
try:
del threat_lists_to_remove[repr(threat_list)]
except KeyError:
pass
for ts in threat_lists_to_remove.values():
self.storage.delete_hash_prefix_list(ts)
self.storage.delete_threat_list(ts)
del threat_lists_to_remove
self.api_client.fair_use_delay()
threat_lists = self.storage.get_threat_lists()
client_state = dict([(t.as_tuple(), s) for t, s in threat_lists])
for response in self.api_client.get_threats_update(client_state):
response_threat_list = ThreatList(response['threatType'],
response['platformType'],
response['threatEntryType'])
if response['responseType'] == 'FULL_UPDATE':
self.storage.delete_hash_prefix_list(response_threat_list)
for r in response.get('removals', []):
self.storage.remove_hash_prefix_indices(
response_threat_list, r['rawIndices']['indices'])
for a in response.get('additions', []):
hash_prefix_list = HashPrefixList(
a['rawHashes']['prefixSize'],
b64decode(a['rawHashes']['rawHashes']))
self.storage.populate_hash_prefix_list(response_threat_list,
hash_prefix_list)
expected_checksum = b64decode(response['checksum']['sha256'])
if self._verify_threat_list_checksum(response_threat_list,
expected_checksum):
log.info('Local cache checksum matches the server: {}'.format(
to_hex(expected_checksum)))
self.storage.update_threat_list_client_state(
response_threat_list, response['newClientState'])
else:
raise Exception(
'Local cache checksum does not match the server: "{}". Consider removing {}'
.format(to_hex(expected_checksum), self.storage.db_path))
def _sync_hash_prefix_cache(self):
self.api_client.fair_use_delay()
client_state = self.storage.get_client_state()
new_client_state = {}
for response in self.api_client.get_threats_update(client_state):
response_threat_list = ThreatList(response['threatType'],
response['platformType'],
response['threatEntryType'])
if response['responseType'] == 'FULL_UPDATE':
self.storage.delete_hash_prefix_list(response_threat_list)
for r in response.get('removals', []):
self.storage.remove_hash_prefix_indices(
response_threat_list, r['rawIndices']['indices'])
for a in response.get('additions', []):
hash_prefix_list = HashPrefixList(
a['rawHashes']['prefixSize'],
b64decode(a['rawHashes']['rawHashes']))
self.storage.populate_hash_prefix_list(response_threat_list,
hash_prefix_list)
expected_checksum = b64decode(response['checksum']['sha256'])
if self._verify_threat_list_checksum(response_threat_list,
expected_checksum):
log.info('Local cache checksum matches the server: {}'.format(
to_hex(expected_checksum)))
new_client_state[response_threat_list] = response[
'newClientState']
else:
raise Exception(
'Local cache checksum does not match the server: "{}". Consider removing {}'
.format(to_hex(expected_checksum), self.storage.db_path))
self.storage.update_threat_list_client_state(new_client_state)
def _sync_full_hashes(self, hash_prefixes):
"""Download full hashes matching hash_prefixes.
Also update cache expiration timetsamps.
"""
client_state = self.storage.get_client_state()
self.api_client.fair_use_delay()
fh_response = self.api_client.get_full_hashes(hash_prefixes,
client_state)
# update negative cache for each hash prefix
# store full hash (insert or update) with positive cache bumped up
for m in fh_response['matches']:
threat_list = ThreatList(m['threatType'], m['platformType'],
m['threatEntryType'])
hash_value = b64decode(m['threat']['hash'])
cache_duration = int(m['cacheDuration'].rstrip('s'))
malware_threat_type = None
for metadata in m['threatEntryMetadata'].get('entries', []):
k = b64decode(metadata['key'])
v = b64decode(metadata['value'])
if k == 'malware_threat_type':
malware_threat_type = v
self.storage.store_full_hash(threat_list, hash_value,
cache_duration, malware_threat_type)
negative_cache_duration = int(
fh_response['negativeCacheDuration'].rstrip('s'))
for prefix_value in hash_prefixes:
self.storage.update_hash_prefix_expiration(
prefix_value, negative_cache_duration)
def _sync_threat_lists(self):
self.api_client.fair_use_delay()
threat_lists_to_remove = dict()
for ts in self.storage.get_threat_lists():
threat_lists_to_remove[repr(ts)] = ts
threat_lists = self.api_client.get_threats_lists()
for entry in threat_lists:
threat_list = ThreatList.from_api_entry(entry)
if self.platforms is None or threat_list.platform_type in self.platforms:
self.storage.add_threat_list(threat_list)
try:
del threat_lists_to_remove[repr(threat_list)]
except KeyError:
pass
for ts in threat_lists_to_remove.values():
self.storage.delete_hash_prefix_list(ts)
self.storage.delete_threat_list(ts)
del threat_lists_to_remove
def _sync_threat_lists(self):
self.api_client.fair_use_delay()
threat_lists_to_remove = dict()
for ts in self.storage.get_threat_lists():
threat_lists_to_remove[repr(ts)] = ts
threat_lists = self.api_client.get_threats_lists()
for entry in threat_lists:
threat_list = ThreatList.from_api_entry(entry)
if self.platforms is None or threat_list.platform_type in self.platforms:
self.storage.add_threat_list(threat_list)
try:
del threat_lists_to_remove[repr(threat_list)]
except KeyError:
pass
for ts in threat_lists_to_remove.values():
self.storage.delete_hash_prefix_list(ts)
self.storage.delete_threat_list(ts)
del threat_lists_to_remove
def update_hash_prefix_cache(self):
"""Update locally cached threat lists.
"""
self.api_client.fair_use_delay()
self.storage.cleanup_full_hashes()
threat_lists_to_remove = dict()
for ts, cs in self.storage.get_threat_lists():
threat_lists_to_remove[repr(ts)] = ts
threat_lists = self.api_client.get_threats_lists()
for entry in threat_lists:
threat_list = ThreatList.from_api_entry(entry)
if self.platforms is None or threat_list.platform_type in self.platforms:
self.storage.add_threat_list(threat_list)
try:
del threat_lists_to_remove[repr(threat_list)]
except KeyError:
pass
for ts in threat_lists_to_remove.values():
self.storage.delete_hash_prefix_list(ts)
self.storage.delete_threat_list(ts)
del threat_lists_to_remove
self.api_client.fair_use_delay()
threat_lists = self.storage.get_threat_lists()
client_state = dict([(t.as_tuple(), s) for t,s in threat_lists])
for response in self.api_client.get_threats_update(client_state):
response_threat_list = ThreatList(response['threatType'], response['platformType'], response['threatEntryType'])
if response['responseType'] == 'FULL_UPDATE':
self.storage.delete_hash_prefix_list(response_threat_list)
for r in response.get('removals', []):
self.storage.remove_hash_prefix_indices(response_threat_list, r['rawIndices']['indices'])
for a in response.get('additions', []):
hash_prefix_list = HashPrefixList(a['rawHashes']['prefixSize'], b64decode(a['rawHashes']['rawHashes']))
self.storage.populate_hash_prefix_list(response_threat_list, hash_prefix_list)
expected_checksum = b64decode(response['checksum']['sha256'])
if self._verify_threat_list_checksum(response_threat_list, expected_checksum):
log.info('Local cache checksum matches the server: {}'.format(to_hex(expected_checksum)))
self.storage.update_threat_list_client_state(response_threat_list, response['newClientState'])
else:
raise Exception('Local cache checksum does not match the server: "{}". Consider removing {}'.format(to_hex(expected_checksum), self.storage.db_path))