def galaxies(self):
if not HAVE_PYGALAXIES:
self.log('error', "Missing dependency, install PyMISPGalaxies (`pip install git+https://github.com/MISP/PyMISPGalaxies.git`)")
return
clusters = Clusters()
if self.args.list:
self.log('table', dict(header=['Name', 'Description'], rows=[(name, cluster.description)
for name, cluster in clusters.items()]))
elif self.args.search:
to_search = ' '.join(self.args.search)
matches = clusters.search(to_search)
if not matches:
self.log('error', 'No matches for "{}" in the clusters.'.format(to_search))
return
self.log('success', 'Clusters matching "{}":'.format(to_search))
for cluster, values in matches:
self.log('success', cluster.name)
for val in values:
_print_cluster_value(self, val)
elif self.args.details:
cluster = clusters.get(self.args.details)
if not cluster:
self.log('error', 'No cluster called "{}".'.format(self.args.details))
return
if not self.args.cluster_value:
# Show all values
self.log('info', cluster.description)
self.log('info', 'Type: ' + cluster.type)
self.log('info', 'Source: ' + cluster.source)
self.log('info', 'Authors: ' + ', '.join(cluster.authors))
self.log('info', 'UUID: ' + cluster.uuid)
self.log('info', 'Version: {}'.format(cluster.version))
self.log('info', 'Values:')
header = ['ID', 'Name', 'Description']
rows = []
i = 1
for name, value in cluster.items():
row = (i, value.value, value.description)
rows.append(row)
i += 1
self.log('table', dict(header=header, rows=rows))
else:
cluster_value = ' '.join(self.args.cluster_value)
# Show meta of a value
c_val = cluster.get(cluster_value)
if not c_val:
self.log('error', 'No cluster value called "{}".'.format(cluster_value))
return
_print_cluster_value(self, c_val)
app = Flask_config.app
cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_serv_tags = Flask_config.r_serv_tags
r_serv_metadata = Flask_config.r_serv_metadata
r_serv_statistics = Flask_config.r_serv_statistics
max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal
bootstrap_label = Flask_config.bootstrap_label
PASTES_FOLDER = Flask_config.PASTES_FOLDER
Tags = Blueprint('Tags', __name__, template_folder='templates')
galaxies = Galaxies()
clusters = Clusters(skip_duplicates=True)
list_all_tags = {}
for name, c in clusters.items(): #galaxy name + tags
list_all_tags[name] = c
list_galaxies = []
for g in galaxies.values():
list_galaxies.append(g.to_json())
list_clusters = []
for c in clusters.values():
list_clusters.append(c.to_json())
# tags numbers in galaxies
total_tags = {}
app = Flask_config.app
cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl
r_serv_tags = Flask_config.r_serv_tags
r_serv_metadata = Flask_config.r_serv_metadata
r_serv_statistics = Flask_config.r_serv_statistics
max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal
bootstrap_label = Flask_config.bootstrap_label
max_tags_result = Flask_config.max_tags_result
PASTES_FOLDER = Flask_config.PASTES_FOLDER
Tags = Blueprint('Tags', __name__, template_folder='templates')
galaxies = Galaxies()
clusters = Clusters(skip_duplicates=True)
list_all_tags = {}
for name, c in clusters.items(): #galaxy name + tags
list_all_tags[name] = c
list_galaxies = []
for g in galaxies.values():
list_galaxies.append(g.to_json())
list_clusters = []
for c in clusters.values():
list_clusters.append(c.to_json())
# tags numbers in galaxies
total_tags = {}
tags = ['false-positive:risk="medium"']
config['impact'] = 3
if 'FP Risk: High' in str(entry):
tags = ['false-positive:risk="high"']
config['impact'] = 1
name = entry.b.string
tags += find_tags(name)
if name:
for link in entry.find_all('a'):
if link.get('href').endswith('iplist.txt'):
path = link.get('href')
if link.get('href').endswith('nsiplist.txt'):
name = f'{name}_NS'
config['name'] = name.replace(' ', '_')
config['url'] = f'{root}{path}'
config['tags'] = tags
yield config
def make_config(config):
filename = re.sub('[^0-9a-zA-Z]+', '_', config['name'])
with open(f'bambenekconsulting_{filename}.json', 'w') as f:
json.dump(config, f, indent=2)
if __name__ == '__main__':
c = Clusters()
for entry in get_paths():
make_config(entry)
def setUp(self):
self.galaxies = Galaxies()
self.clusters = Clusters(skip_duplicates=True)
self.maxDiff = None
class TestPyMISPGalaxies(unittest.TestCase):
def setUp(self):
self.galaxies = Galaxies()
self.clusters = Clusters(skip_duplicates=True)
self.maxDiff = None
def test_searchable(self):
for cluster in self.clusters.values():
all_searchable = []
for c_values in cluster.values():
all_searchable += c_values.searchable
count = Counter(all_searchable)
for k, v in count.items():
if v != 1:
warnings.warn('Duplicate on {}: {}'.format(
cluster.type, k))
def test_duplicates(self):
has_duplicates = False
for name, c in self.clusters.items():
if c.duplicates:
has_duplicates = True
to_print = Counter(c.duplicates)
for entry, counter in to_print.items():
print(counter + 1, entry)
self.assertFalse(has_duplicates)
def test_dump_galaxies(self):
galaxies_from_files = {}
for galaxy_file in glob(
os.path.join(self.galaxies.root_dir_galaxies, '*.json')):
with open(galaxy_file, 'r') as f:
galaxy = json.load(f)
galaxies_from_files[galaxy['name']] = galaxy
for name, g in self.galaxies.items():
out = g.to_dict()
self.assertDictEqual(out, galaxies_from_files[g.name])
def test_dump_clusters(self):
clusters_from_files = {}
for cluster_file in glob(
os.path.join(self.clusters.root_dir_clusters, '*.json')):
with open(cluster_file, 'r') as f:
cluster = json.load(f)
clusters_from_files[cluster['name']] = cluster
for name, c in self.clusters.items():
out = c.to_dict()
self.assertCountEqual(out, clusters_from_files[c.name])
def test_validate_schema_clusters(self):
self.clusters.validate_with_schema()
def test_validate_schema_galaxies(self):
self.galaxies.validate_with_schema()
def test_meta_additional_properties(self):
# All the properties in the meta key of the bundled-in clusters should be known
for c in self.clusters.values():
for cv in c.values():
if cv.meta:
self.assertIsNot(cv.meta.additional_properties, {})
def test_machinetags(self):
self.clusters.all_machinetags()
def test_print(self):
print(self.clusters)
def test_search(self):
self.assertIsNot(len(self.clusters.search('apt')), 0)
def test_revert_machinetag(self):
self.assertEqual(
len(self.clusters.revert_machinetag('misp-galaxy:tool="Babar"')),
2)
with self.assertRaises(UnableToRevertMachinetag):
self.clusters.revert_machinetag('blah')
def test_len(self):
self.assertIsNot(len(self.clusters), 0)
self.assertIsNot(len(self.galaxies), 0)
for c in self.clusters.values():
self.assertIsNot(len(c), 0)
def test_json(self):
for g in self.galaxies.values():
g.to_json()
for c in self.clusters.values():
c.to_json()
class TestPyMISPGalaxies(unittest.TestCase):
def setUp(self):
self.galaxies = Galaxies()
self.clusters = Clusters(skip_duplicates=False)
self.maxDiff = None
def test_searchable(self):
for cluster in self.clusters.values():
all_searchable = []
for c_values in cluster.values():
all_searchable += c_values.searchable
count = Counter(all_searchable)
for k, v in count.items():
if v != 1:
warnings.warn('On search in {}: {} is present multiple times'.format(cluster.type, k))
def test_duplicates(self):
has_duplicates = False
for name, c in self.clusters.items():
if c.duplicates:
has_duplicates = True
to_print = Counter(c.duplicates)
for entry, counter in to_print.items():
print(counter + 1, entry)
self.assertFalse(has_duplicates)
def test_dump_galaxies(self):
galaxies_from_files = {}
for galaxy_file in glob(os.path.join(self.galaxies.root_dir_galaxies, '*.json')):
with open(galaxy_file, 'r') as f:
galaxy = json.load(f)
galaxies_from_files[galaxy['name']] = galaxy
for name, g in self.galaxies.items():
out = g.to_dict()
self.assertDictEqual(out, galaxies_from_files[g.name])
def test_dump_clusters(self):
clusters_from_files = {}
for cluster_file in glob(os.path.join(self.clusters.root_dir_clusters, '*.json')):
with open(cluster_file, 'r') as f:
cluster = json.load(f)
clusters_from_files[cluster['name']] = cluster
for name, c in self.clusters.items():
out = c.to_dict()
print(name, c.name)
self.assertCountEqual(out, clusters_from_files[c.name])
def test_validate_schema_clusters(self):
self.clusters.validate_with_schema()
def test_validate_schema_galaxies(self):
self.galaxies.validate_with_schema()
def test_meta_additional_properties(self):
# All the properties in the meta key of the bundled-in clusters should be known
for c in self.clusters.values():
for cv in c.values():
if cv.meta:
self.assertIsNot(cv.meta.additional_properties, {})
def test_machinetags(self):
self.clusters.all_machinetags()
def test_print(self):
print(self.clusters)
def test_search(self):
self.assertIsNot(len(self.clusters.search('apt')), 0)
def test_revert_machinetag(self):
self.assertEqual(len(self.clusters.revert_machinetag('misp-galaxy:tool="Babar"')), 2)
with self.assertRaises(UnableToRevertMachinetag):
self.clusters.revert_machinetag('blah')
def test_len(self):
self.assertIsNot(len(self.clusters), 0)
self.assertIsNot(len(self.galaxies), 0)
for c in self.clusters.values():
self.assertIsNot(len(c), 0)
def test_json(self):
for g in self.galaxies.values():
g.to_json()
for c in self.clusters.values():
c.to_json()
def test_uuids(self):
all_uuids = defaultdict(list)
for cluster in self.clusters.values():
# Skip deprecated
if self.galaxies[cluster.name].namespace == 'deprecated':
continue
try:
self.assertIsInstance(UUID(cluster.uuid), UUID, f'{cluster.name} - {cluster.uuid}')
except ValueError:
raise Exception(f'{cluster.name} - {cluster.uuid}')
all_uuids[cluster.uuid].append(cluster.name)
for value in cluster.values():
try:
self.assertIsInstance(UUID(value.uuid), UUID, f'{cluster.name} - {value.value} - {value.uuid}')
except ValueError:
raise Exception(f'{cluster.name} - {value.value} - {value.uuid}')
all_uuids[value.uuid].append(f'{cluster.name}|{value.value}')
errors = {}
for uuid, entries in all_uuids.items():
if len(entries) != 1:
errors[uuid] = entries
print(json.dumps(errors, indent=2))
self.assertFalse(errors)
def search_galaxies(query):
clusters = Clusters()
return clusters.search(query)
def revert_tag_from_galaxies(tag):
clusters = Clusters()
try:
return clusters.revert_machinetag(tag)
except Exception:
return []