def test_get_messages(self):
user = models.User(email='*****@*****.**')
so = models.User(email='*****@*****.**')
user.save(password='******')
so.save(password='******')
user.so_id = so.id
so.so_id = user.id
user.save()
so.save()
for idx in range(5):
ping = models.Ping(from_id=user.id, to_id=so.id)
ping.save()
self.assertEqual(user.sent_messages().count(), 5)
response = self.app.post('/login_standard/',
data={
'email': so.email,
'password': '******',
})
response = self.app.get('/get_messages/')
self.assertEqual(response.status_code, 200)
data = json.loads(response.data)
self.assertEqual(len(data), 5)
response = self.app.post('/login_standard/',
data={
'email': user.email,
'password': '******',
})
response = self.app.get('/get_messages/')
data = json.loads(response.data)
self.assertEqual(len(data), 0)
def test_notmember_json(self):
"""
:return:
"""
user = models.User(first_name='not',
last_name='Member',
email='*****@*****.**',
password='******',
confirmed=True)
db.session.add(user)
db.session.commit()
rv = self.login(user.email, 'password')
self.assertEquals(rv.status_code, 200)
result = self.client.get('/organization/' + str(self.organization.id),
follow_redirects=True,
headers={'Accept': 'application/json'})
self.assertEquals(result.status_code, 200)
json_data = json.loads(result.data)
assert 'status' in json_data
assert json_data['status'] == 'error'
assert 'message' in json_data
assert json_data[
'message'] == '403 Access Denied. You must be a member of this organization.'
rv = self.logout()
self.assertEquals(rv.status_code, 200)
def test_notmember_html(self):
"""
Test an endpoint with organization owner protection and html response
:return:
"""
user = models.User(first_name='not',
last_name='Member',
email='*****@*****.**',
password='******',
confirmed=True)
db.session.add(user)
db.session.commit()
rv = self.login(user.email, 'password')
self.assertEquals(rv.status_code, 200)
result = self.client.get('/organization/' + str(self.organization.id),
follow_redirects=True,
headers={'Accept': 'text/html'})
print result.status_code
print result.data
self.assertEquals(result.status_code, 403)
rv = self.logout()
self.assertEquals(rv.status_code, 200)
def create_user(request: schema.User, db: Session = Depends(database.get_db)):
hashed_pass = hashing.get_password_hash(request.password)
new_user = models.User(name=request.name,
username=request.username,
password=hashed_pass)
db.add(new_user)
db.commit()
db.refresh(new_user)
return request
def test_ping_without_so(self):
user = models.User(email='*****@*****.**')
so = models.User(email='*****@*****.**')
user.save(password='******')
so.save(password='******')
user.so_id = so.id
so.so_id = user.id
user.save()
so.save()
self.assertEqual(user.sent_messages().count(), 0)
response = self.app.post('/login_standard/',
data={
'email': user.email,
'password': '******',
})
response = self.app.post('/ping/')
self.assertEqual(response.status_code, 200)
self.assertEqual(user.sent_messages().count(), 1)
def test_incorrect_password(self):
new_user = models.User(email='*****@*****.**')
new_user.save(password='******')
response = self.app.post('/login_standard/',
data={
'email': new_user.email,
'password': '******',
})
self.assertEqual(response.status_code, 400)
def test_update_existing_so(self):
user = models.User(username='******',
email='*****@*****.**')
user.save(password='******')
existing_so = models.User(username='******',
email='*****@*****.**')
existing_so.save(password='******')
existing_so = models.User.query.filter_by(username='******').first()
response = self.app.post('/login/',
data={
'login': user.username,
'password': '******',
})
self.assertEqual(response.status_code, 200)
response = self.app.post('/update_so/',
data={'email': existing_so.email})
self.assertEqual(response.status_code, 200)
user = models.User.query.get(user.id)
self.assertEqual(user.so_id, existing_so.id)
def test_login(self):
new_user = models.User(username='******',
email='*****@*****.**')
new_user.save(password='******')
response = self.app.post('/login/',
data={
'login': new_user.username,
'password': '******',
})
self.assertEqual(response.status_code, 200)
def test_register_temp_user(self):
new_user = models.User(email='*****@*****.**',
password_hash='hash',
is_temp=True)
new_user.save()
so_pending_user = models.User(email='*****@*****.**',
password_hash='hash',
is_temp=False,
so_id=new_user.id)
so_pending_user.save()
response = self.app.post('/register/',
data={
'email': new_user.email,
'password': '******',
})
self.assertEqual(response.status_code, 201)
data = json.loads(response.data)
self.assertFalse(data['created'])
self.assertEqual(data['email'], '*****@*****.**')
self.assertEqual(data['so_pending_id'], so_pending_user.id)
def test_duplicate_info(self):
new_user = models.User(email='*****@*****.**',
password_hash='hash')
new_user.save()
response = self.app.post('/register/',
data={
'email': new_user.email,
'password': '******',
})
self.assertEqual(response.status_code, 400)
def test_ping(self):
user = models.User(email='*****@*****.**')
so = models.User(email='*****@*****.**')
user.save(password='******')
so.save(password='******')
user.so_id = so.id
so.so_id = user.id
user.save()
so.save()
self.assertEqual(user.sent_messages().count(), 0)
response = self.app.post('/login_standard/',
data={
'email': user.email,
'password': '******',
})
response = self.app.post('/ping/')
self.assertEqual(response.status_code, 200)
self.assertEqual(user.sent_messages().count(), 1)
data = json.loads(response.data)
response_sent_at = data['message_sent_at']
self.assertIsNotNone(response_sent_at)
def test_update_existing_so(self):
user = models.User(email='*****@*****.**')
user.save(password='******')
existing_so = models.User(email='*****@*****.**')
existing_so.save(password='******')
existing_so = models.User.query.filter_by(
email='*****@*****.**').first()
response = self.app.post('/login_standard/',
data={
'email': user.email,
'password': '******',
})
self.assertEqual(response.status_code, 200)
response = self.app.post('/update_so/',
data={'email': existing_so.email})
self.assertEqual(response.status_code, 200)
user = models.User.query.get(user.id)
self.assertEqual(user.so_id, existing_so.id)
data = json.loads(response.data)
self.assertEqual(data['id'], user.id)
self.assertEqual(data['so_id'], user.so_id)
self.assertFalse(data['is_so_temp'])
def test_update_new_so(self):
user = models.User(username='******',
email='*****@*****.**')
user.save(password='******')
response = self.app.post('/login/',
data={
'login': user.username,
'password': '******',
})
self.assertEqual(response.status_code, 200)
response = self.app.post('/update_so/',
data={'email': '*****@*****.**'})
self.assertEqual(response.status_code, 200)
so_user = models.User.query.filter_by(
email='*****@*****.**').first()
user = models.User.query.get(user.id)
self.assertEqual(user.so_id, so_user.id)
def test_duplicate_info(self):
new_user = models.User(username='******',
email='*****@*****.**',
password_hash='hash')
db.session.add(new_user)
db.session.commit()
response = self.app.post('/register/',
data={
'username': new_user.username,
'email': '*****@*****.**',
'password': '******',
})
self.assertEqual(response.status_code, 400)
response = self.app.post('/register/',
data={
'username': '******',
'email': new_user.email,
'password': '******',
})
self.assertEqual(response.status_code, 400)
def import_crits_indicators():
""" Imports CRITS indicators from the exported MongoDB JSON """
# Make sure the indicators.json file exists.
if not os.path.exists('./import/indicators.json'):
current_app.logger.error('Could not locate indicators.json for CRITS import')
return
# Validate the indicators JSON.
crits_create_schema = {
'type': 'object',
'properties': {
'bucket_list': {
'type': 'array',
'items': {'type': 'string', 'maxLength': 255}
},
'campaign': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'name': {'type': 'string', 'maxLength': 255}
}
}
},
'confidence': {
'type': 'object',
'properties': {
'rating': {'type': 'string', 'minLength': 1, 'maxLength': 255}
}
},
'created': {
'type': 'object',
'properties': {
'$date': {'type': 'string', 'minLength': 24, 'maxLength': 24}
}
},
'impact': {
'type': 'object',
'properties': {
'rating': {'type': 'string', 'minLength': 1, 'maxLength': 255}
}
},
'modified': {
'type': 'object',
'properties': {
'$date': {'type': 'string', 'minLength': 24, 'maxLength': 24}
}
},
'source': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'instances': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'analyst': {'type': 'string', 'minLength': 1, 'maxLength': 255},
'reference': {'type': 'string', 'maxLength': 512}
}
}
},
'name': {'type': 'string', 'minLength': 1, 'maxLength': 255}
}
}
},
'status': {'type': 'string', 'minLength': 1, 'maxLength': 255},
'type': {'type': 'string', 'minLength': 1, 'maxLength': 255},
'value': {'type': 'string', 'minLength': 1}
},
'required': ['confidence', 'created', 'impact', 'modified', 'source', 'status', 'type', 'value']
}
start = time.time()
# Connect to the database engine to issue faster select statements.
with db.engine.connect() as conn:
# Get the existing values from the database that indicators depend upon.
existing_campaigns = dict()
for x in models.Campaign.query.all():
existing_campaigns[x.name] = x
existing_indicator_confidence = dict()
for x in models.IndicatorConfidence.query.all():
existing_indicator_confidence[x.value] = x
existing_indicator_impact = dict()
for x in models.IndicatorImpact.query.all():
existing_indicator_impact[x.value] = x
existing_indicator_status = dict()
for x in models.IndicatorStatus.query.all():
existing_indicator_status[x.value] = x
existing_indicator_type = dict()
for x in models.IndicatorType.query.all():
existing_indicator_type[x.value] = x
existing_intel_reference = dict() # source: reference: object
for x in models.IntelReference.query.all():
if x.source.value not in existing_intel_reference:
existing_intel_reference[x.source.value] = dict()
existing_intel_reference[x.source.value][x.reference] = x
existing_intel_source = dict()
for x in models.IntelSource.query.all():
existing_intel_source[x.value] = x
existing_tags = dict()
for x in models.Tag.query.all():
existing_tags[x.value] = x
existing_users = dict()
for x in models.User.query.all():
existing_users[x.username] = x
num_new_indicators = 0
unique_indicators = dict() # type: value
line_count = 0
with open('./import/indicators.json') as f:
for line in f:
indicator = json.loads(line)
jsonschema.validate(indicator, crits_create_schema)
line_count += 1
# Skip this indicator if it is a duplicate.
if indicator['type'] not in unique_indicators:
unique_indicators[indicator['type']] = []
if indicator['value'] in unique_indicators[indicator['type']]:
current_app.logger.warning('CRITS IMPORT: Skipping duplicate indicator: {}'.format(indicator['_id']['$oid']))
continue
else:
unique_indicators[indicator['type']].append(indicator['value'])
# Check if the indicator confidence must be created.
if indicator['confidence']['rating'] in existing_indicator_confidence:
indicator_confidence = existing_indicator_confidence[indicator['confidence']['rating']]
else:
new_indicator_confidence = models.IndicatorConfidence(value=indicator['confidence']['rating'])
existing_indicator_confidence[new_indicator_confidence.value] = new_indicator_confidence
indicator_confidence = new_indicator_confidence
db.session.add(new_indicator_confidence)
db.session.flush()
# Check if the indicator impact must be created.
if indicator['impact']['rating'] in existing_indicator_impact:
indicator_impact = existing_indicator_impact[indicator['impact']['rating']]
else:
new_indicator_impact = models.IndicatorImpact(value=indicator['impact']['rating'])
existing_indicator_impact[new_indicator_impact.value] = new_indicator_impact
indicator_impact = new_indicator_impact
db.session.add(new_indicator_impact)
db.session.flush()
# Check if the indicator status must be created.
if indicator['status'] in existing_indicator_status:
indicator_status = existing_indicator_status[indicator['status']]
else:
new_indicator_status = models.IndicatorStatus(value=indicator['status'])
existing_indicator_status[new_indicator_status.value] = new_indicator_status
indicator_status = new_indicator_status
db.session.add(new_indicator_status)
db.session.flush()
# Check if the indicator type must be created.
if indicator['type'] in existing_indicator_type:
indicator_type = existing_indicator_type[indicator['type']]
else:
new_indicator_type = models.IndicatorType(value=indicator['type'])
existing_indicator_type[new_indicator_type.value] = new_indicator_type
indicator_type = new_indicator_type
db.session.add(new_indicator_type)
db.session.flush()
# Check if the user must be created.
username = indicator['source'][0]['instances'][0]['analyst']
if username in existing_users:
user = existing_users[username]
else:
password = ''.join(random.choice(string.ascii_letters + string.punctuation + string.digits) for x in range(20))
new_user = models.User(active=False,
email='{}@unknown'.format(username),
first_name=username,
last_name='Unknown',
password=hash_password(password),
roles=[],
username=username)
existing_users[username] = new_user
user = new_user
db.session.add(new_user)
db.session.flush()
# Create the campaigns list.
campaigns = set()
if 'campaign' in indicator and indicator['campaign']:
for campaign in indicator['campaign']:
campaigns.add(existing_campaigns[campaign['name']])
# Create the intel references list.
references = set()
for s in indicator['source']:
# Check if the source must be created.
if s['name'] in existing_intel_source:
source = existing_intel_source[s['name']]
else:
source = models.IntelSource(value=s['name'])
existing_intel_source[s['name']] = source
db.session.add(source)
db.session.flush()
# Make sure the source exists in the intel reference dictionary.
if source.value not in existing_intel_reference:
existing_intel_reference[source.value] = dict()
for instance in s['instances']:
# Check if the reference must be created.
if 'reference' in instance and source.value in existing_intel_reference and instance['reference'] in existing_intel_reference[source.value]:
references.add(existing_intel_reference[source.value][instance['reference']])
elif 'reference' in instance and instance['reference']:
new_reference = models.IntelReference(reference=instance['reference'],
source=source,
user=user)
existing_intel_reference[source.value][instance['reference']] = new_reference
references.add(new_reference)
db.session.add(new_reference)
db.session.flush()
# Create the tags list.
tags = []
if 'bucket_list' in indicator and indicator['bucket_list']:
for tag in indicator['bucket_list']:
# Check if the tag must be created.
if tag in existing_tags:
tags.append(existing_tags[tag])
else:
new_tag = models.Tag(value=tag)
existing_tags[tag] = new_tag
tags.append(new_tag)
db.session.add(new_tag)
db.session.flush()
# Create the new indicator.
try:
new_indicator = models.Indicator(campaigns=list(campaigns),
case_sensitive=False,
confidence=indicator_confidence,
created_time=parse(indicator['created']['$date']),
impact=indicator_impact,
modified_time=parse(indicator['modified']['$date']),
references=list(references),
status=indicator_status,
substring=False,
tags=tags,
type=indicator_type,
user=user,
value=indicator['value'])
db.session.add(new_indicator)
num_new_indicators += 1
except Exception as e:
current_app.logger.exception("CRITS IMPORT: unable to import indicator {}: {}".format(indicator, e))
#db.session.rollback()
# Save any database changes.
db.session.commit()
current_app.logger.info('CRITS IMPORT: Imported {}/{} indicators in {}'.format(num_new_indicators, line_count, time.time() - start))
def setUp(self):
super(MaterialTestCase, self).__init__()
self.client = Client()
self.user = models.User(first_name='alex')
self.user.save()
