def get_ad_users_json(container, azure_json_path):
"""Pass in the container name and path to a JSON dump of AD users, return parsed JSON.
"""
connect_string = os.environ.get("AZURE_CONNECTION_STRING", None)
if not connect_string:
return None
store = AzureBlobStorage(connect_string, container)
return json.loads(store.get_content(azure_json_path))
def handle(self, *args, **options):
logger = logging.getLogger('organisation')
logger.info('Generating diff between Ascender and on-premise AD data')
discrepancies = ascender_onprem_ad_data_diff()
f = NamedTemporaryFile()
f.write(json.dumps(discrepancies, indent=4).encode('utf-8'))
logger.info('Uploading diff JSON to Azure blob storage')
connect_string = os.environ.get('AZURE_CONNECTION_STRING')
store = AzureBlobStorage(connect_string, options['container'])
store.upload_file(options['path'], f.name)
logger.info('Completed')
def handle(self, *args, **options):
logger = logging.getLogger('organisation')
logger.info(
'Generating discrepancies between department user and Ascender data'
)
users = DepartmentUser.objects.all()
spreadsheet = department_user_ascender_discrepancies(BytesIO(), users)
f = NamedTemporaryFile()
f.write(spreadsheet.getbuffer())
logger.info('Uploading discrepancies to Azure blob storage')
connect_string = os.environ.get('AZURE_CONNECTION_STRING')
store = AzureBlobStorage(connect_string, options['container'])
store.upload_file(options['path'], f.name)
logger.info('Completed')
class TestResourceRepositoryClient(TestResourceRepositoryClientMixin,
unittest.TestCase):
storage = AzureBlobStorage(settings.AZURE_CONNECTION_STRING,
settings.AZURE_CONTAINER)
resource_base_path = "resourcerepository"
def create_resource_repository(self):
return ResourceRepository(self.storage,
self.resource_name,
resource_base_path=self.resource_base_path,
archive=self.archive,
metaname="metadata",
cache=self.cache,
logical_delete=self.logical_delete)
def populate_test_datas(self):
resource_ids = [("/test/2018_05_02_test2.txt", ),
("test2/2019_06_02_test4.txt", ),
("2018_05_01_test1.txt", ),
("test/2020_07_01_test5.txt", ),
("test/2019_06_01_test3.txt", ),
("test2/2020_07_02_test6.txt", )]
testdatas = {}
for resource_id in resource_ids:
testdatas[resource_id] = self.populate_test_data(resource_id)
return testdatas
def host_dependencies():
# Download the list of Nginx host proxy targets.
connect_string = env('AZURE_CONNECTION_STRING')
store = AzureBlobStorage(connect_string, 'analytics')
store.download('nginx_host_proxy_targets.json',
'/tmp/nginx_host_proxy_targets.json')
f = open('/tmp/nginx_host_proxy_targets.json')
targets = json.loads(f.read())
host_ct = ContentType.objects.get(app_label='status', model='host')
# Production / Production (legacy) systems only.
for it in ITSystem.objects.filter(link__isnull=False,
status__in=[0, 2]).exclude(link=''):
# Remove any existing IT System Host dependencies.
for dep in it.dependencies.filter(content_type=host_ct):
it.dependencies.remove(dep)
if it.extra_data is None:
it.extra_data = {}
it.save()
continue
if 'url_synonyms' not in it.extra_data or not it.extra_data[
'url_synonyms']:
# Skip this IT System (no known URL or synonyms).
continue
# Create/update Host dependencies for IT systems as 'proxy targets'.
target = None
for syn in it.extra_data['url_synonyms']:
for t in targets:
if syn == t['host']:
target = t
break
if target:
for p in target["proxy_pass"]:
u = urlparse(p)
host = u.netloc.split(':')[0]
if Host.objects.filter(name=host).exists():
h = Host.objects.filter(name=host).first()
host_dep, created = Dependency.objects.get_or_create(
content_type=host_ct,
object_id=h.pk,
category='Proxy target',
)
# Add the dependency to the IT System.
it.dependencies.add(host_dep)
def get_resource_consume_client():
"""
Return the blob resource client
"""
global _resource_consume_client
if _resource_consume_client is None:
_resource_consume_client = ResourceConsumeClient(
AzureBlobStorage(settings.NGINX_STORAGE_CONNECTION_STRING,
settings.NGINX_CONTAINER),
settings.NGINX_RESOURCE_NAME, settings.RESOURCE_CLIENTID)
return _resource_consume_client
def get_blob_resource(reuse=True):
"""
Return the blob resource client
"""
global _blob_resource
if _blob_resource is None or not reuse:
_blob_resource = ResourceRepository(AzureBlobStorage(
settings.AZURE_CONNECTION_STRING, settings.AZURE_CONTAINER),
settings.RESOURCE_NAME,
archive=False)
return _blob_resource
def get_resource_repository():
"""
Return the blob resource client
"""
global _resource_repository
if _resource_repository is None:
_resource_repository = GroupResourceRepository(
AzureBlobStorage(settings.AZURE_CONNECTION_STRING,settings.AZURE_CONTAINER),
settings.DOCKER_RESOURCE_NAME,
archive=False
)
return _resource_repository
def signal_sciences_feed_csv(feed_str, corp_name, timestamp, upload=True):
"""For a given passed-in Signal Sciences feed string, summarise it to a CSV for analysis.
Upload the CSV to Azure blob storage.
"""
filename = 'sigsci_request_tags_{}_{}.csv'.format(corp_name, timestamp)
tf = open('/tmp/{}'.format(filename), 'w')
writer = csv.writer(tf)
feed_json = json.loads(feed_str)
for entry in feed_json:
for tag in entry['tags']:
writer.writerow(
[entry['timestamp'], entry['serverName'], tag['type']])
tf.close()
if upload:
connect_string = env('AZURE_CONNECTION_STRING')
store = AzureBlobStorage(connect_string, 'http-requests-tagged')
store.upload_file(filename, tf.name)
return
def signal_sciences_upload_feed(from_datetime=None,
minutes=None,
compress=False,
upload=True,
csv=False):
"""For the given datetime and duration, download the Signal Sciences feed and upload the data
to Azure blob storage (optionally compress the file using gzip).
Optionally also upload a CSV summary of tagged requests to blob storage.
"""
if not from_datetime or not minutes:
return False
feed_str = signal_sciences_extract_feed(from_datetime, minutes)
corp_name = env('SIGSCI_CORP_NAME', 'dbca')
if upload and csv:
signal_sciences_feed_csv(feed_str, corp_name,
from_datetime.isoformat())
if compress:
# Conditionally gzip the file.
filename = 'sigsci_feed_{}_{}.json.gz'.format(
corp_name, from_datetime.strftime('%Y-%m-%dT%H%M%S'))
tf = gzip.open('/tmp/{}'.format(filename), 'wb')
tf.write(feed_str.encode('utf-8'))
else:
filename = 'sigsci_feed_{}_{}.json'.format(
corp_name, from_datetime.strftime('%Y-%m-%dT%H%M%S'))
tf = open('/tmp/{}'.format(filename), 'w')
tf.write(feed_str)
tf.close()
if upload:
# Upload the returned feed data to blob storage.
connect_string = env('AZURE_CONNECTION_STRING')
store = AzureBlobStorage(connect_string, 'signalsciences')
store.upload_file(filename, tf.name)
return filename
def get_resource_repository():
"""
Return the blob resource client
"""
global _resource_repository
if _resource_repository is None:
_resource_repository = IndexedGroupResourceRepository(
AzureBlobStorage(settings.AZURE_CONNECTION_STRING,
settings.AZURE_CONTAINER),
settings.LOGGEDPOINT_RESOURCE_NAME,
get_metaname,
archive=False,
index_metaname=index_metaname)
return _resource_repository
def get_blob_resource(reuse=True):
"""
Return the blob resource client
"""
global _blob_resource
if _blob_resource is None or not reuse:
_blob_resource = ResourceRepository(
AzureBlobStorage(settings.AZURE_CONNECTION_STRING,
settings.AZURE_CONTAINER),
settings.RESOURCE_NAME,
archive=False,
resource_base_path="{}/{}".format(settings.RESOURCE_NAME,
settings.RANCHER_CLUSTER),
logical_delete=True)
return _blob_resource
class TestResourceRepository(TestResourceRepositoryMixin, unittest.TestCase):
storage = AzureBlobStorage(settings.AZURE_CONNECTION_STRING,
settings.AZURE_CONTAINER)
resource_base_path = "resourcerepository"
def create_resource_repository(self):
return ResourceRepository(self.storage,
self.resource_name,
resource_base_path=self.resource_base_path,
archive=self.archive,
metaname="metadata",
cache=self.cache,
logical_delete=self.logical_delete)
def get_test_data_keys(self):
return [("/test/2018_05_02_test2.txt", ),
("test2/2019_06_02_test4.txt", ), ("2018_05_01_test1.txt", ),
("test/2020_07_01_test5.txt", ),
("test/2019_06_01_test3.txt", ),
("test2/2020_07_02_test6.txt", )]
def itsystem_risks_traffic(it_systems=None):
"""Set automatic risk assessment for IT system web apps based on the mean of daily HTTP requests.
"""
if not it_systems:
it_systems = ITSystem.objects.all()
# Download the report of HTTP requests.
connect_string = env('AZURE_CONNECTION_STRING')
store = AzureBlobStorage(connect_string, 'analytics')
store.download('host_requests_7_day_count.csv',
'/tmp/host_requests_7_day_count.csv')
counts = csv.reader(open('/tmp/host_requests_7_day_count.csv'))
next(counts) # Skip the header.
report = {}
for row in counts:
try:
report[row[0]] = int(row[1])
except:
# Sometimes the report contains junk rows; just ignore these.
pass
itsystem_ct = ContentType.objects.get(app_label='registers',
model='itsystem')
for it in it_systems:
# First, check if an auto assessment has been created OR if not assessment exists.
# If so, carry on. If not, skip automated assessment (assumes that a manual assessment exists,
# which we don't want to overwrite).
if (RiskAssessment.objects.filter(
content_type=itsystem_ct,
object_id=it.pk,
category='Traffic',
notes__contains='[AUTOMATED ASSESSMENT]').exists()
or not RiskAssessment.objects.filter(
content_type=itsystem_ct,
object_id=it.pk,
category='Traffic').exists()):
if 'url_synonyms' not in it.extra_data or not it.extra_data[
'url_synonyms']:
# Skip this IT System (no known URL or synonyms).
continue
# Get/create a Traffic risk
risk = RiskAssessment.objects.filter(content_type=itsystem_ct,
object_id=it.pk,
category='Traffic').first()
if not risk:
risk = RiskAssessment(content_type=itsystem_ct,
object_id=it.pk,
category='Traffic')
# Total the count of HTTP requests to all URI synonyms for this system.
request_count = 0
for syn in it.extra_data['url_synonyms']:
if syn in report and report[syn]:
request_count += report[syn]
requests_mean = request_count / 7 # Daily average request count.
if requests_mean:
risk = RiskAssessment.objects.filter(
content_type=itsystem_ct,
object_id=it.pk,
category='Traffic').first()
if not risk:
risk = RiskAssessment(content_type=itsystem_ct,
object_id=it.pk,
category='Traffic')
if requests_mean >= 10000:
risk.rating = 3
risk.notes = '[AUTOMATED ASSESSMENT] High traffic of daily HTTP requests'
elif requests_mean >= 1000:
risk.rating = 2
risk.notes = '[AUTOMATED ASSESSMENT] Moderate traffic of daily HTTP requests'
elif requests_mean >= 100:
risk.rating = 1
risk.notes = '[AUTOMATED ASSESSMENT] Low traffic of daily HTTP requests'
else:
risk.rating = 0
risk.notes = '[AUTOMATED ASSESSMENT] Minimal traffic of daily HTTP requests'
risk.save()
else: # Volume of HTTP traffic is too small to assess.
# If any Traffic risk exists, delete it.
if RiskAssessment.objects.filter(content_type=itsystem_ct,
object_id=it.pk,
category='Traffic').exists():
risk = RiskAssessment.objects.filter(
content_type=itsystem_ct,
object_id=it.pk,
category='Traffic').first()
risk.delete()
def itsystem_risks_access(it_systems=None):
"""Set automatic risk assessment for IT system web apps based on whether they require SSO on the root location.
"""
if not it_systems:
it_systems = ITSystem.objects.all()
# Download the list of Nginx host proxy targets.
connect_string = env('AZURE_CONNECTION_STRING')
store = AzureBlobStorage(connect_string, 'analytics')
store.download('nginx_host_proxy_targets.json',
'/tmp/nginx_host_proxy_targets.json')
f = open('/tmp/nginx_host_proxy_targets.json')
targets = json.loads(f.read())
itsystem_ct = ContentType.objects.get(app_label='registers',
model='itsystem')
for it in it_systems:
# First, check if an auto assessment has been created OR if no assessment exists.
# If so, carry on. If not, skip automated assessment (assumes that a manual assessment exists,
# which we don't want to overwrite).
if (RiskAssessment.objects.filter(
content_type=itsystem_ct,
object_id=it.pk,
category='Access',
notes__contains='[AUTOMATED ASSESSMENT]').exists() or
not RiskAssessment.objects.filter(content_type=itsystem_ct,
object_id=it.pk,
category='Access').exists()):
if 'url_synonyms' not in it.extra_data or not it.extra_data[
'url_synonyms']:
# Skip this IT System (no known URL or synonyms).
continue
target = None
# Get/create an access risk
risk = RiskAssessment.objects.filter(content_type=itsystem_ct,
object_id=it.pk,
category='Access').first()
if not risk:
risk = RiskAssessment(content_type=itsystem_ct,
object_id=it.pk,
category='Access')
for syn in it.extra_data['url_synonyms']:
for t in targets:
if syn == t['host']:
target = t
break
if target:
if 'sso_locations' in target:
if '/' in target['sso_locations'] or '^~ /' in target[
'sso_locations'] or '= /' in target[
'sso_locations']:
risk.rating = 0
risk.notes = '[AUTOMATED ASSESSMENT] Web application root location requires SSO'
else:
risk.rating = 1
risk.notes = '[AUTOMATED ASSESSMENT] Web application locations configured to require SSO'
else:
if 'custom/dpaw_subnets' in target['includes']:
risk.rating = 1
risk.notes = '[AUTOMATED ASSESSMENT] Web application root location does not require SSO, but is restricted to internal subnets'
else:
risk.rating = 2
risk.notes = '[AUTOMATED ASSESSMENT] Web application root location does not require SSO and is not restricted to internal subnets'
risk.save()
else:
# If any access risk exists, delete it.
if RiskAssessment.objects.filter(
content_type=itsystem_ct,
object_id=it.pk,
category='Access').exists():
risk = RiskAssessment.objects.filter(
content_type=itsystem_ct,
object_id=it.pk,
category='Access').first()
risk.delete()