def create_resources(self):
# Create Dynamo and user
dynamo_allocator = DynamoAllocator()
self.create_table()
dynamo_table = dynamo_allocator.table('Autobahn')
ec2_client = EC2Allocator().client('eu-west-1')
vpc_response = ec2_client.create_vpc(CidrBlock='10.1.0.0/24',
AmazonProvidedIpv6CidrBlock=False,
DryRun=False,
InstanceTenancy='default')
self.vpc_id = vpc_response['Vpc']['VpcId']
ec2_client.create_tags(DryRun=False,
Resources=[self.vpc_id],
Tags=[{
'Key': 'Name',
'Value': "vpc-test"
}])
acl_response = ec2_client.create_network_acl(DryRun=False,
VpcId=self.vpc_id)
self.acl_id = acl_response['NetworkAcl']['NetworkAclId']
entry_response = ec2_client.create_network_acl_entry(
CidrBlock='244.244.244.244/32',
DryRun=False,
Egress=False,
NetworkAclId=self.acl_id,
Protocol='-1',
RuleAction='deny',
RuleNumber=77)
return self.vpc_id, self.acl_id
def test_dynamo_upsert(self):
vpc_id, acl_id = self.create_resources()
ip_hash_map = {'244.244.244.244': 1}
ec2_client = EC2Allocator().client('eu-west-1')
dynamo_allocator = DynamoAllocator()
dynamo_resource = dynamo_allocator.resource()
table = dynamo_resource.Table(get_dynamo_table())
struct_time = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M")
epoch = UTC_time_to_epoch(struct_time)
table.put_item(
Item={
'ip': '244.244.244.244',
'last_seen': epoch,
'times_seen': 1,
'banned': True
})
DynamoUpserter().upsert_ip_in_table(dynamo_resource, '244.244.244.244')
response = table.query(
KeyConditionExpression=Key('ip').eq('244.244.244.244'))
print(response)
assert len(response['Items']) == 1
assert response['Items'][0]['banned'] == False
def test_ipbanner(self):
vpc_id, acl_id = self.create_resources()
datadog_client = DatadogClient()
ip_hash_map = {'244.244.244.244': 1}
ec2_client = EC2Allocator().client('eu-west-1')
dynamo_allocator = DynamoAllocator()
dynamo_resource = dynamo_allocator.resource()
table = dynamo_resource.Table(get_dynamo_table())
struct_time = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M")
epoch = UTC_time_to_epoch(struct_time)
table.put_item(
Item={
'ip': '244.244.244.244',
'last_seen': epoch,
'times_seen': 1,
'banned': False
})
ip_banner = IpBanner(ec2_client, datadog_client)
ip_banner.ban_ip('244.244.244.244', acl_id, 90)
response = table.query(
KeyConditionExpression=Key('ip').eq('244.244.244.244'))
assert response['Items'][0]['banned'] == True
def process(record):
ip_hash_map = {}
country = record['country']
project = record['project']
dynamo_allocator = DynamoAllocator()
dynamo_resource = dynamo_allocator.resource()
region = get_region_from_country(country)
cloudwatch_allocator = CloudWatchLogsAllocator()
for event_page in cloudwatch_allocator.iterator(
region, country, project):
print(len(event_page['events']))
for event in event_page['events']:
line = event['message']
try:
ip, timestamp, agent, url = ElbEntryParser.parseline(line)
if not GoodUrlDiscarder().is_ignored(url):
ip_hash_map = IpCounter().processIp(ip, ip_hash_map)
except Exception as e:
pass
ip_hash_map = {
k: v
for k, v in ip_hash_map.items() if v >= get_request_treshold()
}
ip_hash_map = GoodBotDiscarder().discard_good_bots(ip_hash_map)
list_of_ips = [k for k, v in ip_hash_map.items()]
for ip in list_of_ips:
DynamoUpserter.upsert_ip_in_table(dynamo_resource, ip)
def tearDown(self):
# Create Dynamo and user
dynamo_allocator = DynamoAllocator()
dynamo_table = dynamo_allocator.table('Autobahn')
ec2_client = EC2Allocator().client('eu-west-1')
try:
acl_response = ec2_client.delete_network_acl(
DryRun=False, NetworkAclId=self.acl_id)
except:
pass
def process(event):
dynamo_db_allocator = DynamoAllocator()
dynamo_resource = dynamo_db_allocator.resource()
dynamo_client = dynamo_db_allocator.client()
unbannable_ips = DynamoGetUnbannable().get_unbannables(dynamo_client, dynamo_resource)
for country in get_countries():
region = get_region_from_country(country)
ec2_client = EC2Allocator().client(region_name=region)
ip_unbanner = IpUnbanner(ec2_client=ec2_client)
vpc_id = VpcFinder().get_vpc(ec2_client, country)
nacl, proposed_rulenumber = VpcNaclElector(
).get_available_nacl_and_proposed_rulenumber(ec2_client, vpc_id, 100)
ip_unbanner.unban_ips(unbannable_ips, nacl, vpc_id)
def test_dynamo_upsert(self):
vpc_id, acl_id = self.create_resources()
ip_hash_map = {'244.244.244.244': 1}
ec2_client = EC2Allocator().client('eu-west-1')
dynamo_allocator = DynamoAllocator()
dynamo_resource = dynamo_allocator.resource()
DynamoUpserter().upsert_ip_in_table(dynamo_resource, '244.244.244.244')
table = dynamo_resource.Table(get_dynamo_table())
response = table.query(
KeyConditionExpression=Key('ip').eq('244.244.244.244'))
print(response)
assert len(response['Items']) == 1
def test_record_processer(self):
print(__name__)
vpc_id, acl_id = self.create_resources()
self.cloudwatchfixtures()
record = {'country': 'dev', 'project': 'test'}
ip_hash_map = {'244.244.244.244': 1}
ec2_client = EC2Allocator().client('eu-west-1')
dynamo_allocator = DynamoAllocator()
dynamo_resource = dynamo_allocator.resource()
RecordProcesser().process(record)
table = dynamo_resource.Table(get_dynamo_table())
response = table.query(
KeyConditionExpression=Key('ip').eq('80.253.202.218'))
print(response)
assert len(response['Items']) == 1
def create_table(self):
dynamo_allocator = DynamoAllocator()
dynamo_resource = dynamo_allocator.resource()
dynamo_resource.create_table(
TableName='Autobahn',
KeySchema=[{
'AttributeName': 'ip',
'KeyType': 'HASH' # Partition key
}],
AttributeDefinitions=[{
'AttributeName': 'ip',
'AttributeType': 'S'
}],
ProvisionedThroughput={
'ReadCapacityUnits': 5,
'WriteCapacityUnits': 5
})
def create_resources(self):
# Create Dynamo and user
dynamo_allocator = DynamoAllocator()
dynamo_resource = dynamo_allocator.resource()
self.create_table()
dynamo_table = dynamo_allocator.table('Autobahn')
ec2_client = EC2Allocator().client('eu-west-1')
vpc_response = ec2_client.create_vpc(CidrBlock='10.1.0.0/24',
AmazonProvidedIpv6CidrBlock=False,
DryRun=False,
InstanceTenancy='default')
self.vpc_id = vpc_response['Vpc']['VpcId']
ec2_client.create_tags(DryRun=False,
Resources=[self.vpc_id],
Tags=[{
'Key': 'Name',
'Value': "vpc-test"
}])
acl_response = ec2_client.create_network_acl(DryRun=False,
VpcId=self.vpc_id)
self.acl_id = acl_response['NetworkAcl']['NetworkAclId']
entry_response = ec2_client.create_network_acl_entry(
CidrBlock='244.244.244.244/32',
DryRun=False,
Egress=False,
NetworkAclId=self.acl_id,
Protocol='-1',
RuleAction='deny',
RuleNumber=77)
struct_time = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M")
epoch = UTC_time_to_epoch(struct_time)
table = dynamo_resource.Table('Autobahn')
table.put_item(
Item={
'ip': '244.244.244.244',
'last_seen': epoch - 99100,
'times_seen': 1,
'banned': True
})
return self.vpc_id, self.acl_id
def ban_ip(self, ip, nacl, proposed_rulenumber):
self.ec2_client.create_network_acl_entry(
CidrBlock="{}/32".format(ip),
Egress=False,
Protocol="-1",
RuleAction="deny",
RuleNumber=proposed_rulenumber,
NetworkAclId=nacl)
dynamo_resource = DynamoAllocator().resource()
DynamoUpserter().upsert_ip_in_table(dynamo_resource, ip)
logging.getLogger().info(
'Marked as Banned in Dynamo : ip {}'.format(ip))
def test_get_unbannables(self):
vpc_id, acl_id = self.create_resources()
ip_hash_map = {'244.244.244.244': 1}
ec2_client = EC2Allocator().client('eu-west-1')
dynamo_allocator = DynamoAllocator()
dynamo_resource = dynamo_allocator.resource()
dynamo_client = dynamo_allocator.client()
table = dynamo_resource.Table(get_dynamo_table())
struct_time = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M")
epoch = UTC_time_to_epoch(struct_time)
table.put_item(
Item={
'ip': '244.244.244.244',
'last_seen': epoch,
'times_seen': 1,
'banned': True
})
unbannables = DynamoGetUnbannable().get_unbannables(
dynamo_client, dynamo_resource)
assert '244.244.244.244' in unbannables
def create_resources(self):
# Create Dynamo and user
dynamo_allocator = DynamoAllocator()
self.create_table()
dynamo_table = dynamo_allocator.table('Autobahn')
ec2_client = EC2Allocator().client('eu-west-1')
vpc_response = ec2_client.create_vpc(CidrBlock='10.1.0.0/24',
AmazonProvidedIpv6CidrBlock=False,
DryRun=False,
InstanceTenancy='default')
self.vpc_id = vpc_response['Vpc']['VpcId']
nacls = ec2_client.describe_network_acls(
Filters=[{
'Name': 'vpc-id',
'Values': [self.vpc_id]
}])
for nacl in nacls['NetworkAcls']:
try:
ec2_client.delete_network_acl(
DryRun=False, NetworkAclId=nacl['NetworkAclId'])
except:
pass
acl_response = ec2_client.create_network_acl(DryRun=False,
VpcId=self.vpc_id)
self.acl_id = acl_response['NetworkAcl']['NetworkAclId']
entry_response = ec2_client.create_network_acl_entry(
CidrBlock='244.244.244.244/32',
DryRun=False,
Egress=False,
NetworkAclId=self.acl_id,
Protocol='-1',
RuleAction='deny',
RuleNumber=77)
return self.vpc_id
def unban_ip(self, ip, nacl, vpc_id):
self.remove_nacl_entry(ip, vpc_id)
dynamo_resource = DynamoAllocator().resource()
DynamoUpserter().upsert_ip_in_table(dynamo_resource, ip)
logging.getLogger().info('Marked as Unbanned in Dynamo : ip {}'.format(ip))
def test_event_processer(self):
print(__name__)
vpc_id, acl_id = self.create_resources()
struct_time = datetime.utcnow()
epoch = UTC_time_to_epoch(struct_time)
record = {'country': 'dev', 'project': 'test'}
ip_hash_map = {'244.244.244.244': 1}
event = {
"eventID": "1",
"eventVersion": "1.0",
"dynamodb": {
"Keys": {
"ip": {
"S": "244.244.244.244"
},
"times_seen": {
"S": "244.244.244.244"
},
"last_seen": {
"N": "{}".format(epoch)
},
"banned": {
"BOOL": False
}
},
"NewImage": {
"ip": {
"S": "244.244.244.244"
},
"times_seen": {
"S": "244.244.244.244"
},
"last_seen": {
"N": "{}".format(epoch)
},
"banned": {
"BOOL": False
}
},
"StreamViewType": "NEW_AND_OLD_IMAGES",
"SequenceNumber": "111",
"SizeBytes": 26
},
"awsRegion": "eu-west-1",
"eventName": "INSERT",
"eventSourceARN": 'testarn',
"eventSource": "aws:dynamodb"
}
ec2_client = EC2Allocator().client('eu-west-1')
dynamo_allocator = DynamoAllocator()
dynamo_resource = dynamo_allocator.resource()
table = dynamo_resource.Table(get_dynamo_table())
table.put_item(
Item={
'ip': '244.244.244.244',
'last_seen': epoch,
'times_seen': 1,
'banned': False
})
EventProcesser().process(event)
response = table.query(
KeyConditionExpression=Key('ip').eq('244.244.244.244'))
datadog_client = DatadogClient()
ip_banner = IpBanner(ec2_client, datadog_client)
assert response['Items'][0]['banned'] == True
assert ip_banner.not_already_banned('244.244.244.244', vpc_id) == False