def get(self, request: HttpRequest) -> JsonResponse:
rpid = request.GET.get('rpid')
lead = Lead.objects.filter(raspberry_pi__rpid=rpid).first()
if not lead:
return JsonResponse({'result': False, 'reason': 'RPID not found'})
EC2Instance.launch_for_lead(lead)
return JsonResponse({'result': True, 'launched': True})
def get(self, request: HttpRequest, rpid: str) -> HttpResponse:
'Get netstat output from EC2. Used as a fallback if RaspberryPi cannot get it by itself'
rpid = rpid.strip()
ec2_instance = EC2Instance.get_by_rpid(rpid)
if not ec2_instance or not ec2_instance.is_running():
return HttpResponse('Not exists')
try:
output = ec2_instance.ssh_execute('netstat -an')
except SSHConnectException:
return HttpResponse('', content_type='text/plain')
result = []
if ec2_instance.TUNNEL_RE.search(output):
result.append('SSH tunnel is UP')
else:
result.append('SSH tunnel is DOWN')
if ec2_instance.REVERSE_TUNNEL_RE.search(output):
result.append('Reverse tunnel is UP')
else:
result.append('Reverse tunnel is DOWN')
if ec2_instance.RDP_RE.search(output):
result.append('RDP session is active')
result.append(output)
return HttpResponse('\n'.join(result), content_type='text/plain')
def create_r53_entry(self, ec2_instance: EC2Instance) -> None:
ec2_resource = self.get_resource('ec2')
route53_client = boto3.client('route53')
hostname = ec2_instance.get_r53_hostname()
elastic_ip = ec2_resource.allocate_address(Domain='vpc')
ec2_resource.associate_address(
InstanceId=ec2_instance.instance_id,
AllocationId=elastic_ip["AllocationId"],
)
route53_client.change_resource_record_sets(
HostedZoneId=settings.AWS_R53_ZONE_ID,
ChangeBatch={
'Comment':
'Add {} instance to Route53'.format(ec2_instance.rpid),
'Changes': [
{
'Action': 'CREATE',
'ResourceRecordSet': {
'Name':
hostname,
'Type':
'A',
'TTL':
60,
'ResourceRecords': [
{
'Value': elastic_ip["PublicIp"],
},
],
}
},
]
},
)
def get(self, request: HttpRequest, rpid: str) -> JsonResponse:
'Start reverse tunnel from EC2. Used as a fallback if RaspberryPi cannot created it by itself'
ec2_instance = EC2Instance.get_by_rpid(rpid)
if not ec2_instance or not ec2_instance.is_running():
return JsonResponse(dict(result=False))
tunnel_up = False
attempts = 0
netstat_output = 'True'
while not tunnel_up and attempts < self.MAX_ATTEMPTS and netstat_output:
attempts += 1
netstat_output = ''
try:
netstat_output = ec2_instance.ssh_execute('netstat -an') or ''
except SSHConnectException:
pass
if ec2_instance.REVERSE_TUNNEL_RE.search(netstat_output):
tunnel_up = True
if attempts:
break
try:
ec2_instance.ssh_execute('ssh -N -D 3808 -p 2046 pi@localhost')
except SSHConnectException:
pass
time.sleep(5)
return JsonResponse(dict(
result=tunnel_up,
attempts=attempts,
))
def launch_essential_instance(
self,
ec2_instance: EC2Instance) -> boto3.resources.base.ServiceResource:
'Start or create AWS EC2 instance for given RPID'
boto_instance = self.get_by_essential_key(ec2_instance.essential_key)
if boto_instance:
return boto_instance
self.get_resource('ec2').create_instances(
ImageId=settings.AWS_IMAGE_AMI_ESSENTIAL,
MinCount=1,
MaxCount=1,
KeyName='AI Farming Key',
InstanceType=ec2_instance.instance_type,
SecurityGroupIds=settings.AWS_SECURITY_GROUP_IDS,
UserData='essential',
TagSpecifications=[
{
'ResourceType':
'instance',
'Tags': [
{
'Key': 'Essential',
'Value': 'true',
},
{
'Key': 'EssentialKey',
'Value': ec2_instance.essential_key,
},
{
'Key': 'Duplicate',
'Value': 'false',
},
]
},
],
)
time.sleep(5)
boto_instance = self.get_by_essential_key(ec2_instance.essential_key)
if not boto_instance:
raise ValueError('Boto did not create isntance. Try again.')
ec2_instance.instance_id = boto_instance.id
ec2_instance.save()
ec2_instance.update_from_boto(boto_instance)
return boto_instance
def convert_to_ec2(self, request, queryset):
for raspberry_pi in queryset:
raspberry_pi.reset_cache()
raspberry_pi.is_proxy_tunnel = False
raspberry_pi.new_config_required = True
raspberry_pi.unassign_tunnel_ports()
raspberry_pi.save()
lead = raspberry_pi.get_lead()
ec2_instance = raspberry_pi.get_ec2_instance()
if ec2_instance:
ec2_instance.start()
else:
EC2Instance.launch_for_lead(lead)
messages.success(request,
f'Device {raspberry_pi.rpid} converted to EC2')
def get_hostname(self, lead: Lead, raspberry_pi: RaspberryPi,
ec2_instance: EC2Instance) -> typing.Optional[str]:
if not lead or not lead.is_active():
return None
if raspberry_pi.is_proxy_tunnel:
return raspberry_pi.proxy_hostname
if ec2_instance and lead.is_active() and ec2_instance.is_running():
return ec2_instance.hostname
return None
def is_ec2_instance_data_consistent(data: typing.Dict,
ec2_instance: EC2Instance) -> bool:
ec2_ip_address = data['ec2_ip_address']
ec2_instance_status = data['ec2_instance_status']
if not ec2_instance:
if ec2_ip_address:
return False
return True
if ec2_instance_status != ec2_instance.status:
return False
if ec2_instance.is_status_temp():
return False
if ec2_ip_address != ec2_instance.ip_address:
return False
return True
def _handler_process_all(self, terminate_stopped, execute):
boto_resource = BotoResource().get_resource()
ec2_instances = EC2Instance.objects.all()
ec2_instances_map = {}
for ec2_instance in ec2_instances:
ec2_instances_map[ec2_instance.instance_id] = ec2_instance
boto_instances = boto_resource.instances.filter(
MaxResults=self.ec2_max_results, )
counter = 0
terminated_rpids = []
deleted_rpids = []
existing_instance_ids = []
for boto_instance in boto_instances:
status = boto_instance.state['Name']
if status == EC2Instance.STATUS_TERMINATED:
continue
counter += 1
existing_instance_ids.append(boto_instance.id)
instance = EC2Instance.upsert_from_boto(
boto_instance, ec2_instances_map.get(boto_instance.id))
if terminate_stopped:
if instance and instance.status == EC2Instance.STATUS_STOPPED and not instance.lead:
if execute:
instance.terminate()
terminated_rpids.append(instance.rpid)
for instance in ec2_instances:
if instance.instance_id not in existing_instance_ids:
deleted_rpids.append(instance.rpid)
instance.lead = None
instance.save()
instance.delete()
return JsonResponse({
'total': counter,
'terminated_rpids': terminated_rpids,
'deleted_rpids': deleted_rpids,
'result': True,
})
def get(self, request, rpid, action=None):
if rpid == 'redirect':
return redirect('rdp_ec2_connect',
rpid=request.GET.get('rpid', ''))
is_ready = False
lead = Lead.objects.filter(raspberry_pi__rpid=rpid).first()
if not lead:
messages.warning(request, 'This RPID has no assigned lead.')
return render(
request, 'rdp/ec2_connect.html',
dict(
rpid=rpid,
ec2_instance=None,
check_connection=False,
is_ready=is_ready,
netstat_url='',
))
if lead.raspberry_pi and lead.raspberry_pi.is_proxy_tunnel:
return redirect('rpi_proxy_tunnel_info', rpid=rpid)
ec2_instance = EC2Instance.get_by_rpid(rpid)
if not ec2_instance:
ec2_instance = EC2Instance.objects.filter(lead=lead).first()
if ec2_instance:
ec2_instance.rpid = rpid
ec2_instance.save()
if not ec2_instance:
ec2_instance = EC2Instance.launch_for_lead(lead)
# ec2_instance = EC2Instance.objects.filter(is_essential=True, rpid__isnull=True, status=EC2Instance.STATUS_RUNNING).first()
# if not ec2_instance:
# ec2_instance = EC2Instance.objects.filter(is_essential=True, rpid__isnull=True).first()
# stopped_essential_ec2 = EC2Instance.objects.filter(is_essential=True, rpid__isnull=True, status=EC2Instance.STATUS_STOPPED).first()
# if stopped_essential_ec2:
# stopped_essential_ec2.start()
# else:
# # EC2Instance.launch_essential()
# if not ec2_instance:
# ec2_instance = EC2Instance.launch_essential()
# messages.info(request, 'New Essential EC2 instance has been launched.')
# return redirect('rpi_proxy_tunnel_info', rpid=rpid)
# ec2_instance.assign_essential(rpid, lead)
if ec2_instance.is_essential:
messages.success(request, 'Using essential EC2.')
raspberry_pi = ec2_instance.get_raspberry_pi()
if not raspberry_pi or not raspberry_pi.online():
messages.warning(
request,
'Assigned RaspberryPi device is offline, please ping support')
if ec2_instance.is_stopped(
) and ec2_instance.instance_type != EC2Instance.INSTANCE_TYPE_M5_LARGE:
client = BotoResource().get_client('ec2')
client.modify_instance_attribute(
InstanceId=ec2_instance.instance_id,
Attribute='instanceType',
Value=EC2Instance.INSTANCE_TYPE_M5_LARGE)
ec2_instance.instance_type = EC2Instance.INSTANCE_TYPE_M5_LARGE
ec2_instance.save()
if action:
self.handle_action(request, ec2_instance, action)
return redirect('rdp_ec2_connect', rpid=ec2_instance.rpid)
ec2_instance.update_from_boto()
if not ec2_instance.is_running():
ec2_instance.start()
ec2_instance.last_rdp_start = timezone.now()
ec2_instance.save()
netstat_output = ''
try:
netstat_output = ec2_instance.ssh_execute('netstat -an', timeout=5)
except SSHConnectException:
messages.warning(request,
'SSH is down, instance is not usable now')
else:
is_ready = True
if not ec2_instance.TUNNEL_RE.search(netstat_output):
messages.warning(
request,
'SSH Tunnel is down, instance has no internet connection yet'
)
elif not ec2_instance.REVERSE_TUNNEL_RE.search(netstat_output):
try:
ec2_instance.ssh_execute(
'ssh -N -D 3808 -p 2046 pi@localhost')
except SSHConnectException:
messages.warning(
request,
'Reverse Tunnel is down, instance has no internet connection yet'
)
else:
messages.info(request, 'Reverse tunnel has been started')
if ec2_instance.is_running():
if ec2_instance.password == settings.EC2_ADMIN_PASSWORD:
try:
ec2_instance.change_password(generate_password(length=12))
except SSHConnectException:
pass
return render(
request, 'rdp/ec2_connect.html',
dict(
rpid=rpid,
ec2_instance=ec2_instance,
check_connection=True,
is_ready=is_ready,
netstat_url=request.build_absolute_uri(
reverse('ec2_ssh_get_netstat', kwargs=dict(rpid=rpid))),
))
def launch_essential_ec2(self, request, queryset):
ec2_instance = EC2Instance.launch_essential()
messages.success(request, 'Essential EC2 {} is started successfully'.format(ec2_instance.essential_key))