def CleanUp(project_id, zone, instance_name):
"""Clean up GCP project.
Remove the instance [instance_name] in the GCP project [project_id] and its
disks that were created as part of the end to end test.
Attributes:
project_id (str): the project id of the GCP project.
zone (str): the zone for the project.
instance_name (str): the name of the analysis VM to remove.
"""
gcp_client = common.GoogleCloudComputeClient(project_id=project_id)
project = gcp_project.GoogleCloudProject(project_id, zone)
disks = compute.GoogleComputeInstance(project.project_id, zone,
instance_name).ListDisks()
# delete the created forensics VMs
log.info('Deleting analysis instance: {0:s}.'.format(instance_name))
gce_instances_client = gcp_client.GceApi().instances()
request = gce_instances_client.delete(project=project.project_id,
zone=project.default_zone,
instance=instance_name)
try:
request.execute()
except HttpError:
# GceOperation triggers a while(True) loop that checks on the
# operation ID. Sometimes it loops one more time right when the
# operation has finished and thus the associated ID doesn't exists
# anymore, throwing an HttpError. We can ignore this.
pass
log.info('Instance {0:s} successfully deleted.'.format(instance_name))
# delete the copied disks
# we ignore the disk that was created for the analysis VM (disks[0]) as
# it is deleted in the previous operation
gce_disks_client = gcp_client.GceApi().disks()
for disk in list(disks.keys())[1:]:
log.info('Deleting disk: {0:s}.'.format(disk))
while True:
try:
request = gce_disks_client.delete(project=project.project_id,
zone=project.default_zone,
disk=disk)
request.execute()
break
except HttpError as exception:
# GceApi() will throw a 400 error until the analysis VM deletion is
# correctly propagated. When the disk is finally deleted, it will
# throw a 404 not found if it looped again after deletion.
if exception.resp.status == 404:
break
if exception.resp.status != 400:
log.warning(
'Could not delete the disk {0:s}: {1:s}'.format(
disk, str(exception)))
# Throttle the requests to one every 10 seconds
time.sleep(10)
log.info('Disk {0:s} successfully deleted.'.format(disk))
def PostprocessDetachDisk(disk_name, local_path):
"""Detaches Google Cloud Disk from an instance.
Args:
disk_name(str): The name of the Cloud Disk to detach.
local_path(str): The local path to the block device to detach.
"""
#TODO: can local_path be something diffferent than the /dev/disk/by-id/google*
if local_path:
path = local_path
else:
path = '/dev/disk/by-id/google-{0:s}'.format(disk_name)
if not IsBlockDevice(path):
log.info('Disk {0:s} already detached!'.format(disk_name))
return
config.LoadConfig()
instance_name = GetLocalInstanceName()
project = gcp_project.GoogleCloudProject(config.TURBINIA_PROJECT,
default_zone=config.TURBINIA_ZONE)
instance = project.compute.GetInstance(instance_name,
zone=config.TURBINIA_ZONE)
disk = instance.GetDisk(disk_name)
log.info('Detaching disk {0:s} from instance {1:s}'.format(
disk_name, instance_name))
instance.DetachDisk(disk)
# Make sure device is Detached
for _ in xrange(RETRY_MAX):
if not os.path.exists(path):
log.info('Block device {0:s} is no longer attached'.format(path))
break
time.sleep(5)
def SetUp(self,
source_project_name,
source_disk_name,
gcs_output_location,
analysis_project_name=None,
exported_disk_name=None):
"""Sets up a Google Cloud Platform (GCP) Disk Export.
This method creates the required objects to initialize
the GoogleCloudDiskExport class attributes.
If the analysis_project_name is not specified it will use the
source_project_name instead.
Args:
source_project_name (str): Source project ID containing
the disk to export.
source_disk_name (str): Disk that needs to be exported.
gcs_output_location (str): Google Cloud Storage parent bucket/folder
path of the exported image.
analysis_project_name (Optional[str]): Project ID where the
disk image is created then exported. If not exit,
source_project_name will be used.
exported_disk_name (Optional[str]): Name of the output file, must comply
with ^[A-Za-z0-9-]*$' and '.tar.gz' will be appended to the name.
If not exist, random name will be generated.
"""
self.source_project = gcp_project.GoogleCloudProject(
source_project_name)
if analysis_project_name:
self.analysis_project = gcp_project.GoogleCloudProject(
analysis_project_name)
else:
self.analysis_project = self.source_project
self.source_disk = self.source_project.compute.GetDisk(
source_disk_name)
self.gcs_output_location = gcs_output_location
self._image_name = '{0:s}-image-df-export-temp'.format(
self.source_disk.name)
if exported_disk_name:
self.exported_disk_name = exported_disk_name
else:
self.exported_disk_name = self._image_name
def PreprocessAttachDisk(disk_name):
"""Attaches Google Cloud Disk to an instance.
Args:
disk_name(str): The name of the Cloud Disk to attach.
Returns:
(str, list(str)): a tuple consisting of the path to the 'disk' block device
and a list of paths to partition block devices. For example:
(
'/dev/disk/by-id/google-disk0',
['/dev/disk/by-id/google-disk0-part1', '/dev/disk/by-id/google-disk0-p2']
)
Raises:
TurbiniaException: If the device is not a block device.
"""
path = '/dev/disk/by-id/google-{0:s}'.format(disk_name)
if IsBlockDevice(path):
log.info('Disk {0:s} already attached!'.format(disk_name))
return (path, sorted(glob.glob('{0:s}-part*'.format(path))))
config.LoadConfig()
instance_name = GetLocalInstanceName()
project = gcp_project.GoogleCloudProject(
config.TURBINIA_PROJECT, default_zone=config.TURBINIA_ZONE)
instance = project.compute.GetInstance(instance_name)
disk = project.compute.GetDisk(disk_name)
log.info(
'Attaching disk {0:s} to instance {1:s}'.format(disk_name, instance_name))
instance.AttachDisk(disk)
# Make sure we have a proper block device
for _ in range(RETRY_MAX):
if IsBlockDevice(path):
log.info('Block device {0:s} successfully attached'.format(path))
break
if os.path.exists(path):
log.info(
'Block device {0:s} mode is {1}'.format(path,
os.stat(path).st_mode))
time.sleep(1)
message = None
if not os.path.exists(path):
turbinia_nonexisting_disk_path.inc()
message = 'Device path {0:s} does not exist'.format(path)
elif not IsBlockDevice(path):
message = 'Device path {0:s} is not a block device'.format(path)
if message:
log.error(message)
raise TurbiniaException(message)
return (path, sorted(glob.glob('{0:s}-part*'.format(path))))
def ListDisks(args: 'argparse.Namespace') -> None:
"""List GCE disks in GCP project.
Args:
args (argparse.Namespace): Arguments from ArgumentParser.
"""
project = gcp_project.GoogleCloudProject(args.project)
disks = project.compute.ListDisks()
logger.info('Disks found:')
for disk in disks:
logger.info('Name: {0:s}, Zone: {1:s}'.format(disk, disks[disk].zone))
def VMRemoveServiceAccount(project_id: str,
instance_name: str,
leave_stopped: bool = False) -> bool:
"""
Remove a service account attachment from a GCP VM.
Service account attachments to VMs allow the VM to obtain credentials
via the instance metadata service to perform API actions. Removing
the service account attachment will prevent credentials being issued.
Note that the instance will be powered down, if it isn't already for
this action.
Args:
project_id (str): Google Cloud Project ID.
instance_name (str): The name of the virtual machine.
leave_stopped (bool): Optional. True to leave the machine powered off.
Returns:
bool: True if the service account was successfully removed, False otherwise.
"""
logger.info('Removing service account attachment from "{0:s}",'
' in project {1:s}'.format(instance_name, project_id))
valid_starting_states = ['RUNNING', 'STOPPING', 'TERMINATED']
project = gcp_project.GoogleCloudProject(project_id)
instance = project.compute.GetInstance(instance_name)
# Get the initial powered state of the instance
initial_state = instance.GetPowerState()
if not initial_state in valid_starting_states:
logger.error('Instance "{0:s}" is currently {1:s} which is an invalid '
'state for this operation'.format(instance_name, initial_state))
return False
try:
# Stop the instance if it is not already (or on the way)....
if not initial_state in ('TERMINATED', 'STOPPING'):
instance.Stop()
# Remove the service account
instance.DetachServiceAccount()
# If the instance was running initially, and the option has been set,
# start up the instance again
if initial_state == 'RUNNING' and not leave_stopped:
instance.Start()
except errors.LCFError as exception:
logger.error('Fatal exception encountered: {0:s}'.format(str(exception)))
return True
def ListInstances(args: 'argparse.Namespace') -> None:
"""List GCE instances in GCP project.
Args:
args (argparse.Namespace): Arguments from ArgumentParser.
"""
project = gcp_project.GoogleCloudProject(args.project)
instances = project.compute.ListInstances()
print('Instances found:')
for instance in instances:
bootdisk = instances[instance].GetBootDisk()
if bootdisk:
print('Name: {0:s}, Bootdisk: {1:s}'.format(instance, bootdisk.name))
def InstanceNetworkQuarantine(project_id: str,
instance_name: str,
exempted_src_ips: Optional[List[str]] = None,
enable_logging: bool = False) -> None:
"""Put a Google Cloud instance in network quarantine.
Network quarantine is imposed via applying deny-all
ingress/egress firewall rules on each network interface.
Args:
project_id (str): Google Cloud Project ID.
instance_name (str): : The name of the virtual machine.
exempted_src_ips (List[str]): List of IPs exempted from the deny-all
ingress firewall rules, ex: analyst IPs.
enable_logging (bool): Optional. Enable firewall logging.
Default is False.
"""
logger.info('Putting instance "{0:s}", in project {1:s}, in network '
'quarantine.'.format(instance_name, project_id))
project = gcp_project.GoogleCloudProject(project_id)
instance = project.compute.GetInstance(instance_name)
get_operation = instance.GetOperation()
network_interfaces = get_operation['networkInterfaces']
target_tags = []
for interface in network_interfaces:
network_url = interface["network"]
# Adding a random suffix to the tag to avoid name collisions,
# tags are used as firewall rule names, which need to be unique.
tag_suffix = random.randint(10**(19),(10**20)-1)
deny_ingress_tag = 'deny-ingress-tag-' + str(tag_suffix)
deny_egress_tag = 'deny-egress-tag-' + str(tag_suffix)
AddDenyAllFirewallRules(
project_id,
network_url,
deny_ingress_tag,
deny_egress_tag,
exempted_src_ips,
enable_logging)
target_tags.append(deny_ingress_tag)
target_tags.append(deny_egress_tag)
instance.SetTags(target_tags)
if exempted_src_ips:
logger.info('From a host with an exempted IP, '
'connect to the quarantined instance using:\n'
'gcloud compute ssh --zone "{0:s}" "{1:s}" --project "{2:s}"\n'
'Connecting from the browser via GCP console will not work.'.format(
instance.zone, instance_name, project_id))
def StartAnalysisVm(
project: str,
vm_name: str,
zone: str,
boot_disk_size: int,
boot_disk_type: str,
cpu_cores: int,
attach_disks: Optional[List[str]] = None,
image_project: str = 'ubuntu-os-cloud',
image_family: str = 'ubuntu-1804-lts') -> Tuple['compute.GoogleComputeInstance', bool]: # pylint: disable=line-too-long
"""Start a virtual machine for analysis purposes.
Args:
project (str): Project id for virtual machine.
vm_name (str): The name of the virtual machine.
zone (str): Zone for the virtual machine.
boot_disk_size (int): The size of the analysis VM boot disk (in GB).
boot_disk_type (str): URL of the disk type resource describing
which disk type to use to create the disk. Use pd-standard for a
standard disk and pd-ssd for a SSD disk.
cpu_cores (int): The number of CPU cores to create the machine with.
attach_disks (List[str]): Optional. List of disk names to attach.
image_project (str): Optional. Name of the project where the analysis VM
image is hosted.
image_family (str): Optional. Name of the image to use to create the
analysis VM.
Returns:
Tuple(GoogleComputeInstance, bool): A tuple with a virtual machine object
and a boolean indicating if the virtual machine was created or not.
"""
proj = gcp_project.GoogleCloudProject(project, default_zone=zone)
logger.info('Starting analysis VM {0:s}'.format(vm_name))
analysis_vm, created = proj.compute.GetOrCreateAnalysisVm(
vm_name,
boot_disk_size,
disk_type=boot_disk_type,
cpu_cores=cpu_cores,
image_project=image_project,
image_family=image_family)
logger.info('VM started.')
for disk_name in (attach_disks or []):
logger.info('Attaching disk {0:s}'.format(disk_name))
analysis_vm.AttachDisk(proj.compute.GetDisk(disk_name))
logger.info('VM ready.')
return analysis_vm, created
def PreprocessAttachDisk(disk_name):
"""Attaches Google Cloud Disk to an instance.
Args:
disk_name(str): The name of the Cloud Disk to attach.
Returns:
(str, list(str)): a tuple consisting of the path to the 'disk' block device
and a list of paths to partition block devices. For example:
(
'/dev/disk/by-id/google-disk0',
['/dev/disk/by-id/google-disk0-part1', '/dev/disk/by-id/google-disk0-p2']
)
"""
path = '/dev/disk/by-id/google-{0:s}'.format(disk_name)
if IsBlockDevice(path):
log.info('Disk {0:s} already attached!'.format(disk_name))
return (path, glob.glob('{0:s}-part*'.format(path)))
config.LoadConfig()
instance_name = GetLocalInstanceName()
project = gcp_project.GoogleCloudProject(config.TURBINIA_PROJECT,
default_zone=config.TURBINIA_ZONE)
instance = project.compute.GetInstance(instance_name,
zone=config.TURBINIA_ZONE)
disk = instance.GetDisk(disk_name)
log.info('Attaching disk {0:s} to instance {1:s}'.format(
disk_name, instance_name))
instance.AttachDisk(disk)
# Make sure we have a proper block device
for _ in xrange(RETRY_MAX):
if IsBlockDevice(path):
log.info('Block device {0:s} successfully attached'.format(path))
break
if os.path.exists(path):
log.info('Block device {0:s} mode is {1}'.format(
path,
os.stat(path).st_mode))
time.sleep(1)
return (path, glob.glob('{0:s}-part*'.format(path)))
def setUpClass(cls):
try:
project_info = utils.ReadProjectInfo(['project_id', 'instance', 'zone'])
except (OSError, RuntimeError, ValueError) as exception:
raise unittest.SkipTest(str(exception))
cls.project_id = project_info['project_id']
cls.instance_to_analyse = project_info['instance']
# Optional: test a disk other than the boot disk
cls.disk_to_forensic = project_info.get('disk', None)
cls.zone = project_info['zone']
cls.gcp = gcp_project.GoogleCloudProject(cls.project_id, cls.zone)
cls.analysis_vm_name = 'new-vm-for-analysis'
# Create and start the analysis VM
cls.analysis_vm, _ = forensics.StartAnalysisVm(project=cls.project_id,
vm_name=cls.analysis_vm_name,
zone=cls.zone,
boot_disk_size=10,
boot_disk_type='pd-ssd',
cpu_cores=4)
def CreateDiskCopy(
src_proj: str,
dst_proj: str,
instance_name: str,
zone: str,
disk_name: Optional[str] = None,
disk_type: str = 'pd-standard') -> 'compute.GoogleComputeDisk':
"""Creates a copy of a Google Compute Disk.
Args:
src_proj (str): Name of project that holds the disk to be copied.
dst_proj (str): Name of project to put the copied disk in.
instance_name (str): Instance using the disk to be copied.
zone (str): Zone where the new disk is to be created.
disk_name (str): Optional. Name of the disk to copy. If None, boot disk
will be copied.
disk_type (str): Optional. URL of the disk type resource describing
which disk type to use to create the disk. Default is pd-standard. Use
pd-ssd to have a SSD disk.
Returns:
GoogleComputeDisk: A Google Compute Disk object.
Raises:
RuntimeError: If there are errors copying the disk
"""
src_project = gcp_project.GoogleCloudProject(src_proj)
dst_project = gcp_project.GoogleCloudProject(dst_proj, default_zone=zone)
instance = src_project.compute.GetInstance(
instance_name) if instance_name else None
try:
if disk_name:
disk_to_copy = src_project.compute.GetDisk(disk_name)
else:
disk_to_copy = instance.GetBootDisk() # type: ignore
common.LOGGER.info('Disk copy of {0:s} started...'.format(
disk_to_copy.name))
snapshot = disk_to_copy.Snapshot()
new_disk = dst_project.compute.CreateDiskFromSnapshot(
snapshot, disk_name_prefix='evidence', disk_type=disk_type)
snapshot.Delete()
common.LOGGER.info('Disk {0:s} successfully copied to {1:s}'.format(
disk_to_copy.name, new_disk.name))
except RefreshError as exception:
error_msg = ('Something is wrong with your gcloud access token: '
'{0:s}.').format(exception)
raise RuntimeError(error_msg)
except DefaultCredentialsError as exception:
error_msg = ('Something is wrong with your Application Default '
'Credentials. '
'Try running:\n $ gcloud auth application-default login')
raise RuntimeError(error_msg)
except HttpError as exception:
if exception.resp.status == 403:
raise RuntimeError(
'Make sure you have the appropriate permissions on the project'
)
if exception.resp.status == 404:
raise RuntimeError(
'GCP resource not found. Maybe a typo in the project / instance / '
'disk name?')
raise RuntimeError(exception)
except RuntimeError as exception:
error_msg = 'Cannot copy disk "{0:s}": {1!s}'.format(
disk_name, exception)
raise RuntimeError(error_msg)
return new_disk
def SetUp(self,
analysis_project_name,
remote_project_name,
incident_id,
zone,
boot_disk_size,
boot_disk_type,
cpu_cores,
remote_instance_name=None,
disk_names=None,
all_disks=False,
image_project='ubuntu-os-cloud',
image_family='ubuntu-1804-lts'):
"""Sets up a Google Cloud Platform(GCP) collector.
This method creates and starts an analysis VM in the analysis project and
selects disks to copy from the remote project.
If disk_names is specified, it will copy the corresponding disks from the
project, ignoring disks belonging to any specific instances.
If remote_instance_name is specified, two behaviors are possible:
* If no other parameters are specified, it will select the instance's boot
disk
* if all_disks is set to True, it will select all disks in the project
that are attached to the instance
disk_names takes precedence over instance_names
Args:
analysis_project_name (str): name of the project that contains
the analysis VM.
remote_project_name (str): name of the remote project where the disks
must be copied from.
incident_id (str): incident identifier on which the name of the analysis
VM will be based.
zone (str): GCP zone in which new resources should be created.
boot_disk_size (float): size of the analysis VM boot disk (in GB).
boot_disk_type (str): Disk type to use [pd-standard, pd-ssd]
cpu_cores (int): number of CPU cores to create the VM with.
remote_instance_name (Optional[str]): name of the instance in
the remote project containing the disks to be copied.
disk_names (Optional[str]): Comma separated disk names to copy.
all_disks (Optional[bool]): True if all disks attached to the source
instance should be copied.
image_project (Optional[str]): name of the project where the analysis
VM image is hosted.
image_family (Optional[str]): name of the image to use to create the
analysis VM.
"""
if not (remote_instance_name or disk_names):
self.state.AddError(
'You need to specify at least an instance name or disks to copy',
critical=True)
return
disk_names = disk_names.split(',') if disk_names else []
self.analysis_project = gcp_project.GoogleCloudProject(
analysis_project_name, default_zone=zone)
self.remote_project = gcp_project.GoogleCloudProject(
remote_project_name)
self.remote_instance_name = remote_instance_name
self.disk_names = disk_names
self.incident_id = incident_id
self.all_disks = all_disks
self._gcp_label = {'incident_id': self.incident_id}
analysis_vm_name = 'gcp-forensics-vm-{0:s}'.format(self.incident_id)
self.logger.info(
'Your analysis VM will be: {0:s}'.format(analysis_vm_name))
self.logger.info('Complimentary gcloud command:')
self.logger.info(
'gcloud compute ssh --project {0:s} {1:s} --zone {2:s}'.format(
self.analysis_project.project_id, analysis_vm_name, zone))
self.state.StoreContainer(
containers.TicketAttribute(
name=self._ANALYSIS_VM_CONTAINER_ATTRIBUTE_NAME,
type_=self._ANALYSIS_VM_CONTAINER_ATTRIBUTE_TYPE,
value=analysis_vm_name))
try:
# TODO: Make creating an analysis VM optional
# pylint: disable=too-many-function-args
# pylint: disable=redundant-keyword-arg
self.analysis_vm, _ = gcp_forensics.StartAnalysisVm(
self.analysis_project.project_id,
analysis_vm_name,
zone,
boot_disk_size,
boot_disk_type,
int(cpu_cores),
image_project=image_project,
image_family=image_family)
self.analysis_vm.AddLabels(self._gcp_label)
self.analysis_vm.GetBootDisk().AddLabels(self._gcp_label)
except (RefreshError, DefaultCredentialsError) as exception:
msg = (
'Something is wrong with your Application Default Credentials. '
'Try running:\n $ gcloud auth application-default login\n')
msg += str(exception)
self.ModuleError(msg, critical=True)
"""Tests the GoogleCloudDiskExport."""
import os
import unittest
import mock
from libcloudforensics.providers.gcp.internal import project as gcp_project
from libcloudforensics.providers.gcp.internal import compute
from dftimewolf import config
from dftimewolf.lib import state
from dftimewolf.lib.containers import containers
from dftimewolf.lib.exporters import gce_disk_export
FAKE_SOURCE_PROJECT = gcp_project.GoogleCloudProject(
'fake-source-project', 'fake-zone')
FAKE_DISK = compute.GoogleComputeDisk(
FAKE_SOURCE_PROJECT.project_id,
'fake_zone',
'fake-source-disk')
FAKE_IMAGE = compute.GoogleComputeImage(
FAKE_SOURCE_PROJECT.project_id,
'fake-zone',
'fake-source-disk-image-df-export-temp')
class GoogleCloudDiskExportTest(unittest.TestCase):
"""Tests for the Google Cloud disk exporter."""
def testInitialization(self):
"""Tests that the disk exporter can be initialized."""
def SetUp(self,
analysis_project_name,
remote_project_name,
incident_id=None,
zone='us-central1-f',
create_analysis_vm=True,
boot_disk_size=50,
boot_disk_type='pd-standard',
cpu_cores=4,
remote_instance_name=None,
disk_names=None,
all_disks=False,
image_project='ubuntu-os-cloud',
image_family='ubuntu-1804-lts'):
"""Sets up a Google Cloud Platform(GCP) collector.
This method creates and starts an analysis VM in the analysis project and
selects disks to copy from the remote project.
If analysis_project_name is not specified, analysis_project will be same
as remote_project.
If disk_names is specified, it will copy the corresponding disks from the
project, ignoring disks belonging to any specific instances.
If remote_instance_name is specified, two behaviors are possible:
* If no other parameters are specified, it will select the instance's boot
disk
* if all_disks is set to True, it will select all disks in the project
that are attached to the instance
disk_names takes precedence over instance_names
Args:
analysis_project_name (str): Optional. name of the project that contains
the analysis VM. Default is None.
remote_project_name (str): name of the remote project where the disks
must be copied from.
incident_id (Optional[str]): Optional. Incident identifier on which the
name of the analysis VM will be based. Default is None, which means
add no label and format VM name as
"gcp-forensics-vm-{TIMESTAMP('%Y%m%d%H%M%S')}".
zone (Optional[str]): Optional. GCP zone in which new resources should
be created. Default is us-central1-f.
create_analysis_vm (Optional[bool]): Optional. Create analysis VM in
the analysis project. Default is True.
boot_disk_size (Optional[float]): Optional. Size of the analysis VM boot
disk (in GB). Default is 50.
boot_disk_type (Optional[str]): Optional. Disk type to use.
Default is pd-standard.
cpu_cores (Optional[int]): Optional. Number of CPU cores to
create the VM with. Default is 4.
remote_instance_name (Optional[str]): Optional. Name of the instance in
the remote project containing the disks to be copied.
disk_names (Optional[str]): Optional. Comma separated disk names to copy.
all_disks (Optional[bool]): Optional. True if all disks attached to the
source instance should be copied.
image_project (Optional[str]): Optional. Name of the project where the
analysis VM image is hosted.
image_family (Optional[str]): Optional. Name of the image to use to
create the analysis VM.
"""
if not (remote_instance_name or disk_names):
self.ModuleError(
'You need to specify at least an instance name or disks to copy',
critical=True)
return
disk_names = disk_names.split(',') if disk_names else []
self.remote_project = gcp_project.GoogleCloudProject(
remote_project_name, default_zone=zone)
if analysis_project_name:
self.analysis_project = gcp_project.GoogleCloudProject(
analysis_project_name, default_zone=zone)
else:
self.analysis_project = self.remote_project
self.remote_instance_name = remote_instance_name
self.disk_names = disk_names
self.all_disks = all_disks
if incident_id:
self.incident_id = incident_id
self._gcp_label = {'incident_id': self.incident_id}
try:
if self.remote_instance_name:
self.remote_project.compute.GetInstance(
self.remote_instance_name)
except ResourceNotFoundError as exception:
self.ModuleError(
message='Instance "{0:s}" not found or insufficient permissions'
.format(self.remote_instance_name),
critical=True)
return
if create_analysis_vm:
if self.incident_id:
analysis_vm_name = 'gcp-forensics-vm-{0:s}'.format(
self.incident_id)
else:
analysis_vm_name = common.GenerateUniqueInstanceName(
'gcp-forensics-vm', common.COMPUTE_NAME_LIMIT)
self.logger.info(
'Your analysis VM will be: {0:s}'.format(analysis_vm_name))
self.logger.info('Complimentary gcloud command:')
self.logger.info(
'gcloud compute ssh --project {0:s} {1:s} --zone {2:s}'.format(
self.analysis_project.project_id, analysis_vm_name,
self.analysis_project.default_zone))
self.state.StoreContainer(
containers.TicketAttribute(
name=self._ANALYSIS_VM_CONTAINER_ATTRIBUTE_NAME,
type_=self._ANALYSIS_VM_CONTAINER_ATTRIBUTE_TYPE,
value=analysis_vm_name))
try:
# pylint: disable=too-many-function-args
# pylint: disable=redundant-keyword-arg
self.analysis_vm, _ = gcp_forensics.StartAnalysisVm(
self.analysis_project.project_id,
analysis_vm_name,
self.analysis_project.default_zone,
boot_disk_size,
boot_disk_type,
int(cpu_cores),
image_project=image_project,
image_family=image_family)
if self._gcp_label:
self.analysis_vm.AddLabels(self._gcp_label)
self.analysis_vm.GetBootDisk().AddLabels(self._gcp_label)
except (RefreshError, DefaultCredentialsError) as exception:
msg = (
'Something is wrong with your Application Default Credentials. '
'Try running:\n $ gcloud auth application-default login\n'
)
msg += str(exception)
self.ModuleError(msg, critical=True)
def SetUp(self,
source_project_name,
gcs_output_location,
analysis_project_name=None,
source_disk_names=None,
remote_instance_name=None,
all_disks=False,
exported_image_name=None):
"""Sets up a Google Cloud Platform (GCP) Disk Export.
This method creates the required objects to initialize
the GoogleCloudDiskExport class attributes.
If the analysis_project_name is not specified it will use the
source_project_name instead.
If source_disk_names is specified, it will copy the corresponding disks from
the project, ignoring disks belonging to any specific instances.
If remote_instance_name is specified, two behaviors are possible:
* If no other parameters are specified, it will select the instance's boot
disk
* if all_disks is set to True, it will select all disks in the project
that are attached to the instance
Args:
source_project_name (str): Source project ID containing
the disk to export.
gcs_output_location (str): Google Cloud Storage parent bucket/folder
path of the exported image.
analysis_project_name (Optional[str]): Optional. Project ID where the
disk image is created then exported. If not specified,
source_project_name will be used.
source_disk_names (Optional[str]): Optional. Comma separated disk names
to copy.Default is None.
remote_instance_name (Optional[str]): Optional. Instance in source
project to export its disks. Default, if not exist, source_disk_names
will be used.
all_disks (Optional[bool]): Optional. True if all disks attached to
the source instance should be copied. Default is False. If False
and remote_instance_name is provided it will select the instance's
boot disk.
exported_image_name (Optional[str]): Optional. Name of the output file,
must comply with ^[A-Za-z0-9-]*$' and '.tar.gz' will be appended to
the name. Default is None, if not exist or if more than one disk
is selected, exported image name as
"exported-image-{TIMESTAMP('%Y%m%d%H%M%S')}".
"""
self.source_project = gcp_project.GoogleCloudProject(
source_project_name)
if analysis_project_name:
self.analysis_project = gcp_project.GoogleCloudProject(
analysis_project_name)
else:
self.analysis_project = self.source_project
if not (remote_instance_name or source_disk_names):
self.ModuleError(
'You need to specify at least an instance name or disks to copy',
critical=True)
self.remote_instance_name = remote_instance_name
if source_disk_names:
source_disk_names = source_disk_names.split(',')
else:
source_disk_names = []
self.source_disk_names = source_disk_names
self.all_disks = all_disks
self.source_disks = self._FindDisksToCopy()
self.gcs_output_location = gcs_output_location
if exported_image_name and len(self.source_disks) == 1:
self.exported_image_name = exported_image_name
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Tests the GoogleCloudCollector."""
import unittest
import mock
from libcloudforensics.providers.gcp.internal import project as gcp_project
from libcloudforensics.providers.gcp.internal import compute
from dftimewolf import config
from dftimewolf.lib import state
from dftimewolf.lib.containers import containers
from dftimewolf.lib.collectors import gcloud
FAKE_PROJECT = gcp_project.GoogleCloudProject('test-target-project-name',
'fake_zone')
FAKE_ANALYSIS_VM = compute.GoogleComputeInstance(FAKE_PROJECT.project_id,
'fake_zone',
'fake-analysis-vm')
FAKE_INSTANCE = compute.GoogleComputeInstance(FAKE_PROJECT.project_id,
'fake_zone', 'fake-instance')
FAKE_DISK = compute.GoogleComputeDisk(FAKE_PROJECT.project_id, 'fake_zone',
'disk1')
FAKE_BOOT_DISK = compute.GoogleComputeDisk(FAKE_PROJECT.project_id,
'fake_zone', 'bootdisk')
FAKE_SNAPSHOT = compute.GoogleComputeSnapshot(FAKE_DISK, 'fake_snapshot')
FAKE_DISK_COPY = compute.GoogleComputeDisk(FAKE_PROJECT.project_id,
'fake_zone', 'disk1-copy')
class GoogleCloudCollectorTest(unittest.TestCase):
# See the License for the specific language governing permissions and
# limitations under the License.
"""GCP mocks used across tests."""
import re
# pylint: disable=line-too-long
from libcloudforensics.providers.gcp.internal import build as gcp_build
from libcloudforensics.providers.gcp.internal import compute
from libcloudforensics.providers.gcp.internal import project as gcp_project
from libcloudforensics.providers.gcp.internal import log as gcp_log
from libcloudforensics.providers.gcp.internal import monitoring as gcp_monitoring
from libcloudforensics.providers.gcp.internal import storage as gcp_storage
# pylint: enable=line-too-long
FAKE_ANALYSIS_PROJECT = gcp_project.GoogleCloudProject('fake-target-project',
'fake-zone')
FAKE_ANALYSIS_VM = compute.GoogleComputeInstance(
FAKE_ANALYSIS_PROJECT.project_id, 'fake-zone', 'fake-analysis-vm')
FAKE_IMAGE = compute.GoogleComputeImage(FAKE_ANALYSIS_PROJECT.project_id, '',
'fake-image')
# Source project with the instance that needs forensicating
FAKE_SOURCE_PROJECT = gcp_project.GoogleCloudProject('fake-source-project',
'fake-zone')
FAKE_INSTANCE = compute.GoogleComputeInstance(FAKE_SOURCE_PROJECT.project_id,
'fake-zone', 'fake-instance')
FAKE_DISK = compute.GoogleComputeDisk(FAKE_SOURCE_PROJECT.project_id,
def CreateDiskFromGCSImage(
project_id: str,
storage_image_path: str,
zone: str,
name: Optional[str] = None) -> Dict[str, Any]:
"""Creates a GCE persistent disk from a image in GCS.
The method supports raw disk images and most virtual disk
file formats. Valid import formats are:
[raw (dd), qcow2, qcow , vmdk, vdi, vhd, vhdx, qed, vpc].
The created GCE disk might be larger than the original raw (dd)
image stored in GCS to satisfy GCE capacity requirements:
https://cloud.google.com/compute/docs/disks/#introduction
However the bytes_count and the md5_hash values of the source
image are returned with the newly created disk.
The md5_hash can be used to verify the integrity of the
created GCE disk, it must be compared with the hash of the
created GCE disk from byte 0 to bytes_count. i.e:
result['md5Hash'] = hash(created_gce_disk,
start_byte=0,
end_byte=result['bytes_count'])
Args:
project_id (str): Google Cloud Project ID.
storage_image_path (str): Path to the source image in GCS.
zone (str): Zone to create the new disk in.
name (str): Optional. Name of the disk to create. Default
is imported-disk-[TIMESTAMP('%Y%m%d%H%M%S')].
Returns:
Dict: A key value describing the imported GCE disk.
Ex: {
'project_id': 'fake-project',
'disk_name': 'fake-imported-disk',
'zone': 'fake-zone',
'bytes_count': '1234' # Content-Length of source image in bytes.
'md5Hash': 'Source Image MD5 hash string in hex'
}
Raises:
InvalidNameError: If the GCE disk name is invalid.
"""
if name:
if not common.REGEX_DISK_NAME.match(name):
raise errors.InvalidNameError(
'Disk name {0:s} does not comply with {1:s}'.format(
name, common.REGEX_DISK_NAME.pattern), __name__)
name = name[:common.COMPUTE_NAME_LIMIT]
else:
name = common.GenerateUniqueInstanceName('imported-disk',
common.COMPUTE_NAME_LIMIT)
project = gcp_project.GoogleCloudProject(project_id)
image_object = project.compute.ImportImageFromStorage(storage_image_path)
disk_object = project.compute.CreateDiskFromImage(
image_object, zone=zone, name=name)
storage_object_md = project.storage.GetObjectMetadata(storage_image_path)
md5_hash_hex = base64.b64decode(storage_object_md['md5Hash']).hex()
result = {
'project_id': disk_object.project_id,
'disk_name': disk_object.name,
'zone': disk_object.zone,
'bytes_count': storage_object_md['size'],
'md5Hash': md5_hash_hex
}
return result
def CreateDiskCopy(
src_proj: str,
dst_proj: str,
zone: str,
instance_name: Optional[str] = None,
disk_name: Optional[str] = None,
disk_type: Optional[str] = None) -> 'compute.GoogleComputeDisk':
"""Creates a copy of a Google Compute Disk.
Args:
src_proj (str): Name of project that holds the disk to be copied.
dst_proj (str): Name of project to put the copied disk in.
zone (str): Zone where the new disk is to be created.
instance_name (str): Optional. Instance using the disk to be copied.
disk_name (str): Optional. Name of the disk to copy. If None,
instance_name must be specified and the boot disk will be copied.
disk_type (str): Optional. URL of the disk type resource describing
which disk type to use to create the disk. The default behavior is to
use the same disk type as the source disk.
Returns:
GoogleComputeDisk: A Google Compute Disk object.
Raises:
ResourceNotFoundError: If the GCP resource is not found.
CredentialsConfigurationError: If the library could not authenticate to GCP.
RuntimeError: If an unknown HttpError is thrown.
ResourceCreationError: If there are errors copying the disk.
ValueError: If both instance_name and disk_name are missing.
"""
if not instance_name and not disk_name:
raise ValueError(
'You must specify at least one of [instance_name, disk_name].')
src_project = gcp_project.GoogleCloudProject(src_proj)
dst_project = gcp_project.GoogleCloudProject(dst_proj, default_zone=zone)
try:
if disk_name:
disk_to_copy = src_project.compute.GetDisk(disk_name)
elif instance_name:
instance = src_project.compute.GetInstance(instance_name)
disk_to_copy = instance.GetBootDisk()
if not disk_type:
disk_type = disk_to_copy.GetDiskType()
logger.info('Disk copy of {0:s} started...'.format(
disk_to_copy.name))
snapshot = disk_to_copy.Snapshot()
logger.debug('Snapshot created: {0:s}'.format(snapshot.name))
new_disk = dst_project.compute.CreateDiskFromSnapshot(
snapshot, disk_name_prefix='evidence', disk_type=disk_type)
logger.info(
'Disk {0:s} successfully copied to {1:s}'.format(
disk_to_copy.name, new_disk.name))
snapshot.Delete()
logger.debug('Snapshot {0:s} deleted.'.format(snapshot.name))
except (RefreshError, DefaultCredentialsError) as exception:
raise errors.CredentialsConfigurationError(
'Something is wrong with your Application Default Credentials. Try '
'running: $ gcloud auth application-default login: {0!s}'.format(
exception), __name__) from exception
except HttpError as exception:
if exception.resp.status == 403:
raise errors.CredentialsConfigurationError(
'Make sure you have the appropriate permissions on the project',
__name__) from exception
if exception.resp.status == 404:
raise errors.ResourceNotFoundError(
'GCP resource not found. Maybe a typo in the project / instance / '
'disk name?', __name__) from exception
raise RuntimeError(exception) from exception
except RuntimeError as exception:
raise errors.ResourceCreationError(
'Cannot copy disk "{0:s}": {1!s}'.format(disk_name, exception),
__name__) from exception
return new_disk
def AddDenyAllFirewallRules(project_id: str,
network: str,
deny_ingress_tag: str,
deny_egress_tag: str,
exempted_src_ips: Optional[List[str]] = None,
enable_logging: bool = False) -> None:
"""Add deny-all firewall rules, of highest priority.
Args:
project_id (str): Google Cloud Project ID.
network (str): URL of the network resource for thesee firewall rules.
deny_ingress_tag (str): Target tag name to apply deny ingress rule,
also used as a deny ingress firewall rule name.
deny_egress_tag (str): Target tag name to apply deny egress rule,
also used as a deny egress firewall rule name.
exempted_src_ips (List[str]): List of IPs exempted from the deny-all
ingress firewall rules, ex: analyst IPs.
enable_logging (bool): Optional. Enable firewall logging.
Default is False.
Raises:
InvalidNameError: If Tag names are invalid.
"""
logger.info('Creating deny-all (ingress/egress) '
'firewall rules in {0:s} network.'.format(network))
project = gcp_project.GoogleCloudProject(project_id)
if not common.COMPUTE_RFC1035_REGEX.match(deny_ingress_tag):
raise errors.InvalidNameError(
'Deny ingress tag name {0:s} does not comply with {1:s}'.format(
deny_ingress_tag, common.COMPUTE_RFC1035_REGEX.pattern), __name__)
if not common.COMPUTE_RFC1035_REGEX.match(deny_egress_tag):
raise errors.InvalidNameError(
'Deny egress tag name {0:s} does not comply with {1:s}'.format(
deny_egress_tag, common.COMPUTE_RFC1035_REGEX.pattern), __name__)
source_range = common.GenerateSourceRange(exempted_src_ips)
deny_ingress = {
'name': deny_ingress_tag,
'network': network,
'direction': 'INGRESS',
'priority': 0,
'targetTags': [
deny_ingress_tag
],
'denied': [
{
'IPProtocol': 'all'
}
],
'logConfig': {
'enable': enable_logging
},
'sourceRanges': source_range
}
deny_egress = {
'name': deny_egress_tag,
'network': network,
'direction': 'EGRESS',
'priority': 0,
'targetTags': [
deny_egress_tag
],
'denied': [
{
'IPProtocol': 'all'
}
],
'logConfig': {
'enable': enable_logging
},
'destinationRanges': [
'0.0.0.0/0'
]
}
project.compute.InsertFirewallRule(body=deny_ingress)
project.compute.InsertFirewallRule(body=deny_egress)
