def ListInstances(self) -> Dict[str, 'GoogleComputeInstance']: """List instances in project. Returns: Dict[str, GoogleComputeInstance]: Dictionary mapping instance names (str) to their respective GoogleComputeInstance object. """ instances = {} gce_instance_client = self.GceApi().instances() responses = common.ExecuteRequest(gce_instance_client, 'aggregatedList', {'project': self.project_id}) for response in responses: for zone in response['items']: try: for instance in response['items'][zone]['instances']: _, zone = instance['zone'].rsplit('/', 1) name = instance['name'] instances[name] = GoogleComputeInstance( self.project_id, zone, name, labels=instance.get('labels')) except KeyError: pass return instances
def setup_publisher(self): """Set up the pubsub publisher.""" config.LoadConfig() # the publisher we will use the pubsub client in googleapiclient.discovery # for more information on using the APIs, see # https://cloud.google.com/pubsub/docs/reference/rest self.pubsub_api_client = gcp_common.CreateService('pubsub', 'v1') self.topic_path = 'projects/{0:s}/topics/{1:s}'.format( config.TURBINIA_PROJECT, self.topic_name) try: log.debug('Trying to create pubsub topic {0:s}'.format( self.topic_path)) topics_client = self.pubsub_api_client.projects().topics() # the ExecuteRequest takes API URI, method name as string and parameters # as a dict, it executes the API call, handles paging and return response. gcp_common.ExecuteRequest(topics_client, 'create', {'name': self.topic_path}) except HttpError as exception: if exception.resp.status == 409: log.debug('PubSub topic {0:s} already exists.'.format( self.topic_path)) else: raise TurbiniaException( 'Unknown error occurred when creating Topic:' ' {0!s}'.format(exception), __name__) from exception log.debug('Setup PubSub publisher at {0:s}'.format(self.topic_path))
def ListDisks(self) -> Dict[str, 'GoogleComputeDisk']: """List disks in project. Returns: Dict[str, GoogleComputeDisk]: Dictionary mapping disk names (str) to their respective GoogleComputeDisk object. """ disks = {} gce_disk_client = self.GceApi().disks() responses = common.ExecuteRequest(gce_disk_client, 'aggregatedList', {'project': self.project_id}) for response in responses: for zone in response['items']: try: for disk in response['items'][zone]['disks']: _, zone = disk['zone'].rsplit('/', 1) name = disk['name'] disks[name] = GoogleComputeDisk( self.project_id, zone, name, labels=disk.get('labels')) except KeyError: pass return disks
def ExecuteQuery(self, qfilter: str) -> List[Dict[str, Any]]: """Query logs in GCP project. Args: qfilter (str): The query filter to use. Returns: List[Dict]: Log entries returned by the query, e.g. [{'projectIds': [...], 'resourceNames': [...]}, {...}] Raises: RuntimeError: If API call failed. """ body = { 'resourceNames': 'projects/' + self.project_id, 'filter': qfilter, 'orderBy': 'timestamp desc', } entries = [] gcl_instance_client = self.GclApi().entries() responses = common.ExecuteRequest(gcl_instance_client, 'list', {'body': body}, throttle=True) for response in responses: for entry in response.get('entries', []): entries.append(entry) return entries
def GetBucketSize(self, bucket: str, timeframe: int = 1) -> Dict[str, int]: """List the size of a Google Storage Bucket in a project (default: last 1 day). Note: This will list the _maximum size_ (in bytes) the bucket had in the timeframe. Ref: https://cloud.google.com/monitoring/api/metrics_gcp#gcp-storage Args: bucket (str): Name of a bucket in GCS. timeframe (int): Optional. The number (in days) for which to measure activity. Default: 1 day. Returns: Dict[str, int]: Dictionary mapping bucket name to its size (in bytes). """ start_time = common.FormatRFC3339(datetime.datetime.utcnow() - datetime.timedelta(days=timeframe)) end_time = common.FormatRFC3339(datetime.datetime.utcnow()) period = timeframe * 24 * 60 * 60 assert self.project_id # Necessary for mypy check gcm = gcp_monitoring.GoogleCloudMonitoring(self.project_id) gcm_api = gcm.GcmApi() gcm_timeseries_client = gcm_api.projects().timeSeries() qfilter = ('metric.type="storage.googleapis.com/storage/total_bytes" ' 'resource.type="gcs_bucket"') qfilter += ' resource.label.bucket_name="{0:s}"'.format(bucket) responses = common.ExecuteRequest( gcm_timeseries_client, 'list', { 'name': 'projects/{0:s}'.format(self.project_id), 'filter': qfilter, 'interval_startTime': start_time, 'interval_endTime': end_time, 'aggregation_groupByFields': 'resource.label.bucket_name', 'aggregation_perSeriesAligner': 'ALIGN_MAX', 'aggregation_alignmentPeriod': '{0:d}s'.format(period), 'aggregation_crossSeriesReducer': 'REDUCE_NONE' }) ret = {} for response in responses: for ts in response.get('timeSeries', []): bucket = ts.get('resource', {}).get('labels', {}).get('bucket_name', '') if bucket: points = ts.get('points', []) for point in points: val = point.get('value', {}).get('doubleValue', 0) if bucket not in ret: ret[bucket] = val elif val > ret[bucket]: ret[bucket] = val return ret
def ListLogs(self) -> List[str]: """List logs in project. Returns: List[str]: The project logs available. Raises: RuntimeError: If API call failed. """ logs = [] gcl_instance_client = self.GclApi().logs() responses = common.ExecuteRequest( gcl_instance_client, 'list', {'parent': 'projects/' + self.project_id}) for response in responses: for logtypes in response.get('logNames', []): logs.append(logtypes) return logs
def ActiveServices(self, timeframe: int = 30) -> Dict[str, int]: """List active services in the project (default: last 30 days). Args: timeframe (int): Optional. The number (in days) for which to measure activity. Returns: Dict[str, int]: Dictionary mapping service name to number of uses. """ start_time = common.FormatRFC3339(datetime.datetime.utcnow() - datetime.timedelta(days=timeframe)) end_time = common.FormatRFC3339(datetime.datetime.utcnow()) period = timeframe * 24 * 60 * 60 service = self.GcmApi() gcm_timeseries_client = service.projects().timeSeries() responses = common.ExecuteRequest( gcm_timeseries_client, 'list', { 'name': 'projects/{0:s}'.format(self.project_id), 'filter': 'metric.type="serviceruntime.googleapis.com/api/request_count"', 'interval_startTime': start_time, 'interval_endTime': end_time, 'aggregation_groupByFields': 'resource.labels.service', 'aggregation_perSeriesAligner': 'ALIGN_SUM', 'aggregation_alignmentPeriod': '{0:d}s'.format(period), 'aggregation_crossSeriesReducer': 'REDUCE_SUM', }) ret = {} for response in responses: for ts in response.get('timeSeries', []): service = ts.get('resource', {}).get('labels', {}).get('service', '') if service: points = ts.get('points', []) if points: val = points[0].get('value', {}).get('int64Value', '') if val: ret[service] = int(val) return ret
def send_message(self, message): """Send a pubsub message. message: The message to send. """ base64_data = base64.b64encode(message.encode('utf-8')) request_body = { "messages": [{ "data": base64_data.decode('utf-8') # base64 encoded string }] } publish_client = self.pubsub_api_client.projects().topics() response = gcp_common.ExecuteRequest(publish_client, 'publish', { 'topic': self.topic_path, 'body': request_body }) # Safe to unpack since response is unpaged. if not response[0]['messageIds']: raise TurbiniaException( 'Message {0:s} was not published to topic {1:s}'.format( message, self.topic_path)) msg_id = response[0]['messageIds'][0] log.info('Published message {0!s} to topic {1!s}'.format( msg_id, self.topic_name))