def to_proto(self) -> FeatureViewProto: """ Converts an feature view object to its protobuf representation. Returns: FeatureViewProto protobuf """ meta = FeatureViewMetaProto( created_timestamp=self.created_timestamp, last_updated_timestamp=self.last_updated_timestamp, materialization_intervals=[], ) for interval in self.materialization_intervals: interval_proto = MaterializationIntervalProto() interval_proto.start_time.FromDatetime(interval[0]) interval_proto.end_time.FromDatetime(interval[1]) meta.materialization_intervals.append(interval_proto) ttl_duration = None if self.ttl is not None: ttl_duration = Duration() ttl_duration.FromTimedelta(self.ttl) spec = FeatureViewSpecProto( name=self.name, entities=self.entities, features=[feature.to_proto() for feature in self.features], tags=self.tags, ttl=(ttl_duration if ttl_duration is not None else None), online=self.online, input=self.input.to_proto(), ) return FeatureViewProto(spec=spec, meta=meta)
def test_list_point_in_time_changes(): """Demonstrate listing assets along with their state changes.""" from google.cloud import securitycenter_v1beta1 as securitycenter from google.protobuf.duration_pb2 import Duration from datetime import timedelta # [START demo_list_assets_changes] client = securitycenter.SecurityCenterClient() # ORGANIZATION_ID is the numeric ID of the organization (e.g. 123213123121) org_name = "organizations/{org_id}".format(org_id=ORGANIZATION_ID) project_filter = ("security_center_properties.resource_type=" + '"google.cloud.resourcemanager.Project"') # List assets and their state change the last 30 days compare_delta = timedelta(days=30) # Convert the timedelta to a Duration duration_proto = Duration() duration_proto.FromTimedelta(compare_delta) # Call the API and print results. asset_iterator = client.list_assets(org_name, filter_=project_filter, compare_duration=duration_proto) for i, asset in enumerate(asset_iterator): print(i, asset) # [END demo_list_assets_changes] assert i > 0
def to_proto(self) -> FeatureViewProto: """ Converts an feature view object to its protobuf representation. Returns: FeatureViewProto protobuf """ meta = FeatureViewMetaProto( created_timestamp=self.created_timestamp, last_updated_timestamp=self.last_updated_timestamp, ) if self.ttl is not None: ttl_duration = Duration() ttl_duration.FromTimedelta(self.ttl) spec = FeatureViewSpecProto( name=self.name, entities=self.entities, features=[feature.to_proto() for feature in self.features], tags=self.tags, ttl=(ttl_duration if ttl_duration is not None else None), online=self.online, input=self.input.to_proto(), ) return FeatureViewProto(spec=spec, meta=meta)
def __init__( self, name: str, entities: List[str], features: List[Feature], tags: Dict[str, str], ttl: Optional[Union[Duration, timedelta]], online: bool, input: BigQuerySource, ): cols = [entity for entity in entities] + [feat.name for feat in features] for col in cols: if input.field_mapping is not None and col in input.field_mapping.keys(): raise ValueError( f"The field {col} is mapped to {input.field_mapping[col]} for this data source. Please either remove this field mapping or use {input.field_mapping[col]} as the Entity or Feature name." ) self.name = name self.entities = entities self.features = features self.tags = tags if isinstance(ttl, timedelta): proto_ttl = Duration() proto_ttl.FromTimedelta(ttl) self.ttl = proto_ttl else: self.ttl = ttl self.online = online self.input = input
def group_findings_and_changes(source_name): """Demonstrates grouping all findings across an organization and associated changes.""" i = 0 # [START group_filtered_findings_with_changes] from datetime import timedelta from google.cloud import securitycenter from google.protobuf.duration_pb2 import Duration # Create a client. client = securitycenter.SecurityCenterClient() # source_name is the resource path for a source that has been # created previously (you can use list_sources to find a specific one). # Its format is: # source_name = "organizations/{organization_id}/sources/{source_id}" # e.g.: # source_name = "organizations/111122222444/sources/1234" # List assets and their state change the last 30 days compare_delta = timedelta(days=30) # Convert the timedelta to a Duration duration_proto = Duration() duration_proto.FromTimedelta(compare_delta) group_result_iterator = client.group_findings( source_name, group_by="state_change", compare_duration=duration_proto) for i, group_result in enumerate(group_result_iterator): print((i + 1), group_result) # [END group_findings_with_changes] return i
def test_create_upload_location(mock_raw_create_upload_location): client = _SynchronousFlyteClient( PlatformConfig.for_endpoint("a.b.com", True)) client.get_upload_signed_url("foo", "bar", bytes(), "baz.qux", timedelta(minutes=42)) duration_pb = Duration() duration_pb.FromTimedelta(timedelta(minutes=42)) create_upload_location_request = _data_proxy_pb2.CreateUploadLocationRequest( project="foo", domain="bar", filename="baz.qux", expires_in=duration_pb) mock_raw_create_upload_location.assert_called_with( create_upload_location_request)
def _create_csr(self): """Create CSR protobuf Returns: CSR protobuf object """ csr = cert_utils.create_csr(self._gateway_key, self._hw_id) duration = Duration() duration.FromTimedelta(datetime.timedelta(days=4)) csr = CSR( id=Identity(gateway=Identity.Gateway(hardware_id=self._hw_id)), valid_time=duration, csr_der=csr.public_bytes(serialization.Encoding.DER), ) return csr
def to_proto(self) -> FeatureViewProto: """ Converts a feature view object to its protobuf representation. Returns: A FeatureViewProto protobuf. """ meta = FeatureViewMetaProto(materialization_intervals=[]) if self.created_timestamp: meta.created_timestamp.FromDatetime(self.created_timestamp) if self.last_updated_timestamp: meta.last_updated_timestamp.FromDatetime( self.last_updated_timestamp) for interval in self.materialization_intervals: interval_proto = MaterializationIntervalProto() interval_proto.start_time.FromDatetime(interval[0]) interval_proto.end_time.FromDatetime(interval[1]) meta.materialization_intervals.append(interval_proto) ttl_duration = None if self.ttl is not None: ttl_duration = Duration() ttl_duration.FromTimedelta(self.ttl) batch_source_proto = self.batch_source.to_proto() batch_source_proto.data_source_class_type = f"{self.batch_source.__class__.__module__}.{self.batch_source.__class__.__name__}" stream_source_proto = None if self.stream_source: stream_source_proto = self.stream_source.to_proto() stream_source_proto.data_source_class_type = f"{self.stream_source.__class__.__module__}.{self.stream_source.__class__.__name__}" spec = FeatureViewSpecProto( name=self.name, entities=self.entities, features=[field.to_proto() for field in self.schema], description=self.description, tags=self.tags, owner=self.owner, ttl=(ttl_duration if ttl_duration is not None else None), online=self.online, batch_source=batch_source_proto, stream_source=stream_source_proto, ) return FeatureViewProto(spec=spec, meta=meta)
def test_group_assets_by_changes(organization_id): """Demonstrates grouping assets by there changes over a period of time.""" # [START group_all_assets_by_change] from datetime import timedelta from google.cloud import securitycenter from google.protobuf.duration_pb2 import Duration client = securitycenter.SecurityCenterClient() duration_proto = Duration() duration_proto.FromTimedelta(timedelta(days=5)) # organization_id is the numeric ID of the organization. # organization_id = "1234567777" org_name = "organizations/{org_id}".format(org_id=organization_id) result_iterator = client.group_assets(org_name, group_by="state_change", compare_duration=duration_proto) for i, result in enumerate(result_iterator): print((i + 1), result) # [END group_all_assets_by_change] # only one asset type is a project assert i >= 0
class Job: def __init__(self, do_not_cache, action_digest, platform_requirements=None, priority=0, name=None, operations=(), cancelled_operations=set(), lease=None, stage=OperationStage.UNKNOWN.value, cancelled=False, queued_timestamp=None, queued_time_duration=None, worker_start_timestamp=None, worker_completed_timestamp=None, done=False, result=None, worker_name=None, n_tries=0): self.__logger = logging.getLogger(__name__) self._name = name or str(uuid.uuid4()) self._priority = priority self._lease = lease self.__execute_response = result if result is None: self.__execute_response = remote_execution_pb2.ExecuteResponse() self.__operation_metadata = remote_execution_pb2.ExecuteOperationMetadata( ) self.__queued_timestamp = Timestamp() if queued_timestamp is not None: self.__queued_timestamp.CopyFrom(queued_timestamp) self.__queued_time_duration = Duration() if queued_time_duration is not None: self.__queued_time_duration.CopyFrom(queued_time_duration) self.__worker_start_timestamp = Timestamp() if worker_start_timestamp is not None: self.__worker_start_timestamp.CopyFrom(worker_start_timestamp) self.__worker_completed_timestamp = Timestamp() if worker_completed_timestamp is not None: self.__worker_completed_timestamp.CopyFrom( worker_completed_timestamp) self.__operations_by_name = {op.name: op for op in operations } # Name to Operation 1:1 mapping self.__operations_cancelled = cancelled_operations self.__lease_cancelled = cancelled self.__job_cancelled = cancelled self.__operation_metadata.action_digest.CopyFrom(action_digest) self.__operation_metadata.stage = stage self._do_not_cache = do_not_cache self._n_tries = n_tries self._platform_requirements = platform_requirements \ if platform_requirements else dict() self._done = done self.worker_name = worker_name def __lt__(self, other): try: return self.priority < other.priority except AttributeError: return NotImplemented def __le__(self, other): try: return self.priority <= other.priority except AttributeError: return NotImplemented def __eq__(self, other): if isinstance(other, Job): return self.name == other.name return False def __ne__(self, other): return not self.__eq__(other) def __gt__(self, other): try: return self.priority > other.priority except AttributeError: return NotImplemented def __ge__(self, other): try: return self.priority >= other.priority except AttributeError: return NotImplemented # --- Public API --- @property def name(self): return self._name @property def cancelled(self): return self.__job_cancelled @property def priority(self): return self._priority def set_priority(self, new_priority, *, data_store): self._priority = new_priority data_store.update_job(self.name, {'priority': new_priority}) @property def done(self): return self._done # --- Public API: REAPI --- @property def platform_requirements(self): return self._platform_requirements @property def do_not_cache(self): return self._do_not_cache @property def action_digest(self): return self.__operation_metadata.action_digest @property def operation_stage(self): return OperationStage(self.__operation_metadata.stage) @property def action_result(self): if self.__execute_response is not None: return self.__execute_response.result else: return None @property def execute_response(self): return self.__execute_response @execute_response.setter def execute_response(self, response): self.__execute_response = response for operation in self.__operations_by_name.values(): operation.response.Pack(self.__execute_response) @property def holds_cached_result(self): if self.__execute_response is not None: return self.__execute_response.cached_result else: return False @property def queued_timestamp(self): return self.__queued_timestamp @property def queued_time_duration(self): return self.__queued_time_duration @property def worker_start_timestamp(self): return self.__worker_start_timestamp @property def worker_completed_timestamp(self): return self.__worker_completed_timestamp def mark_worker_started(self): self.__worker_start_timestamp.GetCurrentTime() def set_action_url(self, url): """Generates a CAS browser URL for the job's action.""" if url.for_message('action', self.__operation_metadata.action_digest): self.__execute_response.message = url.generate() def set_cached_result(self, action_result): """Allows specifying an action result form the action cache for the job. Note: This won't trigger any :class:`Operation` stage transition. Args: action_result (ActionResult): The result from cache. """ self.__execute_response.result.CopyFrom(action_result) self.__execute_response.cached_result = True def n_peers(self, operations_by_peer): return len([ peer for peer, names in operations_by_peer.items() if any(name in self.__operations_by_name for name in names) ]) def n_peers_for_operation(self, operation_name, operations_by_peer): return len([ peer for peer, names in operations_by_peer.items() if any(name == operation_name for name in names) ]) def register_new_operation_peer(self, peer, message_queue, operations_by_peer, peer_message_queues, *, data_store): """Subscribes to a new job's :class:`Operation` stage changes. Args: peer (str): a unique string identifying the client. message_queue (queue.Queue): the event queue to register. Returns: str: The name of the subscribed :class:`Operation`. """ new_operation = operations_pb2.Operation() # Copy state from first existing and non cancelled operation: for operation in self.__operations_by_name.values(): if operation.name not in self.__operations_cancelled: new_operation.CopyFrom(operation) break new_operation.name = str(uuid.uuid4()) self.__logger.debug("Operation created for job [%s]: [%s]", self._name, new_operation.name) self.__operations_by_name[new_operation.name] = new_operation if peer in operations_by_peer: operations_by_peer[peer].add(new_operation.name) else: operations_by_peer[peer] = set([new_operation.name]) if peer in peer_message_queues: peer_message_queues[peer][new_operation.name] = message_queue else: peer_message_queues[peer] = {new_operation.name: message_queue} data_store.create_operation(new_operation, self._name) self._send_operations_updates(peers=[peer], operations_by_peer=operations_by_peer, peer_message_queues=peer_message_queues, data_store=data_store) return new_operation.name def register_operation_peer(self, operation_name, peer, message_queue, operations_by_peer, peer_message_queues, *, data_store): """Subscribes to one of the job's :class:`Operation` stage changes. Args: operation_name (str): an existing operation's name to subscribe to. peer (str): a unique string identifying the client. message_queue (queue.Queue): the event queue to register. Returns: str: The name of the subscribed :class:`Operation`. Raises: NotFoundError: If no operation with `operation_name` exists. """ if operation_name not in self.__operations_by_name: raise NotFoundError( "Operation name does not exist: [{}]".format(operation_name)) if peer in operations_by_peer: operations_by_peer[peer].add(operation_name) else: operations_by_peer[peer] = set([operation_name]) if peer in peer_message_queues: peer_message_queues[peer][operation_name] = message_queue else: peer_message_queues[peer] = {operation_name: message_queue} self._send_operations_updates(peers=[peer], operations_by_peer=operations_by_peer, peer_message_queues=peer_message_queues, data_store=data_store) def unregister_operation_peer(self, operation_name, peer): """Unsubscribes to the job's :class:`Operation` stage change. Args: operation_name (str): an existing operation's name to unsubscribe from. peer (str): a unique string identifying the client. Raises: NotFoundError: If no operation with `operation_name` exists. """ if operation_name not in self.__operations_by_name: raise NotFoundError( "Operation name does not exist: [{}]".format(operation_name)) def list_operations(self): """Lists the :class:`Operation` related to a job. Returns: list: A list of :class:`Operation` names. """ return list(self.__operations_by_name.keys()) def get_operation(self, operation_name): """Returns a copy of the the job's :class:`Operation`. Args: operation_name (str): the operation's name. Raises: NotFoundError: If no operation with `operation_name` exists. """ try: operation = self.__operations_by_name[operation_name] except KeyError: raise NotFoundError( "Operation name does not exist: [{}]".format(operation_name)) return self._copy_operation(operation) def update_operation_stage(self, stage, operations_by_peer, peer_message_queues, *, data_store): """Operates a stage transition for the job's :class:`Operation`. Args: stage (OperationStage): the operation stage to transition to. """ if stage.value == self.__operation_metadata.stage: return changes = {} self.__operation_metadata.stage = stage.value changes["stage"] = stage.value self.__logger.debug("Stage changed for job [%s]: [%s] (operation)", self._name, stage.name) if self.__operation_metadata.stage == OperationStage.QUEUED.value: if self.__queued_timestamp.ByteSize() == 0: self.__queued_timestamp.GetCurrentTime() changes[ "queued_timestamp"] = self.__queued_timestamp.ToDatetime() self._n_tries += 1 changes["n_tries"] = self._n_tries elif self.__operation_metadata.stage == OperationStage.EXECUTING.value: queue_in, queue_out = self.__queued_timestamp.ToDatetime( ), datetime.utcnow() self.__queued_time_duration.FromTimedelta(queue_out - queue_in) changes[ "queued_time_duration"] = self.__queued_time_duration.seconds elif self.__operation_metadata.stage == OperationStage.COMPLETED.value: self._done = True data_store.update_job(self.name, changes) self._send_operations_updates(operations_by_peer=operations_by_peer, peer_message_queues=peer_message_queues, data_store=data_store) def cancel_operation(self, operation_name, operations_by_peer, peer_message_queues, *, data_store): """Triggers a job's :class:`Operation` cancellation. This may cancel any job's :class:`Lease` that may have been issued. Args: operation_name (str): the operation's name. Raises: NotFoundError: If no operation with `operation_name` exists. """ try: operation = self.__operations_by_name[operation_name] except KeyError: raise NotFoundError( "Operation name does not exist: [{}]".format(operation_name)) self.__operations_cancelled.add(operation.name) self.__logger.debug("Operation cancelled for job [%s]: [%s]", self._name, operation.name) ongoing_operations = set(self.__operations_by_name.keys()) # Job is cancelled if all the operation are: self.__job_cancelled = ongoing_operations.issubset( self.__operations_cancelled) if self.__job_cancelled: self.__operation_metadata.stage = OperationStage.COMPLETED.value changes = { "stage": OperationStage.COMPLETED.value, "cancelled": True } data_store.update_job(self.name, changes) if self._lease is not None: self.cancel_lease(data_store=data_store) peers_to_notify = set() # If the job is not cancelled, notify all the peers watching the given # operation; if the job is cancelled, only notify the peers for which # the operation status changed. for peer, names in operations_by_peer.items(): relevant_names = [ n for n in names if n in self.__operations_by_name ] if self.__job_cancelled: if not any(name in self.__operations_cancelled for name in relevant_names): peers_to_notify.add(peer) elif operation_name in relevant_names: peers_to_notify.add(peer) else: if operation_name in relevant_names: peers_to_notify.add(peer) self._send_operations_updates(peers=peers_to_notify, notify_cancelled=True, operations_by_peer=operations_by_peer, peer_message_queues=peer_message_queues, data_store=data_store) # --- Public API: RWAPI --- @property def lease(self): return self._lease @property def lease_state(self): if self._lease is not None: return LeaseState(self._lease.state) else: return None @property def lease_cancelled(self): return self.__lease_cancelled @property def n_tries(self): return self._n_tries def create_lease(self, worker_name, bot_id=None, *, data_store): """Emits a new :class:`Lease` for the job. Only one :class:`Lease` can be emitted for a given job. This method should only be used once, any further calls are ignored. Args: worker_name (string): The name of the worker this lease is for. bot_id (string): The name of the corresponding bot for this job's worker. """ if self._lease is not None: return self._lease elif self.__job_cancelled: return None self._lease = bots_pb2.Lease() self._lease.id = self._name self._lease.payload.Pack(self.__operation_metadata.action_digest) self._lease.state = LeaseState.UNSPECIFIED.value if bot_id is None: bot_id = "UNKNOWN" self.__logger.debug( "Lease created for job [%s]: [%s] (assigned to bot [%s])", self._name, self._lease.id, bot_id) self.update_lease_state(LeaseState.PENDING, skip_lease_persistence=True, data_store=data_store) self.worker_name = worker_name return self._lease def update_lease_state(self, state, status=None, result=None, skip_lease_persistence=False, *, data_store): """Operates a state transition for the job's current :class:`Lease`. Args: state (LeaseState): the lease state to transition to. status (google.rpc.Status, optional): the lease execution status, only required if `state` is `COMPLETED`. result (google.protobuf.Any, optional): the lease execution result, only required if `state` is `COMPLETED`. """ if state.value == self._lease.state: return job_changes = {} lease_changes = {} self._lease.state = state.value lease_changes["state"] = state.value self.__logger.debug("State changed for job [%s]: [%s] (lease)", self._name, state.name) if self._lease.state == LeaseState.PENDING.value: self.__worker_start_timestamp.Clear() self.__worker_completed_timestamp.Clear() job_changes[ "worker_start_timestamp"] = self.__worker_start_timestamp.ToDatetime( ) job_changes[ "worker_completed_timestamp"] = self.__worker_completed_timestamp.ToDatetime( ) self._lease.status.Clear() self._lease.result.Clear() lease_changes["status"] = self._lease.status.code elif self._lease.state == LeaseState.COMPLETED.value: self.__worker_completed_timestamp.GetCurrentTime() job_changes[ "worker_completed_timestamp"] = self.__worker_completed_timestamp.ToDatetime( ) action_result = remote_execution_pb2.ActionResult() # TODO: Make a distinction between build and bot failures! if status.code != code_pb2.OK: self._do_not_cache = True job_changes["do_not_cache"] = True lease_changes["status"] = status.code if result is not None and result.Is(action_result.DESCRIPTOR): result.Unpack(action_result) action_metadata = action_result.execution_metadata action_metadata.queued_timestamp.CopyFrom(self.__queued_timestamp) action_metadata.worker_start_timestamp.CopyFrom( self.__worker_start_timestamp) action_metadata.worker_completed_timestamp.CopyFrom( self.__worker_completed_timestamp) self.__execute_response.result.CopyFrom(action_result) self.__execute_response.cached_result = False self.__execute_response.status.CopyFrom(status) data_store.update_job(self.name, job_changes) if not skip_lease_persistence: data_store.update_lease(self.name, lease_changes) def cancel_lease(self, *, data_store): """Triggers a job's :class:`Lease` cancellation. Note: This will not cancel the job's :class:`Operation`. """ self.__lease_cancelled = True self.__logger.debug("Lease cancelled for job [%s]: [%s]", self._name, self._lease.id) if self._lease is not None: self.update_lease_state(LeaseState.CANCELLED, data_store=data_store) def delete_lease(self): """Discard the job's :class:`Lease`. Note: This will not cancel the job's :class:`Operation`. """ if self._lease is not None: self.__worker_start_timestamp.Clear() self.__worker_completed_timestamp.Clear() self.__logger.debug("Lease deleted for job [%s]: [%s]", self._name, self._lease.id) self._lease = None # --- Public API: Monitoring --- def query_queue_time(self): return self.__queued_time_duration.ToTimedelta() def query_n_retries(self): return self._n_tries - 1 if self._n_tries > 0 else 0 # --- Private API --- def _copy_operation(self, operation): """Simply duplicates a given :class:`Lease` object.""" new_operation = operations_pb2.Operation() new_operation.CopyFrom(operation) new_operation.metadata.Pack(self.__operation_metadata) new_operation.response.Pack(self.__execute_response) return new_operation def _update_operation(self, operation, operation_metadata, execute_response=None, done=False, *, data_store): """Forges a :class:`Operation` message given input data.""" operation.metadata.Pack(operation_metadata) if execute_response is not None: operation.response.Pack(execute_response) operation.done = done changes = {"done": done} data_store.update_operation(operation.name, changes) def _update_cancelled_operation(self, operation, operation_metadata, execute_response=None, *, data_store): """Forges a cancelled :class:`Operation` message given input data.""" cancelled_operation_metadata = remote_execution_pb2.ExecuteOperationMetadata( ) cancelled_operation_metadata.CopyFrom(operation_metadata) cancelled_operation_metadata.stage = OperationStage.COMPLETED.value operation.metadata.Pack(cancelled_operation_metadata) cancelled_execute_response = remote_execution_pb2.ExecuteResponse() if execute_response is not None: cancelled_execute_response.CopyFrom(self.__execute_response) cancelled_execute_response.status.code = code_pb2.CANCELLED cancelled_execute_response.status.message = "Operation cancelled by client." operation.response.Pack(cancelled_execute_response) operation.done = True changes = {"done": True, "cancelled": True} data_store.update_operation(operation.name, changes) def _send_operations_updates(self, peers=None, notify_cancelled=False, operations_by_peer=None, peer_message_queues=None, *, data_store): """Sends :class:`Operation` stage change messages to watchers.""" if operations_by_peer is None: operations_by_peer = {} if peer_message_queues is None: peer_message_queues = {} for operation in self.__operations_by_name.values(): if operation.name in self.__operations_cancelled: self._update_cancelled_operation( operation, self.__operation_metadata, execute_response=self.__execute_response, data_store=data_store) else: self._update_operation( operation, self.__operation_metadata, execute_response=self.__execute_response, done=self._done, data_store=data_store) relevant_queues = { peer: mqs for peer, mqs in peer_message_queues.items() if any(name in self.__operations_by_name for name in mqs) } for peer, message_queues in relevant_queues.items(): if peer not in operations_by_peer: continue elif peers and peer not in peers: continue operations = [ self.__operations_by_name[name] for name in operations_by_peer[peer] if name in self.__operations_by_name ] for operation in operations: # Messages are pairs of (Exception, Operation,): if not notify_cancelled and operation.name in self.__operations_cancelled: continue elif operation.name not in self.__operations_cancelled: message = ( None, self._copy_operation(operation), ) else: message = ( CancelledError("Operation has been cancelled"), self._copy_operation(operation), ) message_queue = message_queues[operation.name] message_queue.put(message)
def duration_from_timedelta(delta): dr = Duration() dr.FromTimedelta(delta) return dr
def Duration_from_timedelta(dt: datetime): pb_d = Duration() pb_d.FromTimedelta(dt) return pb_d