def get_regional_emissions_choropleth_data( self, net_energy_consumed: float, country_iso_code: str) -> List[Dict]: # add country codes here to render for different countries if country_iso_code.upper() not in ["USA"]: return [{"region_code": "", "region_name": "", "emissions": ""}] region_emissions = self._data_source.get_country_emissions_data( country_iso_code.lower()) choropleth_data = [] for region_name in region_emissions.keys(): region_code = region_emissions[region_name]["regionCode"] if region_name not in ["_unit"]: from codecarbon.core.units import Energy energy_consumed = Energy.from_energy(kwh=net_energy_consumed) from codecarbon.external.geography import GeoMetadata emissions = self._emissions.get_region_emissions( energy_consumed, GeoMetadata(country_iso_code=country_iso_code, region=region_name), ) choropleth_data.append({ "region_code": region_code, "region_name": region_name.upper(), "emissions": emissions, }) return choropleth_data
def get_global_emissions_choropleth_data( self, net_energy_consumed: float) -> List[Dict]: def formatted_energy_percentage(energy_type: float, total: float) -> float: return float("{:.1f}".format((energy_type / total) * 100)) global_energy_mix = self._data_source.get_global_energy_mix_data() choropleth_data = [] for country_iso_code in global_energy_mix.keys(): country_name = global_energy_mix[country_iso_code]["country_name"] if country_iso_code not in ["_define", "ATA"]: from codecarbon.core.units import Energy energy_consumed = Energy.from_energy(kWh=net_energy_consumed) from codecarbon.external.geography import GeoMetadata country_emissions = self._emissions.get_country_emissions( energy_consumed, GeoMetadata(country_name=country_name, country_iso_code=country_iso_code), ) total = global_energy_mix[country_iso_code]["total_TWh"] choropleth_data.append({ "iso_code": country_iso_code, "emissions": country_emissions, "country": country_name, "fossil": formatted_energy_percentage( global_energy_mix[country_iso_code]["fossil_TWh"], total), "geothermal": formatted_energy_percentage( global_energy_mix[country_iso_code]["geothermal_TWh"], total), "hydroelectricity": formatted_energy_percentage( global_energy_mix[country_iso_code] ["hydroelectricity_TWh"], total, ), "nuclear": formatted_energy_percentage( global_energy_mix[country_iso_code]["nuclear_TWh"], total), "solar": formatted_energy_percentage( global_energy_mix[country_iso_code]["solar_TWh"], total), "wind": formatted_energy_percentage( global_energy_mix[country_iso_code]["wind_TWh"], total), }) return choropleth_data
def test_get_emissions_PRIVATE_INFRA_USA_WITHOUT_COUNTRYNAME(self): # WHEN emissions = self._emissions.get_private_infra_emissions( Energy.from_energy(kwh=0.3), GeoMetadata(country_iso_code="USA")) # THEN assert isinstance(emissions, float) self.assertAlmostEqual(emissions, 0.20, places=2)
def test_emissions_CLOUD_GCP(self): emissions = self._emissions.get_cloud_emissions( Energy.from_energy(kwh=0.01), CloudMetadata(provider="gcp", region="us-central1"), ) # THEN assert isinstance(emissions, float) self.assertAlmostEqual(emissions, 0.01, places=2)
def setUp(self) -> None: # GIVEN self._energy = Energy.from_energy(kwh=10) self._geo = GeoMetadata( country_iso_code="FRA", country_name="France", region=None, country_2letter_iso_code="FR", )
def test_emissions_CLOUD_AZURE(self): # WHEN emissions = self._emissions.get_cloud_emissions( Energy.from_energy(kwh=1.5), CloudMetadata(provider="azure", region="eastus"), ) # THEN assert isinstance(emissions, float) self.assertAlmostEqual(emissions, 0.55, places=2)
def test_get_emissions_PRIVATE_INFRA_unknown_country(self): """ If we do not know the country we fallback to a default value. """ emissions = self._emissions.get_private_infra_emissions( Energy.from_energy(kWh=1), GeoMetadata(country_iso_code="AAA", country_name="unknown"), ) assert isinstance(emissions, float) self.assertAlmostEqual(emissions, 0.475, places=2)
def test_get_emissions_PRIVATE_INFRA_USA_WITHOUT_REGION(self): # WHEN emissions = self._emissions.get_private_infra_emissions( Energy.from_energy(kWh=0.3), GeoMetadata(country_iso_code="USA", country_name="United States"), ) # THEN assert isinstance(emissions, float) self.assertAlmostEqual(emissions, 0.115, places=2)
def test_get_emissions_PRIVATE_INFRA_CANADA_WITHOUT_REGION(self): # WHEN emissions = self._emissions.get_private_infra_emissions( Energy.from_energy(kwh=3), GeoMetadata(country_iso_code="CAN", country_name="Canada"), ) # THEN assert isinstance(emissions, float) self.assertAlmostEqual(emissions, 1.6, places=2)
def test_get_emissions_CLOUD_AWS(self): # WHEN emissions = self._emissions.get_cloud_emissions( Energy.from_energy(kwh=0.6), CloudMetadata(provider="aws", region="us-east-1"), ) # THEN assert isinstance(emissions, float) self.assertAlmostEqual(emissions, 0.22, places=2)
def __init__( self, project_name: str = "codecarbon", measure_power_secs: int = 15, output_dir: str = ".", save_to_file: bool = True, gpu_ids: Optional[List] = None, ): """ :param project_name: Project name for current experiment run, default name as "codecarbon" :param measure_power_secs: Interval (in seconds) to measure hardware power usage, defaults to 15 :param output_dir: Directory path to which the experiment details are logged in a CSV file called `emissions.csv`, defaults to current directory :param save_to_file: Indicates if the emission artifacts should be logged to a file, defaults to True :param gpu_ids: User-specified known gpu ids to track, defaults to None """ self._project_name: str = project_name self._measure_power_secs: int = measure_power_secs self._start_time: Optional[float] = None self._last_measured_time: float = time.time() self._output_dir: str = output_dir self._total_energy: Energy = Energy.from_energy(kwh=0) self._scheduler = BackgroundScheduler() self._hardware = list() if gpu.is_gpu_details_available(): logger.info("CODECARBON : Tracking Nvidia GPU via pynvml") self._hardware.append(GPU.from_utils(gpu_ids)) if cpu.is_powergadget_available(): logger.info("CODECARBON : Tracking Intel CPU via Power Gadget") self._hardware.append( CPU.from_utils(self._output_dir, "intel_power_gadget")) elif cpu.is_rapl_available(): logger.info("CODECARBON : Tracking Intel CPU via RAPL interface") self._hardware.append( CPU.from_utils(self._output_dir, "intel_rapl")) # Run `self._measure_power` every `measure_power_secs` seconds in a background thread self._scheduler.add_job(self._measure_power, "interval", seconds=measure_power_secs) self._data_source = DataSource() self._emissions: Emissions = Emissions(self._data_source) self.persistence_objs: List[BaseOutput] = list() if save_to_file: self.persistence_objs.append( FileOutput(os.path.join(self._output_dir, "emissions.csv")))
def test_get_emissions_PRIVATE_INFRA_NOR(self): """ Norway utilises hydropower more than any other country around the globe """ # WHEN emissions = self._emissions.get_private_infra_emissions( Energy.from_energy(kWh=1), GeoMetadata(country_iso_code="NOR", country_name="Norway"), ) # THEN assert isinstance(emissions, float) self.assertAlmostEqual(emissions, 33.4 / 1_000, places=2)
def _get_energy_from_cpus(self, delay: Time) -> Energy: """ Get CPU energy deltas from RAPL files :return: energy in kWh """ all_cpu_details: Dict = self._intel_interface.get_cpu_details(delay) energy = 0 for metric, value in all_cpu_details.items(): if re.match(r"^Processor Energy Delta_\d", metric): energy += value # logger.debug(f"_get_energy_from_cpus - MATCH {metric} : {value}") return Energy.from_energy(energy)
def test_get_emissions_PRIVATE_INFRA_FRA(self): """ European country is a specific case as we have there carbon intensity to without computation. """ # WHEN emissions = self._emissions.get_private_infra_emissions( Energy.from_energy(kWh=1), GeoMetadata(country_iso_code="FRA", country_name="France"), ) # THEN assert isinstance(emissions, float) self.assertAlmostEqual(emissions, 0.055, places=2)
def get_global_emissions_choropleth_data( self, net_energy_consumed: float) -> List[Dict]: def formatted_energy_percentage(energy_type: float, total: float) -> float: return float("{:.1f}".format((energy_type / total) * 100)) global_energy_mix = self._data_source.get_global_energy_mix_data() choropleth_data = [] for country_iso_code in global_energy_mix.keys(): country_name = global_energy_mix[country_iso_code]["countryName"] if country_iso_code not in ["_define", "ATA"]: from codecarbon.core.units import Energy energy_consumed = Energy.from_energy(kwh=net_energy_consumed) from codecarbon.external.geography import GeoMetadata country_emissions = self._emissions.get_country_emissions( energy_consumed, GeoMetadata(country_name=country_name, country_iso_code=country_iso_code), ) total = global_energy_mix[country_iso_code]["total"] choropleth_data.append({ "iso_code": country_iso_code, "emissions": country_emissions, "country": country_name, "coal": formatted_energy_percentage( global_energy_mix[country_iso_code]["coal"], total), "petroleum": formatted_energy_percentage( global_energy_mix[country_iso_code]["petroleum"], total), "natural_gas": formatted_energy_percentage( global_energy_mix[country_iso_code]["naturalGas"], total), "low_carbon": formatted_energy_percentage( global_energy_mix[country_iso_code]["lowCarbon"], total), }) return choropleth_data
def get_regional_emissions_choropleth_data( self, net_energy_consumed: float, country_iso_code: str) -> List[Dict]: # add country codes here to render for different countries if country_iso_code.upper() not in ["USA", "CAN"]: return [{"region_code": "", "region_name": "", "emissions": ""}] try: region_emissions = self._data_source.get_country_emissions_data( country_iso_code.lower()) except DataSourceException: # This country has regional data at the energy mix level, not the emissions level country_energy_mix = self._data_source.get_country_energy_mix_data( country_iso_code.lower()) region_emissions = { region: { "regionCode": region } for region, energy_mix in country_energy_mix.items() } choropleth_data = [] for region_name in region_emissions.keys(): region_code = region_emissions[region_name]["regionCode"] if region_name not in ["_unit"]: from codecarbon.core.units import Energy energy_consumed = Energy.from_energy(kWh=net_energy_consumed) from codecarbon.external.geography import GeoMetadata emissions = self._emissions.get_region_emissions( energy_consumed, GeoMetadata(country_iso_code=country_iso_code, region=region_name), ) choropleth_data.append({ "region_code": region_code, "region_name": region_name.upper(), "emissions": emissions, }) return choropleth_data
def test_get_emissions_PRIVATE_INFRA_JOR(self): """ Jordania use fossil energy """ # WHEN emissions = self._emissions.get_private_infra_emissions( Energy.from_energy(kWh=1_000), GeoMetadata(country_iso_code="JOR", country_name="Jordan"), ) # THEN jor_carbon_intensity = 17.19636 / 19.380129999999998 * 635 # fossil jor_carbon_intensity += 0.02277 / 19.380129999999998 * 26 # hydroelectricity jor_carbon_intensity += 1.441 / 19.380129999999998 * 48 # solar jor_carbon_intensity += 0.72 / 19.380129999999998 * 26 # wind jor_carbon_intensity /= 1000 # convert from g_per_kWh to kgs_per_kWh jor_emissions = (jor_carbon_intensity * 1_000 ) # Emissions in Kgs of CO2 For 1 000 kWh of energy assert isinstance(emissions, float) self.assertAlmostEqual(emissions, jor_emissions, places=2)
def __init__( self, project_name: Optional[str] = _sentinel, measure_power_secs: Optional[int] = _sentinel, api_call_interval: Optional[int] = _sentinel, api_endpoint: Optional[str] = _sentinel, api_key: Optional[str] = _sentinel, output_dir: Optional[str] = _sentinel, output_file: Optional[str] = _sentinel, save_to_file: Optional[bool] = _sentinel, save_to_api: Optional[bool] = _sentinel, save_to_logger: Optional[bool] = _sentinel, logging_logger: Optional[LoggerOutput] = _sentinel, gpu_ids: Optional[List] = _sentinel, emissions_endpoint: Optional[str] = _sentinel, experiment_id: Optional[str] = _sentinel, co2_signal_api_token: Optional[str] = _sentinel, tracking_mode: Optional[str] = _sentinel, log_level: Optional[Union[int, str]] = _sentinel, on_csv_write: Optional[str] = _sentinel, logger_preamble: Optional[str] = _sentinel, ): """ :param project_name: Project name for current experiment run, default name as "codecarbon" :param measure_power_secs: Interval (in seconds) to measure hardware power usage, defaults to 15 :param api_call_interval: Occurrence to wait before calling API : -1 : only call api on flush() and at the end. 1 : at every measure 2 : every 2 measure, etc... :param api_endpoint: Optional URL of Code Carbon API endpoint for sending emissions data :param api_key: API key for Code Carbon API, mandatory to use it ! :param output_dir: Directory path to which the experiment details are logged, defaults to current directory :param output_file: Name of output CSV file, defaults to `emissions.csv` :param save_to_file: Indicates if the emission artifacts should be logged to a file, defaults to True :param save_to_api: Indicates if the emission artifacts should be send to the CodeCarbon API, defaults to False :param save_to_logger: Indicates if the emission artifacts should be written to a dedicated logger, defaults to False :param logging_logger: LoggerOutput object encapsulating a logging.logger or a Google Cloud logger :param gpu_ids: User-specified known gpu ids to track, defaults to None :param emissions_endpoint: Optional URL of http endpoint for sending emissions data :param experiment_id: Id of the experiment :param co2_signal_api_token: API token for co2signal.com (requires sign-up for free beta) :param tracking_mode: One of "process" or "machine" in order to measure the power consumptions due to the entire machine or try and isolate the tracked processe's in isolation. Defaults to "machine" :param log_level: Global codecarbon log level. Accepts one of: {"debug", "info", "warning", "error", "critical"}. Defaults to "info". :param on_csv_write: "append" or "update". Whether to always append a new line to the csv when writing or to update the existing `run_id` row (useful when calling`tracker.flush()` manually). Accepts one of "append" or "update". :param logger_preamble: String to systematically include in the logger's. messages. Defaults to "". """ # logger.info("base tracker init") self._external_conf = get_hierarchical_config() self._set_from_conf(api_call_interval, "api_call_interval", 8, int) self._set_from_conf(api_endpoint, "api_endpoint", "https://api.codecarbon.io") self._set_from_conf(co2_signal_api_token, "co2_signal_api_token") self._set_from_conf(emissions_endpoint, "emissions_endpoint") self._set_from_conf(gpu_ids, "gpu_ids") self._set_from_conf(log_level, "log_level", "info") self._set_from_conf(measure_power_secs, "measure_power_secs", 15, int) self._set_from_conf(output_dir, "output_dir", ".") self._set_from_conf(output_file, "output_file", "emissions.csv") self._set_from_conf(project_name, "project_name", "codecarbon") self._set_from_conf(save_to_api, "save_to_api", False, bool) self._set_from_conf(save_to_file, "save_to_file", True, bool) self._set_from_conf(save_to_logger, "save_to_logger", False, bool) self._set_from_conf(logging_logger, "logging_logger") self._set_from_conf(tracking_mode, "tracking_mode", "machine") self._set_from_conf(on_csv_write, "on_csv_write", "append") self._set_from_conf(logger_preamble, "logger_preamble", "") assert self._tracking_mode in ["machine", "process"] set_logger_level(self._log_level) set_logger_format(self._logger_preamble) self._start_time: Optional[float] = None self._last_measured_time: float = time.time() self._total_energy: Energy = Energy.from_energy(kWh=0) self._total_cpu_energy: Energy = Energy.from_energy(kWh=0) self._total_gpu_energy: Energy = Energy.from_energy(kWh=0) self._total_ram_energy: Energy = Energy.from_energy(kWh=0) self._cpu_power: Power = Power.from_watts(watts=0) self._gpu_power: Power = Power.from_watts(watts=0) self._ram_power: Power = Power.from_watts(watts=0) self._cc_api__out = None self._measure_occurrence: int = 0 self._cloud = None self._previous_emissions = None self._conf["os"] = platform.platform() self._conf["python_version"] = platform.python_version() self._conf["cpu_count"] = count_cpus() self._geo = None if isinstance(self._gpu_ids, str): self._gpu_ids: List[int] = parse_gpu_ids(self._gpu_ids) self._conf["gpu_ids"] = self._gpu_ids self._conf["gpu_count"] = len(self._gpu_ids) logger.info("[setup] RAM Tracking...") ram = RAM(tracking_mode=self._tracking_mode) self._conf["ram_total_size"] = ram.machine_memory_GB self._hardware: List[Union[RAM, CPU, GPU]] = [ram] # Hardware detection logger.info("[setup] GPU Tracking...") if gpu.is_gpu_details_available(): logger.info("Tracking Nvidia GPU via pynvml") self._hardware.append(GPU.from_utils(self._gpu_ids)) gpu_names = [n["name"] for n in gpu.get_gpu_static_info()] gpu_names_dict = Counter(gpu_names) self._conf["gpu_model"] = "".join( [f"{i} x {name}" for name, i in gpu_names_dict.items()]) self._conf["gpu_count"] = len(gpu.get_gpu_static_info()) else: logger.info("No GPU found.") logger.info("[setup] CPU Tracking...") if cpu.is_powergadget_available(): logger.info("Tracking Intel CPU via Power Gadget") hardware = CPU.from_utils(self._output_dir, "intel_power_gadget") self._hardware.append(hardware) self._conf["cpu_model"] = hardware.get_model() elif cpu.is_rapl_available(): logger.info("Tracking Intel CPU via RAPL interface") hardware = CPU.from_utils(self._output_dir, "intel_rapl") self._hardware.append(hardware) self._conf["cpu_model"] = hardware.get_model() else: logger.warning( "No CPU tracking mode found. Falling back on CPU constant mode." ) tdp = cpu.TDP() power = tdp.tdp model = tdp.model logger.info(f"CPU Model on constant consumption mode: {model}") self._conf["cpu_model"] = model if tdp: hardware = CPU.from_utils(self._output_dir, "constant", model, power) self._hardware.append(hardware) else: logger.warning("Failed to match CPU TDP constant. " + "Falling back on a global constant.") hardware = CPU.from_utils(self._output_dir, "constant") self._hardware.append(hardware) self._conf["hardware"] = list( map(lambda x: x.description(), self._hardware)) logger.info(">>> Tracker's metadata:") logger.info(f" Platform system: {self._conf.get('os')}") logger.info(f" Python version: {self._conf.get('python_version')}") logger.info( f" Available RAM : {self._conf.get('ram_total_size'):.3f} GB") logger.info(f" CPU count: {self._conf.get('cpu_count')}") logger.info(f" CPU model: {self._conf.get('cpu_model')}") logger.info(f" GPU count: {self._conf.get('gpu_count')}") logger.info(f" GPU model: {self._conf.get('gpu_model')}") # Run `self._measure_power` every `measure_power_secs` seconds in a # background thread self._scheduler = PeriodicScheduler( function=self._measure_power_and_energy, interval=self._measure_power_secs, ) self._data_source = DataSource() cloud: CloudMetadata = self._get_cloud_metadata() if cloud.is_on_private_infra: self._geo = self._get_geo_metadata() self._conf["longitude"] = self._geo.longitude self._conf["latitude"] = self._geo.latitude self._conf["region"] = cloud.region self._conf["provider"] = cloud.provider else: self._conf["region"] = cloud.region self._conf["provider"] = cloud.provider self._emissions: Emissions = Emissions(self._data_source, self._co2_signal_api_token) self.persistence_objs: List[BaseOutput] = list() if self._save_to_file: self.persistence_objs.append( FileOutput( os.path.join(self._output_dir, self._output_file), self._on_csv_write, )) if self._save_to_logger: self.persistence_objs.append(self._logging_logger) if self._emissions_endpoint: self.persistence_objs.append(HTTPOutput(emissions_endpoint)) if self._save_to_api: experiment_id = self._set_from_conf( experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1") self._cc_api__out = CodeCarbonAPIOutput( endpoint_url=self._api_endpoint, experiment_id=experiment_id, api_key=api_key, conf=self._conf, ) self.run_id = self._cc_api__out.run_id self.persistence_objs.append(self._cc_api__out) else: self.run_id = uuid.uuid4()
def __init__( self, project_name: str = "codecarbon", measure_power_secs: int = 15, output_dir: str = ".", save_to_file: bool = True, gpu_ids: Optional[List] = None, emissions_endpoint: Optional[str] = None, co2_signal_api_token: Optional[str] = None, ): """ :param project_name: Project name for current experiment run, default name as "codecarbon" :param measure_power_secs: Interval (in seconds) to measure hardware power usage, defaults to 15 :param output_dir: Directory path to which the experiment details are logged in a CSV file called `emissions.csv`, defaults to current directory :param save_to_file: Indicates if the emission artifacts should be logged to a file, defaults to True :param gpu_ids: User-specified known gpu ids to track, defaults to None :param emissions_endpoint: Optional URL of http endpoint for sending emissions data :param co2_signal_api_token: API token for co2signal.com (requires sign-up for free beta) """ self._project_name: str = project_name self._measure_power_secs: int = measure_power_secs self._start_time: Optional[float] = None self._last_measured_time: float = time.time() self._output_dir: str = output_dir self._total_energy: Energy = Energy.from_energy(kwh=0) self._scheduler = BackgroundScheduler() self._hardware = list() if gpu.is_gpu_details_available(): logger.info("CODECARBON : Tracking Nvidia GPU via pynvml") self._hardware.append(GPU.from_utils(gpu_ids)) if cpu.is_powergadget_available(): logger.info("CODECARBON : Tracking Intel CPU via Power Gadget") self._hardware.append( CPU.from_utils(self._output_dir, "intel_power_gadget")) elif cpu.is_rapl_available(): logger.info("CODECARBON : Tracking Intel CPU via RAPL interface") self._hardware.append( CPU.from_utils(self._output_dir, "intel_rapl")) # Print warning if no supported hardware is found' if not self._hardware: logger.warning( "CODECARBON : No CPU/GPU tracking mode found. This " "may be due to your code running on Windows WSL, or due to " "unsupported hardware (see " "https://github.com/mlco2/codecarbon#infrastructure-support)") # Run `self._measure_power` every `measure_power_secs` seconds in a background thread self._scheduler.add_job(self._measure_power, "interval", seconds=measure_power_secs) self._data_source = DataSource() self._emissions: Emissions = Emissions(self._data_source) self.persistence_objs: List[BaseOutput] = list() if save_to_file: self.persistence_objs.append( FileOutput(os.path.join(self._output_dir, "emissions.csv"))) if emissions_endpoint: self.persistence_objs.append(HTTPOutput(emissions_endpoint)) if co2_signal_api_token: co2_signal.CO2_SIGNAL_API_TOKEN = co2_signal_api_token