def __init__(
self,
*args,
country_iso_code: Optional[str] = None,
region: Optional[str] = None,
cloud_provider: Optional[str] = None,
cloud_region: Optional[str] = None,
country_2letter_iso_code: Optional[str] = None,
**kwargs,
):
"""
:param country_iso_code: 3 letter ISO Code of the country where the
experiment is being run
:param region: The provincial region, for example, California in the US.
Currently, this only affects calculations for the United States and Canada
:param cloud_provider: The cloud provider specified for estimating emissions intensity, defaults to None.
See https://github.com/mlco2/codecarbon/blob/master/codecarbon/data/cloud/impact.csv for a list of cloud providers
:param cloud_region: The region of the cloud data center, defaults to None.
See https://github.com/mlco2/codecarbon/blob/master/codecarbon/data/cloud/impact.csv for a list of cloud regions
Currently, this only affects calculations for the United States
:param country_2letter_iso_code: For use with the CO2Signal emissions API.
See http://api.electricitymap.org/v3/zones for a list of codes and their corresponding locations.
"""
self._cloud_provider: Optional[str] = cloud_provider
self._cloud_region: Optional[str] = cloud_region
self._country_iso_code: Optional[str] = country_iso_code
self._region: Optional[
str] = region if region is None else region.lower()
if self._cloud_provider:
if self._cloud_region is None:
logger.error(
"CODECARBON : Cloud Region must not be None if cloud provider is set"
)
df = DataSource().get_cloud_emissions_data()
if (len(df.loc[(df["provider"] == self._cloud_provider)
& (df["region"] == self._cloud_region)]) == 0):
logger.error("CODECARBON : Cloud Provider/Region "
f"{self._cloud_provider} {self._cloud_region} "
"not found in cloud emissions data.")
if self._country_iso_code:
try:
self._country_name: str = DataSource(
).get_global_energy_mix_data()[
self._country_iso_code]["countryName"]
except KeyError as e:
logger.error(
f"CODECARBON : Does not support country with ISO code {self._country_iso_code} "
f"Exception occured {e}")
self.country_2letter_iso_code: Optional[str] = (
country_2letter_iso_code.upper()
if country_2letter_iso_code else None)
super().__init__(*args, **kwargs)
def __init__(
self,
country_iso_code: str,
*args,
region: Optional[str] = None,
**kwargs,
):
"""
:param country_iso_code: 3 letter ISO Code of the country where the
experiment is being run
:param region: The provincial region, for example, California in the US.
Currently, this only affects calculations for the United States
"""
# TODO: Currently we silently use a default value of Canada.
# Decide if we should fail with missing args.
self._country_iso_code: str = ("CAN" if country_iso_code is None else
country_iso_code)
try:
self._country_name: str = (
DataSource().get_global_energy_mix_data().get(
self._country_iso_code).get("countryName"))
except Exception as e:
logger.error(
f"CODECARBON : Does not support country with ISO code {self._country_iso_code} "
f"Exception occured {e}")
self._region: Optional[
str] = region if region is None else region.lower()
super().__init__(*args, **kwargs)
def get_country_emissions(self, energy: Energy, geo: GeoMetadata) -> float:
"""
Computes emissions for a country on private infra,
given a quantity of power consumed by
using data for the mix of energy sources of that country.
:param energy: Mean power consumption of the process (kWh)
:param geo: Country and region metadata
:return: CO2 emissions in kg
"""
energy_mix = self._data_source.get_global_energy_mix_data()
if geo.country_iso_code not in energy_mix:
logger.warning(
f"We do not have data for {geo.country_iso_code}, using world average."
)
carbon_intensity_per_source = (
DataSource().get_carbon_intensity_per_source_data())
return (EmissionsPerKWh.from_g_per_kWh(
carbon_intensity_per_source.get("world_average")).kgs_per_kWh *
energy.kWh) # kgs
country_energy_mix: Dict = energy_mix[geo.country_iso_code]
emissions_per_kWh = self._global_energy_mix_to_emissions_rate(
country_energy_mix)
logger.debug(
f"We apply an energy mix of {emissions_per_kWh.kgs_per_kWh*1000:.0f}"
+ f" g.CO2eq/kWh for {geo.country_name}")
return emissions_per_kWh.kgs_per_kWh * energy.kWh # kgs
def _get_cpu_power_from_registry(self, cpu_model_raw: str) -> int:
cpu_power_df = DataSource().get_cpu_power_data()
cpu_matching = self._get_matching_cpu(cpu_model_raw, cpu_power_df)
if cpu_matching:
power = self._get_cpu_constant_power(cpu_matching, cpu_power_df)
return power
else:
return None
def __init__(
self,
project_name: str = "codecarbon",
measure_power_secs: int = 15,
output_dir: str = ".",
save_to_file: bool = True,
gpu_ids: Optional[List] = None,
):
"""
:param project_name: Project name for current experiment run, default name
as "codecarbon"
:param measure_power_secs: Interval (in seconds) to measure hardware power
usage, defaults to 15
:param output_dir: Directory path to which the experiment details are logged
in a CSV file called `emissions.csv`, defaults to current
directory
:param save_to_file: Indicates if the emission artifacts should be logged to a
file, defaults to True
:param gpu_ids: User-specified known gpu ids to track, defaults to None
"""
self._project_name: str = project_name
self._measure_power_secs: int = measure_power_secs
self._start_time: Optional[float] = None
self._last_measured_time: float = time.time()
self._output_dir: str = output_dir
self._total_energy: Energy = Energy.from_energy(kwh=0)
self._scheduler = BackgroundScheduler()
self._hardware = list()
if gpu.is_gpu_details_available():
logger.info("CODECARBON : Tracking Nvidia GPU via pynvml")
self._hardware.append(GPU.from_utils(gpu_ids))
if cpu.is_powergadget_available():
logger.info("CODECARBON : Tracking Intel CPU via Power Gadget")
self._hardware.append(
CPU.from_utils(self._output_dir, "intel_power_gadget"))
elif cpu.is_rapl_available():
logger.info("CODECARBON : Tracking Intel CPU via RAPL interface")
self._hardware.append(
CPU.from_utils(self._output_dir, "intel_rapl"))
# Run `self._measure_power` every `measure_power_secs` seconds in a background thread
self._scheduler.add_job(self._measure_power,
"interval",
seconds=measure_power_secs)
self._data_source = DataSource()
self._emissions: Emissions = Emissions(self._data_source)
self.persistence_objs: List[BaseOutput] = list()
if save_to_file:
self.persistence_objs.append(
FileOutput(os.path.join(self._output_dir, "emissions.csv")))
def _get_power_from_constant(self) -> int:
"""
Get CPU power from constant mode
:return: power in Watt
"""
cpu_info = cpuinfo.get_cpu_info()
if cpu_info:
model_raw = cpu_info.get("brand_raw", "")
model = parse_cpu_model(model_raw)
cpu_power_df = DataSource().get_cpu_power_data()
cpu_power_df_model = cpu_power_df[cpu_power_df["Name"] == model]
if len(cpu_power_df_model) > 0:
power = cpu_power_df_model["TDP"].tolist()[0]
return power
return None
def _global_energy_mix_to_emissions_rate(
energy_mix: Dict) -> EmissionsPerKWh:
"""
Convert a mix of electricity sources into emissions per kWh.
:param energy_mix: A dictionary that breaks down the electricity produced into
energy sources, with a total value. Format will vary, but must have keys for "total_TWh"
:return: an EmissionsPerKwh object representing the average emissions rate
in Kgs.CO2 / kWh
"""
# If we have the chance to have the carbon intensity for this country
if energy_mix.get("carbon_intensity"):
return EmissionsPerKWh.from_g_per_kWh(
energy_mix.get("carbon_intensity"))
# Else we compute it from the energy mix.
# Read carbon_intensity from the json data file.
carbon_intensity_per_source = (
DataSource().get_carbon_intensity_per_source_data())
carbon_intensity = 0
energy_sum = energy_mix["total_TWh"]
energy_sum_computed = 0
# Iterate through each source of energy in the country
for energy_type, energy_per_year in energy_mix.items():
if "_TWh" in energy_type:
# Compute the carbon intensity ratio of this source for this country
carbon_intensity_for_type = carbon_intensity_per_source.get(
energy_type[:-len("_TWh")])
if carbon_intensity_for_type: # to ignore "total_TWh"
carbon_intensity += (energy_per_year / energy_sum
) * carbon_intensity_for_type
energy_sum_computed += energy_per_year
# Sanity check
if energy_sum_computed != energy_sum:
logger.error(
f"We find {energy_sum_computed} TWh instead of {energy_sum} TWh for {energy_mix.get('official_name_en')}, using world average."
)
return EmissionsPerKWh.from_g_per_kWh(
carbon_intensity_per_source.get("world_average"))
return EmissionsPerKWh.from_g_per_kWh(carbon_intensity)
def __init__(
self,
*args,
country_iso_code: Optional[str] = _sentinel,
region: Optional[str] = _sentinel,
cloud_provider: Optional[str] = _sentinel,
cloud_region: Optional[str] = _sentinel,
country_2letter_iso_code: Optional[str] = _sentinel,
**kwargs,
):
"""
:param country_iso_code: 3 letter ISO Code of the country where the
experiment is being run
:param region: The provincial region, for example, California in the US.
Currently, this only affects calculations for the United States
and Canada
:param cloud_provider: The cloud provider specified for estimating emissions
intensity, defaults to None.
See https://github.com/mlco2/codecarbon/
blob/master/codecarbon/data/cloud/impact.csv
for a list of cloud providers
:param cloud_region: The region of the cloud data center, defaults to None.
See https://github.com/mlco2/codecarbon/
blob/master/codecarbon/data/cloud/impact.csv
for a list of cloud regions.
:param country_2letter_iso_code: For use with the CO2Signal emissions API.
See http://api.electricitymap.org/v3/zones for
a list of codes and their corresponding
locations.
"""
self._external_conf = get_hierarchical_config()
self._set_from_conf(cloud_provider, "cloud_provider")
self._set_from_conf(cloud_region, "cloud_region")
self._set_from_conf(country_2letter_iso_code,
"country_2letter_iso_code")
self._set_from_conf(country_iso_code, "country_iso_code")
self._set_from_conf(region, "region")
logger.info("offline tracker init")
if self._region is not None:
assert isinstance(self._region, str)
self._region: str = self._region.lower()
if self._cloud_provider:
if self._cloud_region is None:
logger.error("Cloud Region must be provided " +
" if cloud provider is set")
df = DataSource().get_cloud_emissions_data()
if (len(df.loc[(df["provider"] == self._cloud_provider)
& (df["region"] == self._cloud_region)]) == 0):
logger.error("Cloud Provider/Region "
f"{self._cloud_provider} {self._cloud_region} "
"not found in cloud emissions data.")
if self._country_iso_code:
try:
self._country_name: str = DataSource(
).get_global_energy_mix_data()[
self._country_iso_code]["country_name"]
except KeyError as e:
logger.error("Does not support country" +
f" with ISO code {self._country_iso_code} "
f"Exception occurred {e}")
if self._country_2letter_iso_code:
assert isinstance(self._country_2letter_iso_code, str)
self._country_2letter_iso_code: str = self._country_2letter_iso_code.upper(
)
super().__init__(*args, **kwargs)
def __init__(
self,
project_name: Optional[str] = _sentinel,
measure_power_secs: Optional[int] = _sentinel,
api_call_interval: Optional[int] = _sentinel,
api_endpoint: Optional[str] = _sentinel,
api_key: Optional[str] = _sentinel,
output_dir: Optional[str] = _sentinel,
output_file: Optional[str] = _sentinel,
save_to_file: Optional[bool] = _sentinel,
save_to_api: Optional[bool] = _sentinel,
save_to_logger: Optional[bool] = _sentinel,
logging_logger: Optional[LoggerOutput] = _sentinel,
gpu_ids: Optional[List] = _sentinel,
emissions_endpoint: Optional[str] = _sentinel,
experiment_id: Optional[str] = _sentinel,
co2_signal_api_token: Optional[str] = _sentinel,
tracking_mode: Optional[str] = _sentinel,
log_level: Optional[Union[int, str]] = _sentinel,
on_csv_write: Optional[str] = _sentinel,
logger_preamble: Optional[str] = _sentinel,
):
"""
:param project_name: Project name for current experiment run, default name
as "codecarbon"
:param measure_power_secs: Interval (in seconds) to measure hardware power
usage, defaults to 15
:param api_call_interval: Occurrence to wait before calling API :
-1 : only call api on flush() and at the end.
1 : at every measure
2 : every 2 measure, etc...
:param api_endpoint: Optional URL of Code Carbon API endpoint for sending
emissions data
:param api_key: API key for Code Carbon API, mandatory to use it !
:param output_dir: Directory path to which the experiment details are logged,
defaults to current directory
:param output_file: Name of output CSV file, defaults to `emissions.csv`
:param save_to_file: Indicates if the emission artifacts should be logged to a
file, defaults to True
:param save_to_api: Indicates if the emission artifacts should be send to the
CodeCarbon API, defaults to False
:param save_to_logger: Indicates if the emission artifacts should be written
to a dedicated logger, defaults to False
:param logging_logger: LoggerOutput object encapsulating a logging.logger
or a Google Cloud logger
:param gpu_ids: User-specified known gpu ids to track, defaults to None
:param emissions_endpoint: Optional URL of http endpoint for sending emissions
data
:param experiment_id: Id of the experiment
:param co2_signal_api_token: API token for co2signal.com (requires sign-up for
free beta)
:param tracking_mode: One of "process" or "machine" in order to measure the
power consumptions due to the entire machine or try and
isolate the tracked processe's in isolation.
Defaults to "machine"
:param log_level: Global codecarbon log level. Accepts one of:
{"debug", "info", "warning", "error", "critical"}.
Defaults to "info".
:param on_csv_write: "append" or "update". Whether to always append a new line
to the csv when writing or to update the existing `run_id`
row (useful when calling`tracker.flush()` manually).
Accepts one of "append" or "update".
:param logger_preamble: String to systematically include in the logger's.
messages. Defaults to "".
"""
# logger.info("base tracker init")
self._external_conf = get_hierarchical_config()
self._set_from_conf(api_call_interval, "api_call_interval", 8, int)
self._set_from_conf(api_endpoint, "api_endpoint",
"https://api.codecarbon.io")
self._set_from_conf(co2_signal_api_token, "co2_signal_api_token")
self._set_from_conf(emissions_endpoint, "emissions_endpoint")
self._set_from_conf(gpu_ids, "gpu_ids")
self._set_from_conf(log_level, "log_level", "info")
self._set_from_conf(measure_power_secs, "measure_power_secs", 15, int)
self._set_from_conf(output_dir, "output_dir", ".")
self._set_from_conf(output_file, "output_file", "emissions.csv")
self._set_from_conf(project_name, "project_name", "codecarbon")
self._set_from_conf(save_to_api, "save_to_api", False, bool)
self._set_from_conf(save_to_file, "save_to_file", True, bool)
self._set_from_conf(save_to_logger, "save_to_logger", False, bool)
self._set_from_conf(logging_logger, "logging_logger")
self._set_from_conf(tracking_mode, "tracking_mode", "machine")
self._set_from_conf(on_csv_write, "on_csv_write", "append")
self._set_from_conf(logger_preamble, "logger_preamble", "")
assert self._tracking_mode in ["machine", "process"]
set_logger_level(self._log_level)
set_logger_format(self._logger_preamble)
self._start_time: Optional[float] = None
self._last_measured_time: float = time.time()
self._total_energy: Energy = Energy.from_energy(kWh=0)
self._total_cpu_energy: Energy = Energy.from_energy(kWh=0)
self._total_gpu_energy: Energy = Energy.from_energy(kWh=0)
self._total_ram_energy: Energy = Energy.from_energy(kWh=0)
self._cpu_power: Power = Power.from_watts(watts=0)
self._gpu_power: Power = Power.from_watts(watts=0)
self._ram_power: Power = Power.from_watts(watts=0)
self._cc_api__out = None
self._measure_occurrence: int = 0
self._cloud = None
self._previous_emissions = None
self._conf["os"] = platform.platform()
self._conf["python_version"] = platform.python_version()
self._conf["cpu_count"] = count_cpus()
self._geo = None
if isinstance(self._gpu_ids, str):
self._gpu_ids: List[int] = parse_gpu_ids(self._gpu_ids)
self._conf["gpu_ids"] = self._gpu_ids
self._conf["gpu_count"] = len(self._gpu_ids)
logger.info("[setup] RAM Tracking...")
ram = RAM(tracking_mode=self._tracking_mode)
self._conf["ram_total_size"] = ram.machine_memory_GB
self._hardware: List[Union[RAM, CPU, GPU]] = [ram]
# Hardware detection
logger.info("[setup] GPU Tracking...")
if gpu.is_gpu_details_available():
logger.info("Tracking Nvidia GPU via pynvml")
self._hardware.append(GPU.from_utils(self._gpu_ids))
gpu_names = [n["name"] for n in gpu.get_gpu_static_info()]
gpu_names_dict = Counter(gpu_names)
self._conf["gpu_model"] = "".join(
[f"{i} x {name}" for name, i in gpu_names_dict.items()])
self._conf["gpu_count"] = len(gpu.get_gpu_static_info())
else:
logger.info("No GPU found.")
logger.info("[setup] CPU Tracking...")
if cpu.is_powergadget_available():
logger.info("Tracking Intel CPU via Power Gadget")
hardware = CPU.from_utils(self._output_dir, "intel_power_gadget")
self._hardware.append(hardware)
self._conf["cpu_model"] = hardware.get_model()
elif cpu.is_rapl_available():
logger.info("Tracking Intel CPU via RAPL interface")
hardware = CPU.from_utils(self._output_dir, "intel_rapl")
self._hardware.append(hardware)
self._conf["cpu_model"] = hardware.get_model()
else:
logger.warning(
"No CPU tracking mode found. Falling back on CPU constant mode."
)
tdp = cpu.TDP()
power = tdp.tdp
model = tdp.model
logger.info(f"CPU Model on constant consumption mode: {model}")
self._conf["cpu_model"] = model
if tdp:
hardware = CPU.from_utils(self._output_dir, "constant", model,
power)
self._hardware.append(hardware)
else:
logger.warning("Failed to match CPU TDP constant. " +
"Falling back on a global constant.")
hardware = CPU.from_utils(self._output_dir, "constant")
self._hardware.append(hardware)
self._conf["hardware"] = list(
map(lambda x: x.description(), self._hardware))
logger.info(">>> Tracker's metadata:")
logger.info(f" Platform system: {self._conf.get('os')}")
logger.info(f" Python version: {self._conf.get('python_version')}")
logger.info(
f" Available RAM : {self._conf.get('ram_total_size'):.3f} GB")
logger.info(f" CPU count: {self._conf.get('cpu_count')}")
logger.info(f" CPU model: {self._conf.get('cpu_model')}")
logger.info(f" GPU count: {self._conf.get('gpu_count')}")
logger.info(f" GPU model: {self._conf.get('gpu_model')}")
# Run `self._measure_power` every `measure_power_secs` seconds in a
# background thread
self._scheduler = PeriodicScheduler(
function=self._measure_power_and_energy,
interval=self._measure_power_secs,
)
self._data_source = DataSource()
cloud: CloudMetadata = self._get_cloud_metadata()
if cloud.is_on_private_infra:
self._geo = self._get_geo_metadata()
self._conf["longitude"] = self._geo.longitude
self._conf["latitude"] = self._geo.latitude
self._conf["region"] = cloud.region
self._conf["provider"] = cloud.provider
else:
self._conf["region"] = cloud.region
self._conf["provider"] = cloud.provider
self._emissions: Emissions = Emissions(self._data_source,
self._co2_signal_api_token)
self.persistence_objs: List[BaseOutput] = list()
if self._save_to_file:
self.persistence_objs.append(
FileOutput(
os.path.join(self._output_dir, self._output_file),
self._on_csv_write,
))
if self._save_to_logger:
self.persistence_objs.append(self._logging_logger)
if self._emissions_endpoint:
self.persistence_objs.append(HTTPOutput(emissions_endpoint))
if self._save_to_api:
experiment_id = self._set_from_conf(
experiment_id, "experiment_id",
"5b0fa12a-3dd7-45bb-9766-cc326314d9f1")
self._cc_api__out = CodeCarbonAPIOutput(
endpoint_url=self._api_endpoint,
experiment_id=experiment_id,
api_key=api_key,
conf=self._conf,
)
self.run_id = self._cc_api__out.run_id
self.persistence_objs.append(self._cc_api__out)
else:
self.run_id = uuid.uuid4()
def get_test_data_source() -> DataSource:
return DataSource()
def __init__(self):
self._data_source = DataSource()
self._emissions = Emissions(self._data_source)
def __init__(
self,
project_name: str = "codecarbon",
measure_power_secs: int = 15,
output_dir: str = ".",
save_to_file: bool = True,
gpu_ids: Optional[List] = None,
emissions_endpoint: Optional[str] = None,
co2_signal_api_token: Optional[str] = None,
):
"""
:param project_name: Project name for current experiment run, default name
as "codecarbon"
:param measure_power_secs: Interval (in seconds) to measure hardware power
usage, defaults to 15
:param output_dir: Directory path to which the experiment details are logged
in a CSV file called `emissions.csv`, defaults to current
directory
:param save_to_file: Indicates if the emission artifacts should be logged to a
file, defaults to True
:param gpu_ids: User-specified known gpu ids to track, defaults to None
:param emissions_endpoint: Optional URL of http endpoint for sending emissions data
:param co2_signal_api_token: API token for co2signal.com (requires sign-up for free beta)
"""
self._project_name: str = project_name
self._measure_power_secs: int = measure_power_secs
self._start_time: Optional[float] = None
self._last_measured_time: float = time.time()
self._output_dir: str = output_dir
self._total_energy: Energy = Energy.from_energy(kwh=0)
self._scheduler = BackgroundScheduler()
self._hardware = list()
if gpu.is_gpu_details_available():
logger.info("CODECARBON : Tracking Nvidia GPU via pynvml")
self._hardware.append(GPU.from_utils(gpu_ids))
if cpu.is_powergadget_available():
logger.info("CODECARBON : Tracking Intel CPU via Power Gadget")
self._hardware.append(
CPU.from_utils(self._output_dir, "intel_power_gadget"))
elif cpu.is_rapl_available():
logger.info("CODECARBON : Tracking Intel CPU via RAPL interface")
self._hardware.append(
CPU.from_utils(self._output_dir, "intel_rapl"))
# Print warning if no supported hardware is found'
if not self._hardware:
logger.warning(
"CODECARBON : No CPU/GPU tracking mode found. This "
"may be due to your code running on Windows WSL, or due to "
"unsupported hardware (see "
"https://github.com/mlco2/codecarbon#infrastructure-support)")
# Run `self._measure_power` every `measure_power_secs` seconds in a background thread
self._scheduler.add_job(self._measure_power,
"interval",
seconds=measure_power_secs)
self._data_source = DataSource()
self._emissions: Emissions = Emissions(self._data_source)
self.persistence_objs: List[BaseOutput] = list()
if save_to_file:
self.persistence_objs.append(
FileOutput(os.path.join(self._output_dir, "emissions.csv")))
if emissions_endpoint:
self.persistence_objs.append(HTTPOutput(emissions_endpoint))
if co2_signal_api_token:
co2_signal.CO2_SIGNAL_API_TOKEN = co2_signal_api_token
def test_get_carbon_intensity_per_source_data(self):
# pytest tests/test_emissions.py::TestEmissions::test_get_carbon_intensity_per_source_data
carbon_intensity = DataSource().get_carbon_intensity_per_source_data()
self.assertEqual(len(carbon_intensity.keys()), 21)
self.assertGreater(carbon_intensity["coal"], 800)
self.assertLess(carbon_intensity["wind"], 80)