def stop(self) -> Optional[float]:
"""
Stops tracking the experiment
:return: CO2 emissions in kgs
"""
if self._start_time is None:
logger.error("Need to first start the tracker")
return None
if self._scheduler:
self._scheduler.stop()
# Run to calculate the power used from last
# scheduled measurement to shutdown
self._measure_power_and_energy()
emissions_data = self._prepare_emissions_data()
for persistence in self.persistence_objs:
if isinstance(persistence, CodeCarbonAPIOutput):
emissions_data = self._prepare_emissions_data(delta=True)
persistence.out(emissions_data)
self.final_emissions_data = emissions_data
self.final_emissions = emissions_data.emissions
return emissions_data.emissions
def _fetch_rapl_files(self):
"""
Fetches RAPL files from the RAPL directory
"""
# consider files like `intel-rapl:$i`
files = list(filter(lambda x: ":" in x,
os.listdir(self._lin_rapl_dir)))
i = 0
for file in files:
path = os.path.join(self._lin_rapl_dir, file, "name")
with open(path) as f:
name = f.read().strip()
# Fake the name used by Power Gadget
if "package" in name:
name = f"Processor Energy Delta_{i}(kWh)"
i += 1
rapl_file = os.path.join(self._lin_rapl_dir, file, "energy_uj")
try:
# Try to read the file to be sure we can
with open(rapl_file, "r") as f:
_ = float(f.read())
self._rapl_files.append(RAPLFile(name=name,
path=rapl_file))
logger.debug(
f"We will read Intel RAPL files at {rapl_file}")
except PermissionError as e:
logger.error(
"Unable to read Intel RAPL files for CPU power, we will use a constant for your CPU power."
+
" Please view https://github.com/mlco2/codecarbon/issues/244"
+ f" for workarounds : {e}")
return
def _log_error(self, url, payload, response):
logger.error(
f"ApiClient Error when calling the API on {url} with : {json.dumps(payload)}"
)
logger.error(
f"ApiClient API return http code {response.status_code} and answer : {response.text}"
)
def get_private_infra_emissions(self, energy: Energy,
geo: GeoMetadata) -> float:
"""
Computes emissions for private infra
:param energy: Mean power consumption of the process (kWh)
:param geo: Country and region metadata
:return: CO2 emissions in kg
"""
if self._co2_signal_api_token:
try:
return co2_signal.get_emissions(energy, geo,
self._co2_signal_api_token)
except Exception as e:
logger.error("co2_signal.get_emissions: " + str(e) +
" >>> Using CodeCarbon's data.")
compute_with_regional_data: bool = (geo.region is not None) and (
geo.country_iso_code.upper() in ["USA", "CAN"])
if compute_with_regional_data:
try:
return self.get_region_emissions(energy, geo)
except Exception as e:
logger.error(e, exc_info=True)
logger.warning("Regional emissions retrieval failed." +
" Falling back on country emissions.")
return self.get_country_emissions(energy, geo)
def out(self, data: EmissionsData):
try:
payload = dataclasses.asdict(data)
payload["user"] = getpass.getuser()
resp = requests.post(self.endpoint_url, json=payload, timeout=10)
if resp.status_code != 201:
logger.warning(
"HTTP Output returned an unexpected status code: ",
resp,
)
except Exception as e:
logger.error(e, exc_info=True)
def add_emission(self, carbon_emission: dict):
assert self.experiment_id is not None
self._previous_call = time.time()
if self.run_id is None:
# TODO : raise an Exception ?
logger.debug(
"ApiClient.add_emission need a run_id : the initial call may "
+ "have failed. Retrying..."
)
self._create_run(self.experiment_id)
if self.run_id is None:
logger.error(
"ApiClient.add_emission still no run_id, aborting for this time !"
)
return False
if carbon_emission["duration"] < 1:
logger.warning(
"ApiClient : emissions not sent because of a duration smaller than 1."
)
return False
emission = EmissionCreate(
timestamp=get_datetime_with_timezone(),
run_id=self.run_id,
duration=int(carbon_emission["duration"]),
emissions_sum=carbon_emission["emissions"],
emissions_rate=carbon_emission["emissions_rate"],
cpu_power=carbon_emission["cpu_power"],
gpu_power=carbon_emission["gpu_power"],
ram_power=carbon_emission["ram_power"],
cpu_energy=carbon_emission["cpu_energy"],
gpu_energy=carbon_emission["gpu_energy"],
ram_energy=carbon_emission["ram_energy"],
energy_consumed=carbon_emission["energy_consumed"],
)
try:
payload = dataclasses.asdict(emission)
url = self.url + "/emission"
r = requests.post(url=url, json=payload, timeout=2)
if r.status_code != 201:
self._log_error(url, payload, r)
return False
logger.debug(f"ApiClient - Successful upload emission {payload} to {url}")
except Exception as e:
logger.error(e, exc_info=True)
return False
return True
def _global_energy_mix_to_emissions_rate(
energy_mix: Dict) -> EmissionsPerKWh:
"""
Convert a mix of electricity sources into emissions per kWh.
:param energy_mix: A dictionary that breaks down the electricity produced into
energy sources, with a total value. Format will vary, but must have keys for "total_TWh"
:return: an EmissionsPerKwh object representing the average emissions rate
in Kgs.CO2 / kWh
"""
# If we have the chance to have the carbon intensity for this country
if energy_mix.get("carbon_intensity"):
return EmissionsPerKWh.from_g_per_kWh(
energy_mix.get("carbon_intensity"))
# Else we compute it from the energy mix.
# Read carbon_intensity from the json data file.
carbon_intensity_per_source = (
DataSource().get_carbon_intensity_per_source_data())
carbon_intensity = 0
energy_sum = energy_mix["total_TWh"]
energy_sum_computed = 0
# Iterate through each source of energy in the country
for energy_type, energy_per_year in energy_mix.items():
if "_TWh" in energy_type:
# Compute the carbon intensity ratio of this source for this country
carbon_intensity_for_type = carbon_intensity_per_source.get(
energy_type[:-len("_TWh")])
if carbon_intensity_for_type: # to ignore "total_TWh"
carbon_intensity += (energy_per_year / energy_sum
) * carbon_intensity_for_type
energy_sum_computed += energy_per_year
# Sanity check
if energy_sum_computed != energy_sum:
logger.error(
f"We find {energy_sum_computed} TWh instead of {energy_sum} TWh for {energy_mix.get('official_name_en')}, using world average."
)
return EmissionsPerKWh.from_g_per_kWh(
carbon_intensity_per_source.get("world_average"))
return EmissionsPerKWh.from_g_per_kWh(carbon_intensity)
def _create_run(self, experiment_id):
"""
Create the experiment for project_id
# TODO : Allow to give an existing experiment_id
"""
if self.experiment_id is None:
# TODO : raise an Exception ?
logger.error("ApiClient FATAL The API _create_run needs an experiment_id !")
return None
try:
run = RunCreate(
timestamp=get_datetime_with_timezone(),
experiment_id=experiment_id,
os=self.conf.get("os"),
python_version=self.conf.get("python_version"),
cpu_count=self.conf.get("cpu_count"),
cpu_model=self.conf.get("cpu_model"),
gpu_count=self.conf.get("gpu_count"),
gpu_model=self.conf.get("gpu_model"),
longitude=self.conf.get("longitude"),
latitude=self.conf.get("latitude"),
region=self.conf.get("region"),
provider=self.conf.get("provider"),
ram_total_size=self.conf.get("ram_total_size"),
tracking_mode=self.conf.get("tracking_mode"),
)
payload = dataclasses.asdict(run)
url = self.url + "/run"
r = requests.post(url=url, json=payload, timeout=2)
if r.status_code != 201:
self._log_error(url, payload, r)
return None
self.run_id = r.json()["id"]
logger.info(
"ApiClient Successfully registered your run on the API.\n\n"
+ f"Run ID: {self.run_id}\n"
+ f"Experiment ID: {self.experiment_id}\n"
)
return self.run_id
except Exception as e:
logger.error(e, exc_info=True)
def flush(self) -> Optional[float]:
"""
Write emission to disk or call the API depending on the configuration
but keep running the experiment.
:return: CO2 emissions in kgs
"""
if self._start_time is None:
logger.error("Need to first start the tracker")
return None
# Run to calculate the power used from last
# scheduled measurement to shutdown
self._measure_power_and_energy()
emissions_data = self._prepare_emissions_data()
for persistence in self.persistence_objs:
if isinstance(persistence, CodeCarbonAPIOutput):
emissions_data = self._prepare_emissions_data(delta=True)
persistence.out(emissions_data)
return emissions_data.emissions
def __init__(
self,
*args,
country_iso_code: Optional[str] = _sentinel,
region: Optional[str] = _sentinel,
cloud_provider: Optional[str] = _sentinel,
cloud_region: Optional[str] = _sentinel,
country_2letter_iso_code: Optional[str] = _sentinel,
**kwargs,
):
"""
:param country_iso_code: 3 letter ISO Code of the country where the
experiment is being run
:param region: The provincial region, for example, California in the US.
Currently, this only affects calculations for the United States
and Canada
:param cloud_provider: The cloud provider specified for estimating emissions
intensity, defaults to None.
See https://github.com/mlco2/codecarbon/
blob/master/codecarbon/data/cloud/impact.csv
for a list of cloud providers
:param cloud_region: The region of the cloud data center, defaults to None.
See https://github.com/mlco2/codecarbon/
blob/master/codecarbon/data/cloud/impact.csv
for a list of cloud regions.
:param country_2letter_iso_code: For use with the CO2Signal emissions API.
See http://api.electricitymap.org/v3/zones for
a list of codes and their corresponding
locations.
"""
self._external_conf = get_hierarchical_config()
self._set_from_conf(cloud_provider, "cloud_provider")
self._set_from_conf(cloud_region, "cloud_region")
self._set_from_conf(country_2letter_iso_code,
"country_2letter_iso_code")
self._set_from_conf(country_iso_code, "country_iso_code")
self._set_from_conf(region, "region")
logger.info("offline tracker init")
if self._region is not None:
assert isinstance(self._region, str)
self._region: str = self._region.lower()
if self._cloud_provider:
if self._cloud_region is None:
logger.error("Cloud Region must be provided " +
" if cloud provider is set")
df = DataSource().get_cloud_emissions_data()
if (len(df.loc[(df["provider"] == self._cloud_provider)
& (df["region"] == self._cloud_region)]) == 0):
logger.error("Cloud Provider/Region "
f"{self._cloud_provider} {self._cloud_region} "
"not found in cloud emissions data.")
if self._country_iso_code:
try:
self._country_name: str = DataSource(
).get_global_energy_mix_data()[
self._country_iso_code]["country_name"]
except KeyError as e:
logger.error("Does not support country" +
f" with ISO code {self._country_iso_code} "
f"Exception occurred {e}")
if self._country_2letter_iso_code:
assert isinstance(self._country_2letter_iso_code, str)
self._country_2letter_iso_code: str = self._country_2letter_iso_code.upper(
)
super().__init__(*args, **kwargs)
def _measure_power_and_energy(self) -> None:
"""
A function that is periodically run by the `BackgroundScheduler`
every `self._measure_power_secs` seconds.
:return: None
"""
last_duration = time.time() - self._last_measured_time
warning_duration = self._measure_power_secs * 3
if last_duration > warning_duration:
warn_msg = ("Background scheduler didn't run for a long period" +
" (%ds), results might be inaccurate")
logger.warning(warn_msg, last_duration)
for hardware in self._hardware:
h_time = time.time()
# Compute last_duration again for more accuracy
last_duration = time.time() - self._last_measured_time
power, energy = hardware.measure_power_and_energy(
last_duration=last_duration)
self._total_energy += energy
if isinstance(hardware, CPU):
self._total_cpu_energy += energy
self._cpu_power = power
logger.info(
f"Energy consumed for all CPUs : {self._total_cpu_energy.kWh:.6f} kWh"
+ f". All CPUs Power : {self._cpu_power.W} W")
elif isinstance(hardware, GPU):
self._total_gpu_energy += energy
self._gpu_power = power
logger.info(
f"Energy consumed for all GPUs : {self._total_gpu_energy.kWh:.6f} kWh"
+ f". All GPUs Power : {self._gpu_power.W} W")
elif isinstance(hardware, RAM):
self._total_ram_energy += energy
self._ram_power = power
logger.info(
f"Energy consumed for RAM : {self._total_ram_energy.kWh:.6f} kWh"
+ f". RAM Power : {self._ram_power.W} W")
else:
logger.error(
f"Unknown hardware type: {hardware} ({type(hardware)})")
h_time = time.time() - h_time
logger.debug(
f"{hardware.__class__.__name__} : {hardware.total_power().W:,.2f} "
+
f"W during {last_duration:,.2f} s [measurement time: {h_time:,.4f}]"
)
logger.info(
f"{self._total_energy.kWh:.6f} kWh of electricity used since the begining."
)
self._last_measured_time = time.time()
self._measure_occurrence += 1
if self._cc_api__out is not None and self._api_call_interval != -1:
if self._measure_occurrence >= self._api_call_interval:
emissions = self._prepare_emissions_data(delta=True)
logger.info(
f"{emissions.emissions_rate:.6f} g.CO2eq/s mean an estimation of "
+
f"{emissions.emissions_rate*3600*24*365/1000:,} kg.CO2eq/year"
)
self._cc_api__out.out(emissions)
self._measure_occurrence = 0
logger.debug(
f"last_duration={last_duration}\n------------------------")
def out(self, data: EmissionsData):
try:
payload = dataclasses.asdict(data)
self.logger.log_struct(payload, severity=self.logging_severity)
except Exception as e:
logger.error(e, exc_info=True)
def out(self, data: EmissionsData):
try:
payload = dataclasses.asdict(data)
self.logger.log(self.logging_severity, msg=json.dumps(payload))
except Exception as e:
logger.error(e, exc_info=True)
def out(self, data: EmissionsData):
try:
self.api.add_emission(dataclasses.asdict(data))
except Exception as e:
logger.error(e, exc_info=True)