def test_read_confs_and_parse_envs(self):
global_conf = dedent(
"""\
[codecarbon]
no_overwrite=path/to/somewhere
local_overwrite=ERROR:not overwritten
syntax_test_key= no/space= problem2
env_overwrite=ERROR:not overwritten
"""
)
local_conf = dedent(
"""\
[codecarbon]
local_overwrite=SUCCESS:overwritten
local_new_key=cool value
env_overwrite=ERROR:not overwritten
"""
)
with patch(
"builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf)
):
conf = dict(get_hierarchical_config())
target = {
"no_overwrite": "path/to/somewhere",
"local_overwrite": "SUCCESS:overwritten",
"env_overwrite": "SUCCESS:overwritten",
"syntax_test_key": "no/space= problem2",
"local_new_key": "cool value",
"env_new_key": "cool value",
}
self.assertDictEqual(conf, target)
def test_empty_conf(self):
global_conf = ""
local_conf = ""
with patch(
"builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf)
):
conf = dict(get_hierarchical_config())
target = {}
self.assertDictEqual(conf, target)
def __init__(
self,
*args,
country_iso_code: Optional[str] = _sentinel,
region: Optional[str] = _sentinel,
cloud_provider: Optional[str] = _sentinel,
cloud_region: Optional[str] = _sentinel,
country_2letter_iso_code: Optional[str] = _sentinel,
**kwargs,
):
"""
:param country_iso_code: 3 letter ISO Code of the country where the
experiment is being run
:param region: The provincial region, for example, California in the US.
Currently, this only affects calculations for the United States
and Canada
:param cloud_provider: The cloud provider specified for estimating emissions
intensity, defaults to None.
See https://github.com/mlco2/codecarbon/
blob/master/codecarbon/data/cloud/impact.csv
for a list of cloud providers
:param cloud_region: The region of the cloud data center, defaults to None.
See https://github.com/mlco2/codecarbon/
blob/master/codecarbon/data/cloud/impact.csv
for a list of cloud regions.
:param country_2letter_iso_code: For use with the CO2Signal emissions API.
See http://api.electricitymap.org/v3/zones for
a list of codes and their corresponding
locations.
"""
self._external_conf = get_hierarchical_config()
self._set_from_conf(cloud_provider, "cloud_provider")
self._set_from_conf(cloud_region, "cloud_region")
self._set_from_conf(country_2letter_iso_code,
"country_2letter_iso_code")
self._set_from_conf(country_iso_code, "country_iso_code")
self._set_from_conf(region, "region")
logger.info("offline tracker init")
if self._region is not None:
assert isinstance(self._region, str)
self._region: str = self._region.lower()
if self._cloud_provider:
if self._cloud_region is None:
logger.error("Cloud Region must be provided " +
" if cloud provider is set")
df = DataSource().get_cloud_emissions_data()
if (len(df.loc[(df["provider"] == self._cloud_provider)
& (df["region"] == self._cloud_region)]) == 0):
logger.error("Cloud Provider/Region "
f"{self._cloud_provider} {self._cloud_region} "
"not found in cloud emissions data.")
if self._country_iso_code:
try:
self._country_name: str = DataSource(
).get_global_energy_mix_data()[
self._country_iso_code]["country_name"]
except KeyError as e:
logger.error("Does not support country" +
f" with ISO code {self._country_iso_code} "
f"Exception occurred {e}")
if self._country_2letter_iso_code:
assert isinstance(self._country_2letter_iso_code, str)
self._country_2letter_iso_code: str = self._country_2letter_iso_code.upper(
)
super().__init__(*args, **kwargs)
def __init__(
self,
project_name: Optional[str] = _sentinel,
measure_power_secs: Optional[int] = _sentinel,
api_call_interval: Optional[int] = _sentinel,
api_endpoint: Optional[str] = _sentinel,
api_key: Optional[str] = _sentinel,
output_dir: Optional[str] = _sentinel,
output_file: Optional[str] = _sentinel,
save_to_file: Optional[bool] = _sentinel,
save_to_api: Optional[bool] = _sentinel,
save_to_logger: Optional[bool] = _sentinel,
logging_logger: Optional[LoggerOutput] = _sentinel,
gpu_ids: Optional[List] = _sentinel,
emissions_endpoint: Optional[str] = _sentinel,
experiment_id: Optional[str] = _sentinel,
co2_signal_api_token: Optional[str] = _sentinel,
tracking_mode: Optional[str] = _sentinel,
log_level: Optional[Union[int, str]] = _sentinel,
on_csv_write: Optional[str] = _sentinel,
logger_preamble: Optional[str] = _sentinel,
):
"""
:param project_name: Project name for current experiment run, default name
as "codecarbon"
:param measure_power_secs: Interval (in seconds) to measure hardware power
usage, defaults to 15
:param api_call_interval: Occurrence to wait before calling API :
-1 : only call api on flush() and at the end.
1 : at every measure
2 : every 2 measure, etc...
:param api_endpoint: Optional URL of Code Carbon API endpoint for sending
emissions data
:param api_key: API key for Code Carbon API, mandatory to use it !
:param output_dir: Directory path to which the experiment details are logged,
defaults to current directory
:param output_file: Name of output CSV file, defaults to `emissions.csv`
:param save_to_file: Indicates if the emission artifacts should be logged to a
file, defaults to True
:param save_to_api: Indicates if the emission artifacts should be send to the
CodeCarbon API, defaults to False
:param save_to_logger: Indicates if the emission artifacts should be written
to a dedicated logger, defaults to False
:param logging_logger: LoggerOutput object encapsulating a logging.logger
or a Google Cloud logger
:param gpu_ids: User-specified known gpu ids to track, defaults to None
:param emissions_endpoint: Optional URL of http endpoint for sending emissions
data
:param experiment_id: Id of the experiment
:param co2_signal_api_token: API token for co2signal.com (requires sign-up for
free beta)
:param tracking_mode: One of "process" or "machine" in order to measure the
power consumptions due to the entire machine or try and
isolate the tracked processe's in isolation.
Defaults to "machine"
:param log_level: Global codecarbon log level. Accepts one of:
{"debug", "info", "warning", "error", "critical"}.
Defaults to "info".
:param on_csv_write: "append" or "update". Whether to always append a new line
to the csv when writing or to update the existing `run_id`
row (useful when calling`tracker.flush()` manually).
Accepts one of "append" or "update".
:param logger_preamble: String to systematically include in the logger's.
messages. Defaults to "".
"""
# logger.info("base tracker init")
self._external_conf = get_hierarchical_config()
self._set_from_conf(api_call_interval, "api_call_interval", 8, int)
self._set_from_conf(api_endpoint, "api_endpoint",
"https://api.codecarbon.io")
self._set_from_conf(co2_signal_api_token, "co2_signal_api_token")
self._set_from_conf(emissions_endpoint, "emissions_endpoint")
self._set_from_conf(gpu_ids, "gpu_ids")
self._set_from_conf(log_level, "log_level", "info")
self._set_from_conf(measure_power_secs, "measure_power_secs", 15, int)
self._set_from_conf(output_dir, "output_dir", ".")
self._set_from_conf(output_file, "output_file", "emissions.csv")
self._set_from_conf(project_name, "project_name", "codecarbon")
self._set_from_conf(save_to_api, "save_to_api", False, bool)
self._set_from_conf(save_to_file, "save_to_file", True, bool)
self._set_from_conf(save_to_logger, "save_to_logger", False, bool)
self._set_from_conf(logging_logger, "logging_logger")
self._set_from_conf(tracking_mode, "tracking_mode", "machine")
self._set_from_conf(on_csv_write, "on_csv_write", "append")
self._set_from_conf(logger_preamble, "logger_preamble", "")
assert self._tracking_mode in ["machine", "process"]
set_logger_level(self._log_level)
set_logger_format(self._logger_preamble)
self._start_time: Optional[float] = None
self._last_measured_time: float = time.time()
self._total_energy: Energy = Energy.from_energy(kWh=0)
self._total_cpu_energy: Energy = Energy.from_energy(kWh=0)
self._total_gpu_energy: Energy = Energy.from_energy(kWh=0)
self._total_ram_energy: Energy = Energy.from_energy(kWh=0)
self._cpu_power: Power = Power.from_watts(watts=0)
self._gpu_power: Power = Power.from_watts(watts=0)
self._ram_power: Power = Power.from_watts(watts=0)
self._cc_api__out = None
self._measure_occurrence: int = 0
self._cloud = None
self._previous_emissions = None
self._conf["os"] = platform.platform()
self._conf["python_version"] = platform.python_version()
self._conf["cpu_count"] = count_cpus()
self._geo = None
if isinstance(self._gpu_ids, str):
self._gpu_ids: List[int] = parse_gpu_ids(self._gpu_ids)
self._conf["gpu_ids"] = self._gpu_ids
self._conf["gpu_count"] = len(self._gpu_ids)
logger.info("[setup] RAM Tracking...")
ram = RAM(tracking_mode=self._tracking_mode)
self._conf["ram_total_size"] = ram.machine_memory_GB
self._hardware: List[Union[RAM, CPU, GPU]] = [ram]
# Hardware detection
logger.info("[setup] GPU Tracking...")
if gpu.is_gpu_details_available():
logger.info("Tracking Nvidia GPU via pynvml")
self._hardware.append(GPU.from_utils(self._gpu_ids))
gpu_names = [n["name"] for n in gpu.get_gpu_static_info()]
gpu_names_dict = Counter(gpu_names)
self._conf["gpu_model"] = "".join(
[f"{i} x {name}" for name, i in gpu_names_dict.items()])
self._conf["gpu_count"] = len(gpu.get_gpu_static_info())
else:
logger.info("No GPU found.")
logger.info("[setup] CPU Tracking...")
if cpu.is_powergadget_available():
logger.info("Tracking Intel CPU via Power Gadget")
hardware = CPU.from_utils(self._output_dir, "intel_power_gadget")
self._hardware.append(hardware)
self._conf["cpu_model"] = hardware.get_model()
elif cpu.is_rapl_available():
logger.info("Tracking Intel CPU via RAPL interface")
hardware = CPU.from_utils(self._output_dir, "intel_rapl")
self._hardware.append(hardware)
self._conf["cpu_model"] = hardware.get_model()
else:
logger.warning(
"No CPU tracking mode found. Falling back on CPU constant mode."
)
tdp = cpu.TDP()
power = tdp.tdp
model = tdp.model
logger.info(f"CPU Model on constant consumption mode: {model}")
self._conf["cpu_model"] = model
if tdp:
hardware = CPU.from_utils(self._output_dir, "constant", model,
power)
self._hardware.append(hardware)
else:
logger.warning("Failed to match CPU TDP constant. " +
"Falling back on a global constant.")
hardware = CPU.from_utils(self._output_dir, "constant")
self._hardware.append(hardware)
self._conf["hardware"] = list(
map(lambda x: x.description(), self._hardware))
logger.info(">>> Tracker's metadata:")
logger.info(f" Platform system: {self._conf.get('os')}")
logger.info(f" Python version: {self._conf.get('python_version')}")
logger.info(
f" Available RAM : {self._conf.get('ram_total_size'):.3f} GB")
logger.info(f" CPU count: {self._conf.get('cpu_count')}")
logger.info(f" CPU model: {self._conf.get('cpu_model')}")
logger.info(f" GPU count: {self._conf.get('gpu_count')}")
logger.info(f" GPU model: {self._conf.get('gpu_model')}")
# Run `self._measure_power` every `measure_power_secs` seconds in a
# background thread
self._scheduler = PeriodicScheduler(
function=self._measure_power_and_energy,
interval=self._measure_power_secs,
)
self._data_source = DataSource()
cloud: CloudMetadata = self._get_cloud_metadata()
if cloud.is_on_private_infra:
self._geo = self._get_geo_metadata()
self._conf["longitude"] = self._geo.longitude
self._conf["latitude"] = self._geo.latitude
self._conf["region"] = cloud.region
self._conf["provider"] = cloud.provider
else:
self._conf["region"] = cloud.region
self._conf["provider"] = cloud.provider
self._emissions: Emissions = Emissions(self._data_source,
self._co2_signal_api_token)
self.persistence_objs: List[BaseOutput] = list()
if self._save_to_file:
self.persistence_objs.append(
FileOutput(
os.path.join(self._output_dir, self._output_file),
self._on_csv_write,
))
if self._save_to_logger:
self.persistence_objs.append(self._logging_logger)
if self._emissions_endpoint:
self.persistence_objs.append(HTTPOutput(emissions_endpoint))
if self._save_to_api:
experiment_id = self._set_from_conf(
experiment_id, "experiment_id",
"5b0fa12a-3dd7-45bb-9766-cc326314d9f1")
self._cc_api__out = CodeCarbonAPIOutput(
endpoint_url=self._api_endpoint,
experiment_id=experiment_id,
api_key=api_key,
conf=self._conf,
)
self.run_id = self._cc_api__out.run_id
self.persistence_objs.append(self._cc_api__out)
else:
self.run_id = uuid.uuid4()
def viz(port: int = 8051, debug: bool = False) -> None:
conf = get_hierarchical_config()
df = Data.get_data_from_api(conf.get("api_endpoint", "http://localhost:8000"))
app = render_app(df)
app.run_server(port=port, debug=debug)