def build(self): ds = self.graph self.context = {"ce": "https://raw.githubusercontent.com/Vocamp/ComputationalActivity/master/pattern/ComputationalEnvironment.jsonld"} CE = Namespace("http://dase.cs.wright.edu/ontologies/ComputationalEnvironment#") CA = Namespace("http://dase.cs.wright.edu/ontologies/ComputationalActivity#") DOCKER = Namespace("http://w3id.org/daspos/docker#") info = cpuinfo.get_cpu_info() # ISSUES: We want if the architecture URI's to be created only once on # build or initial commit. Otherwise, we want to re-read the URI's # from the original graph. There are imm ds.bind("ce", CE) ceuri = URIRef(str(uuid.uuid4())) ds.add((ceuri, RDF.type, CE.ComputationalEnvironment)) osUri = URIRef(str(uuid.uuid4())) ds.add((ceuri, CE.hasOperatingSystem, osUri)) ds.add((osUri, RDFS.label, Literal("linux"))) processorUri = URIRef(str(uuid.uuid4())) ds.add((ceuri, CE.hasHardware, processorUri)) archUri = URIRef(str(uuid.uuid4())) ds.add((processorUri, CE.hasArchitecture, archUri)) ds.add((archUri, RDFS.label, Literal("amd64"))) ds.add((processorUri, CE.hasNumberOfCores, Literal("4", datatype=XSD.nonNegativeInteger)))
def collect_system_information(trainer): import psutil mem = psutil.virtual_memory() trainer.set_job_info('memory_total', mem.total) # at this point, theano is already initialised through KerasLogger from theano.sandbox import cuda trainer.set_job_info('cuda_available', cuda.cuda_available) if cuda.cuda_available: trainer.on_gpu = cuda.use.device_number is not None trainer.set_job_info('cuda_device_number', cuda.active_device_number()) trainer.set_job_info('cuda_device_name', cuda.active_device_name()) if cuda.cuda_ndarray.cuda_ndarray.mem_info: gpu = cuda.cuda_ndarray.cuda_ndarray.mem_info() trainer.set_job_info('cuda_device_max_memory', gpu[1]) free = gpu[0]/1024/1024/1024 total = gpu[1]/1024/1024/1024 used = total-free print("%.2fGB GPU memory used of %.2fGB" %(used, total)) trainer.set_job_info('on_gpu', trainer.on_gpu) import cpuinfo cpu = cpuinfo.get_cpu_info() trainer.set_job_info('cpu_name', cpu['brand']) trainer.set_job_info('cpu', [cpu['hz_actual_raw'][0], cpu['count']])
def get_cpu_info(): import cpuinfo all_info = cpuinfo.get_cpu_info() all_info = all_info or {} info = {} for key in ('vendor_id', 'hardware', 'brand'): info[key] = all_info.get(key, 'unknown') return info
def _collect_cpu_info(run_info): """Collect the CPU information for the local environment.""" cpu_info = {} cpu_info["num_cores"] = multiprocessing.cpu_count() info = cpuinfo.get_cpu_info() cpu_info["cpu_info"] = info["brand"] cpu_info["mhz_per_cpu"] = info["hz_advertised_raw"][0] / 1.0e6 run_info["cpu_info"] = cpu_info
def SysInfoString(self): SysInfo = '<b>System Information</b>' SysInfo += '<br>OS : {0}'.format(platform.platform()) SysInfo += '<br>CPU : {0}'.format(cpuinfo.get_cpu_info()['brand']) SysInfo += '<br>CPU Utilization : ' CPUPercent = psutil.cpu_percent(interval = 0, percpu = True) for a in CPUPercent: SysInfo += 'CPU#{1} : {0}% '.format(a, CPUPercent.index(a)) MemInfo = psutil.virtual_memory() SysInfo += '<br>RAM Total : {0:.2} GB, Used : {1} %'.format(MemInfo.total / math.pow(1024,3), MemInfo.percent) return SysInfo
def _collect_cpu_info(run_info): """Collect the CPU information for the local environment.""" cpu_info = {} cpu_info["num_cores"] = multiprocessing.cpu_count() # Note: cpuinfo is not installed in the TensorFlow OSS tree. # It is installable via pip. import cpuinfo # pylint: disable=g-import-not-at-top info = cpuinfo.get_cpu_info() cpu_info["cpu_info"] = info["brand"] cpu_info["mhz_per_cpu"] = info["hz_advertised_raw"][0] / 1.0e6 run_info["machine_config"]["cpu_info"] = cpu_info
def gather_cpu_info(): """Gather CPU Information. Assumes all CPUs are the same.""" cpu_info = test_log_pb2.CPUInfo() cpu_info.num_cores = multiprocessing.cpu_count() # Gather num_cores_allowed try: with gfile.GFile('/proc/self/status', 'rb') as fh: nc = re.search(r'(?m)^Cpus_allowed:\s*(.*)$', fh.read()) if nc: # e.g. 'ff' => 8, 'fff' => 12 cpu_info.num_cores_allowed = ( bin(int(nc.group(1).replace(',', ''), 16)).count('1')) except errors.OpError: pass finally: if cpu_info.num_cores_allowed == 0: cpu_info.num_cores_allowed = cpu_info.num_cores # Gather the rest info = cpuinfo.get_cpu_info() cpu_info.cpu_info = info['brand'] cpu_info.num_cores = info['count'] cpu_info.mhz_per_cpu = info['hz_advertised_raw'][0] / 1.0e6 l2_cache_size = re.match(r'(\d+)', str(info.get('l2_cache_size', ''))) if l2_cache_size: # If a value is returned, it's in KB cpu_info.cache_size['L2'] = int(l2_cache_size.group(0)) * 1024 # Try to get the CPU governor try: cpu_governors = set([ gfile.GFile(f, 'r').readline().rstrip() for f in glob.glob( '/sys/devices/system/cpu/cpu*/cpufreq/scaling_governor') ]) if cpu_governors: if len(cpu_governors) > 1: cpu_info.cpu_governor = 'mixed' else: cpu_info.cpu_governor = list(cpu_governors)[0] except errors.OpError: pass return cpu_info
def reset(self): self.__data = {} now = datetime.datetime.now() self.__data['seq'] = int(round(time.time() * 1000)) self.__data['timestamp-client'] = now.isoformat() cinfo = cpuinfo.get_cpu_info() self.__data['cpu'] = format(cinfo['brand']) osinfo = OsInfo() oinfo = osinfo.getOsInfo(cinfo) self.__data['os'] = oinfo['os'] self.__data['os-dist'] = oinfo['dist'] self.__data['os-version'] = oinfo['version'] self.__data['os-arch'] = oinfo['arch'] self.__data['os-kernel'] = oinfo['kernel'] self.__data['cpu-temp'] = cputemp.get_cpu_temp() l = Load() self.__data['cpu-load'] = l.getCpuLoad() self.__data['storage'] = l.getStorageStatus() self.__data['network'] = netinfo.get_network_interfaces()
def display_cpu_information(): """ Display CPU information. Assumes all CPUs are the same. """ import cpuinfo output_string = '\n-- INFORMATION: CPU -------------------\n' cpu_info = cpuinfo.get_cpu_info() try: output_string = add_param_to_output(output_string, 'brand', cpu_info['brand']) output_string = add_param_to_output(output_string, 'vendor id', cpu_info['vendor_id']) output_string = add_param_to_output(output_string, 'model', cpu_info['model']) output_string = add_param_to_output(output_string, 'family', cpu_info['family']) output_string = add_param_to_output(output_string, 'bits', cpu_info['bits']) output_string = add_param_to_output(output_string, 'architecture', cpu_info['arch']) output_string = add_param_to_output(output_string, 'cores', cpu_info['count']) output_string = add_param_to_output(output_string, 'advertised Hz', cpu_info['hz_advertised']) output_string = add_param_to_output(output_string, 'actual Hz', cpu_info['hz_actual']) output_string = add_param_to_output(output_string, 'l2 cache size', cpu_info['l2_cache_size']) except Exception: output_string += 'Some CPU information cannot be displayed\n' output_string += '----------------------------------------' neon_logger.display(output_string)
def _collect_cpu_info(run_info): """Collect the CPU information for the local environment.""" cpu_info = {} cpu_info["num_cores"] = multiprocessing.cpu_count() try: # Note: cpuinfo is not installed in the TensorFlow OSS tree. # It is installable via pip. import cpuinfo # pylint: disable=g-import-not-at-top info = cpuinfo.get_cpu_info() cpu_info["cpu_info"] = info["brand"] cpu_info["mhz_per_cpu"] = info["hz_advertised_raw"][0] / 1.0e6 run_info["machine_config"]["cpu_info"] = cpu_info except ImportError: tf.compat.v1.logging.warn( "'cpuinfo' not imported. CPU info will not be logged.")
def get_job(self): """ Gets job description from a file or from server. :return: job description. """ log.info("Trying to get job description.") job_desc = self.try_get_json_file(self.args.job_description) if job_desc is None: log.info("Job description is not saved locally. Asking server.") cpu_info = cpuinfo.get_cpu_info() mem_info = psutil.virtual_memory() disk_space = float(psutil.disk_usage(".").total) / 1024. / 1024. # diskSpace = min(diskSpace, 14336) # I doubt this is necessary, so RM data = { 'cpu': float(cpu_info['hz_actual_raw'][0]) / 1000000., 'mem': float(mem_info.total) / 1024. / 1024., 'node': self.node_name, 'diskSpace': disk_space, 'getProxyKey': False, # do we need it? 'computingElement': self.args.queue, 'siteName': self.args.queue, 'workingGroup': '', # do we need it? 'prodSourceLabel': self.args.job_tag } _str = self.curl_query("https://%s:%d/server/panda/updateJob" % (self.args.jobserver, self.args.jobserver_port), ssl=True, body=urllib.urlencode(data)) # log.debug("Got from server: "+_str) try: job_desc = json.loads(_str) except ValueError: log.error("JSON parser failed.") log.error("Got from server: "+_str) raise log.info("Got job description.") from job import Job job = Job(self, description_fixer(job_desc)) return job
def reset(self): a = 1 self.__data = {} from os.path import expanduser idFileName = expanduser("~") + '/.hidevid' fileExists = os.path.exists(idFileName) if ( fileExists ): idFileHandle = open(idFileName,'r') deviceId = idFileHandle.readline() deviceId = deviceId.strip() self.__data['id'] = deviceId idFileHandle.close now = datetime.datetime.now() self.__data['seq'] = int(round(time.time() * 1000)) self.__data['tsClient'] = now.isoformat() cinfo = cpuinfo.get_cpu_info() self.__data['cpu'] = format(cinfo['brand']) self.__data['cpuCount'] = cinfo['count'] osinfo = OsInfo() oinfo = osinfo.getOsInfo(cinfo) self.__data['os'] = oinfo['os'] self.__data['osDist'] = oinfo['dist'] self.__data['osVersion'] = oinfo['version'] self.__data['osArch'] = oinfo['arch'] self.__data['osKernel'] = oinfo['kernel'] self.__data['cpuTemp'] = cputemp.get_cpu_temp() l = Load() self.__data['cpuLoad'] = l.getCpuLoad() MemInfo = meminfo.getMemoryStatus() self.__data['memAvail'] = MemInfo['memAvail'] / 1024 self.__data['memUsed'] = MemInfo['memUsed'] / 1024 self.__data['swapAvail'] = MemInfo['swpAvail'] / 1024 self.__data['swapUsed'] = MemInfo['swpUsed'] / 1024 self.__data['storage'] = l.getStorageStatus() self.__data['network'] = netinfo.get_network_interfaces() CpuTimes = cputimes.get_cpu_times() self.__data['cpuUser'] = CpuTimes['user'] self.__data['cpuSystem'] = CpuTimes['system'] self.__data['cpuIdle'] = CpuTimes['idle'] self.__data['ioWait'] = CpuTimes['iowt'] self.__data['UpTime'] = CpuTimes['uptime']
def _rsmanifest(): from pykern import pkcollections from pykern import pkjson from pykern.pkcli import rsmanifest import cpuinfo import datetime import os import subprocess m = rsmanifest.read_all() m['sim'] = { 'run': { 'datetime': datetime.datetime.utcnow().isoformat(), 'cpu_info': cpuinfo.get_cpu_info(), 'pyenv': _pyenv_version(), #TODO(robnagler) can't include because of auth/credential # values in environment variables #'environ': pkcollections.Dict(os.environ), }, } pkjson.dump_pretty(m, filename=rsmanifest.BASENAME)
def collect_system_information(self): values = {} mem = psutil.virtual_memory() values['memory_total'] = mem.total import cpuinfo cpu = cpuinfo.get_cpu_info() values['resources_limit'] = self.resources_limit values['cpu_name'] = cpu['brand'] values['cpu'] = [cpu['hz_advertised_raw'][0], cpu['count']] values['nets'] = {} values['disks'] = {} values['gpus'] = {} values['boot_time'] = psutil.boot_time() try: for gpu_id, gpu in enumerate(aetros.cuda_gpu.get_ordered_devices()): gpu['available'] = gpu_id in self.enabled_gpus values['gpus'][gpu_id] = gpu except aetros.cuda_gpu.CudaNotImplementedException: pass for disk in psutil.disk_partitions(): try: name = self.get_disk_name(disk[1]) values['disks'][name] = psutil.disk_usage(disk[1]).total except Exception: # suppress Operation not permitted pass try: for id, net in psutil.net_if_stats().items(): if 0 != id.find('lo') and net.isup: self.nets.append(id) values['nets'][id] = net.speed or 1000 except Exception: # suppress Operation not permitted pass return values
def hdInfo(): """ Informacion del hardware que posee la pc """ hardware = {} hardware['uname'] = {} hardware['cpu'] = {} hardware['disk'] = {} hardware['memory'] = {} #Informacion del equipo uname = platform.uname() hardware['uname']['system'] = uname[0] hardware['uname']['name'] = uname[1] hardware['uname']['kernel'] = uname[2] hardware['uname']['arquitecture'] = uname[4] #Informacion del Procesador cpu = cpuinfo.get_cpu_info() hardware['cpu']['brand'] = cpu.get('brand') hardware['cpu']['count'] = cpu.get('count') #Informacion del disco duro disk = psutil.disk_usage('/') hardware['disk']['total'] = sizeof_fmt(disk.total) hardware['disk']['free'] = sizeof_fmt(disk.free) hardware['disk']['used'] = sizeof_fmt(disk.used) #Informacion de la memoria memory = psutil.virtual_memory() hardware['memory']['total'] = sizeof_fmt(memory.total) hardware['memory']['used'] = sizeof_fmt(memory.active) hardware['memory']['free'] = sizeof_fmt(memory.inactive) return hardware
def get_hardware_id(): """ Get an unique hardware ID, based on CPU info. All the times called, the value will be the same. >>> get_hardware_id() 'f8ef57d64aae6f3c45200b39di422bd6ca625d9a79655cb3aa6e171ef6f93013aa16c2df2f2b0359dfaf1782ba6fda94300506cdd9b21fdaf1264fbd0e47abb89' :return: string with the hardware ID :rtype: str """ import cpuinfo _ci = cpuinfo.get_cpu_info() cpu_inf = ("%s%s%s%s" % (_ci['hz_actual'], _ci['brand'], "".join(_ci['flags']), _ci['arch'])).replace(" ", "") d = hashlib.sha512() d.update(cpu_inf.encode(errors="ignore")) return d.hexdigest()
def version_table(print2screen=True): """ This function returns the version numbers of the various pieces of software with which this module was tested. Notes ----- In order for Hyperopt 0.1 to work, ``networkx`` had to be downgraded by running ``pip install networkx==1.11``. This is due to a bug that arises with Hyperopt when version 2.0 of ``networkx`` is installed. Also include: - conda install plotly Parameters ---------- print2screen : bool Print the version table to screen (``True``) or return it as a dictionary (``False``)? Returns ------- version_table : dict Dictionary containing the version table """ # TODO: Add Keras version_table = { 'Python': ('3.6.6', '.'.join(str(v) for v in version_info[0:3])), 'Keras': ('2.1.5', ke_version), 'TensorFlow.': ('1.6.0', tf_version), 'NumPy': ('1.14.5', np_version), 'matplotlib': ('2.2.2', plt_version), 'sklearn': ('0.20.1', sk_version), 'PyQt5': ('5.6.2', None), 'pandas': ('0.23.3', pd_version), 'Hyperopt': ('0.1', hp_version), 'OS': ('Linux-4.13.0-17-generic-x86_64-with-debian-stretch-sid', platform.platform()), 'CPU': ('Intel(R) Core(TM) i7-7500U CPU @ 2.70GHz', cpuinfo.get_cpu_info()['brand']), 'CUDA': ('8.0.44', None), 'GPU': ('NVIDIA GeForce GTX', None) } if print2screen: # Maximum length of the software names pad = max(map(lambda x: len(x), version_table)) # Print the table. print('software'.rjust(pad), ': baseline', sep='') print(''.rjust(pad), ' current', sep='') for k in sorted(version_table.keys()): print(k.rjust(pad), ': ', version_table[k][0], sep='') print(''.rjust(pad), ' ', version_table[k][1], sep='') return version_table
import os import sys import logging import ConfigParser import cpuinfo import cv2 from flask import Flask import pyttsx import vlc if "X86_64" == cpuinfo.get_cpu_info()["arch"]: import getchar as interface config = ConfigParser.ConfigParser() tts_engine = None face_cascade = None video = None vlc_instance = None player = None stay_alive = True # Keep the program running? speaking = False # Am I talking when I detect someone else prescence def initialize(): """Initialize program""" global video global vlc_instance global tts_engine global player global face_cascade
def get_cpu_info(): # Obtain textual cpu info try: with open('/proc/cpuinfo', 'r') as f: lines = f.readlines() except FileNotFoundError: lines = [] cpu_info = {} # Extract CPU flags and branch if lines: try: get = lambda k: [i for i in lines if i.startswith(k)][0].split(':')[1].strip() cpu_info['flags'] = get('flags').split() cpu_info['brand'] = get('model name') except IndexError: # The /proc/cpuinfo format doesn't follow a standard, and on some # more or less exotic combinations of OS and platform it might not # be what we expect, hence ending up here pass if not all(i in cpu_info for i in ('flags', 'brand')): # Fallback ci = cpuinfo.get_cpu_info() cpu_info['flags'] = ci.get('flags') cpu_info['brand'] = ci.get('brand') # Detect number of logical cores logical = psutil.cpu_count(logical=True) if not logical: # Never bumped into a platform that make us end up here, yet # But we try to cover this case anyway, with `lscpu` try: logical = lscpu()['CPU(s)'] except KeyError: warning("Logical core count autodetection failed") logical = 1 cpu_info['logical'] = logical # Detect number of physical cores # TODO: on multi-socket systems + unix, can't use psutil due to # `https://github.com/giampaolo/psutil/issues/1558` mapper = {} if lines: # Copied and readapted from psutil current_info = {} for i in lines: line = i.strip().lower() if not line: # New section if ('physical id' in current_info and 'cpu cores' in current_info): mapper[current_info['physical id']] = current_info[ 'cpu cores'] current_info = {} else: # Ongoing section if (line.startswith('physical id') or line.startswith('cpu cores')): key, value = line.split('\t:', 1) current_info[key] = int(value) physical = sum(mapper.values()) if not physical: # Fallback 1: it should now be fine to use psutil physical = psutil.cpu_count(logical=False) if not physical: # Fallback 2: we might end up here on more exotic platforms such a Power8 # Hopefully we can rely on `lscpu` try: physical = lscpu()['Core(s) per socket'] * lscpu()['Socket(s)'] except KeyError: warning("Physical core count autodetection failed") physical = 1 cpu_info['physical'] = physical return cpu_info
cpu = cpuinfo() platform = platforminfo() if __name__ == "__main__": _VALUE_MAPPING = {"CPUInfoBase__get_nbits": "nbits", "getNCPUs": "ncpus"} cpu_type = {} for name in dir(cpuinfo): if name[0] == "_" and name[1] != "_": name = name[1:] # Avoid leading _ r = getattr(cpu, name)() cpu_type[_VALUE_MAPPING.get(name, name)] = r cpu_info = other_cpu_info.get_cpu_info() cpu_info.pop("python_version") cpu_features = {} keys = frozenset( ("hz_advertised", "hz_actual", "hz_advertised_raw", "hz_actual_raw", "l3_cache_size", "l2_cache_size", "l1_data_cache_size", "l1_instruction_cache_size", "flags")) for key in keys: cpu_features[key] = cpu_info.pop(key, None) dir_name = os.path.dirname(_OUTPUT_JSON_PATH) if dir_name: os.makedirs(dir_name, exist_ok=True) with open(_OUTPUT_JSON_PATH, 'w') as hw_info_file:
# Aceita alguma conexão (conexao, address) = server.accept() print('Conectado a:', str(address)) while True: # Recebe requisicão do cliente data = '' data = conexao.recv(2048) if data.decode('utf-8') == '1': request = data.decode('utf-8') print('Requisição da opção', request, 'recebida com sucesso!') print('Processando...') list_percent_cpu = psutil.cpu_percent(interval=1, percpu=True) dict_info_cpu = cpuinfo.get_cpu_info() num_cores = psutil.cpu_count(logical=True) # Cria a lista de resposta response = [] response.append(list_percent_cpu) response.append(dict_info_cpu) response.append(num_cores) # Prepara a lista para o envio bytes_resp = pickle.dumps(response) # Envia os dados conexao.send(bytes_resp) print('Resposta da opção', request, 'enviada com sucesso!\n') if data.decode('utf-8') == '2': request = data.decode('utf-8') print('Requisição da opção', request, 'recebida com sucesso!')
def cpu_model(): """ :return: The CPU model name. """ return cpuinfo.get_cpu_info()["brand"] if cpuinfo_available else "Unknown"
def test_all(self): os_type = helpers.get_os_type() if os_type == 'BeOS': self.assertEqual(None, cpuinfo.get_cpu_info_from_registry()) self.assertEqual(None, cpuinfo.get_cpu_info_from_proc_cpuinfo()) self.assertEqual(None, cpuinfo.get_cpu_info_from_sysctl()) self.assertEqual(None, cpuinfo.get_cpu_info_from_kstat()) self.assertEqual(None, cpuinfo.get_cpu_info_from_dmesg()) self.assertHasResult(cpuinfo.get_cpu_info_from_sysinfo()) self.assertHasResult(cpuinfo.get_cpu_info_from_cpuid()) self.assertHasResult(cpuinfo.get_cpu_info()) elif os_type == 'BSD': self.assertEqual(None, cpuinfo.get_cpu_info_from_registry()) self.assertEqual(None, cpuinfo.get_cpu_info_from_proc_cpuinfo()) self.assertEqual(None, cpuinfo.get_cpu_info_from_sysctl()) self.assertEqual(None, cpuinfo.get_cpu_info_from_kstat()) self.assertHasResult(cpuinfo.get_cpu_info_from_dmesg()) self.assertEqual(None, cpuinfo.get_cpu_info_from_sysinfo()) # FIXME: This fails by segfaulting for some reason self.assertEqual(None, cpuinfo.get_cpu_info_from_cpuid()) self.assertHasResult(cpuinfo.get_cpu_info()) elif os_type == 'Cygwin': self.assertEqual(None, cpuinfo.get_cpu_info_from_registry()) self.assertHasResult(cpuinfo.get_cpu_info_from_proc_cpuinfo()) self.assertEqual(None, cpuinfo.get_cpu_info_from_sysctl()) self.assertEqual(None, cpuinfo.get_cpu_info_from_kstat()) self.assertEqual(None, cpuinfo.get_cpu_info_from_dmesg()) self.assertEqual(None, cpuinfo.get_cpu_info_from_sysinfo()) # FIXME: This fails by segfaulting for some reason self.assertEqual(None, cpuinfo.get_cpu_info_from_cpuid()) self.assertHasResult(cpuinfo.get_cpu_info()) elif os_type == 'MacOS': self.assertEqual(None, cpuinfo.get_cpu_info_from_registry()) self.assertEqual(None, cpuinfo.get_cpu_info_from_proc_cpuinfo()) self.assertHasResult(cpuinfo.get_cpu_info_from_sysctl()) self.assertEqual(None, cpuinfo.get_cpu_info_from_kstat()) self.assertEqual(None, cpuinfo.get_cpu_info_from_dmesg()) self.assertEqual(None, cpuinfo.get_cpu_info_from_sysinfo()) # FIXME: This fails by segfaulting for some reason self.assertEqual(None, cpuinfo.get_cpu_info_from_cpuid()) self.assertHasResult(cpuinfo.get_cpu_info()) elif os_type == 'Linux': self.assertEqual(None, cpuinfo.get_cpu_info_from_registry()) self.assertHasResult(cpuinfo.get_cpu_info_from_proc_cpuinfo()) self.assertEqual(None, cpuinfo.get_cpu_info_from_sysctl()) self.assertEqual(None, cpuinfo.get_cpu_info_from_kstat()) self.assertEqual(None, cpuinfo.get_cpu_info_from_dmesg()) self.assertEqual(None, cpuinfo.get_cpu_info_from_sysinfo()) self.assertHasResult(cpuinfo.get_cpu_info_from_cpuid()) self.assertHasResult(cpuinfo.get_cpu_info()) elif os_type == 'Solaris': self.assertEqual(None, cpuinfo.get_cpu_info_from_registry()) self.assertEqual(None, cpuinfo.get_cpu_info_from_proc_cpuinfo()) self.assertEqual(None, cpuinfo.get_cpu_info_from_sysctl()) self.assertHasResult(cpuinfo.get_cpu_info_from_kstat()) self.assertEqual(None, cpuinfo.get_cpu_info_from_dmesg()) self.assertEqual(None, cpuinfo.get_cpu_info_from_sysinfo()) # FIXME: This fails by segfaulting for some reason self.assertEqual(None, cpuinfo.get_cpu_info_from_cpuid()) self.assertHasResult(cpuinfo.get_cpu_info()) elif os_type == 'Windows': self.assertHasResult(cpuinfo.get_cpu_info_from_registry()) self.assertEqual(None, cpuinfo.get_cpu_info_from_proc_cpuinfo()) self.assertEqual(None, cpuinfo.get_cpu_info_from_sysctl()) self.assertEqual(None, cpuinfo.get_cpu_info_from_kstat()) self.assertEqual(None, cpuinfo.get_cpu_info_from_dmesg()) self.assertEqual(None, cpuinfo.get_cpu_info_from_sysinfo()) self.assertHasResult(cpuinfo.get_cpu_info_from_cpuid()) self.assertHasResult(cpuinfo.get_cpu_info()) else: raise AssertionError('Unexpected OS type "{0}".'.format(os_type)) self.assertEqual(True, True)
def cpu_model(): """ :return: The CPU model name. """ return cpuinfo.get_cpu_info()["brand"]
def start_command(logger, job_backend, env_overwrite=None, volumes=None, cpus=1, memory=1, gpu_devices=None, offline=False): home_config = read_home_config() env = {} if env_overwrite: env.update(env_overwrite) start_time = time.time() env['AETROS_MODEL_NAME'] = job_backend.model_name env['AETROS_JOB_ID'] = str(job_backend.job_id) env['AETROS_OFFLINE'] = '1' if offline else '' env['AETROS_GIT_INDEX_FILE'] = job_backend.git.index_path env['DEBUG'] = os.getenv('DEBUG', '') env['PYTHONUNBUFFERED'] = os.getenv('PYTHONUNBUFFERED', '1') env['PYTHONIOENCODING'] = os.getenv('PYTHONIOENCODING', 'UTF-8') env['AETROS_ATTY'] = '1' env['AETROS_GIT'] = job_backend.git.get_base_command() env['PATH'] = os.getenv('PATH', '') if 'PYTHONPATH' not in env: env['PYTHONPATH'] = os.getenv('PYTHONPATH', '') if os.getenv('AETROS_SSH_KEY_BASE64'): env['AETROS_SSH_KEY_BASE64'] = os.getenv('AETROS_SSH_KEY_BASE64') elif get_ssh_key_for_host(home_config['host']): # we need to read the key into env so the docker container can connect to AETROS env['AETROS_SSH_KEY_BASE64'] = open(get_ssh_key_for_host(home_config['host']), 'r').read() job_config = job_backend.job['config'] job = job_backend.get_job_model() if 'command' not in job_config: job_backend.fail('No "command" given. See Configuration section in the documentation.') job_commands = job_config['command'] docker_image = job_config['image'] if job_backend.is_simple_model(): if docker_image: simple_command = ['python'] else: simple_command = [sys.executable] simple_command += ['-m', 'aetros', 'start-simple', job_backend.model_name + '/' + job_backend.job_id] job_commands = {'run': ' '.join(simple_command)} if job_commands is None: raise Exception('No command specified.') if not isinstance(job_commands, list) and not isinstance(job_commands, dict): job_commands = [job_commands] # replace {{batch_size}} parameters if isinstance(job_config['parameters'], dict): for key, value in six.iteritems(flatten_parameters(job_config['parameters'])): if isinstance(job_commands, list): for k, v in enumerate(job_commands): if isinstance(job_commands[k], six.string_types): job_commands[k] = job_commands[k].replace('{{' + key + '}}', simplejson.dumps(value)) elif isinstance(job_commands, dict): for k, v in six.iteritems(job_commands): if isinstance(job_commands[k], six.string_types): job_commands[k] = job_commands[k].replace('{{' + key + '}}', simplejson.dumps(value)) job_backend.set_system_info('commands', job_commands) os.chdir(job_backend.git.work_tree) docker_image_built = False if docker_image and (job_config['dockerfile'] or job_config['install']): rebuild_image = job_config['rebuild_image'] if 'rebuild_image' in job_config else False docker_image = docker_build_image(logger, home_config, job_backend, rebuild_image) docker_image_built = True job_backend.collect_device_information(gpu_devices) state = {'last_process': None} job_backend.set_system_info('processRunning', False, True) def pause(): if not state['last_process'] or state['last_process'].poll() is not None: # no running process return if docker_image: if docker_pause(logger, home_config, job_backend): job_backend.set_paused(True) else: os.killpg(os.getpgid(state['last_process'].pid), signal.SIGSTOP) job_backend.set_paused(True) def cont(): if not state['last_process'] or state['last_process'].poll() is not None: # no running process return job_backend.set_paused(False) if docker_image: docker_continue(logger, home_config, job_backend) else: os.killpg(os.getpgid(state['last_process'].pid), signal.SIGCONT) job_backend.on_pause = pause job_backend.on_continue = cont if docker_image: env['AETROS_GIT_INDEX_FILE'] = '/aetros/' + job_backend.model_name + '.git/' + os.path.basename(env['AETROS_GIT_INDEX_FILE']) with job_backend.git.batch_commit('JOB_SYSTEM_INFORMATION'): aetros_environment = {'aetros_version': __version__, 'variables': env.copy()} if 'AETROS_SSH_KEY' in aetros_environment['variables']: del aetros_environment['variables']['AETROS_SSH_KEY'] if 'AETROS_SSH_KEY_BASE64' in aetros_environment['variables']: del aetros_environment['variables']['AETROS_SSH_KEY_BASE64'] job_backend.set_system_info('environment', aetros_environment) job_backend.set_system_info('memory_total', memory * 1024 * 1024 * 1024) import cpuinfo cpu = cpuinfo.get_cpu_info() job_backend.set_system_info('cpu_name', cpu['brand']) job_backend.set_system_info('cpu', [cpu['hz_actual_raw'][0], cpus]) job_backend.start_monitoring(cpu_cores=cpus, gpu_devices=gpu_devices, docker_container=job_backend.job_id) if not docker_image_built: docker_pull_image(logger, home_config, job_backend) docker_image_information(logger, home_config, job_backend) # make sure old container is removed subprocess.Popen([home_config['docker'], 'rm', job_backend.job_id], stderr=subprocess.PIPE).wait() command = docker_command_wrapper(logger, home_config, job_backend, volumes, cpus, memory, gpu_devices, env) # since linux doesnt handle SIGINT when pid=1 process has no signal listener, # we need to make sure, we attached one to the pid=1 process trap = 'trapIt () { "$@"& pid="$!"; trap "kill -INT $pid" INT TERM; ' \ 'while kill -0 $pid > /dev/null 2>&1; do wait $pid; ec="$?"; done; exit $ec;};' command.append(docker_image) command += ['/bin/sh', '-c', trap + 'trapIt /bin/sh /job/aetros/command.sh'] else: # non-docker # env['PYTHONPATH'] += ':' + os.getcwd() job_backend.collect_system_information() job_backend.collect_environment(env) job_backend.start_monitoring(gpu_devices=gpu_devices) command = ['/bin/sh', job_backend.git.work_tree + '/aetros/command.sh'] logger.debug("$ %s " % (' '.join([simplejson.dumps(a) for a in command]))) job_backend.set_system_info('image/name', str(docker_image)) p = None exited = False last_return_code = None state['last_process'] = None all_done = False command_stats = None files = job_backend.file_list() def clean(): # clear working tree shutil.rmtree(job_backend.git.work_tree) def on_force_exit(): # make sure the process dies clean() with open(os.devnull, 'r+b', 0) as DEVNULL: if docker_image: # docker run does not proxy INT signals to the docker-engine, # so we need to do it on our own directly. subprocess.Popen(args=[home_config['docker'], 'kill', job_backend.job_id], stdout=DEVNULL, stderr=DEVNULL).wait() elif not exited and state['last_process'] and state['last_process'].poll() is None: # wait for last command os.killpg(os.getpgid(state['last_process'].pid), signal.SIGKILL) job_backend.on_force_exit = on_force_exit try: job_backend.set_status('STARTED', add_section=False) # logger.warning("$ %s " % (str(command),)) # make sure maxTime limitation is correctly calculated job_backend.monitoring_thread.handle_max_time = True job_backend.monitoring_thread.handle_max_time_time = time.time() # Since JobBackend sends SIGINT to its current process group, it sends also to its parents when same pg. # We need to change the process group of the process, so this won't happen. # If we don't this, the master process (server command e.g.) receives the SIGINT as well. kwargs = {} if os.name == 'nt': kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP else: kwargs['preexec_fn'] = os.setsid # only use full env when no image used command_env = env if not docker_image: command_env = os.environ.copy() command_env.update(env) if os.environ.get('LD_LIBRARY_PATH', None): command_env['LD_LIBRARY_PATH_ORI'] = command_env['LD_LIBRARY_PATH'] def write_command_sh(job_command): f = open(job_backend.git.work_tree + '/aetros/command.sh', 'w+') if not docker_image: # new shells unset LD_LIBRARY_PATH automatically, so we make sure it will be there again f.write('export LD_LIBRARY_PATH=$LD_LIBRARY_PATH_ORI;\n') if job.get_working_dir(): f.write('cd %s;\n' % (job.get_working_dir(),)) f.write(job_command) f.close() def read_line(line): handled, filtered_line, failed = extract_api_calls(line, job_backend.handle_stdout_api, logger=logger) if is_debug(): for call in handled: logger.debug('STDOUT API CALL: ' + str(call)) for fail in failed: logger.warning("API call failed '%s': %s %s" % (str(fail['line']), str(type(fail['exception']).__name__), str(fail['exception']))) return filtered_line def exec_command(index, command, job_command): write_command_sh(job_command) print('%s $ %s' % ('/' + job.get_working_dir(), job_command.strip())) args = command logger.debug('$ ' + ' '.join([simplejson.dumps(a) for a in args])) command_stats[index]['started'] = time.time() - start_time job_backend.set_system_info('command_stats', command_stats, True) # important to prefix it, otherwise name='master' would reset all stats in controller backend command_env['AETROS_JOB_NAME'] = 'command_' + str(index) state['last_process'] = subprocess.Popen( args=args, bufsize=0, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=command_env, **kwargs ) job_backend.set_system_info('processRunning', True, True) wait_stdout = sys.stdout.attach(state['last_process'].stdout, read_line=read_line) wait_stderr = sys.stderr.attach(state['last_process'].stderr) state['last_process'].wait() command_stats[index]['rc'] = last_return_code command_stats[index]['ended'] = time.time() - start_time job_backend.set_system_info('command_stats', command_stats, True) job_backend.set_system_info('processRunning', True, False) wait_stdout() wait_stderr() # make sure a new line is printed after a command print("") return state['last_process'] done = 0 total = len(job_commands) job_backend.set_system_info('command_stats', command_stats, True) if isinstance(job_commands, list): command_stats = [{'rc': None, 'started': None, 'ended': None} for x in job_commands] for k, job_command in enumerate(job_commands): job_backend.set_status('Command ' + str(k+1)) p = exec_command(k, command, job_command) last_return_code = p.poll() if last_return_code == 0: done += 1 else: # one failed, so exit and don't execute next break if isinstance(job_commands, dict): command_stats = {} for name, job_command in six.iteritems(job_commands): command_stats[name] = {'rc': None, 'started': None, 'ended': None} for name, job_command in six.iteritems(job_commands): job_backend.set_status('Command ' + name) p = exec_command(name, command, job_command) last_return_code = p.poll() if last_return_code == 0: done += 1 else: # one failed, so exit and don't execute next break all_done = done == total exited = True if state['last_process']: sys.exit(state['last_process'].poll()) else: sys.exit(1) except SystemExit: # since we started the command in a new process group, a SIGINT or CTRL+C on this process won't affect # our actual command process. So we need to take care that we stop everything. logger.debug("SystemExit, exited=%s, all-done=%s, has-last-process=%s, pid=%s" %( str(exited), str(all_done), state['last_process'] is not None, state['last_process'].poll() if state['last_process'] else None )) # make sure the process dies if docker_image: # docker run does not proxy INT signals to the docker-engine, # so we need to do it on our own directly. p = subprocess.Popen(args=[home_config['docker'], 'inspect', job_backend.job_id], stderr=subprocess.PIPE, stdout=subprocess.PIPE) p.wait() if p.poll() == 0: subprocess.Popen(args=[home_config['docker'], 'kill', job_backend.job_id]).wait() elif not exited and state['last_process'] and state['last_process'].poll() is None: # wait for last command os.killpg(os.getpgid(state['last_process'].pid), signal.SIGINT) state['last_process'].wait() if 'output' in job_config and job_config['output']: upload_output_files(job_backend, job_config['output']) if exited: if all_done: job_backend.stop(progress=JOB_STATUS.PROGRESS_STATUS_DONE) else: job_backend.stop(progress=JOB_STATUS.PROGRESS_STATUS_FAILED) else: # master received SIGINT before the all job commands exited. if not job_backend.in_early_stop: # in_early_stop indicates whether we want to have a planned stop (maxTime limitation for example), # which should mark the job as done, not as abort(). # if this is not set, we the master received a SIGINT without early_stop, so mark as aborted. job_backend.abort() else: # let the on_shutdown listener handle the rest pass clean()
def make_report_backend(folder, as_df=False): """ Looks into a folder for dumped files after the unit tests. :param folder: dump folder, it should contain files *.bench* :param as_df: returns a dataframe instread of a list of dictionary :return: time execution """ import onnx import onnxruntime import cpuinfo res = {} benched = 0 files = os.listdir(folder) for name in files: if name.endswith(".expected.pkl"): model = name.split(".")[0] if model not in res: res[model] = {} res[model]["_tested"] = True elif ".backend." in name: bk = name.split(".backend.")[-1].split(".")[0] model = name.split(".")[0] if model not in res: res[model] = {} res[model][bk] = True elif name.endswith(".err"): model = name.split(".")[0] fullname = os.path.join(folder, name) with open(fullname, "r", encoding="utf-8") as f: content = f.read() error = content.split("\n")[0].strip("\n\r ") if model not in res: res[model] = {} res[model]["stderr"] = error elif name.endswith(".model.pkl"): model = name.split(".")[0] if model not in res: res[model] = {} res[model].update(stat_model_skl(os.path.join(folder, name))) elif name.endswith(".model.onnx"): model = name.split(".")[0] if model not in res: res[model] = {} res[model].update(stat_model_onnx(os.path.join(folder, name))) elif name.endswith(".bench"): model = name.split(".")[0] fullname = os.path.join(folder, name) df = pandas.read_csv(fullname, sep=",") if model not in res: res[model] = {} for index, row in df.iterrows(): name = row["name"] ave = row["average"] std = row["deviation"] size = row["input_size"] res[model]["{0}_time".format(name)] = ave res[model]["{0}_std".format(name)] = std res[model]["input_size"] = size benched += 1 if benched == 0: raise RuntimeError("No benchmark files in '{0}', found:\n{1}".format( folder, "\n".join(files))) def dict_update(d, u): d.update(u) return d aslist = [dict_update(dict(_model=k), v) for k, v in res.items()] if as_df: from pandas import DataFrame df = DataFrame(aslist).sort_values(["_model"]) df["numpy-version"] = numpy.__version__ df["onnx-version"] = onnx.__version__ df["onnxruntime-version"] = onnxruntime.__version__ cols = list(df.columns) if "stderr" in cols: ind = cols.index("stderr") del cols[ind] cols += ["stderr"] df = df[cols] for col in ["onnxrt_time", "original_time"]: if col not in df.columns: raise RuntimeError("Column '{0}' is missing from {1}".format( col, ", ".join(df.columns))) df["ratio"] = df["onnxrt_time"] / df["original_time"] df["ratio_nodes"] = df["nb_onnx_nodes"] / df["nb_estimators"] df["CPU"] = platform.processor() df["CPUI"] = cpuinfo.get_cpu_info()["brand"] return df else: cpu = cpuinfo.get_cpu_info()["brand"] proc = platform.processor() for row in aslist: try: row["ratio"] = row["onnxrt_time"] / row["original_time"] except KeyError: # execution failed pass try: row["ratio_nodes"] = (row["nb_onnx_nodes"] / row["nb_estimators"]) except KeyError: # execution failed pass row["CPU"] = proc row["CPUI"] = cpu row["numpy-version"] = numpy.__version__ row["onnx-version"] = onnx.__version__ row["onnxruntime-version"] = onnxruntime.__version__ return aslist
timeout = 5 # Socket timeout resourcesFolder = "PCMiner_resources" shares = [0, 0] diff = 0 last_hash_count = 0 khash_count = 0 hash_count = 0 hash_mean = [] donatorrunning = False debug = True serveripfile = "https://mhcoin.s3.filebase.com/Pool.txt" # Serverip file config = configparser.ConfigParser() autorestart = 0 donationlevel = 0 freeze_support() # If not used, pyinstaller hangs when checking cpuinfo cpu = cpuinfo.get_cpu_info() # Processor info if not os.path.exists(resourcesFolder): os.mkdir(resourcesFolder) # Create resources folder if it doesn't exist def debugOutput(text): if debug == "True": print(now().strftime(Style.DIM + "%H:%M:%S.%f ") + "DEBUG: " + text) def title(title): if os.name == 'nt': os.system("title "+title) else: print('\33]0;'+title+'\a', end='') sys.stdout.flush()
def handleInfoCommand(self): info = cpuinfo.get_cpu_info() response = self.build_message(info) self.write_message(response)
def update(self): """ Gets the latest data and updates the state. """ from cpuinfo import cpuinfo self.info = cpuinfo.get_cpu_info() self._state = round(float(self.info['hz_actual_raw'][0])/10**9, 2)
with open(sys.argv[1], "rb") as fd: # tediously derived from the monster struct defined in <hdreg.h> # see comment at end of file to verify hd_driveid_format_str = "@ 10H 20s 3H 8s 40s 2B H 2B H 4B 6H 2B I 36H I Q 152H" # Also from <hdreg.h> HDIO_GET_IDENTITY = 0x030d # How big a buffer do we need? sizeof_hd_driveid = struct.calcsize(hd_driveid_format_str) # ensure our format string is the correct size # 512 is extracted using sizeof(struct hd_id) in the c code assert sizeof_hd_driveid == 512 # Call native function buf = fcntl.ioctl(fd, HDIO_GET_IDENTITY, " " * sizeof_hd_driveid) fields = struct.unpack(hd_driveid_format_str, buf) hdd = fields[15].strip() cpu = cpuinfo.get_cpu_info()['brand'] print(cpu) print("Hard Drive Model: {0}".format(hdd)) info = { 'CPU': cpu, 'HDD': hdd, } with open('hardware.pkl', 'w') as pkl: pickle.dump(info, pkl)
def benchmark_against_others(repo_root: pathlib.Path, overwrite: bool) -> None: """Run benchmars against other libraries and include them in the Readme.""" script_rel_paths = [ "benchmarks/compare_with_others.py", ] if not overwrite: for i, script_rel_path in enumerate(script_rel_paths): if i > 0: print() subprocess.check_call([sys.executable, str(repo_root / script_rel_path)]) else: out = ["The following scripts were run:\n\n"] for script_rel_path in script_rel_paths: out.append( "* `{0} <https://github.com/Parquery/icontract/tree/master/{0}>`_\n".format( script_rel_path ) ) out.append("\n") out.append( ( "The benchmarks were executed on {}.\nWe used Python {}, " "icontract {}, deal 4.4.0 and dpcontracts 0.6.0.\n\n" ).format( cpuinfo.get_cpu_info()["brand"], platform.python_version(), icontract.__version__, ) ) out.append("The following tables summarize the results.\n\n") stdouts = [] # type: List[str] for script_rel_path in script_rel_paths: stdout = subprocess.check_output( [sys.executable, str(repo_root / script_rel_path)] ).decode() stdouts.append(stdout) out.append(stdout) out.append("\n") readme_path = repo_root / "README.rst" readme = readme_path.read_text(encoding="utf-8") marker_start = ".. Benchmark report starts." marker_end = ".. Benchmark report ends." lines = readme.splitlines() try: index_start = lines.index(marker_start) except ValueError as exc: raise ValueError( "Could not find the marker for the benchmarks in the {}: {}".format( readme_path, marker_start ) ) from exc try: index_end = lines.index(marker_end) except ValueError as exc: raise ValueError( "Could not find the start marker for the benchmarks in the {}: {}".format( readme_path, marker_end ) ) from exc assert ( index_start < index_end ), "Unexpected end marker before start marker for the benchmarks." lines = ( lines[: index_start + 1] + ["\n"] + ("".join(out)).splitlines() + ["\n"] + lines[index_end:] ) readme_path.write_text("\n".join(lines) + "\n", encoding="utf-8") # This is necessary so that the benchmarks do not complain on a Windows machine if the console encoding has not # been properly set. sys.stdout.buffer.write(("\n\n".join(stdouts) + "\n").encode("utf-8"))
import sys # Import cpuinfo.py from up one directory sys.path.append('../cpuinfo') if __name__ == '__main__': from cpuinfo import get_cpu_info print(get_cpu_info())
def run_unit_tests(path=PISA_PATH, allow_missing=OPTIONAL_MODULES, verbosity=Levels.WARN): """Run all tests found at `path` (or recursively below if `path` is a directory). Each module is imported and each test function is run initially with `set_verbosity(verbosity)`, but if an exception is caught, the module is re-imported or the test function is re-run with `set_verbosity(Levels.TRACE)`, then the traceback from the (original) exception emitted is displayed. Parameters ---------- path : str Path to file or directory allow_missing : None or sequence of str verbosity : int in pisa.utils.log.Levels Raises ------ Exception If any import or test fails not in `allow_missing` """ set_verbosity(verbosity) logging.info("%sPlatform information:", PFX) logging.info("%s HOSTNAME = %s", PFX, socket.gethostname()) logging.info("%s FQDN = %s", PFX, socket.getfqdn()) logging.info("%s OS = %s %s", PFX, platform.system(), platform.release()) for key, val in cpuinfo.get_cpu_info().items(): logging.info("%s %s = %s", PFX, key, val) logging.info(PFX) logging.info("%sModule versions:", PFX) for module_name in REQUIRED_MODULES + OPTIONAL_MODULES: try: module = import_module(module_name) except ImportError: if module_name in REQUIRED_MODULES: raise ver = "optional module not installed or not import-able" else: if hasattr(module, "__version__"): ver = module.__version__ else: ver = "?" logging.info("%s %s : %s", PFX, module_name, ver) logging.info(PFX) path = expand(path, absolute=True, resolve_symlinks=True) if allow_missing is None: allow_missing = [] elif isinstance(allow_missing, str): allow_missing = [allow_missing] tests = find_unit_tests(path) module_pypaths_succeeded = [] module_pypaths_failed = [] module_pypaths_failed_ignored = [] test_pypaths_succeeded = [] test_pypaths_failed = [] test_pypaths_failed_ignored = [] for rel_file_path, test_func_names in tests.items(): pypath = ["pisa"] + rel_file_path[:-3].split("/") parent_pypath = ".".join(pypath[:-1]) module_name = pypath[-1].replace(".", "_") module_pypath = f"{parent_pypath}.{module_name}" try: set_verbosity(verbosity) logging.info(PFX + f"importing {module_pypath}") set_verbosity(Levels.WARN) module = import_module(module_pypath, package=parent_pypath) except Exception as err: if (isinstance(err, ImportError) and hasattr(err, "name") and err.name in allow_missing # pylint: disable=no-member ): err_name = err.name # pylint: disable=no-member module_pypaths_failed_ignored.append(module_pypath) logging.warning( f"{PFX}module {err_name} failed to import wile importing" f" {module_pypath}, but ok to ignore") continue module_pypaths_failed.append(module_pypath) set_verbosity(verbosity) msg = f"<< FAILURE IMPORTING : {module_pypath} >>" logging.error(PFX + "=" * len(msg)) logging.error(PFX + msg) logging.error(PFX + "=" * len(msg)) # Reproduce the failure with full output set_verbosity(Levels.TRACE) try: import_module(module_name, package=parent_pypath) except Exception: pass set_verbosity(Levels.TRACE) logging.exception(err) set_verbosity(verbosity) logging.error(PFX + "#" * len(msg)) continue else: module_pypaths_succeeded.append(module_pypath) for test_func_name in test_func_names: test_pypath = f"{module_pypath}.{test_func_name}" try: set_verbosity(verbosity) logging.debug(PFX + f"getattr({module}, {test_func_name})") set_verbosity(Levels.WARN) test_func = getattr(module, test_func_name) # Run the test function set_verbosity(verbosity) logging.info(PFX + f"{test_pypath}()") set_verbosity(Levels.WARN) test_func() except Exception as err: if (isinstance(err, ImportError) and hasattr(err, "name") and err.name in allow_missing # pylint: disable=no-member ): err_name = err.name # pylint: disable=no-member test_pypaths_failed_ignored.append(module_pypath) logging.warning( PFX + f"{test_pypath} failed because module {err_name} failed to" + f" load, but ok to ignore") continue test_pypaths_failed.append(test_pypath) set_verbosity(verbosity) msg = f"<< FAILURE RUNNING : {test_pypath} >>" logging.error(PFX + "=" * len(msg)) logging.error(PFX + msg) logging.error(PFX + "=" * len(msg)) # Reproduce the error with full output set_verbosity(Levels.TRACE) try: test_func = getattr(module, test_func_name) with np.printoptions( precision=np.finfo(pisa.FTYPE).precision + 2, floatmode="fixed", sign=" ", linewidth=200, ): test_func() except Exception: pass set_verbosity(Levels.TRACE) logging.exception(err) set_verbosity(verbosity) logging.error(PFX + "#" * len(msg)) else: test_pypaths_succeeded.append(test_pypath) finally: # remove references to the test function, e.g. to remove refs # to pycuda / numba.cuda contexts so these can be closed try: del test_func except NameError: pass # NOTE: Until we get all GPU code into Numba, need to unload pycuda # and/or numba.cuda contexts before a module requiring the other one is # to be imported. # NOTE: the following causes a traceback to be emitted at the very end # of the script, regardless of the exception catching here. if (pisa.TARGET == "cuda" and pycuda is not None and hasattr(pycuda, "autoinit") and hasattr(pycuda.autoinit, "context")): try: pycuda.autoinit.context.detach() except Exception: pass # Attempt to unload the imported module # TODO: pipeline, etc. fail as isinstance(service, (Stage, PiStage)) is False #if module_pypath in sys.modules and module_pypath != "pisa": # del sys.modules[module_pypath] #del module # TODO: crashes program; subseqeunt calls in same shell crash(!?!?) # if pisa.TARGET == 'cuda' and nbcuda is not None: # try: # nbcuda.close() # except Exception: # pass # Summarize results n_import_successes = len(module_pypaths_succeeded) n_import_failures = len(module_pypaths_failed) n_import_failures_ignored = len(module_pypaths_failed_ignored) n_test_successes = len(test_pypaths_succeeded) n_test_failures = len(test_pypaths_failed) n_test_failures_ignored = len(test_pypaths_failed_ignored) set_verbosity(verbosity) logging.info( PFX + f"<< IMPORT TESTS : {n_import_successes} imported," f" {n_import_failures} failed," f" {n_import_failures_ignored} failed to import but ok to ignore >>") logging.info(PFX + f"<< UNIT TESTS : {n_test_successes} succeeded," f" {n_test_failures} failed," f" {n_test_failures_ignored} failed but ok to ignore >>") # Exit with error if any failures (import or unit test) if module_pypaths_failed or test_pypaths_failed: msgs = [] if module_pypaths_failed: msgs.append( f"{n_import_failures} module(s) failed to import:\n " + ", ".join(module_pypaths_failed)) if test_pypaths_failed: msgs.append(f"{n_test_failures} unit test(s) failed:\n " + ", ".join(test_pypaths_failed)) # Note the extra newlines before the exception to make it stand out; # and newlines after the exception are due to the pycuda error message # that is emitted when we call pycuda.autoinit.context.detach() sys.stdout.flush() sys.stderr.write("\n\n\n") raise Exception("\n".join(msgs) + "\n\n\n")
def __init__(self, server, git_version): print('Logs will be written to %s' % self.LOG_FOLDER) self.USERNAME = DEFAULT_USERNAME self.PASSWORD = DEFAULT_PASSWORD if not os.path.exists(self.LOG_FOLDER): os.makedirs(self.LOG_FOLDER) self.log_folder = self.LOG_FOLDER self.server = server self.hostname = socket.gethostname() self.ip = socket.gethostbyname(socket.gethostname()) self.container = JobContainer(self.log_folder, server) self.status_changed = True self.info = cpuinfo.get_cpu_info() self.s = requests.Session() self.first_update = True self.need_restart = False self.cpu_percent = 0.0 self.git_version = git_version # Log File Logger self.logger = logging.getLogger('JobClientLogger') self.logger.setLevel(logging.DEBUG) handler = logging.FileHandler( os.path.join(self.log_folder, 'JobClient.log')) formatter = logging.Formatter( "%(asctime)s - %(levelname)s - %(message)s") handler.setFormatter(formatter) self.logger.addHandler(handler) # Console handler ch = logging.StreamHandler() ch.setLevel(logging.INFO) self.logger.addHandler(ch) self.logger.info('Launching JobClient (Code Version %d)' % VERSION) try: import vhtrack self.cuda_device_count = vhtrack.query_cuda_device_count() if self.cuda_device_count > 0: di = vhtrack.DeviceInfo(0) if di.isCompatible(): total_memory = di.totalMemory() vram_in_GB = total_memory >> 30 if vram_in_GB < 4 or di.majorVersion() < 5: self.cuda_device_count = 0 self.logger.info( '%d CUDA GPU Detected %s with %d GB VRAM' % (self.cuda_device_count, di.name(), vram_in_GB)) else: self.cuda_device_count = 0 except: self.cuda_device_count = 0 if self.cuda_device_count == 0: self.logger.info('No CUDA GPU Detected') self.available_job_classes = create_available_job_list(self.logger)
args = parser.parse_args() # ---------------- PARAMS -------------------- # if args.model_type == 'Detection': args.models_dir = 'detection_models/' else: args.models_dir = 'general_models/' if args.avx: suffix_AVX = 'AVX' else: suffix_AVX = 'NOAVX' aux = cpuinfo.get_cpu_info() if args.device == 'CPU': if 'brand' in aux.keys(): args.device_name = cpuinfo.get_cpu_info()['brand'] else: args.device_name = cpuinfo.get_cpu_info()['brand_raw'] else: args.device_name = args.device + ' X' models = [ name for name in os.listdir(args.models_dir) if os.path.isdir(os.path.join(args.models_dir, name)) ] images = [ name for name in os.listdir(args.images_dir) if name.endswith('png') or name.endswith('jpg')
def blosc_extension(): print('[zarr] Setting up Blosc extension') # setup blosc extension blosc_sources = [] extra_compile_args = [] include_dirs = [] define_macros = [] # generic setup blosc_sources += [f for f in glob('c-blosc/blosc/*.c') if 'avx2' not in f and 'sse2' not in f] blosc_sources += glob('c-blosc/internal-complibs/lz4*/*.c') blosc_sources += glob('c-blosc/internal-complibs/snappy*/*.cc') blosc_sources += glob('c-blosc/internal-complibs/zlib*/*.c') blosc_sources += glob('c-blosc/internal-complibs/zstd*/common/*.c') blosc_sources += glob('c-blosc/internal-complibs/zstd*/compress/*.c') blosc_sources += glob('c-blosc/internal-complibs/zstd*/decompress/*.c') blosc_sources += glob('c-blosc/internal-complibs/zstd*/dictBuilder/*.c') include_dirs += [os.path.join('c-blosc', 'blosc')] include_dirs += [d for d in glob('c-blosc/internal-complibs/*') if os.path.isdir(d)] include_dirs += [d for d in glob('c-blosc/internal-complibs/*/*') if os.path.isdir(d)] define_macros += [('HAVE_LZ4', 1), ('HAVE_SNAPPY', 1), ('HAVE_ZLIB', 1), ('HAVE_ZSTD', 1)] # define_macros += [('CYTHON_TRACE', '1')] # determine CPU support for SSE2 and AVX2 cpu_info = cpuinfo.get_cpu_info() # SSE2 if 'sse2' in cpu_info['flags']: print('[zarr] SSE2 detected') extra_compile_args.append('-DSHUFFLE_SSE2_ENABLED') blosc_sources += [f for f in glob('c-blosc/blosc/*.c') if 'sse2' in f] if os.name == 'posix': extra_compile_args.append('-msse2') elif os.name == 'nt': define_macros += [('__SSE2__', 1)] # AVX2 if 'avx2' in cpu_info['flags']: print('[zarr] AVX2 detected') extra_compile_args.append('-DSHUFFLE_AVX2_ENABLED') blosc_sources += [f for f in glob('c-blosc/blosc/*.c') if 'avx2' in f] if os.name == 'posix': extra_compile_args.append('-mavx2') elif os.name == 'nt': define_macros += [('__AVX2__', 1)] # workaround lack of support for "inline" in MSVC when building for Python # 2.7 64-bit if PY2 and os.name == 'nt': extra_compile_args.append('-Dinline=__inline') if have_cython: sources = ['zarr/blosc.pyx'] else: sources = ['zarr/blosc.c'] # define extension module extensions = [ Extension('zarr.blosc', sources=sources + blosc_sources, include_dirs=include_dirs, define_macros=define_macros, extra_compile_args=extra_compile_args, ), ] if have_cython: extensions = cythonize(extensions) return extensions
import cpuinfo import platform import psutil processador = cpuinfo.get_cpu_info()['brand'] arquitectura = cpuinfo.get_cpu_info()['arch'] bits = cpuinfo.get_cpu_info()['bits'] frequencia_disponivel = cpuinfo.get_cpu_info()['hz_advertised'] frequencia_usada = cpuinfo.get_cpu_info()['hz_actual'] #platform kernel = platform.system() kernel_versao = platform.release() utilizador = platform.node() #psutil nr_nucleos = psutil.cpu_count() bateria = round(psutil.sensors_battery().percent)
def get_cpuinfo(self): return cpuinfo.get_cpu_info()
__author__ = "Alexander [Amper] Marshalov" __email__ = "*****@*****.**" __icq__ = "87-555-3" __jabber__ = "*****@*****.**" __twitter__ = "amper" __url__ = "http://amper.github.com/cityhash" from setuptools import setup from setuptools.extension import Extension from setuptools.dist import Distribution from os.path import join, dirname from cpuinfo import get_cpu_info cpu_info = get_cpu_info() have_sse42 = 'sse4.2' in cpu_info['flags'] try: from Cython.Distutils import build_ext except ImportError: USE_CYTHON = False else: USE_CYTHON = True class BinaryDistribution(Distribution): """ Subclass the setuptools Distribution to flip the purity flag to false. See http://lucumr.pocoo.org/2014/1/27/python-on-wheels/ """ def is_pure(self): # TODO: check if this is still necessary with Python v2.7
def brand(): return cpuinfo.get_cpu_info()['brand']
import logging import os import socket from typing import Optional import cpuinfo import psutil import pydantic import yaml __all__ = [ "get_config", "get_provenance_augments", "global_repr", "NodeDescriptor" ] # Start a globals dictionary with small starting values CPUINFO = cpuinfo.get_cpu_info() # We want physical cores if hasattr(psutil.Process(), "cpu_affinity"): cpu_cnt = len(psutil.Process().cpu_affinity()) else: cpu_cnt = psutil.cpu_count(logical=False) if cpu_cnt is None: cpu_cnt = psutil.cpu_count(logical=True) CPUINFO["count"] = cpu_cnt # Generic globals GLOBALS = {} GLOBALS["hostname"] = socket.gethostname() GLOBALS["memory"] = round(psutil.virtual_memory().available / (1024**3), 3) GLOBALS["cpu"] = CPUINFO["brand"]
def hz_actual(): return cpuinfo.get_cpu_info()['hz_actual']
''' This script is used to ask for the CPU flags on Windows. We use this instead of importing the cpuinfo package, because recent versions of py-cpuinfo use the multiprocessing module, and any import of cpuinfo that is not within a `if __name__ == '__main__':` block will lead to the script being executed twice. The CPU flags are printed to stdout encoded as JSON. ''' from __future__ import absolute_import from __future__ import print_function import json if __name__ == '__main__': import cpuinfo flags = cpuinfo.get_cpu_info()['flags'] print(json.dumps(flags))
def family(): return cpuinfo.get_cpu_info()['family']
def environment(): info = get_cpu_info() return render_template('environment.html',name=current_user.name, info = info)
__author__ = "Alexander [Amper] Marshalov" __email__ = "*****@*****.**" __icq__ = "87-555-3" __jabber__ = "*****@*****.**" __twitter__ = "amper" __url__ = "http://amper.github.com/cityhash" from setuptools import setup from setuptools.extension import Extension from setuptools.dist import Distribution from os.path import join, dirname try: from cpuinfo import get_cpu_info cpu_info = get_cpu_info() have_sse42 = 'sse4.2' in cpu_info['flags'] except Exception: have_sse42 = False try: from Cython.Distutils import build_ext except ImportError: USE_CYTHON = False else: USE_CYTHON = True class BinaryDistribution(Distribution): """ Subclass the setuptools Distribution to flip the purity flag to false. See http://lucumr.pocoo.org/2014/1/27/python-on-wheels/
def _cpu(command): _cpuinfo = cpuinfo.get_cpu_info() _cpuinfo["usage"] = psutil.cpu_percent() return _cpuinfo[command]
def getMaxCpus(self): info = cpuinfo.get_cpu_info() return info['count']
import psutil import time import cpuinfo data = "" cpui = cpuinfo.get_cpu_info() data += "\t\"cpu\": {\n" data += "\t\t\"threads\": \"" + str(psutil.cpu_count(logical=True)) + "\",\n" data += "\t\t\"cores\": \"" + str(psutil.cpu_count(logical=False)) + "\",\n" stats = psutil.cpu_stats() data += "\t\t\"context_switches\": \"" + str(stats.ctx_switches) + "\",\n" data += "\t\t\"interrupts\": \"" + str(stats.interrupts) + "\",\n" data += "\t\t\"soft_interrupts\": \"" + str(stats.soft_interrupts) + "\",\n" data += "\t\t\"architecture\": \"" + cpui["arch"] + "\",\n" data += "\t\t\"vendor\": \"" + cpui["brand"] + "\",\n" data += "\t\t\"frequency\": \"" + str(cpui["hz_actual_raw"][0]) + "\",\n" data += "\t\t\"base_frequency\": \"" + str( cpui["hz_advertised_raw"][0]) + "\",\n" data += "\t\t\"vendor_id\": \"" + cpui["vendor_id"] + "\",\n" data += "\t\t\"bits\": \"" + str(cpui["bits"]) + "\",\n" data += "\t\t\"cache\": \"" + str(cpui["l2_cache_size"]) + "\",\n" data += "\t\t\"cores\": {\n" counter = 0 cores = psutil.cpu_times_percent(interval=1, percpu=True) for times in cores: data += "\t\t\t\"" + str(counter) + "\": {\n" data += "\t\t\t\t\"nice\": \"" + str(times.nice) + "\",\n" data += "\t\t\t\t\"iowait\": \"" + str(times.iowait) + "\",\n" data += "\t\t\t\t\"irq\": \"" + str(times.irq) + "\",\n" data += "\t\t\t\t\"softirq\": \"" + str(times.softirq) + "\",\n"
def is_vm(): flags = cpuinfo.get_cpu_info()['flags'] return('hypervisor' in flags )
def vendor(): return cpuinfo.get_cpu_info()['vendor_id']
def _get_facts(*, kit=None): """ Get facts Facts are immutable global variables set at the very end of variable resolution. :param kit: A kit from which facts are going to be extracted :return: A dictionary with a bunch of scavenged variables """ # these are the facts facts = {} # General facts that are available for every platform _create_fact(facts, 'version', pkg_version) _create_fact(facts, 'install_prefix', os.getenv('HOME')) # General system-related facts _create_fact(facts, 'sys_uid', os.getuid()) _create_fact(facts, 'sys_gid', os.getgid()) # A collection of current environment variables is held in here _create_fact(facts, 'env', dict(os.environ)) # Facts for *nix operating systems _create_fact(facts, 'sys_path', os.getenv("PATH").split(":")) if os.name == 'posix': _create_fact(facts, 'sys_user', os.getenv('USER')) _create_fact(facts, 'sys_user_home', os.getenv('HOME')) #################################################### # System-related facts: # --------------------- # These facts collect characteristics of the current # platform zenfig is running on #################################################### # Operating System facts _system = platform.system() _create_fact(facts, 'system', _system) _create_fact(facts, 'sys_node', platform.node()) # These are exclusive to linux-based systems if _system == 'Linux': linux_distro = platform.linux_distribution() _create_fact(facts, 'linux_dist_name', linux_distro[0]) _create_fact(facts, 'linux_dist_version', linux_distro[1]) _create_fact(facts, 'linux_dist_id', linux_distro[2]) # kernel version _create_fact(facts, 'linux_release', platform.release()) # OSX-specific facts if _system == 'Darwin': _create_fact(facts, 'osx_ver', platform.mac_ver()) # Hardware-related facts _create_fact(facts, 'sys_machine', platform.machine()) # Low level CPU information (thanks to cpuinfo) _cpu_info = cpuinfo.get_cpu_info() _create_fact(facts, 'cpu_vendor_id', _cpu_info['vendor_id']) _create_fact(facts, 'cpu_brand', _cpu_info['brand']) _create_fact(facts, 'cpu_cores', _cpu_info['count']) _create_fact(facts, 'cpu_hz', _cpu_info['hz_advertised_raw'][0]) _create_fact(facts, 'cpu_arch', _cpu_info['arch']) _create_fact(facts, 'cpu_bits', _cpu_info['bits']) # RAM information _create_fact(facts, 'mem_total', psutil.virtual_memory()[0]) #################### # Python information #################### _py_ver = platform.python_version_tuple() _create_fact(facts, 'python_implementation', platform.python_revision()) _create_fact(facts, 'python_version', platform.python_version()) _create_fact(facts, 'python_version_major', _py_ver[0]) _create_fact(facts, 'python_version_minor', _py_ver[1]) _create_fact(facts, 'python_version_patch', _py_ver[2]) # Kit index variables are taken as well as facts # so they can be referenced by other variables, also # this means that index variables from a kit can reference # other variables as well, because all these variables get # rendered as part of variable resolution. if kit is not None: for key, value in kit.index_data.items(): _create_fact(facts, key, value, prefix="{}_{}".format(pkg_name, "kit")) # Give those variables already! return facts
def hz_advertised(): return cpuinfo.get_cpu_info()['hz_advertised']
def printWelcome(logo, version='beta', color_d='blue', color_out='magenta', attributes2=['bold']): print(logo) print( colored('+------------------------------------------+', color_out)) print(colored('beerware software.', color_d, attrs=attributes2)) s = getpass.getuser() + '@' + platform.node() print(colored('Computer: ', color_d, attrs=attributes2), colored(s, color_out)) print(colored('Script: ', color_d, attrs=attributes2), colored(Welcome.getScript(), color_out)) print(colored('Api version: ', color_d, attrs=attributes2), colored(sys.api_version, color_out)) print(colored('Path: ', color_d, attrs=attributes2), colored(sys.executable, color_out)) print(colored('Native Compiler: ', color_d, attrs=attributes2), colored(platform.python_compiler(), color_out)) print(colored('Architecture: ', color_d, attrs=attributes2), colored(platform.processor(), color_out)) s = platform.machine() + ' ' + platform.system( ) + ' Kernel version ' + platform.release() print(colored('Kernel: ', color_d, attrs=attributes2), colored(s, color_out)) print(colored('CPU Info: ', color_d, attrs=attributes2), colored(cpuinfo.get_cpu_info()['brand'], color_out)) print(colored('Python Version: ', color_d, attrs=attributes2), colored(platform.python_version(), color_out)) print(colored('Processors: ', color_d, attrs=attributes2), colored(psutil.cpu_count(logical=False), color_out)) print(colored('Terminal: ', color_d, attrs=attributes2), colored(Welcome.getTerminal(), color_out)) print(colored('User: '******'Current process: ', color_d, attrs=attributes2), colored(psutil.Process().pid, color_out)) print(colored('Code version: ', color_d, attrs=attributes2), colored(version, color_out)) mem = psutil.virtual_memory() print(colored('Total Memory: ', color_d, attrs=attributes2), colored(Welcome.sizeof_fmt(mem.total), color_out)) print(colored('Available Memory: ', color_d, attrs=attributes2), colored(Welcome.sizeof_fmt(mem.available), color_out)) print(colored('Free Memory: ', color_d, attrs=attributes2), colored(Welcome.sizeof_fmt(mem.free), color_out)) print(colored('Used Memory: ', color_d, attrs=attributes2), colored(Welcome.sizeof_fmt(mem.used), color_out)) print(colored('Active Memory: ', color_d, attrs=attributes2), colored(Welcome.sizeof_fmt(mem.active), color_out)) print(colored('Inactive Memory: ', color_d, attrs=attributes2), colored(Welcome.sizeof_fmt(mem.inactive), color_out)) print(colored('Wired Memory: ', color_d, attrs=attributes2), colored(Welcome.sizeof_fmt(mem.wired), color_out)) print(colored('Current path: ', color_d, attrs=attributes2), colored(os.getcwd(), color_out)) print(colored('Current date: ', color_d, attrs=attributes2), colored(datetime.datetime.now(), color_out)) print(colored('Current time: ', color_d, attrs=attributes2), colored(datetime.datetime.now(), color_out)) print( colored('+------------------------------------------+', color_out))
def bit(): return cpuinfo.get_cpu_info()['bits']
def get_cpu_info(self): """ Returns metadata about current CPU the computation is executed on. """ cpu_info = get_cpu_info() return {'brand': cpu_info['brand'], 'hz_actual': cpu_info['hz_actual']}
def model(): return cpuinfo.get_cpu_info()['model']
def get_processor_name() -> str: return cpuinfo.get_cpu_info()['brand_raw']
sources += [f for f in glob('c-blosc/blosc/*.c') if 'avx2' not in f and 'sse2' not in f] sources += glob('c-blosc/internal-complibs/lz4*/*.c') sources += glob('c-blosc/internal-complibs/snappy*/*.cc') sources += glob('c-blosc/internal-complibs/zlib*/*.c') sources += glob('c-blosc/internal-complibs/zstd*/*/*.c') inc_dirs += [os.path.join('c-blosc', 'blosc')] inc_dirs += [d for d in glob('c-blosc/internal-complibs/*') if os.path.isdir(d)] inc_dirs += [d for d in glob('c-blosc/internal-complibs/zstd*/*') if os.path.isdir(d)] def_macros += [('HAVE_LZ4', 1), ('HAVE_SNAPPY', 1), ('HAVE_ZLIB', 1), ('HAVE_ZSTD', 1)] # Guess SSE2 or AVX2 capabilities cpu_info = cpuinfo.get_cpu_info() # SSE2 if 'sse2' in cpu_info['flags']: print('SSE2 detected') CFLAGS.append('-DSHUFFLE_SSE2_ENABLED') sources += [f for f in glob('c-blosc/blosc/*.c') if 'sse2' in f] if os.name == 'posix': CFLAGS.append('-msse2') elif os.name == 'nt': def_macros += [('__SSE2__', 1)] # AVX2 if 'avx2' in cpu_info['flags']: print('AVX2 detected') CFLAGS.append('-DSHUFFLE_AVX2_ENABLED') sources += [f for f in glob('c-blosc/blosc/*.c') if 'avx2' in f]
def argparse_function( version: str, release: str) -> typing.Tuple[str, str, str, str, bool, bool]: conf = config.getInstance() parser = argparse.ArgumentParser( epilog=f"Load Config file '{conf.ini_path}'.") parser.add_argument("AnalysisPath", default='', nargs='?', help="Analysis folder or single file.") parser.add_argument("-p", "--path", default='', nargs='?', help="Analysis folder.") parser.add_argument("-s", "--success-output-folder", default='', nargs='?', help="Success output folder.") parser.add_argument("-f", "--failed-output-folder", default='', nargs='?', help="Failed output folder.") parser.add_argument( "-m", "--main-mode", default='', nargs='?', help="Main mode. 1:Scraping 2:Organizing 3:Scraping in analysis folder" ) # parser.add_argument("-C", "--config", default='config.ini', nargs='?', help="The config file Path.") parser.add_argument( "-L", "--link-mode", default='', nargs='?', help= "Create movie file link. 0:moving movie file, do not create link 1:soft link 2:try hard link first" ) default_logdir = str(Path.home() / '.avlogs') parser.add_argument( "-o", "--log-dir", dest='logdir', default=default_logdir, nargs='?', help= f"""Duplicate stdout and stderr to logfiles in logging folder, default on. default folder for current user: '******'. Change default folder to an empty file, or use --log-dir= to turn log off.""") parser.add_argument("-q", "--regex-query", dest='regexstr', default='', nargs='?', help="python re module regex filepath filtering.") parser.add_argument("-d", "--nfo-skip-days", dest='days', default='', nargs='?', help="Override nfo_skip_days value in config.") parser.add_argument("-c", "--stop-counter", dest='cnt', default='', nargs='?', help="Override stop_counter value in config.") parser.add_argument( "-R", "--rerun-delay", dest='delaytm', default='', nargs='?', help= "Delay (eg. 1h10m30s or 60 (second)) time and rerun, until all movies proceed. Note: stop_counter value in config or -c must none zero." ) parser.add_argument("-i", "--ignore-failed-list", action="store_true", help="Ignore failed list '{}'".format( os.path.join(os.path.abspath(conf.failed_folder()), 'failed_list.txt'))) parser.add_argument("-a", "--auto-exit", action="store_true", help="Auto exit after program complete") parser.add_argument( "-g", "--debug", action="store_true", help="Turn on debug mode to generate diagnostic log for issue report.") parser.add_argument( "-N", "--no-network-operation", action="store_true", help= "No network query, do not get metadata, for cover cropping purposes, only takes effect when main mode is 3." ) parser.add_argument("-w", "--website", dest='site', default='', nargs='?', help="Override [priority]website= in config.") parser.add_argument( "-D", "--download-images", dest='dnimg', action="store_true", help= "Override [common]download_only_missing_images=0 force invoke image downloading." ) parser.add_argument( "-C", "--config-override", dest='cfgcmd', action='append', nargs=1, help= "Common use config override. Grammar: section:key=value[;[section:]key=value] eg. 'de:s=1' or 'debug_mode:switch=1' override[debug_mode]switch=1 Note:this parameters can be used multiple times" ) parser.add_argument( "-z", "--zero-operation", dest='zero_op', action="store_true", help= """Only show job list of files and numbers, and **NO** actual operation is performed. It may help you correct wrong numbers before real job.""") verrel = version if not len(release) else f"{version}-{release}" parser.add_argument("-v", "--version", action="version", version=verrel) args = parser.parse_args() def set_natural_number_or_none(sk, value): if isinstance(value, str) and value.isnumeric() and int(value) >= 0: conf.set_override(f'{sk}={value}') def set_str_or_none(sk, value): if isinstance(value, str) and len(value): conf.set_override(f'{sk}={value}') def set_bool_or_none(sk, value): if isinstance(value, bool) and value: conf.set_override(f'{sk}=1') set_natural_number_or_none("common:main_mode", args.main_mode) set_natural_number_or_none("common:link_mode", args.link_mode) set_str_or_none("common:source_folder", args.path) set_str_or_none("common:success_output_folder", args.success_output_folder) set_str_or_none("common:failed_output_folder", args.failed_output_folder) set_bool_or_none("common:auto_exit", args.auto_exit) set_natural_number_or_none("common:nfo_skip_days", args.days) set_natural_number_or_none("common:stop_counter", args.cnt) set_bool_or_none("common:ignore_failed_list", args.ignore_failed_list) set_str_or_none("common:rerun_delay", args.delaytm) set_str_or_none("priority:website", args.site) if isinstance(args.dnimg, bool) and args.dnimg: conf.set_override("common:download_only_missing_images=0") set_bool_or_none("debug_mode:switch", args.debug) regexstr = args.regexstr if isinstance(args.AnalysisPath, str) and len(args.AnalysisPath): ana_path = Path(args.AnalysisPath) if ana_path.is_file(): sourcedir = ana_path.parent if sourcedir.is_dir(): conf.set_override(f'common:source_folder={str(sourcedir)}') regexstr = get_number(False, ana_path.stem) elif ana_path.is_dir(): conf.set_override(f"common:source_folder={str(ana_path)}") if isinstance(args.cfgcmd, list): for cmd in args.cfgcmd: conf.set_override(cmd[0]) no_net_op = False if conf.main_mode() == 3: no_net_op = args.no_network_operation if no_net_op: conf.set_override( "common:stop_counter=0;rerun_delay=0s;face:aways_imagecut=1") return args.logdir, regexstr, args.zero_op, no_net_op, verrel, get_cpu_info( )