def __init__(self, workload, sys_config, dispatcher, resource_manager=None, reader=None, job_factory=None, additional_data=[], simulator_config=None, overwrite_previous=True, scheduling_output=True, pprint_output=False, benchmark_output=False, statistics_output=True, save_parameters=None, show_statistics=True, **kwargs): """ Constructor of the HPC Simulator class. :param workload: Filepath to the workload, it is used by the reader. If a reader is not given, the default one is used. :param sys_config: Filepath to the synthetic system configuration. Used by the resource manager to create the system. :param dispatcher: Dispatching method :param resource_manager: Optional. Instantiation of the resource_manager class. :param reader: Optional. Instantiation of the reader class. :param job_factory: Optional. Instantiation of the job_factory class. :param additional_data: Optional. Array of Objects or Classes of AdditionalData class. :param simulator_config: Optional. Filepath to the simulator config. For replacing the misc.DEFAULT_SIMULATION parameters. :param overwrite_previous: Default True. Overwrite previous results. :param scheduling_output: Default True. Dispatching plan output. Format modificable in DEFAULT_SIMULATION :param pprint_output: Default False. Dispatching plan output in pretty print version. Format modificable in DEFAULT_SIMULATION :param benchmark_output: Default False. Measurement of the simulator and dispatcher performance. :param statistics_output: Default True. Statistic of the simulation. :param save_parameters: List of simulation name paremeters to be saved in the target results folder. None or empty for not saving the parameters. :param show_statistics: Default True. Show Statistic after finishing the simulation. :param \*\*kwargs: Optional parameters to be included in the Constants. """ assert (version_info >= ( 3, 5, )), 'Unsupported python version. Try with 3.5 or newer.' kwargs['OVERWRITE_PREVIOUS'] = overwrite_previous kwargs['SYS_CONFIG_FILEPATH'] = sys_config kwargs['WORKLOAD_FILEPATH'] = workload kwargs['SCHEDULING_OUTPUT'] = scheduling_output kwargs['PPRINT_OUTPUT'] = pprint_output kwargs['BENCHMARK_OUTPUT'] = benchmark_output kwargs['STATISTICS_OUTPUT'] = statistics_output kwargs['SHOW_STATISTICS'] = show_statistics _uargs = [] if not resource_manager: resource_manager, equiv, start_time = self.generate_enviroment( sys_config) kwargs['equivalence'] = equiv kwargs['start_time'] = start_time if not job_factory: kwargs['job_mapper'] = DEFAULT_SWF_MAPPER if not kwargs.get('EXTENDED_JOB_DESCRIPTION', False): kwargs['job_attrs'] = self.default_job_description() else: kwargs['job_attrs'] = self.extended_job_description() _jf_arguments = ['job_class', 'job_attrs', 'job_mapper'] args = self.prepare_arguments(_jf_arguments, kwargs) _uargs += _jf_arguments job_factory = JobFactory(resource_manager, **args) if workload and not reader: _reader_arguments = [ 'max_lines', 'tweak_function', 'equivalence', 'start_time' ] args = self.prepare_arguments(_reader_arguments, kwargs) reader = self.set_workload_input(workload, job_factory=job_factory, **args) _uargs += _reader_arguments for _u in _uargs: kwargs.pop(_u, None) SimulatorBase.__init__(self, config_file=simulator_config, **kwargs) if not isinstance(additional_data, list): assert ( isinstance(additional_data, AdditionalData) or issubclass(additional_data, AdditionalData) ), 'Only subclasses of AdditionalData class are acepted as additional_data argument ' additional_data = [additional_data] assert (isinstance(resource_manager, ResourceManager)) self.resource_manager = resource_manager assert (isinstance(dispatcher, SchedulerBase)) dispatcher.set_resource_manager(resource_manager) # self.dispatcher = dispatcher assert (isinstance(reader, Reader)) self.reader = reader assert (isinstance(job_factory, JobFactory)) self.job_factory = job_factory additional_data = self.additional_data_init(additional_data) self.mapper = EventManager(self.resource_manager, dispatcher, additional_data) if save_parameters: self._save_parameters(save_parameters) if benchmark_output: self._usage_writer = AsyncWriter( path=path.join( self.constants.RESULTS_FOLDER_PATH, self.constants.BENCHMARK_PREFIX + self.constants.WORKLOAD_FILENAME), pre_process_fun=Simulator.usage_metrics_preprocessor) self._process_obj = Process(getpid()) else: self._usage_writer = None self._process_obj = None self.start_simulation_time = None self.end_simulation_time = None self.max_sample = 2 self.daemons = {} self.loaded_jobs = 0 self.dispatched_jobs = 0 self.rejected_jobs = 0
def test_should_start_worker_process_even_if_no_job(self): self.receive_get_configuration_and_send_it_to_broker() broker_pid = self.broker.pid children_pid = [process.pid for process in \ Process(broker_pid).get_children()] self.assertEqual(len(children_pid), self.cpus)
def gerar_processo(PID): # Função que retorna informações de um processo pelo PID (em JSON) process = Process(PID) conn = process.connections() return json.dumps(conn)
def test_sigint(self): ''' Setup test server sleep 0.01 for each request Start spider in separate python shell (untill sigin or max 200 requests) Wait 1 sec (~100 requests, in reality less because of process start-up time) Send SIGINT to the process Check it returned with 13 or 139 codes 139 code means segfault (yeah...o_O) But as I see from logs it segfaults after successfully processing the SIGINT and this is all I need from this test ''' #logging.error('step-0') # pylint: disable=no-member self.server.response['sleep'] = 0.01 # pylint: enable=no-member with temp_file() as path: with open(path, 'w') as out: # pylint: disable=no-member out.write(self.script_tpl % ('', self.server.get_url())) # pylint: enable=no-member ret_codes = [] for _ in range(10): #logging.error('step-1') proc = Popen('python %s' % path, shell=True) #logging.error('step-2') parent = Process(proc.pid) #logging.error('step-3') time.sleep(1) #logging.error('killing children') for child in parent.children(): #logging.error('CHILD: %s', child.pid) # Sending multiple SIGINTs # because in very rare cases the only # sigint signals is ignored :-/ # do not send too fast for _ in range(1): try: #logging.error('sending sigint') child.send_signal(SIGNAL_INT) except NoSuchProcess: break else: time.sleep(1) if platform.system() == 'Darwin': # On OSX the Popen(shell=True) spawns only # one process, no child #logging.error('Killing parent') #logging.error('PARENT: %s', parent.pid) # Sending multiple SIGINTs # because in very rare cases the only # sigint signals is ignored :-/ # do not send too fast for _ in range(1): try: #logging.error('sending sigint') parent.send_signal(SIGNAL_INT) except NoSuchProcess: break else: time.sleep(1) #logging.error('step-4') ret = None for _ in range(20): #print('before proc-poll-%d' % step) ret = proc.poll() if ret is not None: break time.sleep(0.1) else: #logging.error('CHILD PROCESS DID NOT RETURN') #raise Exception('Child process did not return') # try to clean processes try: for child in parent.children(): child.send_signal(signal.SIGTERM) except NoSuchProcess: pass time.sleep(0.5) try: parent.send_signal(signal.SIGTERM) except NoSuchProcess: pass #logging.error('step-5') # FIXME: find out the reasonf of segfault # the 130 signal means the program was terminated by ctrl-c #print('RET CODE: %s' % ret) ret_codes.append(ret) # Could fail in 10% (1 of 10) # pylint: disable=no-member self.assertTrue(sum(1 for x in ret_codes if x in (13, 130, 139)) >= 9)
async def handle_message(self, message): message = json.loads(str(message["data"], "utf-8")) data = message["data"] type = message["type"] nonce = message["nonce"] original_cluster = message.get("original_cluster") waiting_for = message.get("waiting_for") cluster_id = message.get("cluster_id") extras = message.get("extras", {}) if type == "IDENTIFY": # we're syncing this cluster with ourselves, and send back our clusters if original_cluster == CLUSTER_ID: if isinstance(data, int): self.clusters.add(data) else: for x in data: self.clusters.add(x) else: self.clusters.add(original_cluster) data = json.dumps({ "nonce": None, "cluster_id": CLUSTER_ID, "data": list(self.clusters), "type": "IDENTIFY", "original_cluster": original_cluster, "waiting_for": waiting_for }) await self.redis.publish( f"{RELEASE}:CLUSTER_{original_cluster}", data) elif type == "VERIFICATION": discord_id = int(data["discordID"]) guild_id = int(data["guildID"]) roblox_id = data["robloxID"] #roblox_accounts = data["robloxAccounts"] guild = Bloxlink.get_guild(guild_id) if guild: member = guild.get_member(discord_id) if not member: try: member = await guild.fetch_member(discord_id) except NotFound: return roblox_user, _ = await get_user(roblox_id=roblox_id) try: added, removed, nickname, errors, roblox_user = await guild_obligations( member, guild=guild, roles=True, nickname=True, roblox_user=roblox_user, cache=False, dm=False, exceptions=("Blacklisted", "BloxlinkBypass", "RobloxAPIError", "RobloxDown", "PermissionError")) except Blacklisted as b: blacklist_text = "" if str(b): blacklist_text = f"You have an active restriction for: ``{b}``" else: blacklist_text = f"You have an active restriction from Bloxlink." try: await member.send( f"Failed to update you in the server: ``{blacklist_text}``" ) except Forbidden: pass except BloxlinkBypass: try: await member.send( f"You have the ``Bloxlink Bypass`` role, so I am unable to update you in the server." ) except Forbidden: pass except RobloxAPIError: try: await member.send( "An unknown Roblox API error occured, so I was unable to update you in the server. Please try again later." ) except Forbidden: pass except RobloxDown: try: await member.send( "Roblox appears to be down, so I was unable to retrieve your Roblox information. Please try again later." ) except Forbidden: pass except PermissionError as e: try: await member.send( f"A permission error occured, so I was unable to update you in the server: ``{e}``" ) except Forbidden: pass except CancelCommand: pass else: try: await member.send( f"Your account was successfully updated to **{roblox_user.username}** in the server **{guild.name}.**" ) except Forbidden: pass guild_data = await self.r.table("guilds").get(str( guild.id)).run() or {} # FIXME: use cache await post_event( guild, guild_data, "verification", f"{member.mention} has **verified** as ``{roblox_user.username}``.", GREEN_COLOR) elif type == "EVAL": res = (await eval(data, codeblock=False)).description data = json.dumps({ "nonce": nonce, "cluster_id": CLUSTER_ID, "data": res, "type": "CLIENT_RESULT", "original_cluster": original_cluster, "waiting_for": waiting_for }) await self.redis.publish(f"{RELEASE}:CLUSTER_{original_cluster}", data) elif type == "CLIENT_RESULT": task = self.pending_tasks.get(nonce) if task: task[1][cluster_id] = data task[2] += 1 waiting_for = message["waiting_for"] or len(self.clusters) if task[2] == waiting_for: if not task[0].done(): task[0].set_result(True) elif type == "DM": if 0 in SHARD_RANGE: try: message_ = await Bloxlink.wait_for( "message", check=lambda m: m.author.id == data and not m.guild, timeout=PROMPT["PROMPT_TIMEOUT"]) except asyncio.TimeoutError: message_ = "cancel (timeout)" data = json.dumps({ "nonce": nonce, "cluster_id": CLUSTER_ID, "data": getattr(message_, "content", message_), "type": "CLIENT_RESULT", "original_cluster": original_cluster, "waiting_for": waiting_for }) await self.redis.publish( f"{RELEASE}:CLUSTER_{original_cluster}", data) elif type == "STATS": seconds = floor(time() - STARTED) m, s = divmod(seconds, 60) h, m = divmod(m, 60) d, h = divmod(h, 24) days, hours, minutes, seconds = None, None, None, None if d: days = f"{d}d" if h: hours = f"{h}h" if m: minutes = f"{m}m" if s: seconds = f"{s}s" uptime = f"{days or ''} {hours or ''} {minutes or ''} {seconds or ''}".strip( ) process = Process(getpid()) mem = floor(process.memory_info()[0] / float(2**20)) data = json.dumps({ "nonce": nonce, "cluster_id": CLUSTER_ID, "data": (len(self.client.guilds), mem, uptime), "type": "CLIENT_RESULT", "original_cluster": original_cluster, "waiting_for": waiting_for }) await self.redis.publish(f"{RELEASE}:CLUSTER_{original_cluster}", data) elif type == "PLAYING_STATUS": presence_type = extras.get("presence_type", "normal") playing_status = extras.get("status", PLAYING_STATUS).format(prefix=PREFIX) if presence_type == "normal": await Bloxlink.change_presence(status=Status.online, activity=Game(playing_status)) elif presence_type == "streaming": stream_url = extras.get("stream_url", "https://twitch.tv/blox_link") await Bloxlink.change_presence( activity=Streaming(name=playing_status, url=stream_url)) data = json.dumps({ "nonce": nonce, "cluster_id": CLUSTER_ID, "data": True, "type": "CLIENT_RESULT", "original_cluster": original_cluster, "waiting_for": waiting_for }) await self.redis.publish(f"{RELEASE}:CLUSTER_{original_cluster}", data)
def _get_children_pids(self, pid): return Process(pid).children(recursive=True)
def test_infinite_loop(self): # Given user_answer = dedent("""\ #include<stdio.h> int main(void){ while(0==0){ printf("abc");} }""") hook_code = dedent("""\ def check_answer(user_answer): with open("Test.c", "w+") as f: f.write(user_answer) import subprocess success = False err = "Incorrect Answer" mark_fraction = 0.0 def _run_command(cmd): proc = subprocess.Popen("{}".format(cmd), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) stdout,stderr = proc.communicate() return stdout,stderr cmds = ["gcc Test.c", "./a.out"] for cmd in cmds: stdout, stderr = _run_command(cmd) if stdout.decode("utf-8") == "Hello, world!": success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """) test_case_data = [{ "test_case_type": "hooktestcase", "hook_code": hook_code, "weight": 1.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) self.assert_correct_output(self.timeout_msg, result.get("error")[0]["message"]) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) self.assertFalse(any(children_procs.children(recursive=True)))
import logging import sys from pulsar.utils.log import dictConfig, NullHandler # We can't log memory info without psutil try: # pragma nocover from psutil import Process from os import getpid _p = Process(getpid()) except: # pragma nocover _p = None def get_mem_rss(): if _p: return _p.get_memory_info().rss/1024 else: return 0.0 class ProcessInfoLogger(logging.Logger): """Custom logger that allows process information to be logged. Supported items: *:mem_rss: resident set size, in KB """ def makeRecord(self, *args, **kwargs): rv = logging.Logger.makeRecord(self, *args, **kwargs) if _p: rv.mem_rss = get_mem_rss() return rv
try: from bottombar import BottomBar except ImportError: BottomBar = None try: from resource import getrusage, RUSAGE_SELF except ImportError: try: from psutil import Process except ImportError: _rss_memory = None else: _rss_memory = lambda: Process().memory_info().rss else: _rss_memory = lambda: getrusage(RUSAGE_SELF).ru_maxrss << 10 def _format(uri, t0, width): if len(uri) + 8 <= width: # space for full uri, maybe stats minutes, seconds = divmod(int(time.perf_counter() - t0), 60) hours, minutes = divmod(minutes, 60) runtime = ' runtime: {}:{:02d}:{:02d}'.format(hours, minutes, seconds) memory = ' memory: {:,}M |'.format( _rss_memory() >> 20) if _rss_memory else '' prefix = 'writing log to ' if len(prefix) + len(uri) + len(memory) + len(runtime) > width: memory = memory[8:] runtime = runtime[9:]
def get_info(component_path, format, _now=datetime.utcnow): component_details = open(os.path.join(component_path, ZATO_INFO_FILE)).read() out = { 'component_details': component_details, 'component_full_path': component_path, 'component_host': current_host(), 'component_running': False, 'current_time': datetime.now().isoformat(), 'current_time_utc': datetime.utcnow().isoformat(), 'master_proc_connections': None, 'master_proc_pid': None, 'master_proc_name': None, 'master_proc_create_time': None, 'master_proc_create_time_utc': None, 'master_proc_username': None, 'master_proc_workers_no': None, 'master_proc_workers_pids': None, } master_proc_pid = None try: master_proc_pid = int( open(os.path.join(component_path, MISC.PIDFILE)).read()) except (IOError, ValueError): # Ok, no such file or it's empty pass if master_proc_pid: out['component_running'] = True master_proc = Process(master_proc_pid) workers_pids = sorted(elem.pid for elem in master_proc.children()) now = datetime.fromtimestamp(time(), UTC) mater_proc_create_time = master_proc.create_time() mater_proc_create_time_utc = datetime.fromtimestamp( mater_proc_create_time, UTC) out['mater_proc_uptime'] = now - mater_proc_create_time_utc out['mater_proc_uptime_seconds'] = int( out['mater_proc_uptime'].total_seconds()) out['master_proc_connections'] = format_connections( master_proc.connections(), format) out['master_proc_pid'] = master_proc.pid out['master_proc_create_time'] = datetime.fromtimestamp( mater_proc_create_time).isoformat() out['master_proc_create_time_utc'] = mater_proc_create_time_utc.isoformat( ) out['master_proc_username'] = master_proc.username() out['master_proc_name'] = master_proc.name() out['master_proc_workers_no'] = len(workers_pids) out['master_proc_workers_pids'] = workers_pids for pid in workers_pids: worker = Process(pid) worker_create_time = worker.create_time() worker_create_time_utc = datetime.fromtimestamp( worker_create_time, UTC) out['worker_{}_uptime'.format(pid)] = now - worker_create_time_utc out['worker_{}_uptime_seconds'.format(pid)] = int( out['worker_{}_uptime'.format(pid)].total_seconds()) out['worker_{}_create_time'.format(pid)] = datetime.fromtimestamp( worker_create_time).isoformat() out['worker_{}_create_time_utc'.format( pid)] = worker_create_time_utc.isoformat() out['worker_{}_connections'.format(pid)] = format_connections( worker.connections(), format) return out
def memory_usage_psutil(): # return the memory usage in MB return Process(os.getpid()).memory_info()[0] / float(2 ** 20)
def get_worker_pids(component_path): """ Returns PIDs of all workers of a given server, which must be already started. """ master_proc_pid = int( open(os.path.join(component_path, MISC.PIDFILE)).read()) return sorted(elem.pid for elem in Process(master_proc_pid).children())
def bot(worker_name): """ :return: """ global locks, drivers, video_url, cls, driver, watch_length, api_key proxies = Proxies(proto="http", worker_name=worker_name, api_key=api_key) first_run = True visits = 0 while True: try: LOG.INFO("{} Starting bot".format(worker_name)) if not first_run: proxies.rotate_proxy() else: with locks[0]: proxies.generate_proxies() first_run = False ua = generate_user_agent(os=('win', 'android')) if driver == "chrome": chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--proxy-server={}'.format( proxies.get_ip())) chrome_options.add_argument('--user-agent={}'.format(ua)) chrome_options.add_argument('--mute-audio') chrome_options.add_argument('--headless') if system() == 'Windows': executable_path = path_join(DRIVER_BIN, 'chromedriver.exe') elif system() == "Darwin": executable_path = path_join(DRIVER_BIN, 'chromedriver_mac') else: executable_path = path_join(DRIVER_BIN, 'chromedriver_linux') driver = webdriver.Chrome(options=chrome_options, executable_path=executable_path) else: firefox_profile = webdriver.FirefoxProfile() firefox_profile.set_preference("media.volume_scale", '0.0') firefox_profile.set_preference('general.useragent.override', ua) firefox_profile.set_preference('network.proxy.type', 1) firefox_profile.set_preference('network.proxy.http', proxies.get_ip()) firefox_profile.set_preference('network.proxy.http_port', int(proxies.get_port())) firefox_profile.set_preference('network.proxy.ssl', proxies.get_ip()) firefox_profile.set_preference('network.proxy.ssl_port', int(proxies.get_port())) firefox_profile.update_preferences() options = Options() options.headless = True if system() == 'Windows': executable_path = path_join(DRIVER_BIN, 'geckodriver_win64.exe') elif system() == "Darwin": if "geckodriver" in os.environ["PATH"]: executable_path = "/usr/local/bin/geckodriver" else: Exception("Geckdriver not found in /usr/local/bin/") else: executable_path = path_join(DRIVER_BIN, 'geckodriver_linux64') driver = webdriver.Firefox(firefox_profile=firefox_profile, options=options, executable_path=executable_path) # snag the PID of the driver process = driver.service.process pid = process.pid child_pids = [x.pid for x in Process(pid).children()] pids = [pid] + child_pids # Begin watching driver.set_page_load_timeout(45) driver.get(video_url) # Sometimes pop ups appear we should think about handling them if driver.title.endswith('YouTube'): play_button = driver.find_element_by_class_name( 'ytp-play-button') if play_button.get_attribute('title') == 'Play (k)': play_button.click() if play_button.get_attribute('title') == 'Play (k)': raise ElementClickInterceptedException if watch_length is None: video_current = driver.find_element_by_class_name( 'ytp-time-current').get_attribute('innerHTML') video_duration = driver.find_element_by_class_name( 'ytp-time-duration').get_attribute('innerHTML') duration = calc_dur_time(current_pos=video_current, duration=video_duration) status(worker_name, proxies, visits, pid) sleep(duration) else: status(worker_name, proxies, visits, pid) sleep(watch_length) LOG.INFO("{} finished watching video".format(worker_name)) visits += 1 except WebDriverException as e: LOG.WARN('[{}] - [{}] {}'.format(worker_name, id, e.__class__.__name__)) except KeyboardInterrupt: exit(0) except: exit(1) finally: LOG.WARN('[{}][{}] Quitting webdriver!'.format(worker_name, id)) try: driver except NameError: pass else: driver.quit() with locks[2]: try: pids except NameError: pass else: for pid in pids: try: drivers.remove(pid) except: pass
def extract_compiler(): """ Take the output of psutil (if available) or pstree and find the compiler """ result = "unknown" cmdline = "unknown" try: from psutil import Process p = Process(pid=int(os.getpid())) ignore_programs = ['ld', 'collect2', 'bash', 'Python'] def p_parent(): """ Determine parent of Process instance. Check whether parent is a function (psutil > v2) or a property (psutil < v2) """ return (callable(p.parent) and p.parent()) or p.parent def p_parent_name(): """ Determine the name of a parent of Process instance. Check whether parent is a function (psutil > v2) or a property (psutil < v2) """ return (callable(p.parent) and p.parent().name()) or p.parent.name def p_parent_exe(): """ Determine the absolute path of the executable. Check whether parent is a function (psutil > v2) or a property (psutil < v2) """ return (callable(p.parent) and p.parent().exe()) or p.parent.exe() def p_parent_cmdline(): """ Determine the command line of the the parent process. Check whether parent is a function (psutil > v2) or a property (psutil < v2) """ return (callable(p.parent) and p.parent().cmdline()) or p.parent.cmdline() while p_parent(): if p_parent_name() not in ignore_programs: path = p_parent_exe() result = p_parent_name() cmdline = p_parent_cmdline() break p = p_parent() except ImportError as e: ignore_programs = ['pstree', 'ld', 'collect2', 'python', 'sh'] pstree_bin = "@path_to_pstree@" pstree = capture("%s -l -sp %d" % (pstree_bin, os.getpid())).strip() if (pstree == "unknown"): return result a = pstree.split("---") for cmd in reversed(a): idx = cmd.find('(') pid = cmd[idx + 1:cmd.find(')')] cmd = cmd[:idx] if cmd not in ignore_programs: result = cmd path = 'unknown' proc_path = "/proc/%s/exe" % pid if os.path.exists(proc_path): path = os.path.realpath(proc_path) try: cmdline = capture("ps -o cmd -p %s" % pid).strip().split("\n")[1] except: cmdline = 'unknown' pass break return result, path, cmdline
def _get_shell(): try: shell = Process(os.getpid()).parent().name() except TypeError: shell = Process(os.getpid()).parent.name return shells[shell]
url_string = url.split('/') if len(url_string) > 2: return (url_string[2]) elif len(url_string) == 2: return (url_string[1]) else: return (url_string[0]) # Defining dictionary for storing values app_dictionary = {} start = time.time() last_window_id = GetForegroundWindow() pid = GetWindowThreadProcessId(last_window_id) application_name = Process(pid[-1]).name() application_name = application_name.replace(".exe", "") print(application_name) while True: current_window_id = GetForegroundWindow() if (current_window_id != last_window_id) & (len( GetWindowText(current_window_id)) != 0): end = time.time() duration, time_tuple = printTime(end - start) #print(current_window_id) #print(len(GetWindowText(current_window_id))) if round(end - start) != 0: print(duration) try:
def __init__(self): self.process = Process() self.peak_mem_res = 0 self.low_mem_warning = False
def memory_monitor(slot=0.1): """ Decorator: memory monitor. Args: slot(float): sampling frequency. Returns: func, memory: function output and memory usage. """ def _get_memory(monitor): """ Get memory usage. """ info = monitor.memory_full_info() return info.uss / 1024. / 1024. class _MonitorEngine(object): """ Monitor engine. """ def __init__(self, container=None, monitor=None, active=False, timeout=0.5): self._container = container self._monitor = monitor self._active = active self._thread = Thread(target=self._run, kwargs={'timeout': timeout}) @property def container(self): """ Container. """ return self._container def _run(self, timeout=1): """ Running function. """ while self._active: current_time = str(datetime.now()) self._container[current_time] = _get_memory(self._monitor) time.sleep(timeout) def start(self): """ Start. """ self._active = True self._thread.start() def stop(self): """ Stop. """ self._active = False self._thread.join() monitor_engine = _MonitorEngine(container=OrderedDict(), monitor=Process(os.getpid()), timeout=slot) def decorator(func): """ Outer Decorator. """ def _decorator(*args, **kwargs): """ Inner decorator. """ monitor_engine.start() result = func(*args, **kwargs) monitor_engine.stop() return result, monitor_engine.container return _decorator return decorator
def dump_coverage(self, timeout=15): assert SIGUSR1 is not None pid = self._puppet.get_pid() if pid is None or not self._puppet.is_healthy(): LOG.debug("Skipping coverage dump (target is not in a good state)") return # If at this point, the browser is in a good state, i.e. no crashes # or hangs, so signal the browser to dump coverage. try: for child in Process(pid).children(recursive=True): LOG.debug("Sending SIGUSR1 to %d (child)", child.pid) try: kill(child.pid, SIGUSR1) except OSError: LOG.warning("Failed to send SIGUSR1 to pid %d", child.pid) except (AccessDenied, NoSuchProcess): # pragma: no cover pass LOG.debug("Sending SIGUSR1 to %d (parent)", pid) try: kill(pid, SIGUSR1) except OSError: LOG.warning("Failed to send SIGUSR1 to pid %d", pid) start_time = time() gcda_found = False delay = 0.1 # wait for processes to write .gcno files # this should typically take less than 1 second while True: for proc in process_iter(attrs=["pid", "ppid", "open_files"]): # check if proc is the target or child process if pid in (proc.info["pid"], proc.info["ppid"]): if proc.info["open_files"] is None: continue if any( x.path.endswith(".gcda") for x in proc.info["open_files"]): gcda_found = True # get the pid of the process that has the file open gcda_open = proc.info["pid"] break else: gcda_open = None elapsed = time() - start_time if gcda_found: if gcda_open is None: # success LOG.debug("gcda dump took %0.2fs", elapsed) break if elapsed >= timeout: # timeout failure LOG.warning("gcda file open by pid %d after %0.2fs", gcda_open, elapsed) try: kill(gcda_open, SIGABRT) except OSError: pass sleep(1) self.close() break if delay < 1.0: # increase delay to a maximum of 1 second delay = min(1.0, delay + 0.1) elif elapsed >= 3: # assume we missed the process writing .gcno files LOG.warning("No gcda files seen after %0.2fs", elapsed) break if not self._puppet.is_healthy(): LOG.warning("Browser failure during dump_coverage()") break sleep(delay)
async def botinfo(self, ctx): """Viser info om meg""" dev = await self.bot.fetch_user(170506717140877312) start = perf_counter() status_msg = await ctx.send('Beregner ping...') end = perf_counter() ping = int((end - start) * 1000) now = time() diff = int(now - self.bot.uptime) days, remainder = divmod(diff, 24 * 60 * 60) hours, remainder = divmod(remainder, 60 * 60) minutes, seconds = divmod(remainder, 60) process = Process(getpid()) memory_usage = round(process.memory_info().rss / 1000000, 1) cpu_percent = process.cpu_percent() total_members = [] online_members = [] idle_members = [] dnd_members = [] offline_members = [] for guild in self.bot.guilds: for member in guild.members: if member.id in total_members: continue total_members.append(member.id) if str(member.status) == 'online': online_members.append(member.id) elif str(member.status) == 'idle': idle_members.append(member.id) elif str(member.status) == 'dnd': dnd_members.append(member.id) elif str(member.status) == 'offline': offline_members.append(member.id) embed = discord.Embed(color=ctx.me.color, url=self.bot.misc['website']) embed.set_author(name=dev.name, icon_url=dev.avatar_url) embed.set_thumbnail(url=self.bot.user.avatar_url) embed.add_field(name='Dev', value=f'{dev.mention}\n{dev.name}#{dev.discriminator}') embed.add_field(name='Oppetid', value=f'{days}d {hours}t {minutes}m {seconds}s') embed.add_field( name='Ping', value= f'Ekte ping: {ping} ms\nWebsocket ping: {int(self.bot.latency * 1000)} ms' ) embed.add_field(name='Servere', value=len(self.bot.guilds)) embed.add_field(name='Discord.py Versjon', value=discord.__version__) embed.add_field(name='Python Versjon', value=platform.python_version()) embed.add_field(name='Ressursbruk', value=f'RAM: {memory_usage} MB\nCPU: {cpu_percent}%') embed.add_field(name='Maskin', value=f'{platform.system()} {platform.release()}') embed.add_field( name=f'Brukere ({len(total_members)})', value=f'{self.bot.emoji["online"]}{len(online_members)} ' + f'{self.bot.emoji["idle"]}{len(idle_members)} ' + f'{self.bot.emoji["dnd"]}{len(dnd_members)} ' + f'{self.bot.emoji["offline"]}{len(offline_members)}') embed.add_field(name='Lenker', value=f'[Nettside]({self.bot.misc["website"]}) ' + f'| [Kildekode]({self.bot.misc["source_code"]})') await Defaults.set_footer(ctx, embed) await status_msg.edit(embed=embed, content=None)
def workers(): return [ p for p in Process().children(recursive=True) if "Worker(sleep_long" in p.name() ]
def bot(id): global args, locks, urls, user_agents, referers, proxies, drivers, watched_videos while True: try: url = choice(urls) with locks[0]: if len(proxies) == 0: proxies.extend(get_proxies()) proxy = choice(proxies) proxies.remove(proxy) log('[INFO][%d] Connecting to %s' % (id, proxy)) user_agent = choice( user_agents) if args.user_agent else user_agents( os=('win', 'android')) log('[INFO][%d] Setting user agent to %s' % (id, user_agent)) if args.slow_start: locks[1].acquire() if system() == 'Windows': executable_dir = path_join(environ['APPDATA'], 'DeBos', 'drivers') else: executable_dir = path_join(environ['HOME'], '.DeBos', 'drivers') seleniumwire_options = { 'proxy': { 'http': 'http://%s' % proxy, 'https': 'https://%s' % proxy, 'no_proxy': 'localhost,127.0.0.1' } } if args.driver == 'chrome': chrome_options = webdriver.ChromeOptions() chrome_options.add_argument( '--user-agent={}'.format(user_agent)) chrome_options.add_argument('--mute-audio') chrome_options.add_experimental_option('excludeSwitches', ['enable-logging']) if args.headless: chrome_options.add_argument('--headless') if is_root(): chrome_options.add_argument('--no-sandbox') if system() == 'Windows': executable_path = path_join(executable_dir, 'chromedriver.exe') else: executable_path = path_join(executable_dir, 'chromedriver') driver = webdriver.Chrome( options=chrome_options, seleniumwire_options=seleniumwire_options, executable_path=executable_path) else: firefox_options = webdriver.FirefoxOptions() firefox_options.preferences.update({ 'media.volume_scale': '0.0', 'general.useragent.override': user_agent }) if args.headless: firefox_options.add_argument('--headless') if system() == 'Windows': executable_path = path_join(executable_dir, 'geckodriver.exe') else: executable_path = path_join(executable_dir, 'geckodriver') driver = webdriver.Firefox( options=firefox_options, seleniumwire_options=seleniumwire_options, service_log_path=devnull, executable_path=executable_path) driver.header_overrides = {'Referer': choice(referers)} process = driver.service.process pid = process.pid cpids = [x.pid for x in Process(pid).children()] pids = [pid] + cpids drivers.extend(pids) if args.slow_start: locks[1].release() log('[INFO][%d] Successully started webdriver!' % id) driver.set_page_load_timeout(45) log('[INFO][%d] Opening %s' % (id, url)) driver.get(url) if driver.title.endswith('YouTube'): log('[INFO][%d] Video successfully loaded!' % id) try: WebDriverWait(driver, 3).until( EC.element_to_be_clickable( (By.CLASS_NAME, 'ytp-large-play-button'))).click() except: pass if args.duration: sleep(args.duration) else: video = WebDriverWait(driver, 3).until( EC.presence_of_element_located( (By.CLASS_NAME, 'html5-main-video'))) video_duration = driver.execute_script( 'return arguments[0].getDuration()', video) sleep(float(video_duration) * uniform(0.35, 0.85)) log('[INFO][%d] Video successfully viewed!' % id) watched_videos += 1 else: log('[INFO][%d] Dead proxy eliminated!' % id) except WebDriverException as e: log('[WARNING][%d] %s' % (id, e.__class__.__name__)) except NoSuchProcess: log('[WARNING][%d] NoSuchProcess' % id) except KeyboardInterrupt: exit(0) except: exit(1) finally: log('[INFO][%d] Quitting webdriver!' % id) try: driver except NameError: pass else: driver.quit() with locks[2]: try: pids except NameError: pass else: for pid in pids: try: drivers.remove(pid) except: pass
def get_process(self, pid): if not pid in self.DProcesses: self.DProcesses[pid] = Process(pid) return self.DProcesses[pid]
def ensure_no_pidfile(self, log_file_marker): pidfile = abspath(join(self.component_dir, 'pidfile')) # Pidfile exists .. if exists(pidfile): # .. but raise an error only if the PID it points to belongs # to an already running component. Otherwise, it must be a stale pidfile # that we can safely delete. pid = open(pidfile).read().strip() try: pid = int(pid) except ValueError: raise Exception( 'Could not parse pid value `{}` as an integer ({})'.format( pid, pidfile)) else: try: info = get_info(self.component_dir, INFO_FORMAT.DICT) except AccessDenied: # This could be another process /or/ it can be our own component started by another user, # so to be on the safe side, indicate an error instead of deleting the pidfile raise Exception( 'Access denied to PID `{}` found in `{}`'.format( pid, pidfile)) except NoSuchProcess: # This is fine, there is no process of that PID, # which means that this PID does not belong to our component # (because it doesn't belong to any process), so we may just delete this pidfile safely .. os.remove(pidfile) # .. but, if the component is load-balancer, we also need to delete its agent's pidfile. # The assumption is that if the load-balancer is not running then so isn't its agent. if log_file_marker == 'lb-agent': lb_agent_pidfile = abspath( join(self.component_dir, 'zato-lb-agent.pid')) os.remove(lb_agent_pidfile) else: # # This PID exists, but it still still possible that it belongs to another process # that took over a PID previously assigned to a Zato component, # in which case we can still delete the pidfile. # # We decide that a process is actually an already running Zato component if it has # opened log files that should belong that kind of component, as indicated by log_file_marker, # otherwise we assume this PID belongs to a completely different process and we can delete pidfile. # has_log = False has_lock = False log_path = abspath( join(self.component_dir, 'logs', '{}.log'.format(log_file_marker))) lock_path = abspath( join(self.component_dir, 'logs', '{}.lock'.format(log_file_marker))) for name in Process(pid).open_files(): if name.path == log_path: has_log = True elif name.path == lock_path: has_lock = True # Both files exist - this is our component and it's running so we cannot continue if has_log and has_lock: raise Exception( 'Cannot proceed, found pidfile `{}`'.format( pidfile)) # This must be an unrelated process, so we can delete pidfile .. os.remove(pidfile) # .. again, if the component is load-balancer, we also need to delete its agent's pidfile. # The assumption is that if the load-balancer is not running then so isn't its agent. if log_file_marker == 'lb-agent': lb_agent_pidfile = abspath( join(self.component_dir, 'zato-lb-agent.pid')) os.remove(lb_agent_pidfile) if self.show_output: self.logger.info('No such pidfile `%s`, OK', pidfile)
def _get_memory_usage(pid, force_gc=False): if force_gc: gc.collect() return Process(pid).memory_info().rss
] get_thread_ident = get_original(thread_mod_name, 'get_ident') start_new_thread = get_original(thread_mod_name, 'start_new_thread') thread_sleep = get_original('time', 'sleep') try: # The standard library 'resource' module doesn't provide # a standard way to get the RSS measure, only the maximum. # You might be tempted to try to compute something by adding # together text and data sizes, but on many systems those come back # zero. So our only option is psutil. from psutil import Process, AccessDenied # Make sure it works (why would we be denied access to our own process?) try: Process().memory_full_info() except AccessDenied: # pragma: no cover Process = None except ImportError: Process = None class MonitorWarning(RuntimeWarning): """The type of warnings we emit.""" class _MonitorEntry(object): __slots__ = ('function', 'period', 'last_run_time') def __init__(self, function, period):
def get_info(process=None, interval=0, with_childs=False): """Return information about a process. (can be an pid or a Process object) If process is None, will return the information about the current process. """ if process is None or isinstance(process, int): if process is None: pid = os.getpid() else: pid = process if pid in _PROCS: process = _PROCS[pid] else: _PROCS[pid] = process = Process(pid) info = {} try: mem_info = process.get_memory_info() info['mem_info1'] = bytes2human(mem_info[0]) info['mem_info2'] = bytes2human(mem_info[1]) except AccessDenied: info['mem_info1'] = info['mem_info2'] = "N/A" try: info['cpu'] = process.get_cpu_percent(interval=interval) except AccessDenied: info['cpu'] = "N/A" try: info['mem'] = round(process.get_memory_percent(), 1) except AccessDenied: info['mem'] = "N/A" try: cpu_times = process.get_cpu_times() ctime = timedelta(seconds=sum(cpu_times)) ctime = "%s:%s.%s" % ( ctime.seconds // 60 % 60, str( (ctime.seconds % 60)).zfill(2), str(ctime.microseconds)[:2]) except AccessDenied: ctime = "N/A" info['ctime'] = ctime try: info['pid'] = process.pid except AccessDenied: info['pid'] = 'N/A' try: info['username'] = process.username except AccessDenied: info['username'] = '******' try: info['nice'] = process.nice except AccessDenied: info['nice'] = 'N/A' except NoSuchProcess: info['nice'] = 'Zombie' try: cmdline = os.path.basename(shlex.split(process.cmdline[0])[0]) except (AccessDenied, IndexError): cmdline = "N/A" info['cmdline'] = cmdline info['children'] = [] if with_childs: for child in process.get_children(): info['children'].append(get_info(child, interval=interval)) return info
def panel_status(): time.sleep(1) panel_path = '/www/server/panel' pool = 'http://' if os.path.exists(panel_path + '/data/ssl.pl'): pool = 'https://' port = '8888' if os.path.exists(panel_path + '/data/port.pl'): port = public.readFile(panel_path + '/data/port.pl').strip() panel_url = pool + '127.0.0.1:' + port + '/service_status' panel_pid = get_panel_pid() n = 0 s = 0 v = 0 while True: time.sleep(5) check_session() if not panel_pid: panel_pid = get_panel_pid() if not panel_pid: service_panel('start') try: f = Process(panel_pid).cmdline()[-1] if f.find('runserver') == -1 and f.find('BT-Panel') == -1: service_panel('start') time.sleep(3) panel_pid = get_panel_pid() continue except: service_panel('start') time.sleep(3) panel_pid = get_panel_pid() continue n += 1 v += 1 if v > 10: v = 0 log_path = panel_path + '/logs/error.log' if os.path.exists(log_path): e_body = public.GetNumLines(log_path, 10) if e_body: if e_body.find( 'PyWSGIServer.do_close') != -1 or e_body.find( 'Expected GET method:') != -1 or e_body.find( 'Invalid HTTP method:' ) != -1 or e_body.find('table session') != -1: result = HttpGet(panel_url) if result != 'True': if e_body.find('table session') != -1: sess_file = '/dev/shm/session.db' if os.path.exists(sess_file): os.remove(sess_file) public.ExecShell( "bash /www/server/panel/init.sh reload &") time.sleep(10) result = HttpGet(panel_url) if result == 'True': public.WriteLog( 'TYPE_SOFE', 'Checked to panel service exception, has been automatically restored!' ) if n > 18000: n = 0 result = HttpGet(panel_url) if result == 'True': time.sleep(10) continue update_panel() result = HttpGet(panel_url) if result == 'True': public.WriteLog( 'TYPE_SOFE', 'Checked to panel service exception, has been automatically restored!' ) time.sleep(10) continue
def bot(id): global args, locks, urls, user_agents, referers, proxies, drivers, watched_ads while True: try: url = choice(urls) with locks[0]: if len(proxies) == 0: proxies.extend(get_proxies()) proxy = choice(proxies) proxies.remove(proxy) log('[INFO][%d] Connecting to %s' % (id, proxy)) user_agent = choice( user_agents) if args.user_agent else user_agents() log('[INFO][%d] Setting user agent to %s' % (id, user_agent)) if args.slow_start: locks[1].acquire() if system() == 'Windows': executable_dir = path_join(environ['APPDATA'], 'DeBos', 'drivers') else: executable_dir = path_join(environ['HOME'], '.DeBos', 'drivers') seleniumwire_options = { 'proxy': { 'http': 'http://%s' % proxy, 'https': 'https://%s' % proxy, 'no_proxy': 'localhost,127.0.0.1' } } if args.driver == 'chrome': chrome_options = webdriver.ChromeOptions() chrome_options.add_argument( '--user-agent={}'.format(user_agent)) chrome_options.add_argument('--mute-audio') chrome_options.add_argument("--disable-extensions") chrome_options.add_argument("--disable-gpu") chrome_options.add_argument("--disable-dev-shm-usage") chrome_options.add_argument("--no-sandbox") chrome_options.add_experimental_option('excludeSwitches', ['enable-logging']) if args.headless: chrome_options.add_argument('--headless') if is_root(): chrome_options.add_argument('--no-sandbox') if system() == 'Windows': executable_path = path_join(executable_dir, 'chromedriver.exe') else: executable_path = path_join(executable_dir, 'chromedriver') driver = webdriver.Chrome( options=chrome_options, seleniumwire_options=seleniumwire_options, executable_path=executable_path) else: firefox_options = webdriver.FirefoxOptions() firefox_options.preferences.update({ 'media.volume_scale': '0.0', 'general.useragent.override': user_agent }) if args.headless: firefox_options.add_argument('--headless') if system() == 'Windows': executable_path = path_join(executable_dir, 'geckodriver.exe') else: executable_path = path_join(executable_dir, 'geckodriver') driver = webdriver.Firefox( options=firefox_options, seleniumwire_options=seleniumwire_options, service_log_path=devnull, executable_path=executable_path) driver.header_overrides = {'Referer': choice(referers)} process = driver.service.process pid = process.pid cpids = [x.pid for x in Process(pid).children()] pids = [pid] + cpids drivers.extend(pids) if args.slow_start: locks[1].release() log('[INFO][%d] Successully started webdriver!' % id) driver.set_page_load_timeout(60) log('[INFO][%d] Opening %s' % (id, url)) driver.get(url) if driver.title == 'Shrink your URLs and get paid!': log('[INFO][%d] Website successfully loaded!' % id) WebDriverWait(driver, 10).until( EC.element_to_be_clickable( (By.ID, 'skip_bu2tton'))).click() log('[INFO][%d] Ad successfully viewed!' % id) if not args.verbose: watched_ads += 1 else: log('[WARNING][%d] Dead proxy eliminated!' % id) except WebDriverException as e: log('[WARNING][%d] %s' % (id, e.__class__.__name__)) except KeyboardInterrupt: exit(0) except: exit(1) finally: log('[INFO][%d] Quitting webdriver!' % id) try: driver except NameError: pass else: driver.quit() with locks[2]: try: pids except NameError: pass else: for pid in pids: try: drivers.remove(pid) except: pass
async def show_bot_info(self, ctx, patreon_status): embed = Embed( title="Doob Info <:doob:754762131085459498>", colour=ctx.author.colour, timestamp=datetime.utcnow(), ) bot_version = self.bot.VERSION proc = Process() with proc.oneshot(): uptime = timedelta(seconds=time() - proc.create_time()) cpu_time = timedelta(seconds=(cpu := proc.cpu_times()).system + cpu.user) # pylint: disable=used-before-assignment mem_total = virtual_memory().total / (1025**2) mem_of_total = proc.memory_percent() mem_usg = mem_total * (mem_of_total / 100) fields = [ ("Name", "Doob <:doob:754762131085459498>", False), ( "Description", "The multipurpose Discord Bot with global leveling and powerful logging tools for your server.", False, ), ("Developers", "<@308000668181069824>", False), ("Doob's Server Count", f"{str(len(self.bot.guilds))}", True), ("Doob's Member Count", f"{str(len(self.bot.users))}", True), ( "The ping for Doob is...", f" :ping_pong: {round(self.bot.latency * 1000)} ms", False, ), ("Python Version", python_version(), True), ("Uptime", uptime, True), ("CPU Time", cpu_time, True), ( "Memory Usage", f"{mem_usg:,.3f} MiB / {mem_total:,.0f} MiB ({mem_of_total:.0f}%)", True, ), ("Library", f"discord.py {discord_version}", True), ( "Bot Version", f"{self.bot.VERSION} - [Changelog](https://github.com/doobdev/doob/blob/master/CHANGELOG.md#v{bot_version.replace('.', '')})", True, ), ("Top.gg Link", "https://top.gg/bot/680606346952966177", False), ( "Invite Link", "[Invite Link Here](https://discordapp.com/oauth2/authorize?client_id=680606346952966177&scope=bot&permissions=271674430)", True, ), ( "GitHub Repository", "[Click Here](https://github.com/doobdev/doob)", True, ), ] for name, value, inline in fields: embed.add_field(name=name, value=value, inline=inline) embed.set_thumbnail(url=ctx.guild.me.avatar_url) embed.set_footer( text=f"{ctx.author.name} requested Doob's information", icon_url=ctx.author.avatar_url, ) if patreon_status == True: embed.add_field( name="Patreon", value= f"Thanks for [Donating](https://patreon.com/doobdev) {ctx.author.display_name}! :white_check_mark:", inline=False, ) await ctx.reply(embed=embed) if patreon_status == False: embed.add_field( name="Patreon", value="[Click Here for Patreon](https://patreon.com/doobdev)", inline=False, ) await ctx.reply(embed=embed)