def start(self) -> None:
"""
Downloads the save, starts the Minecraft server, and uploads the save
upon termination
"""
try:
self._create_minecraft_path()
self.cloud.get_save(self._minecraft_path)
self._create_minecraft_eula()
command = self._build_minecraft_cmd()
print("Starting Minecraft server...")
process = psutil.Popen(
args=shlex.split(command, posix=os.name != "nt"),
stdin=sys.stdin,
stdout=sys.stdout,
cwd=self._minecraft_path,
)
process.wait()
finally:
print("Minecraft server stopped.")
def restore_dump(client: pymongo.MongoClient,
http_session: requests.Session) -> None:
"""Осуществляет восстановление данных по дивидендам."""
if not config.MONGO_DUMP.exists():
logging.info(
f"Файлы с данными о дивидендах отсутствуют - начинается загрузка")
path = config.MONGO_DUMP / SOURCE_DB
path.mkdir(parents=True)
for name, url in DIV_DATA_URL.items():
with http_session.get(url, stream=True) as respond:
with open(path / name, "wb") as fin:
fin.write(respond.content)
logging.info(f"Файлы с данными о дивидендах загружены")
if SOURCE_DB not in client.list_database_names():
logging.info(f"Начато восстановление данных с дивидендами")
mongo_restore = ["mongorestore", config.MONGO_DUMP]
process = psutil.Popen(mongo_restore)
status = process.wait()
logging.info(
f"Восстановление данных с дивидендами завершен со статусом {status}"
)
def spawnVLC(cmd, delay=0):
try:
if AceConfig.osplatform == 'Windows' and AceConfig.vlcuseaceplayer:
import _winreg
import os.path
reg = _winreg.ConnectRegistry(None, _winreg.HKEY_CURRENT_USER)
try:
key = _winreg.OpenKey(reg, 'Software\AceStream')
except:
print "Can't find AceStream!"
sys.exit(1)
dir = _winreg.QueryValueEx(key, 'InstallDir')
playerdir = os.path.dirname(dir[0] + '\\player\\')
cmd[0] = playerdir + '\\' + cmd[0]
stdout = None if AceConfig.loglevel == logging.DEBUG else DEVNULL
stderr = None if AceConfig.loglevel == logging.DEBUG else DEVNULL
AceStuff.vlc = psutil.Popen(cmd, stdout=stdout, stderr=stderr)
gevent.sleep(delay)
return True
except:
return False
def run_popen(self):
self.stdout_stderr_file = "%s/%s_%s.stdout_stderr" % (
self.ts.data_dir, self.obs, self.task)
self.remove_file_if_exists(self.stdout_stderr_file)
stdout_stderr_buf = open(self.stdout_stderr_file, "w")
process = psutil.Popen(
['%s/do_%s.sh' % (self.path_to_do_scripts, self.task)] + self.args,
cwd=self.cwd,
env=self.full_env,
stdout=stdout_stderr_buf,
stderr=subprocess.STDOUT)
try:
process.nice(
2
) # Jon : I want to set all the processes evenly so they don't compete against core OS functionality (ssh, cron etc..) slowing things down.
if PLATFORM != "Darwin": # Jon : cpu_affinity doesn't exist for the mac, testing on a mac... yup... good story.
process.cpu_affinity(range(psutil.cpu_count()))
except:
logger.exception("Could not set cpu affinity")
return process
def execute(self, command, cwd):
self._killMiner()
self.clearAll()
self.process = psutil.Popen(command,
cwd=cwd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
#self.process = psutil.Popen(command, stdout=subprocess.PIPE, shell=True)
#self.process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
self.killSignal = False
self.threadStreams = threading.Thread(target=self.__runStreamThreads)
self.threadStreams.start()
time.sleep(1)
return self.__isProcessRunning()
def start_process(script_name):
"""
Starts a process in the background and writes a PID file
returns integer: pid
http://stackoverflow.com/questions/7989922/opening-a-process-with-popen-and-getting-the-pid
Popen.pid The process ID of the child process.
Note that if you set the shell argument to True, this is the process ID of the spawned shell.
"""
# /tmp/xxx.py -> xxx
script_name = os.path.basename(script_name)
name = os.path.splitext(script_name)[0]
# Check if the process is already running
status, pid = get_process_status(name)
print 'process-%s: %s' % (status, pid)
if pid is not None and status in [psutil.STATUS_RUNNING, psutil.STATUS_SLEEPING]:
# must wait for subprocess thread ready
print 'process-%s exist: %s' % (pid, status)
return pid
# start script xxx.py without shell
# process_shell = subprocess.Popen(path + ' > /dev/null 2> /dev/null &', shell=True)
# process = subprocess.Popen(['python', script_name], shell=False, stderr=subprocess.STDOUT)
process = psutil.Popen(['python', script_name], shell=False, stderr=subprocess.STDOUT)
proc = psutil.Process(process.pid)
# must wait for subprocess thread ready
while proc.status() != psutil.STATUS_RUNNING:
print 'wait process-%s: %s' % (process.pid, proc.status())
time.sleep(0.3)
else:
# record pid to xxx.pid
file_path = os.path.join(PID_PATH, '%s.pid' % name)
with open(file_path, 'w') as pidfile:
pidfile.write(str(process.pid))
print 'process-%s started.' % process.pid
return process.pid
def check_disks():
disk_usage = {}
counters = {}
for partition in psutil.disk_partitions(all=False):
try:
usage = psutil.disk_usage(partition.mountpoint)
disk = re.sub(" ", "_", partition.mountpoint).replace(':', '').replace('\\', '').lower()
if os.name == 'nt':
counters[disk] = "LogicalDisk(%s)\Current Disk Queue Length" % partition.device.replace('\\', '')
if 'cdrom' in partition.opts or partition.fstype == '':
continue
if 'Volumes' in partition.mountpoint:
continue
if 'libc.so' in partition.mountpoint:
continue
disk_usage['disk.' + disk + '.percent_used'] = "%d%%" % int(usage.percent)
disk_usage['disk.' + disk + '.percent_free'] = "%d%%" % int(100 - usage.percent)
disk_usage['disk.' + disk + '.free'] = "%sb" % usage.free
disk_usage['disk.' + disk + '.used'] = "%sb" % usage.used
used_gb = _bytes_to_gb(usage.used)
disk_usage['disk.' + disk + '.used_gb'] = "%sGb" % used_gb
free_gb = _bytes_to_gb(usage.free)
disk_usage['disk.' + disk + '.free_gb'] = "%sGb" % free_gb
total_gb = _bytes_to_gb(usage.total)
disk_usage['disk.' + disk + '.total_gb'] = "%sGb" % total_gb
except OSError:
continue
if os.name == 'nt':
command = ['typeperf.exe', '-sc', '1']
counters_list = counters.values()
p = psutil.Popen(command + counters_list, stdout=subprocess.PIPE)
output = p.communicate()[0]
i = 1
for disk, counter in counters.iteritems():
value = output.splitlines()[2].split(',')[i].replace('"','').strip()
disk_usage['disk.' + disk + '.current_disk_queue_length'] = round(float(value), 2)
i += 1
return disk_usage
def __run_executable(self, args, exe_path, prog_env, prog_cwd, runtime):
process = psutil.Popen(args,
executable=exe_path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=prog_env,
cwd=prog_cwd)
def _msg(*text):
print('[' + str(process.pid) + '] ', *text)
# Run executable. stderr is redirected to stdout.
_msg('RUNNING: ', safe_repr(exe_path), ', args: ', safe_repr(args))
# 'psutil' allows to use timeout in waiting for a subprocess.
# If not timeout was specified then it is 'None' - no timeout, just waiting.
# Runtime is useful mostly for interactive tests.
try:
timeout = runtime if runtime else _EXE_TIMEOUT
stdout, stderr = process.communicate(timeout=timeout)
retcode = process.returncode
except (psutil.TimeoutExpired, subprocess.TimeoutExpired):
if runtime:
# When 'runtime' is set then expired timeout is a good sing
# that the executable was running successfully for a specified time.
# TODO Is there a better way return success than 'retcode = 0'?
retcode = 0
else:
# Exe is running and it is not interactive. Fail the test.
retcode = 1
_msg('TIMED OUT!')
# Kill the subprocess and its child processes.
for p in list(process.children(recursive=True)) + [process]:
with suppress(psutil.NoSuchProcess):
p.kill()
stdout, stderr = process.communicate()
sys.stdout.buffer.write(stdout)
sys.stderr.buffer.write(stderr)
return retcode
def start_ffmpeg(self, ffmpeg_args, via_wrapper=False):
ffmpeg_cmd = '{} {} -bsf dump_extra -an -f rtp rtp://{}:8004?pkt_size=1300'.format(
FFMPEG, ffmpeg_args, JANUS_SERVER)
_logger.debug('Popen: {}'.format(ffmpeg_cmd))
FNULL = open(os.devnull, 'w')
self.ffmpeg_proc = psutil.Popen(ffmpeg_cmd.split(' '),
stdin=subprocess.PIPE,
stdout=FNULL,
stderr=subprocess.PIPE)
self.ffmpeg_proc.nice(10)
cpu_watch_dog(self.ffmpeg_proc,
max=80,
interval=20,
streaming_status=self.plugin.streaming_status)
def monitor_ffmpeg_process(
): # It's pointless to restart ffmpeg without calling pi_camera.record with the new input. Just capture unexpected exits not to see if it's a big problem
ring_buffer = deque(maxlen=50)
while True:
err = to_unicode(self.ffmpeg_proc.stderr.readline())
if not err: # EOF when process ends?
if self.shutting_down:
return
returncode = self.ffmpeg_proc.wait()
msg = 'STDERR:\n{}\n'.format('\n'.join(ring_buffer))
_logger.error(msg)
self.sentry.captureMessage(
'ffmpeg quit! This should not happen. Exit code: {}'.
format(returncode),
tags=get_tags())
return
else:
ring_buffer.append(err)
ffmpeg_thread = Thread(target=monitor_ffmpeg_process)
ffmpeg_thread.daemon = True
ffmpeg_thread.start()
def start(self, start_msgs=None):
"""
Запустить дижек
:param start_msgs: Сообщение в логе, которое нужно проверить после запуска движка
"""
self._stats_receiver.start()
cmd = [self._engine_bin]
for option, value in self._options.items():
if value is not None:
cmd.append(option)
if value is not True:
cmd.append(str(value))
if self._binlog_path:
cmd.append(self._binlog_path)
self._log_file_write_fd = open(self._log_file, 'ab')
self._log_file_read_fd = open(self._log_file, 'r')
print("\nStarting engine: [{}]".format(cyan(" ".join(cmd))))
self._engine_process = psutil.Popen(cmd,
stdout=self._log_file_write_fd,
stderr=subprocess.STDOUT,
cwd=self._working_dir)
if os.waitpid(self._engine_process.pid, os.WNOHANG)[1] != 0:
raise RuntimeError("Can't start the engine process")
self.assert_log(["Connected to statsd"],
message="Can't find statsd connection message")
self._stats_receiver.wait_next_stats()
if start_msgs:
self.assert_log(
start_msgs,
message="Engine expected start message cannot be found!")
else:
self._assert_no_errors()
self._assert_availability()
atexit.register(self.stop)
def buildIntrinsics(self, node, additionalViews=()):
""" Build intrinsics from node current views and optional additional views
Args:
node: the CameraInit node instance to build intrinsics for
additionalViews: (optional) the new views (list of path to images) to add to the node's viewpoints
Returns:
The updated views and intrinsics as two separate lists
"""
assert isinstance(node.nodeDesc, CameraInit)
if node.graph:
# make a copy of the node outside the graph
# to change its cache folder without modifying the original node
node = node.graph.copyNode(node)[0]
tmpCache = tempfile.mkdtemp()
node.updateInternals(tmpCache)
try:
os.makedirs(os.path.join(tmpCache, node.internalFolder))
self.createViewpointsFile(node, additionalViews)
cmd = self.buildCommandLine(node.chunks[0])
# logging.debug(' - commandLine:', cmd)
proc = psutil.Popen(cmd, stdout=None, stderr=None, shell=True)
stdout, stderr = proc.communicate()
# proc.wait()
if proc.returncode != 0:
raise RuntimeError(
'CameraInit failed with error code {}.\nCommand was: "{}".\n'
.format(proc.returncode, cmd))
# Reload result of aliceVision_cameraInit
cameraInitSfM = node.output.value
return readSfMData(cameraInitSfM)
except Exception:
raise
finally:
shutil.rmtree(tmpCache)
def start(startupArgs=[]):
"""Start the Yadoms server"""
# Set logLevel to none if not provided
if not startupArgs or all("logLevel=" not in arg for arg in startupArgs):
startupArgs.append("logLevel=none")
cmdLine = []
cmdLine.append(os.path.join(binaryPath(), executableName()))
for arg in startupArgs:
cmdLine.append("--" + arg)
print 'Start server ...'
print ' ', ' '.join(str(item) for item in cmdLine)
serverProcess = psutil.Popen(cmdLine)
if waitServerStarted():
return serverProcess
print 'Server failed to start'
stop(serverProcess)
assert False
def _validate_notarization(self, enterprise):
"""check whether this binary is notarized"""
if not enterprise and self.enterprise:
return
if IS_MAC:
cmd = ["codesign", "--verify", "--verbose", str(self.path)]
check_strings = [
b"valid on disk", b"satisfies its Designated Requirement"
]
with psutil.Popen(cmd,
bufsize=-1,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE) as proc:
(_, codesign_str) = proc.communicate()
if proc.returncode:
raise Exception("codesign exited nonzero " + str(cmd) +
"\n" + str(codesign_str))
if codesign_str.find(
check_strings[0]) < 0 or codesign_str.find(
check_strings[1]) < 0:
raise Exception("codesign didn't find signature: " +
str(codesign_str))
def mount(self, password):
""" Mount the encrypted volume self.filename, using password, at self.mountpoint. """
if not os.path.exists(self.mountpoint):
os.makedirs(self.mountpoint)
command = [
"hdiutil",
"attach",
"-mountpoint",
self.mountpoint,
"-passphrase",
password,
self.filename,
]
logging.debug(
"Attempting to attach encrypted disk using command: {0}".format(
" ".join(command)))
p = psutil.Popen(command)
p.wait(timeout=60)
return
def main():
fastanames = os.listdir(args.indir)
print('Analyzing {} sequences'.format(len(fastanames)))
result = []
for name in fastanames:
#print("Analyzing {}... ".format(name), end='')
ifname = os.path.join(args.indir, name)
ofname = os.path.join(args.outdir, name.replace('.dp', '.dat'))
p_args = [
'nice', args.rnasubopt, '-e',
str(args.deltaenergy), '-i', ifname
]
print(' '.join(p_args), end=' ')
sys.stdout.flush()
with io.open(ofname, 'w') as of:
start = time.time()
p = psutil.Popen(p_args, stdout=of, stderr=sys.stderr)
pid, status, res = os.wait4(p.pid, 0)
elapsed = time.time() - start
p.wait()
print('Status: {}, user: {:5.1f}, sys: {:5.1f}, maxrss: {:6d}'.format(
status, res.ru_utime, res.ru_stime, res.ru_maxrss))
result.append(
(name, res.ru_utime, res.ru_stime, res.ru_utime + res.ru_stime,
elapsed, res.ru_maxrss, countlines(ofname) - 2))
#break
with io.open(os.path.join(args.outdir, 'stats.dat'), 'w',
encoding='utf-8') as f:
f.write(
'# Name UserTime SystTime CPUTime WallTime MaxRSS Solutions\n'
)
for r in result:
f.write(
'{:<15s} {:8.2f} {:8.2f} {:8.2f} {:8.2f} {:8d} {:9d}\n'.format(
*r))
def start(self):
""" Starts a new datastore server. """
with (yield self._management_lock.acquire()):
if self.state == ServerStates.RUNNING:
return
self.state = ServerStates.STARTING
start_cmd = ['appscale-datastore',
'--type', self.DATASTORE_TYPE,
'--port', str(self.port)]
if self._verbose:
start_cmd.append('--verbose')
log_file = os.path.join(LOG_DIR,
'datastore_server-{}.log'.format(self.port))
self._stdout = open(log_file, 'a')
self.process = psutil.Popen(
['cgexec', '-g', ':'.join(DATASTORE_CGROUP)] + start_cmd,
stdout=self._stdout, stderr=subprocess.STDOUT)
yield self._wait_for_service(timeout=self.START_TIMEOUT)
self.state = ServerStates.RUNNING
def launch(self, cmd: str, **kwargs: Dict) -> None:
"""Launches the RPC client.
Args:
cmd: command string to execute as subprocess"""
if self.is_active():
raise SystemError("RPC is already active.")
if sys.platform == "win32" and not cmd.split(" ")[0].endswith(".cmd"):
if " " in cmd:
cmd = cmd.replace(" ", ".cmd ", 1)
else:
cmd += ".cmd"
kwargs.setdefault("evm_version", EVM_DEFAULT) # type: ignore
for key, value in [(k, v) for k, v in kwargs.items() if v]:
cmd += f" {CLI_FLAGS[key]} {value}"
print(f"Launching '{cmd}'...")
self._time_offset = 0
self._snapshot_id = False
self._reset_id = False
out = DEVNULL if sys.platform == "win32" else PIPE
self._rpc = psutil.Popen(cmd.split(" "), stdin=DEVNULL, stdout=out, stderr=out)
# check that web3 can connect
if not web3.provider:
_notify_registry(0)
return
uri = web3.provider.endpoint_uri if web3.provider else None
for i in range(100):
if web3.isConnected():
self._reset_id = self._snap()
_notify_registry(0)
return
time.sleep(0.1)
if type(self._rpc) is psutil.Popen:
self._rpc.poll()
if not self._rpc.is_running():
self.kill(False)
raise RPCProcessError(cmd, self._rpc, uri)
self.kill(False)
raise RPCConnectionError(cmd, self._rpc, uri)
def run_uftp_server(gateway_ids, target_file, retries=2):
session_list = []
success = None
hostlist = ','.join(gateway_id for gateway_id in gateway_ids)
for attempts in range(retries):
message = ''
status_data = {}
try:
with psutil.Popen(constants.uftp.UFTP_SERVER_CMD +
[hostlist, target_file]) as uftp_server:
return_code = uftp_server.wait(
timeout=constants.uftp.PROCESS_TIMEOUT)
print("UFTP Server returned " + str(return_code))
if return_code in [1, 2, 3, 4, 5, 6, 9]:
message = 'An error occurred!'
elif return_code in [7, 8]:
message = 'No Clients responded!'
elif return_code in [0, 10]:
message = 'Session Complete!'
status_data = parse_status_file()
success = len([
val for val in status_data['FAILED'].values()
if len(val) > 0
]) == 0
if success is True:
break
else:
message = 'Something wrong occured!'
except psutil.TimeoutExpired:
message = 'UFTP Server execution timed out!'
print(message)
finally:
session_list.append({'info': message, **status_data})
return success, session_list
def submit_jobs(NewQueue):
for k, v in NewQueue.items():
#create temporary directory
tempdir = '/'.join([TEMPFOLDER, FOLDERFORMAT.format(k)])
os.mkdir(tempdir)
#create files
Global.createfile(name='/'.join([tempdir,
SUBMITSCRNAME.format(k)]),
data=SUBMITSCR.format(v.execution_script_name, k,
JOBDONE),
stats=Global.MyStats(st_mode=0o774))
for name, info in v.execution_script.items():
Global.createfile(name='/'.join([tempdir, name]),
data=info[1],
stats=info[0])
for name, info in v.input_files.items():
Global.createfile(name='/'.join([tempdir, name]),
data=info[1],
stats=info[0])
for name, info in v.output_files.items():
Global.createfile(name='/'.join([tempdir, name]),
data=info[1],
stats=info[0])
#submit job
proc = psutil.Popen('/'.join([tempdir,
SUBMITSCRNAME.format(k)]),
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
start_new_session=True,
cwd=tempdir)
#update queues
v.status_key = 'r'
proc.set_nice(MyConfigs.niceness)
MyQueue.update({k: v})
jobid2proc.update({k: proc})
#create job file for, if necessary, later loading
Global.createfile(name='/'.join([tempdir,
JOBFILE.format(proc.pid)]),
data=repr(v))
def execute_async_cmd(cmd_list, cwd, env=None, out=PIPE, err=PIPE):
"""Execute an asynchronous command
This function executes an asynchronous command and returns a
popen subprocess object wrapped with psutil.
:param cmd_list: list of command with arguments
:type cmd_list: list of str
:param cwd: current working directory
:type cwd: str
:param env: environment variables to set
:type env: dict
:return: the subprocess object
:rtype: psutil.Popen
"""
global verbose_shell
if verbose_shell:
logger.debug(f"Executing command: {' '.join(cmd_list)}")
try:
popen_obj = psutil.Popen(cmd_list,
cwd=cwd,
stdout=out,
stderr=err,
env=env,
close_fds=True)
time.sleep(0.2)
popen_obj.poll()
if not popen_obj.is_running() and popen_obj.returncode != 0:
output, error = popen_obj.communicate()
err_msg = ""
if output:
err_msg += output.decode("utf-8") + " "
if error:
err_msg += error.decode("utf-8")
raise ShellError("Command failed immediately", err_msg, cmd_list)
except OSError as e:
raise ShellError("Failed to run command", e, cmd_list) from None
return popen_obj
def Popen(self, cargs, attempts=10, shell=False, **kwargs):
"""Каждая попытка выполняется раз в половину секунды."""
opened = self.get_opened()
proc = psutil.Popen(cargs, shell=shell, **kwargs)
cmd = ' '.join(proc.cmdline())
def find(latest):
for hwnd in latest:
pid = self.get_pid_by_hwnd(hwnd)
p = psutil.Process(pid)
if shell:
if psutil.pid_exists(proc.pid):
for child in proc.children():
if child.pid == p.pid:
return hwnd, proc
name = Path.basename(' '.join(p.cmdline()))
if cmd.find(name) != -1:
return hwnd, None
else:
if pid == proc.pid:
return hwnd, proc
if p.name() == proc.name():
return hwnd, None
while attempts:
result = find(self.get_last_opened(opened))
if result:
return result
attempts -= 1
sleep(0.5)
raise NoSuchWindowException(t('started_process_without_gui'))
def start(self):
if sys.platform == 'win32':
args = [os.path.join(self.working_dir, 'run.bat')]
# things break otherwise
env = dict(os.environ)
else:
args = ['sh', os.path.join(self.working_dir, 'run.sh')]
# Path needs to be passed through, otherwise some compilers (e.g gcc) can get confused and not find things
env = {'PATH': os.environ['PATH']}
env['PLAYER_KEY'] = str(self.player_key)
env['RUST_BACKTRACE'] = '1'
env['BC_PLATFORM'] = self._detect_platform()
if isinstance(self.socket_file, tuple):
# tcp port
env['TCP_PORT'] = str(self.socket_file[1])
else:
env['SOCKET_FILE'] = self.socket_file
cwd = self.working_dir
self.process = psutil.Popen(args, env=env, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=-1)
def _popen_process_while1(pid):
# TODO tasket's cpu ????
# mymachine = options.mymachine
cpuindex = 0
# if mymachine.get('taskset', 0) == 1:
# syscpucount = 1
# myservers = options.myservers
# cpucount = psutil.cpu_count() - syscpucount # 逻辑核数量, 刨除第一个核给系统调用
# if cpucount > 0 :
# i = myservers.index(pid)
# cpuindex = (i % cpucount) + syscpucount
redis_conf = redis_config.split(':')
os.chmod(bin_path + '/while1.sh', 00777)
cmdline = [
'./while1.sh', pid, redis_conf[0], redis_conf[1], redis_conf[2],
str(cpuindex), 'pypy'
]
if pid.find('GT7') >= 0 and os.path.isfile('/bin/pypy24'):
cmdline[-1] = '/bin/pypy24'
# LINUX WIN32
if platform.system() == 'Windows':
raise Exception('WIN32 system start up not test !!')
else:
cmdline.insert(0, 'nohup')
cmdline.append('&')
mylog('cmd-->', bin_path, cmdline)
cmdline = ' '.join(cmdline)
_, slave = pty.openpty()
psutil.Popen(cmdline,
cwd=bin_path,
stdin=subprocess.PIPE,
stdout=slave,
stderr=slave,
close_fds=True,
shell=True)
def init_project(project_name):
print "Init project " + project_name + "..."
projects = get_available_projects()
if not project_name in projects:
print "Error: Unknown project " + project_name + "!"
return
try:
project_info = do_api_request(projects[project_name]["setup_url"])
except ApiException as e:
print "Init of project failed!"
print e
return
if is_current_project(project_name):
print "WARNING: continuing will remove your progress for this project!"
time.sleep(3)
shutil.rmtree(get_project_path(project_name))
os.mkdir(get_project_path(project_name))
os.mkdir(get_project_meta_path(project_name))
os.mkdir(get_project_meta_path(project_name) + "/submission")
os.mkdir(get_project_path(project_name) + "/input")
os.mkdir(get_project_path(project_name) + "/output")
os.mkdir(get_project_path(project_name) + "/output/import")
os.mkdir(get_project_path(project_name) + "/output/import/queue")
store_current_project_info(project_name, project_info)
update_project_files(project_name)
if os.path.exists(get_project_path(project_name) + "/init.sh"):
os.chmod(get_project_path(project_name) + "/init.sh", 0744)
psutil.Popen("./init.sh", shell=True, cwd=get_project_path(project_name)).wait()
print "Project " + project_name + " is now ready to run!"
def _factorint(n, **kwargs):
try:
s = str(n)
if "." in s:
raise TypeError
if abs(int(s)) < 1 << 64:
raise ValueError
except (TypeError, ValueError):
return sympy.factorint(n, **kwargs)
args = ["misc/ecm.exe", s]
proc = psutil.Popen(args, stdout=subprocess.PIPE)
proc.wait()
data = proc.stdout.read().decode("utf-8", "replace").replace(" ", "")
if "<li>" not in data:
if not data:
raise RuntimeError("no output found.")
raise RuntimeError(data)
data = data[data.index("<li>") + 4:]
data = data[:data.index("</li>")]
while "(" in data:
i = data.index("(")
try:
j = data.index(")")
except ValueError:
break
data = data[:i] + data[j + 1:]
if data.endswith("isprime"):
data = data[:-7]
else:
data = data[data.rindex("=") + 1:]
factors = {}
for factor in data.split("*"):
if "^" in factor:
k, v = factor.split("^")
else:
k, v = factor, 1
factors[int(k)] = int(v)
return factors
def submit(self, tid, nice=os.nice(0)):
actionlogger.debug("Submitting task " + str(self.idtotask[tid]) +
" with nice value " + str(nice))
c = self.workflowspec['stages'][tid]['cmd']
workdir = self.workflowspec['stages'][tid]['cwd']
if not workdir == '':
if os.path.exists(workdir) and not os.path.isdir(workdir):
actionlogger.error(
'Cannot create working dir ... some other resource exists already'
)
return None
if not os.path.isdir(workdir):
os.makedirs(workdir)
self.procstatus[tid] = 'Running'
if args.dry_run:
drycommand = "echo \' " + str(
self.scheduling_iteration) + " : would do " + str(
self.workflowspec['stages'][tid]['name']) + "\'"
return subprocess.Popen(['/bin/bash', '-c', drycommand],
cwd=workdir)
taskenv = os.environ.copy()
# add task specific environment
if self.workflowspec['stages'][tid].get('env') != None:
taskenv.update(self.workflowspec['stages'][tid]['env'])
p = psutil.Popen(['/bin/bash', '-c', c], cwd=workdir, env=taskenv)
try:
p.nice(nice)
self.nicevalues[tid] = nice
except (psutil.NoSuchProcess, psutil.AccessDenied):
actionlogger.error('Couldn\'t set nice value of ' + str(p.pid) +
' to ' + str(nice) + ' -- current value is ' +
str(p.nice()))
self.nicevalues[tid] = os.nice(0)
return p
def start(startupArgs=[]):
"""Start the Yadoms server"""
# Set logLevel to none if not provided
if not startupArgs or all("logLevel=" not in arg for arg in startupArgs):
startupArgs.append("logLevel=none")
cmdLine = []
cmdLine.append(os.path.join(binaryPath(), executableName()))
for arg in startupArgs:
cmdLine.append("--" + arg)
print 'Start server ...'
print ' ', ' '.join(str(item) for item in cmdLine)
serverProcess = psutil.Popen(cmdLine)
# DEBUG
import datetime
print datetime.datetime.now(
), ': [DEBUG] server process status =', serverProcess.status() #TODO virer
import imp
try:
imp.find_module('wmi')
import wmi
c = wmi.WMI()
for process in c.Win32_Process():
print process.ProcessId, process.Name
except ImportError:
print 'Unable to print running processes, "wmi" module not installed'
# END DEBUG
if waitServerStarted() == True:
return serverProcess
print 'Server failed to start'
stop(serverProcess)
assert False
def measure_benchmark(self, cmd_and_args, err, out, working_directory):
samples = []
fields = [
"cpu_times", "memory_info", "num_threads", "num_fds", "io_counters"
]
start = time.perf_counter()
with psutil.Popen(cmd_and_args,
cwd=working_directory,
stdout=out,
stderr=err,
text=True) as p:
self.measure_process(p, fields, samples)
end = time.perf_counter()
duration = end - start
rss = max([x["memory_info"].rss for x in samples])
vms = max([x["memory_info"].vms for x in samples])
num_threads = max([x["num_threads"] for x in samples])
num_fds = max([x["num_fds"] for x in samples])
user = samples[-1]["cpu_times"].user
system = samples[-1]["cpu_times"].system
read_count = samples[-1]["io_counters"].read_count
write_count = samples[-1]["io_counters"].write_count
read_bytes = samples[-1]["io_counters"].read_bytes
write_bytes = samples[-1]["io_counters"].write_bytes
result = {
"duration": duration,
"rss": rss,
"vms": vms,
"usr": user,
"sys": system,
"read_count": read_count,
"write_count": write_count,
"read_bytes": read_bytes,
"write_bytes": write_bytes,
"num_threads": num_threads,
"num_fds": num_fds,
}
return result
def start(self, snapshot=None):
super(ExternalEngine, self).start(snapshot)
self._traj_num += 1
self.frame_num = 0
self.n_frames_since_start = 0
self.set_filenames(self._traj_num)
self.write_frame_to_file(self.input_file, self.current_snapshot, "w")
self.prepare()
cmd = shlex.split(self.engine_command())
self.start_time = time.time()
try:
logger.info(self.engine_command())
# TODO: add the ability to have handlers for stdin and stdout
self.proc = psutil.Popen(shlex.split(self.engine_command()),
preexec_fn=os.setsid)
except OSError: # pragma: no cover
raise #TODO: need to handle this, but do what?
else:
logger.info("Started engine: " + str(self.proc))
if self.first_frame_in_file:
_ = self.generate_next_frame() # throw away repeat first frame
def graceful(self):
"""If the service supports it, run graceful restart.
Currently this assumes that the pid file is externally managed, for example
by apachectl.
"""
if not self.graceful_cmd:
print('{} does not support graceful restart, skipping.'.format(self.name))
else:
proc = self._get_running_process_if_exists(delete_stale_pidfiles=True)
pidfile_name = self._get_pidfile()
if proc is not None:
self._ensure_stdout_dirs_exist()
with protected_file_path.ProtectedFilePath(pidfile_name):
print('Gracefully restarting {} with:\n{}'.format(
self.name, self.graceful_cmd))
with open(self.stdout, 'a') as stdout:
# pylint: disable=unused-variable
graceful_proc = psutil.Popen(args=self.graceful_cmd,
stdout=stdout,
stderr=stdout,
env=self.env,
cwd=self.cwd)