def publish_site(top, meta_dir):
cur_dir = os.getcwd()
os.chdir(top)
# run publish script from site top directory
subprocess_call(os.path.join(top, meta_dir, "publish"))
os.chdir(cur_dir)
print("Finished running publish script.")
def change_shell():
if current_shell()[0] != "zsh":
try:
subprocess_call(f"chsh -s $(which zsh) {current_user()}",
shell=True)
except KeyboardInterrupt:
printer("Canceled by user", foreground=FG.WARNING)
def editor_run(editor, config):
if shutil_which(editor):
get_editor = environ.get("EDITOR", editor)
with open(config) as f:
subprocess_call([get_editor, f.name])
exit(0)
return
async def download_from_video_url(video_url: str):
start_time: Optional[float] = None
overall_num_bytes_downloaded = 0
def make_per_downloaded_segment_callback(media_type: str):
num_segments_downloaded = 0
num_bytes_downloaded = 0
def wrapped_per_downloaded_segment_callback(_: int,
total_num_segments: int,
__: str,
segment_byte_size: int):
nonlocal start_time, overall_num_bytes_downloaded, num_segments_downloaded, num_bytes_downloaded
num_segments_downloaded += 1
num_bytes_downloaded += segment_byte_size
overall_num_bytes_downloaded += segment_byte_size
write_message(
json_dumps({
'progress': {
'num_segments_downloaded': num_segments_downloaded,
'num_bytes_downloaded': num_bytes_downloaded,
'overall_num_bytes_downloaded':
overall_num_bytes_downloaded,
'total_num_segments': total_num_segments,
'elapsed_seconds': time() - start_time,
'media_type': media_type
}
}))
return wrapped_per_downloaded_segment_callback
async with ClientSession() as session:
start_time = time()
video_streams, audio_streams, _ = await get_streams_from_video_url(
video_url=video_url, session=session)
video_streams.sort(key=lambda video_stream: video_stream.resolution,
reverse=True)
video_data = await video_streams[0].download(
session=session,
per_downloaded_segment_callback=
make_per_downloaded_segment_callback(media_type='video'))
audio_data = await audio_streams[0].download(
session=session,
per_downloaded_segment_callback=
make_per_downloaded_segment_callback(media_type='audio'))
with NamedTemporaryFile() as video_file, NamedTemporaryFile(
) as audio_file:
video_file.write(video_data)
audio_file.write(audio_data)
subprocess_call([
'ffmpeg', '-i', video_file.name, '-i', audio_file.name, '-c',
'copy', f'{PurePath(urlparse(video_url).path).name}.mp4'
])
def local_after_install(options, home_dir):
global tmp_dir
subprocess_call([
path_join('bin', 'pip'),
'install', '-r', 'devel-requirements.txt'
])
rmtree(tmp_dir)
def keyOK(self):
try:
print '[PlugLoad] is enabling Plugins'
#system('/jzk/etc/PlugLoad.sh py')
#Console().ePopen('/jzk/etc/PlugLoad.sh py')
subprocess_call('/jzk/etc/PlugLoad.sh py', shell=True)
print '[PlugLoad] is loading Plugins'
plugins.readPluginList('/usr/lib/enigma2/python/Plugins/')
if path.exists('/usr/lib/enigma2/python/Plugins/Extensions/WebInterface') is True:
woWebIf(self.session)
self.session.open(MessageBox, _('The plugins were reloaded successfully!'), MessageBox.TYPE_INFO, timeout=3)
self.close()
except:
print_exc()
def main():
# Parse input Dockerfile
if os.path.isfile(input_file):
docker_files.append(Docker(input_file, '.'))
else:
print('File "' + input_file + '" does not exist. Exiting...')
exit()
# Clone the official-images library
link = 'https://github.com/docker-library/official-images.git'
subprocess_call(['git', 'clone', link], stdout=PIPE, stderr=PIPE)
# Recursively get all the repos from FROM statements
get_repos_with_FROM(docker_files[0].FROM)
# Write all of the ansible files
for x in range(0, len(docker_files)):
docker_file = docker_files[x]
docker_file.parse_docker()
ansible_file = Ansible(docker_file.ansible_file,
docker_file.all_env_vars)
# Location and file name
if x == 0:
repo_tasks.append(output_file)
file_name = 'roles/' + output_file + '/tasks/main'
else:
# x-1: input not included (docker_files[0])
repo_task = repos[x - 1] + repo_versions[x - 1]
repo_tasks.append(repo_task)
file_name = 'roles/' + repo_task + '/tasks/main'
ansible_file.write_to_file(file_name)
# Want roles above this line to run in reverse order
repo_tasks.reverse()
# Make the site file
make_ansible_role_file(repo_tasks)
# Generates the ansible.cfg file for ssh timeout
make_ansible_config_file()
# Get rid of all the cloned git repos
remove_all_repos()
# print ansible command to run the generated code
print('ansible-playbook site.yml -u <user> -i <host>,')
def __init__(self, name, source, is_cplusplus=False):
self.name = name
self.source = source
if is_cplusplus:
self.srcname = '%s.cc' % self.name
compiler = 'g++'
else:
self.srcname = '%s.c' % self.name
compiler = 'gcc'
f = open(self.srcname, 'w')
f.write(source)
f.close()
c = subprocess_call([
compiler,
# We want debug information:
'-g',
# Name of the binary:
'-o',
self.name,
# The source file:
self.srcname
])
# Check exit status:
assert (c == 0)
# Check that the binary exists:
assert (os.path.exists(self.name))
def __init__(self, name, source, is_cplusplus=False):
self.name = name
self.source = source
if is_cplusplus:
self.srcname = '%s.cc' % self.name
compiler = 'g++'
else:
self.srcname = '%s.c' % self.name
compiler = 'gcc'
f = open(self.srcname, 'w')
f.write(source)
f.close()
c = subprocess_call([compiler,
# We want debug information:
'-g',
# Name of the binary:
'-o', self.name,
# The source file:
self.srcname])
# Check exit status:
assert(c == 0)
# Check that the binary exists:
assert(os.path.exists(self.name))
def on_btn_set_delay_clicked(self, btn):
try:
self.delay = int(self.entry_delay.get_text())
except ValueError:
self.__info_dialog("Delay range problem",
"Delay should be a number")
return -1
if (self.delay < 0 or self.delay > 255):
self.__info_dialog("Delay range problem",
"Delay should be in range [0, 255]")
return -1
ret = subprocess_call([
"{}build/{}".format(const.MAIN_PROG_FOLDER, const.FPGA_PROG), "-d",
str(self.delay)
])
if ret != 0:
err_msg = "Set delay error! Return code {}.".format(ret)
self.statusbar.push(
self.statusbar.get_context_id("error in set delay"), err_msg)
else:
self.statusbar.push(self.statusbar.get_context_id("ok set delay"),
"Set delay OK.")
print("delay = {}".format(self.delay))
def message_received_event(self, transaction_id, uuid, data):
if uuid.get_hex() != self.settings.uuid:
logging.debug(
"Ignoring appdata from unknown sender (%s)" %
data.uuid.get_hex())
return
assert (1 in data), "Missing key on data structure"
assert (2 in data), "Missing key on data structure"
key_id = chr(data[2]) + "," + chr(data[1])
if key_id not in self.settings.key_mappings:
logging.warning("Ignoring key press '%s' of unknown id. " % key_id)
return
to_emulate = self.settings.key_mappings[key_id]
logging.debug("Command called: '%s' " % to_emulate)
subprocess_call(to_emulate, shell=True)
def run(self):
try:
print '[PlugLoad] is enabling Plugins'
#Console().ePopen('/jzk/etc/PlugLoad.sh py')
#system('/jzk/etc/PlugLoad.sh py')
subprocess_call('/usr/lib/enigma2/python/Plugins/Extensions/PlugLoad/PlugLoad.sh py', shell=True)
print '[PlugLoad] is loading Plugins'
plugins.readPluginList('/usr/lib/enigma2/python/Plugins/')
if path.exists('/usr/lib/enigma2/python/Plugins/Extensions/WebInterface') is True:
woWebIf(self.session)
with open("/proc/sys/vm/drop_caches", "w") as f: f.write("1\n")
Console().ePopen('(/bin/run-parts /jzk/cron/post_E2) &')
#system('(/bin/run-parts /jzk/cron/post_E2) &')
#subprocess_call('(/bin/run-parts /jzk/cron/post_GUI) &', shell=True) odpalone przez vfd
except:
print '[PlugLoad] exception handled'
print_exc()
def register_suite(reg, sdir):
"""Registers the suite located in sdir with the registration name reg."""
cmd = ['cylc', 'register', reg, sdir]
print '$ ' + ' '.join(cmd)
if not subprocess_call(cmd, stdout=PIPE, env=CLEAN_ENV):
return True
print '\tFailed'
return False
def register_suite(reg, sdir):
"""Registers the suite located in sdir with the registration name reg."""
cmd = ['cylc', 'register', reg, sdir]
print '$ ' + ' '.join(cmd)
if not subprocess_call(cmd, stdout=PIPE, env=CLEAN_ENV):
return True
print '\tFailed'
return False
def keyOK(self):
try:
print '[PlugLoad] is enabling Plugins'
#system('/jzk/etc/PlugLoad.sh py')
#Console().ePopen('/jzk/etc/PlugLoad.sh py')
subprocess_call('/jzk/etc/PlugLoad.sh py', shell=True)
print '[PlugLoad] is loading Plugins'
plugins.readPluginList('/usr/lib/enigma2/python/Plugins/')
if path.exists(
'/usr/lib/enigma2/python/Plugins/Extensions/WebInterface'
) is True:
woWebIf(self.session)
self.session.open(MessageBox,
_('The plugins were reloaded successfully!'),
MessageBox.TYPE_INFO,
timeout=3)
self.close()
except:
print_exc()
def get_repos_with_FROM(FROM):
"""
Recursively go up the chain of turtles until an os image is found (no repo)
"""
stripped_FROM = ''.join(FROM.split()[1:])
split_FROM = stripped_FROM.split(':')
repo = split_FROM[0]
tags = split_FROM[1]
dirs = tags.split('-')
# Make a string to find the correct directory
dir_str = ''
version = ''
for _dir in dirs:
version += '_' + _dir
dir_str += '/' + _dir
# Check if dir exists
link = 'https://github.com/docker-library/' + repo + '.git'
try:
urllib.request.urlopen(link)
except:
print('Docker used image:\n ' + stripped_FROM)
return
# Keep cloning repos until it finds on that is an image
# Must be an image if there is no repo there
repos.append(repo)
repo_versions.append(version)
# clone the repo
subprocess_call(['git', 'clone', link], stdout=PIPE, stderr=PIPE)
dir_str = '/' + get_repo_dir_from_docker_lib(repo, tags)
dependencies_copy(repo, dir_str)
# instantiate a new Docker object
docker_file = repo + dir_str + '/Dockerfile'
docker_files.append(Docker(docker_file, dir_str))
# recursively call on next FROM statement
get_repos_with_FROM(docker_files[-1].FROM)
def on_btn_reset_clicked(self, btn):
ret = subprocess_call([
"{}build/{}".format(const.MAIN_PROG_FOLDER, const.FPGA_PROG), "-r"
])
if ret != 0:
err_msg = "Reset error! Return code {}.".format(ret)
self.statusbar.push(
self.statusbar.get_context_id("error in reset"), err_msg)
else:
self.statusbar.push(self.statusbar.get_context_id("ok reset"),
"Reset OK.")
def run(self):
try:
print '[PlugLoad] is enabling Plugins'
#Console().ePopen('/jzk/etc/PlugLoad.sh py')
#system('/jzk/etc/PlugLoad.sh py')
subprocess_call(
'/usr/lib/enigma2/python/Plugins/Extensions/PlugLoad/PlugLoad.sh py',
shell=True)
print '[PlugLoad] is loading Plugins'
plugins.readPluginList('/usr/lib/enigma2/python/Plugins/')
if path.exists(
'/usr/lib/enigma2/python/Plugins/Extensions/WebInterface'
) is True:
woWebIf(self.session)
with open("/proc/sys/vm/drop_caches", "w") as f:
f.write("1\n")
Console().ePopen('(/bin/run-parts /jzk/cron/post_E2) &')
#system('(/bin/run-parts /jzk/cron/post_E2) &')
#subprocess_call('(/bin/run-parts /jzk/cron/post_GUI) &', shell=True) odpalone przez vfd
except:
print '[PlugLoad] exception handled'
print_exc()
def _call(cmd, echo=True, stop=True, shell=True):
if echo:
print_ok(cmd)
try:
code = subprocess_call(cmd, shell=shell)
except OSError as e:
code = -1
print_er(e)
if code:
print_er("error=%i" % code)
if stop:
# raise
exit(code)
return code
def on_btn_coinc_clicked(self, btn):
txt_btn = btn.get_label()
if txt_btn == "Coinc on":
self.coinc = 1
ret = subprocess_call([
"{}build/{}".format(const.MAIN_PROG_FOLDER, const.FPGA_PROG),
"-c"
])
elif txt_btn == "Coinc off":
self.coinc = 0
ret = subprocess_call([
"{}build/{}".format(const.MAIN_PROG_FOLDER, const.FPGA_PROG),
"-n"
])
if ret != 0:
err_msg = "Coinc ON/OFF error! Return code {}.".format(ret)
self.statusbar.push(
self.statusbar.get_context_id("error in coinc"), err_msg)
else:
self.statusbar.push(
self.statusbar.get_context_id("ok coinc on/off"),
"Coinc ON/OFF OK.")
def call(cmd, popen=False, **kwargs):
cmd_strings = [str(x) for x in cmd]
print(" ".join(cmd_strings))
out_basename = os.path.join(
kwargs.get("cwd", "."), str_to_fname(chain.from_iterable(cmd_strings))
)
with open(out_basename + "-stdout.txt", "w") as out, open(
out_basename + "-stderr.txt", "w"
) as err:
kwargs.setdefault("stdout", out)
kwargs.setdefault("stderr", err)
if popen:
return Popen(cmd_strings, **kwargs)
assert subprocess_call(cmd_strings, **kwargs) == 0, "subprocess.call failed"
return None
def on_btn_set_porog_clicked(self, btn):
#check for errors for all porogs!!!
for num_btn in range(0, const.DET_NUM):
self.porog[num_btn] = int(self.entry_porog[num_btn].get_text())
if (self.porog[num_btn] < 0 or self.porog[num_btn] >= 8192):
self.__info_dialog("Porog #{} range problem".format(num_btn),
"Porog should be in range [0:8K]")
return -1
#execl C prog for set porog #num_btn
str_porog = "{:d} {:d} {:d} {:d}".format(self.porog[0], self.porog[1],
self.porog[2], self.porog[3])
print("str_porog = {}".format(str_porog))
ret = subprocess_call([
"{}build/{}".format(const.MAIN_PROG_FOLDER, const.FPGA_PROG), "-p",
str_porog
])
if ret != 0:
err_msg = "Set porog error! Return code {}.".format(ret)
self.statusbar.push(
self.statusbar.get_context_id("error in set porog"), err_msg)
else:
self.statusbar.push(self.statusbar.get_context_id("ok set porog"),
"Set porog OK.")
def run(self):
stop = False
for folder in ['trash', 'plain', 'hash']:
if stop:
break
# Let's convert the single date to a path
arg_dir = '/'.join(self.parentArgs.dir.split('-'))
review_folder = self.settings['data_dir'] + '/organized/' + folder + "/" + arg_dir
if not os_path.exists(review_folder):
print 'Folder "' + folder + '" not found, skipping...'
continue
files = [os_path.join(w_path, filename)
for w_path, dirs, files in os_walk(review_folder)
for filename in files
if not filename.endswith(".csv")]
idx = 0
while idx < len(files):
rfile = files[idx]
# Clear the screen before displaying the text
system('cls' if name == 'nt' else 'clear')
# Let's the terminal size, so I can fill it with the file text
cols, rows = get_terminal_size()
print colorama.Fore.YELLOW + rfile
print("")
with open(rfile) as tfile:
i = 0
for line in tfile:
i += 1
if i >= (rows - 6):
break
if len(line) <= cols:
print line.strip('\n\r')
else:
print line[0:cols].strip('\n\r')
files_left = len(files) - idx
print("")
print colorama.Fore.YELLOW + "Folder: " + folder + " " + colorama.Fore.CYAN + str(files_left) + colorama.Fore.YELLOW + " files left"
input_descr = colorama.Fore.MAGENTA + "[o]"
input_descr += colorama.Fore.CYAN + "open "
input_descr += colorama.Fore.MAGENTA + "[s]"
input_descr += colorama.Fore.CYAN + "kip folder "
input_descr += colorama.Fore.MAGENTA + "[n]"
input_descr += colorama.Fore.CYAN + "ext "
input_descr += colorama.Fore.MAGENTA + "[p]"
input_descr += colorama.Fore.CYAN + "revious "
input_descr += colorama.Fore.MAGENTA + "[q]"
input_descr += colorama.Fore.CYAN + "uit=> "
sys_stdout.write(input_descr)
sys_stdout.flush()
answer = getch()
while answer == '':
pass
idx += 1
# Opening a file with the default application AND being cross platform is a PITA...
if answer == 'o':
current_os = platform_system()
if current_os == 'Windows':
from os import startfile as os_startfile
os_startfile(rfile)
elif current_os == 'Linux':
subprocess_call(["xdg-open", rfile])
elif current_os == 'Darwin':
system("open " + rfile)
# Let's start the loop again to read the new key
answer = getch()
while answer == '':
pass
if answer == 'n':
print("")
elif answer == 'p':
if idx >= 2:
idx -= 2
continue
elif answer == 's':
break
elif answer == 'q':
print("")
stop = True
break
else:
print("")
print(colorama.Fore.GREEN + "Review completed")
if args.domain_check:
GANDI_API_KEY = args.domain_check
target_hostname = args.target_hostname
enumerate_nameservers( target_hostname )
# Render graph image
grapher = pgv.AGraph(
draw_graph_from_cache( target_hostname )
)
output_graph_file = "./output/" + target_hostname + "_trust_tree_graph."
export_formats = []
if args.export_formats:
export_parts = args.export_formats.split( "," )
for part in export_parts:
export_formats.append( part.strip() )
else:
export_formats.append( "png" )
for export_format in export_formats:
file_name = output_graph_file + export_format
grapher.draw( file_name, prog="dot" )
if args.open:
print( "[ STATUS ] Opening final graph..." )
subprocess_call( [ "open", file_name ])
print( "[ SUCCESS ] Finished generating graph!" )
def unix_killall_grep(name):
cmd = "kill -9 `ps -A | grep %s | grep -oe '^ *[0-9]\\+'`" % name
# print "KILLCMD: %s" % cmd
subprocess_call(cmd, shell=True, stderr=open('/dev/null', 'w'))
def unix_have_command(cmd):
return 0 == subprocess_call(['which', cmd], stdout=open('/dev/null', 'w'))
def WLANConnect(self):
subprocess_call('D:\TinyCMD\wifi_setting.bat')
def sed_replace(file, replace_dic):
for key, value in replace_dic.items():
subprocess_call(['sed', '-i', 's/{}/{}/g'.format(key, value), file])
def run(cmd):
print(cmd)
subprocess_call(cmd, shell=True)
def call(command):
LOGGER.debug('Executing command:')
LOGGER.debug(' \\\n '.join(command))
return subprocess_call(command)
def run(cmd):
print(cmd)
subprocess_call(cmd, shell=True)
def call_shell_command(self, params: list): with open(os_devnull, 'w') as FNULL: return subprocess_call(params, stdout=FNULL);
# Standard library imports
from subprocess import call as subprocess_call
from utility_036 import fileexists
from time import sleep as time_sleep
from datetime import datetime
mount_try = 1
mounted = False
start_time = datetime.now()
when = datetime.now()
while (mount_try < 30) and not (mounted):
try:
mounted = fileexists("/home/pi/mycloud/Tests/test2.zip")
if not (mounted):
print("Needs mounting this is try number: ", mount_try)
subprocess_call(["sudo", "mount", "-a"])
mount_try += 1
mounted = fileexists("/home/pi/mycloud/Tests/test2.zip")
if mounted:
when = datetime.now()
when = round((when - start_time).total_seconds(), 2)
print("Success at :", when, " secs from start")
except:
print("Count: ", count, " error lines 12 to 19")
time_sleep(1)
while True:
time_sleep(20000)
def unregister_suite(reg):
"""Unregisters the suite reg."""
cmd = ['cylc', 'unregister', reg]
print '$ ' + ' '.join(cmd)
subprocess_call(cmd, stdout=PIPE, env=CLEAN_ENV)
def _gettrainingdata(self):
files = [path.join(w_path, filename)
for w_path, dirs, files in walk(self.settings['data_dir'] + "/" + 'raw')
for filename in files
if not filename.endswith(".csv")]
while 1:
rfile = random_choice(files)
trash = len(listdir(self.settings['data_dir'] + "/" + 'training/trash'))
plain = len(listdir(self.settings['data_dir'] + "/" + 'training/plain'))
hashes = len(listdir(self.settings['data_dir'] + "/" + 'training/hash'))
# Clear the screen before displaying the text
system('cls' if name == 'nt' else 'clear')
# Let's the terminal size, so I can fill it with the file text
cols, rows = get_terminal_size()
print colorama.Fore.YELLOW + rfile
print("")
with open(rfile) as tfile:
i = 0
for line in tfile:
i += 1
if i >= (rows - 6):
break
if len(line) <= cols:
print line.strip('\n\r')
else:
print line[0:cols].strip('\n\r')
print("")
print colorama.Fore.YELLOW + "Trash: " + str(trash) + " Plain: " + str(plain) + " Hash: " + str(hashes)
input_descr = colorama.Fore.MAGENTA + "[o]"
input_descr += colorama.Fore.CYAN + "open "
input_descr += colorama.Fore.MAGENTA + "[t]"
input_descr += colorama.Fore.CYAN + "rash "
input_descr += colorama.Fore.MAGENTA + "[p]"
input_descr += colorama.Fore.CYAN + "lain "
input_descr += colorama.Fore.MAGENTA + "[h]"
input_descr += colorama.Fore.CYAN + "ash "
input_descr += colorama.Fore.MAGENTA + "[s]"
input_descr += colorama.Fore.CYAN + "kip "
input_descr += colorama.Fore.MAGENTA + "[q]"
input_descr += colorama.Fore.CYAN + "uit=> "
sys_stdout.write(input_descr)
sys_stdout.flush()
answer = getch()
while answer == '':
pass
# Opening a file with the default application AND being cross platform is a PITA...
if answer == 'o':
current_os = platform_system()
if current_os == 'Windows':
from os import startfile as os_startfile
os_startfile(rfile)
elif current_os == 'Linux':
subprocess_call(["xdg-open", rfile])
elif current_os == 'Darwin':
system("open " + rfile)
# Let's start the loop again to read the new key
answer = getch()
while answer == '':
pass
if answer == 't':
shutil_copyfile(rfile, self.settings['data_dir'] + "/" + 'training/trash/' + path.basename(rfile))
elif answer == 'p':
shutil_copyfile(rfile, self.settings['data_dir'] + "/" + 'training/plain/' + path.basename(rfile))
elif answer == 'h':
shutil_copyfile(rfile, self.settings['data_dir'] + "/" + 'training/hash/' + path.basename(rfile))
elif answer == 's':
print("")
continue
elif answer == 'q':
print("")
print(colorama.Fore.GREEN + "Training complete")
break
else:
print("")
continue
def run_suite(reg, options, out_file, profile_modes, mode='live',
conf_path=''):
"""Runs cylc run / cylc validate on the provided suite with the requested
profiling options.
Arguments:
reg (str): The registration of the suite to run.
options (list): List of jinja2 setting=value pairs.
out_file (str): The file to redirect stdout to.
profile_modes (list): List of profileing systems to employ
(i.e. cylc, time).
mode (str - optional): The mode to run the suite in, simulation, dummy,
live or validate.
Returns:
str - The path to the suite stderr if any is present.
"""
cmds = []
env = cylc_env(cylc_conf_path=conf_path)
# Cylc profiling, echo command start time.
if PROFILE_MODE_CYLC in profile_modes:
cmds += ['echo', SUITE_STARTUP_STRING, r'$(date +%s.%N)', '&&']
# /usr/bin/time profiling.
if PROFILE_MODE_TIME in profile_modes:
if sys.platform == 'darwin': # MacOS
cmds += ['/usr/bin/time', '-lp']
else: # Assume Linux
cmds += ['/usr/bin/time', '-v']
# Run using `sh -c` to enable the redirecton of output (darwins
# /usr/bin/time command does not have a -o option).
cmds += ['sh', '-c', "'"]
# Cylc run.
run_cmds = []
if mode == 'validate':
run_cmds = ['cylc', 'validate']
elif mode == 'profile-simulation':
# In simulation mode task scripts are manually replaced with sleep 1.
run_cmds = ['cylc', 'run', '--mode', 'live']
else:
run_cmds = ['cylc', 'run', '--mode', mode]
run_cmds += [reg]
cmds += run_cmds
# Jinja2 params.
jinja2_params = ['-s {0}'.format(option) for option in options]
if mode == 'profile-simulation':
# Add namespaces jinja2 param (list of task names).
tmp = ['-s namespaces=root']
namespaces = Popen(
['cylc', 'list', reg] + jinja2_params + tmp, stdout=PIPE,
env=env).communicate()[0].split() + ['root']
jinja2_params.append(
'-s namespaces={0}'.format(','.join(namespaces)))
cmds.extend(jinja2_params)
# Cylc profiling.
if PROFILE_MODE_CYLC in profile_modes:
if mode == 'validate':
sys.exit('ERROR: profile_mode "cylc" not possible in validate '
'mode')
else:
cmds += ['--profile']
# No-detach mode.
if mode != 'validate':
cmds += ['--no-detach']
# Redirect output.
cmd_out = out_file + PROFILE_FILES['cmd-out']
cmd_err = out_file + PROFILE_FILES['cmd-err']
time_err = out_file + PROFILE_FILES['time-err']
startup_file = out_file + PROFILE_FILES['startup']
cmds += ['>', cmd_out, '2>', cmd_err]
if PROFILE_MODE_TIME in profile_modes:
cmds += ["'"] # Close shell.
# Execute.
print '$ ' + ' '.join(cmds)
try:
proc = Popen(' '.join(cmds), shell=True, stderr=open(time_err, 'w+'),
stdout=open(startup_file, 'w+'), env=env)
if proc.wait():
raise SuiteFailedException(run_cmds, cmd_out, cmd_err)
except KeyboardInterrupt:
kill_cmd = ['cylc', 'stop', '--kill', reg]
print '$ ' + ' '.join(kill_cmd)
subprocess_call(kill_cmd, env=env)
raise ProfilingKilledException(run_cmds, cmd_out, cmd_err)
# Return cylc stderr if present.
try:
if os.path.getsize(cmd_err) > 0:
return cmd_err
except OSError:
pass
return None
def call(command):
LOGGER.debug('Executing command:')
LOGGER.debug(' \\\n '.join(command))
return subprocess_call(command)
def LANConnect(self):
subprocess_call('D:\TinyCMD\origin_setting.bat')
def run_suite(reg,
options,
out_file,
profile_modes,
mode='live',
conf_path=''):
"""Runs cylc run / cylc validate on the provided suite with the requested
profiling options.
Arguments:
reg (str): The registration of the suite to run.
options (list): List of jinja2 setting=value pairs.
out_file (str): The file to redirect stdout to.
profile_modes (list): List of profileing systems to employ
(i.e. cylc, time).
mode (str - optional): The mode to run the suite in, simulation, dummy,
live or validate.
Returns:
str - The path to the suite stderr if any is present.
"""
cmds = []
env = cylc_env(cylc_conf_path=conf_path)
# Cylc profiling, echo command start time.
if PROFILE_MODE_CYLC in profile_modes:
cmds += ['echo', SUITE_STARTUP_STRING, r'$(date +%s.%N)', '&&']
# /usr/bin/time profiling.
if PROFILE_MODE_TIME in profile_modes:
if sys.platform == 'darwin': # MacOS
cmds += ['/usr/bin/time', '-lp']
else: # Assume Linux
cmds += ['/usr/bin/time', '-v']
# Run using `sh -c` to enable the redirecton of output (darwins
# /usr/bin/time command does not have a -o option).
cmds += ['sh', '-c', "'"]
# Cylc run.
run_cmds = []
if mode == 'validate':
run_cmds = ['cylc', 'validate']
elif mode == 'profile-simulation':
# In simulation mode task scripts are manually replaced with sleep 1.
run_cmds = ['cylc', 'run', '--mode', 'live']
else:
run_cmds = ['cylc', 'run', '--mode', mode]
run_cmds += [reg]
cmds += run_cmds
# Jinja2 params.
jinja2_params = ['-s {0}'.format(option) for option in options]
if mode == 'profile-simulation':
# Add namespaces jinja2 param (list of task names).
tmp = ['-s namespaces=root']
namespaces = Popen(['cylc', 'list', reg] + jinja2_params + tmp,
stdout=PIPE,
env=env).communicate()[0].split() + ['root']
jinja2_params.append('-s namespaces={0}'.format(','.join(namespaces)))
cmds.extend(jinja2_params)
# Cylc profiling.
if PROFILE_MODE_CYLC in profile_modes:
if mode == 'validate':
sys.exit('ERROR: profile_mode "cylc" not possible in validate '
'mode')
else:
cmds += ['--profile']
# No-detach mode.
if mode != 'validate':
cmds += ['--no-detach']
# Redirect output.
cmd_out = out_file + PROFILE_FILES['cmd-out']
cmd_err = out_file + PROFILE_FILES['cmd-err']
time_err = out_file + PROFILE_FILES['time-err']
startup_file = out_file + PROFILE_FILES['startup']
cmds += ['>', cmd_out, '2>', cmd_err]
if PROFILE_MODE_TIME in profile_modes:
cmds += ["'"] # Close shell.
# Execute.
print '$ ' + ' '.join(cmds)
try:
proc = Popen(' '.join(cmds),
shell=True,
stderr=open(time_err, 'w+'),
stdout=open(startup_file, 'w+'),
env=env)
if proc.wait():
raise SuiteFailedException(run_cmds, cmd_out, cmd_err)
except KeyboardInterrupt:
kill_cmd = ['cylc', 'stop', '--kill', reg]
print '$ ' + ' '.join(kill_cmd)
subprocess_call(kill_cmd, env=env)
raise ProfilingKilledException(run_cmds, cmd_out, cmd_err)
# Return cylc stderr if present.
try:
if os.path.getsize(cmd_err) > 0:
return cmd_err
except OSError:
pass
return None
def __init__(self):
# путь к самому приложению (flibrowser2[.py|.pyz])
self.appFilePath = os.path.abspath(sys.argv[0])
self.appDir = os.path.split(self.appFilePath)[0]
# мы в жо... в зипе?
self.appIsZIP = is_zipfile(self.appFilePath)
# код определения ОС скопипащен из stackoverflow.com
# сильно менять не стал, т.к. как ведет себя макось - не знаю
fnamepfx = ''
if sys.platform.startswith('darwin'): # MacOS?
self.shell_open_func = lambda fp: subprocess_call(('open', fp))
configdir = os.path.expanduser('~')
fnamepfx = '.'
elif os.name == 'nt': # For Windows
self.shell_open_func = lambda fp: os.startfile(fp)
# XPюндель про LOCALAPPDATA не знает!
ES_LAD = 'LOCALAPPDATA'
configdir = os.environ[
ES_LAD] if ES_LAD in os.environ else os.environ['APPDATA']
else: # os.name == 'posix': # For Linux, BSD(?), etc.
self.shell_open_func = lambda fp: subprocess_call(('xdg-open', fp))
configdir = os.path.expanduser('~/.config')
# правильнее было бы через модуль xdg, но он в стандартную питонью RTL не входит
self.path = os.path.join(configdir, '%sbtfm.cfg' % fnamepfx)
self.cfg = ConfigParser()
# значения по умолчанию
self.currentPath = os.path.expanduser('~')
# вспомогательный параметр, нужен для правильного сохранения настроек,
# когда currentPath is None
self.currentDir = os.path.split(self.currentPath)
self.viewEncoding = DEFAULT_ENCODING
self.saveEncoding = DEFAULT_ENCODING
self.fileOpsNoConfirmation = False
self.viewFileStartOnly = False
self.mainWindowMaximized = False
self.mainWindowPanePosition = 128
self.searchText = ''
self.autocorrectionDict = OrderedDict()
if os.path.exists(self.path):
self.cfg.read(self.path)
#
# [settings]
#
self.currentPath = self.cfg.get(self.SEC_SETTINGS,
self.VAR_CURRENT_PATH,
fallback=self.currentPath)
if self.currentPath:
self.currentDir = os.path.split(self.currentPath)[0]
self.viewEncoding = self.cfg.get(self.SEC_SETTINGS,
self.VAR_VIEW_ENCODING,
fallback=self.viewEncoding)
self.saveEncoding = self.cfg.get(self.SEC_SETTINGS,
self.VAR_SAVE_ENCODING,
fallback=self.saveEncoding)
self.fileOpsNoConfirmation = self.cfg.getboolean(
self.SEC_SETTINGS,
self.VAR_FILE_OPS_NO_CONFIRM,
fallback=self.fileOpsNoConfirmation)
self.viewFileStartOnly = self.cfg.getboolean(
self.SEC_SETTINGS,
self.VAR_VIEW_FILE_START_ONLY,
fallback=self.viewFileStartOnly)
self.mainWindowMaximized = self.cfg.getboolean(
self.SEC_SETTINGS,
self.VAR_MAIN_WINDOW_MAXIMIZED,
fallback=self.mainWindowMaximized)
self.mainWindowPanePosition = self.cfg.getint(
self.SEC_SETTINGS,
self.VAR_MAIN_WINDOW_PANE_POS,
fallback=self.mainWindowPanePosition)
self.searchText = self.cfg.get(self.SEC_SETTINGS,
self.VAR_SEARCH_TEXT,
fallback=self.searchText)
#
# [autocorrection]
#
E_CONFIG = 'Ошибка в параметре "%s" секции "%s" файле "%s" - %s'
if self.cfg.has_section(self.SEC_AUTOCORRECTION):
for vname in self.cfg.options(self.SEC_AUTOCORRECTION):
# придётся или извращаться вот так с JSON, или переходить
# вместо обычного инишника на sqlite
try:
rawval = json.loads(
self.cfg.get(self.SEC_AUTOCORRECTION,
vname,
raw=True,
fallback=''))
except Exception as ex:
raise SyntaxError(E_CONFIG %
(vname, self.SEC_AUTOCORRECTION,
self.path, str(ex)))
if len(rawval) != 2:
raise SyntaxError(
E_CONFIG %
(vname, self.SEC_AUTOCORRECTION, self.path,
'неправильный синтаксис значения параметра'))
vname, vval = rawval
if not isinstance(vname, str) or not isinstance(vval, str):
raise SyntaxError(
E_CONFIG %
(vname, self.SEC_AUTOCORRECTION, self.path,
'неправильное значение параметра'))
self.autocorrectionDict[vname] = vval
def finalizer(): subprocess_call(['docker', 'stop', 'test-db'], stdout=DEVNULL) subprocess_call(['docker', 'rm', 'test-db'], stdout=DEVNULL)
def unix_killall(procname):
subprocess_call(['killall', procname], stderr=open('/dev/null', 'w'))
def unregister_suite(reg):
"""Unregisters the suite reg."""
cmd = ['cylc', 'unregister', reg]
print '$ ' + ' '.join(cmd)
subprocess_call(cmd, stdout=PIPE, env=CLEAN_ENV)
def _gettrainingdata(self):
files = [
path.join(w_path, filename)
for w_path, dirs, files in walk(self.settings['data_dir'] + "/" +
'raw') for filename in files
if not filename.endswith(".csv")
]
while 1:
rfile = random_choice(files)
trash = len(
listdir(self.settings['data_dir'] + "/" + 'training/trash'))
plain = len(
listdir(self.settings['data_dir'] + "/" + 'training/plain'))
hashes = len(
listdir(self.settings['data_dir'] + "/" + 'training/hash'))
# Clear the screen before displaying the text
system('cls' if name == 'nt' else 'clear')
# Let's the terminal size, so I can fill it with the file text
cols, rows = get_terminal_size()
print colorama.Fore.YELLOW + rfile
print("")
with open(rfile) as tfile:
i = 0
for line in tfile:
i += 1
if i >= (rows - 6):
break
if len(line) <= cols:
print line.strip('\n\r')
else:
print line[0:cols].strip('\n\r')
print("")
print colorama.Fore.YELLOW + "Trash: " + str(
trash) + " Plain: " + str(plain) + " Hash: " + str(hashes)
input_descr = colorama.Fore.MAGENTA + "[o]"
input_descr += colorama.Fore.CYAN + "open "
input_descr += colorama.Fore.MAGENTA + "[t]"
input_descr += colorama.Fore.CYAN + "rash "
input_descr += colorama.Fore.MAGENTA + "[p]"
input_descr += colorama.Fore.CYAN + "lain "
input_descr += colorama.Fore.MAGENTA + "[h]"
input_descr += colorama.Fore.CYAN + "ash "
input_descr += colorama.Fore.MAGENTA + "[s]"
input_descr += colorama.Fore.CYAN + "kip "
input_descr += colorama.Fore.MAGENTA + "[q]"
input_descr += colorama.Fore.CYAN + "uit=> "
sys_stdout.write(input_descr)
sys_stdout.flush()
answer = getch()
while answer == '':
pass
# Opening a file with the default application AND being cross platform is a PITA...
if answer == 'o':
current_os = platform_system()
if current_os == 'Windows':
from os import startfile as os_startfile
os_startfile(rfile)
elif current_os == 'Linux':
subprocess_call(["xdg-open", rfile])
elif current_os == 'Darwin':
system("open " + rfile)
# Let's start the loop again to read the new key
answer = getch()
while answer == '':
pass
if answer == 't':
shutil_copyfile(
rfile, self.settings['data_dir'] + "/" +
'training/trash/' + path.basename(rfile))
elif answer == 'p':
shutil_copyfile(
rfile, self.settings['data_dir'] + "/" +
'training/plain/' + path.basename(rfile))
elif answer == 'h':
shutil_copyfile(
rfile, self.settings['data_dir'] + "/" + 'training/hash/' +
path.basename(rfile))
elif answer == 's':
print("")
continue
elif answer == 'q':
print("")
print(colorama.Fore.GREEN + "Training complete")
break
else:
print("")
continue
def do_command(cmnd):
subprocess_call(['./' + cmnd])
return []
