def setup_server(server, key):
client = ParallelSSHClient([server],
user='******',
pkey=key,
allow_agent=False)
output = client.run_command('rm -rf ~/server_config')
utils.print_pssh_output(output)
output = client.run_command('mkdir ~/server_config')
utils.print_pssh_output(output)
for f in os.listdir(SERVER_CONFIG_DIR):
dest_path = get_server_config_path(f.split('/')[-1])
local_path = os.path.join(SERVER_CONFIG_DIR, f)
fileUpload.upload_file_no_pssh([server], key, local_path, dest_path)
print(f'Wrote file {local_path} to {server}')
output = client.run_command('bash ~/server_config/setup_new_server.sh')
utils.print_pssh_output(output)
with open(utils.get_project_path('all_servers.txt')) as f:
hostnames = {host.strip() for host in f.readlines()}
hostnames.add(server)
hostnames = sorted(hostnames)
with open(utils.get_project_path('all_servers.txt'), 'w') as f:
f.write('\n'.join(hostnames) + '\n')
print(
f"Added {server} to the fleet. Don't forget to commit autogenerated changes to all_servers.txt."
)
def pragram_error(filepath):
with open(filepath) as f:
is_mutli_note = False
fname = filepath.replace(utils.get_project_path(), '')
for index, line in enumerate(f):
line = line.strip()
if '/*' in line:
is_mutli_note = True
if '*/' in line:
is_mutli_note = False
if is_mutli_note:
continue
if len(line) == 0 or line == '*/':
continue
if re.findall(r'^/+', line):
continue
regx = r'^".*s?"\s*=\s*".*?";$'
matchs = re.findall(regx, line)
if not matchs:
result = fname + ':line[' + str(index) + '] : ' + line
print(filepath)
print(result)
def init_args():
parser = argparse.ArgumentParser(description="Script to make intention recognition dataset")
parser.add_argument('--root_data_dir', type=str, default=get_project_path() + "/data",
help='Directory to save subdirectories, needs to be an absolute path')
parser.add_argument('--complete_data_dir', type=str, default="complete_data",
help='Subdirectory with complete data')
parser.add_argument('--incomplete_data_dir', type=str, default="incomplete_data_tfidf_lower_0.8_noMissingTag",
help='Subdirectory with incomplete data')
parser.add_argument('--results_dir', type=str, default="incomplete_data_tfidf_lower_with_target_0.8_noMissingTag",
help='Subdirectory to save Joint Complete and Incomplete data')
return parser.parse_args()
def __init__(self, data_dir="", text_filename="test.tsv"):
""" Initialize handler
Args:
text_filename (str): name of text file to read sentences from, where each line is a CONTENT (news/abstract)
"""
print("Initializing Text Handler for Intent Corpus")
self.project_dir = utils.get_project_path()
self.data_dir = data_dir
self.text_filename = text_filename
# Needed to separate sentences in CONTENT
try:
nltk.data.find('tokenizers/punkt')
except LookupError:
nltk.download('punkt') # downloaded to /home/gwena/nltk_data
def set_up_dir(self):
project_path = pjoin("Experiments/%s_%s" % (self.args.project_name, self.ExpID))
if hasattr(self.args, 'resume_ExpID') and self.args.resume_ExpID:
project_path = get_project_path(self.args.resume_ExpID)
if self.args.debug: # debug has the highest priority. If debug, all the things will be saved in Debug_dir
project_path = "Debug_Dir"
self.weights_path = pjoin(project_path, "weights")
self.gen_img_path = pjoin(project_path, "gen_img")
self.cache_path = pjoin(project_path, ".caches")
self.log_path = pjoin(project_path, "log")
self.logplt_path = pjoin(project_path, "log", "plot")
self.logtxt_path = pjoin(project_path, "log", "log.txt")
mkdirs(self.weights_path, self.gen_img_path, self.logplt_path, self.cache_path)
self.logtxt = open(self.logtxt_path, "a+")
self.script_hist = open('.script_history', 'a+') # save local script history, for convenience of check
def open_project(self, args):
#if no arguments were given, don't execute
if not args:
utils.print_missing_arg("project name")
return
#if there were more than one additional argument, don't execute
if len(args) > 1:
utils.print_invalid_arg(args[1])
return
project_name = args[0]
project_path = utils.get_project_path(project_name)
if not project_path.exists():
print(f"Could not find project {project_name}")
return
editmode = EditMode(project_path)
print("Opening project " + project_name + "... ")
editmode.enter()
# is drawing scene a sane thing to do here?
self.draw()
return
import os, sys
from utils import (get_target_path, get_ruby_files, get_ruby_targets_path, get_valid_dev,
get_dev, create_internal_dev_list, create_external_dev_list, get_project_path)
project_path = get_project_path(sys)
# Get all ruby files from the project path in an array
Ruby_Files = get_ruby_files(project_path)
# Get the absolute target path of each ruby file in an array
Ruby_files_target = get_ruby_targets_path(Ruby_Files)
# Get the dependency files of each ruby file as a dictionary
dev = get_dev(Ruby_files_target)
# Formatting the dependency file
new_dev = get_valid_dev(Ruby_files_target, dev, "'", "")
valid_dev = get_valid_dev(Ruby_files_target, new_dev, "/", "\\")
# creating the list of internal and external file
create_internal_dev_list(sys.argv[1], valid_dev)
create_external_dev_list(sys.argv[1], valid_dev)
if is_mutli_note:
continue
if len(line) == 0 or line == '*/':
continue
if re.findall(r'^/+', line):
continue
regx = r'^".*s?"\s*=\s*".*?";$'
matchs = re.findall(regx, line)
if not matchs:
result = fname + ':line[' + str(index) + '] : ' + line
print(filepath)
print(result)
def find_from_file(path):
paths = os.listdir(path)
for a in paths:
a_path = os.path.join(path, a)
if os.path.isdir(a_path):
find_from_file(a_path)
elif os.path.isfile(a_path) and os.path.splitext(
a_path)[1] == '.strings':
pragram_error(a_path)
if __name__ == '__main__':
find_from_file(utils.get_project_path())
print('已完成')
import utils
import local_config
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='See all live PlanetLab nodes.')
parser.add_argument('-v',
'--verbose',
help='Print errors',
action='store_true')
args = parser.parse_args()
key = local_config.get_planetlab_key()
all_nodes = utils.collect_hostnames(
utils.get_project_path('all_servers.txt'))
successes, failures = utils.run_command(all_nodes,
'echo test',
'ubc_cpen431_5',
key,
timeout=5)
if args.verbose:
fmt_len = max((len(h) for h in failures.keys()))
for host, stderr in failures.items():
print(f'ERROR [{host+" "*(fmt_len-len(host))}] {stderr}'.strip())
print('\nLIVE NODES:')
for host in successes.keys():
print(host)
import sys
import argparse
import os
import subprocess
from pssh.clients import ParallelSSHClient
from pssh.utils import load_private_key
from pssh.utils import enable_host_logger
import utils
import fileUpload
import local_config
SERVER_CONFIG_DIR = utils.get_project_path('server_config')
def get_server_config_path(file_rel_from_server_config):
return f"/home/ubc_cpen431_5/server_config/{file_rel_from_server_config}"
def setup_server(server, key):
client = ParallelSSHClient([server],
user='******',
pkey=key,
allow_agent=False)
output = client.run_command('rm -rf ~/server_config')
utils.print_pssh_output(output)
output = client.run_command('mkdir ~/server_config')
utils.print_pssh_output(output)
for f in os.listdir(SERVER_CONFIG_DIR):
dest_path = get_server_config_path(f.split('/')[-1])
local_path = os.path.join(SERVER_CONFIG_DIR, f)
import sys
import argparse
import os
import json
import threading
from pssh.clients import ParallelSSHClient
import utils
import deploy
import fileUpload
import local_config
ALL_SERVERS_PATH = utils.get_project_path('all_servers.txt')
PROMETHEUS_HOSTNAME = 'ec2-15-222-1-199.ca-central-1.compute.amazonaws.com'
PROMETHEUS_DIR = '~/prometheus-2.16.0-rc.0.linux-386'
PROMETHEUS_YAML_PATH = f'{PROMETHEUS_DIR}/prometheus.yml'
PROMETHEUS_EXEC_PATH = f'{PROMETHEUS_DIR}/prometheus'
PROMETHEUS_USER = '******'
if __name__ == '__main__':
key = local_config.get_planetlab_key()
prometheus_key = local_config.get_prometheus_key()
hostnames = utils.collect_hostnames(ALL_SERVERS_PATH)
successes, fails = utils.run_command(hostnames, 'netstat -nlp', 'ubc_cpen431_5', key)
node_exporter_urls = []
jmx_exporter_urls = []
for host, stdout in successes.items():
print(host)
for line in stdout.split('\n'):
def main():
"""
Main method of Image-Generator.
:return:
"""
# setup logging
utils.setup_logging()
# setup argument parsing
parser = argparse.ArgumentParser()
parser.add_argument('data_xml',
default='data.xml',
help="relative path to 'data.xml'")
parser.add_argument('-yv',
'--yolo_version',
help='set desired version of YOLO',
type=int,
default=2)
parser.add_argument('-b',
'--bounding_boxes',
help='draw bounding boxes into output images',
action='store_true')
parser.add_argument('-ny',
'--no_yolo_output',
help='don\'t generate YOLO output files',
action='store_true')
args = parser.parse_args()
if not args.data_xml.endswith('.xml'): # given file must be XML
logging.error(
"Error parsing program argument: name of \'data.xml\' must end with \'.xml\'"
)
sys.exit()
if len(args.data_xml) < 5: # XML file must contain a file name
logging.error(
"Error parsing program argument: XML file doesn\'t contain a file name"
)
sys.exit()
if args.yolo_version not in YOLO_VERSIONS: # check valid YOLO version
logging.error(
"Error parsing program argument: YOLO version can only be %s" %
list(YOLO_VERSIONS))
# set xml file names
class_xml_name = 'class.xml'
# read data.xml
data_xml_name = os.path.join(utils.get_project_path(), args.data_xml)
general_struct, filter_list = xml.read_data_xml(data_xml_name)
# read class.xml inside object directory
class_xml_path = os.path.join(general_struct.object_path, class_xml_name)
class_id_to_name_dict, img_name_to_class_id_dict = xml.read_class_xml(
class_xml_path)
# generate images
img_gen.generate(general_struct,
filter_list,
class_id_to_name_dict,
img_name_to_class_id_dict,
args.yolo_version,
draw_bounding_boxes=args.bounding_boxes,
no_yolo_output=args.no_yolo_output)
"zh-Hant",
"en",
]
def find_localizable_strings():
'''
找多语言路径
'''
path_dict = {}
for i in range(0, len(SUPPORT_LOCALIZATIIONN)):
path_dict[SUPPORT_LOCALIZATIIONN[i]] = utils.get_localizable_path(
) + SUPPORT_LOCALIZATIIONN[i] + '.lproj/' + utils.Localizable_strings
return path_dict
if __name__ == '__main__':
print("开始读取Excel")
localizeable_r_excel.read_excel_xls(utils.get_excel_path())
path_dict = find_localizable_strings()
print("开始写入多语言文件")
# 追加
localizeable_r_excel.write_localizable_strings_add(path_dict)
# 覆盖
# localizeable_r_excel.write_localizable_strings_cover(path_dict)
# 检查多语言文件是否有错误
print("开始检查多语言格式")
localizable_error.find_from_file(utils.get_project_path())
print("多语言格式检查结束")
import os
import utils
LOCAL_CONF_DIR = utils.get_project_path('.local_config')
def __get_key(key_name):
if not os.path.exists(LOCAL_CONF_DIR):
os.mkdir(LOCAL_CONF_DIR)
key_path_file = os.path.join(LOCAL_CONF_DIR, key_name)
if not os.path.exists(key_path_file):
print(f'Authorization required. Do not have key for {key_name}.')
path = input(f'Please enter the path for key "{key_name}":')
with open(key_path_file, 'w') as f:
f.write(path.strip())
with open(key_path_file, 'r') as f:
return f.read()
def get_planetlab_key():
return __get_key('PlanetLab')
def get_prometheus_key():
return __get_key('Prometheus')