def test_file_modes(tmpdir, opener, encoding, file_data, mode, newline):
# check common combinations of various methods of opening files with different
# encodings and line-endings
if encoding is None:
mode += 'b'
expected_props = {u'a': u'\u0100'}
if newline:
file_data = file_data.replace(b'\n', newline.encode('ascii'))
open_path = lambda path: opener(
str(path),
mode,
encoding=encoding,
newline=newline,
)
if 'r' in mode:
read_path = tmpdir.join('reading.properties')
read_path.write_binary(file_data)
with open_path(read_path) as fp:
actual_props = jprops.load_properties(fp)
assert actual_props == expected_props
else:
write_path = tmpdir.join('writing.properties')
with open_path(write_path) as fp:
jprops.store_properties(fp, expected_props, timestamp=False)
actual_data = write_path.read_binary()
assert actual_data == file_data
def modify_properties_file(fold):
with open(CARSKIT_ORIGINAL_CONF_FILE) as read_file:
properties = jprops.load_properties(read_file, collections.OrderedDict)
recommender = Constants.CARSKIT_RECOMMENDERS
ratings_fold_folder = CARSKIT_RATINGS_FOLD_FOLDER % fold
if not os.path.exists(ratings_fold_folder):
os.makedirs(ratings_fold_folder)
modified_file = CARSKIT_MODIFIED_CONF_FILE % (fold, recommender)
properties['recommender'] = recommender
properties['dataset.ratings.lins'] = \
ratings_fold_folder + 'carskit_train.csv'
test_file = ratings_fold_folder + 'carskit_test.csv'
properties['evaluation.setup'] = \
'test-set -f %s --rand-seed 1 --test-view all' % test_file
records = ETLUtils.load_json_file(
Constants.RECSYS_CONTEXTUAL_PROCESSED_RECORDS_FILE)
num_items = \
len(extractor.get_groupby_list(records, Constants.ITEM_ID_FIELD))
extra_items = num_items
# extra_items = 10
if Constants.CARSKIT_ITEM_RANKING:
properties['item.ranking'] = 'on -topN %d' % extra_items
else:
properties['item.ranking'] = 'off'
with open(modified_file, 'w') as write_file:
jprops.store_properties(write_file, properties)
def test_file_modes(tmpdir, opener, encoding, file_data, mode, newline):
# check common combinations of various methods of opening files with different
# encodings and line-endings
if encoding is None:
mode += 'b'
expected_props = {u'a': u'\u0100'}
if newline:
file_data = file_data.replace(b'\n', newline.encode('ascii'))
open_path = lambda path: opener(
str(path),
mode,
encoding=encoding,
newline=newline,
)
if 'r' in mode:
read_path = tmpdir.join('reading.properties')
read_path.write_binary(file_data)
with open_path(read_path) as fp:
actual_props = jprops.load_properties(fp)
assert actual_props == expected_props
else:
write_path = tmpdir.join('writing.properties')
with open_path(write_path) as fp:
jprops.store_properties(fp, expected_props, timestamp=False)
actual_data = write_path.read_binary()
assert actual_data == file_data
def generate_report(self):
#给报告中添加执行环境信息
env_dict = {}
env = self.env.conf
env_dict.update(dict(env.appium))
env_dict.update(dict(env.devices))
env_dict.update(dict(env.path))
env_dict.update(self.conf.info)
env_properties = {}
for key0, value0 in env_dict.items():
if (isinstance(value0, dict)):
for key, value in value0.items():
env_properties['{}.{}'.format(key0, key)] = str(value)
else:
env_properties['{}'.format(key0)] = str(value0)
try:
with open(self.properties_path, 'w', encoding='utf-8') as fp:
jprops.store_properties(fp, env_properties)
except:
log.error('配置环境未输出到报告中')
#执行生成报告命令
cmd = 'allure generate %s -o %s --clean' % (self.result_path,
self.html_report_path)
try:
Shell.invoke(cmd)
log.info("测试报告成功生成")
except:
log.error("Html测试报告生成失败,确保已经安装了Allure-Commandline")
def __init__(self):
try:
self.robohome = os.environ['ROBOHOME']
except KeyError:
print("Error in installation. $ROBOHOME does not exist")
raise
self.confdir = os.path.join(self.robohome, "conf")
self.logdir = os.path.join(self.robohome, "log")
self.imagedir = os.path.join(self.robohome, "img")
self.sysprops = os.path.join(self.confdir, "robot.properties")
f = 0
try:
f = open(self.sysprops, 'rb')
except FileNotFoundError:
self.props = {'home_lat': '59.1234', 'home_lon': '5.5678'}
f = open(self.sysprops, 'wb')
jprops.store_properties(f, self.props)
f.close()
f = open(self.sysprops, 'rb')
self.props = jprops.load_properties(f)
f.close()
self.refresh()
self.conf = self.confdir
def generate_report(self):
#给报告中添加执行环境信息
env_dict={}
env=self.env.conf
#修改把yaml格式改成对应的键值对
devices=env.get('devices')
env.pop('devices')
new_env=dict(env, **devices)
env_dict.update(new_env)
env_dict.update(self.conf.info)
env_properties = {}
for key0,value0 in env_dict.items():
for key,value in value0.items():
env_properties['{}.{}'.format(key0,key)]=str(value)
try:
with open( self.properties_path,'w',encoding='utf-8') as fp:
jprops.store_properties(fp, env_properties)
except:
log.error('配置环境未输出到报告中')
#执行生成报告命令
cmd = 'allure generate %s -o %s --clean' % (self.xml_report_path, self.html_report_path)
try:
Shell.invoke(cmd)
log.info("测试报告成功生成")
except:
log.error("Html测试报告生成失败,确保已经安装了Allure-Commandline")
def createUniqueFloodlightPropertiesFile(self):
"""
Creates a unique properties file for the particular Floodlight instance.
Each file is put in the 'properties' folder in the floodlight directory.
Static class attributes keep track of the current port number to use.
:return: None
"""
# The path to the properties file to be copied and the name of the file
old_path = Floodlight.fl_root_dir + 'src/main/resources/'
old_file = 'floodlightdefault.properties'
# The path where the new properties file will be located and the name of the file
new_path = Floodlight.fl_root_dir + 'properties/'
new_file = 'floodlight' + str(Floodlight.controller_number) + '.properties'
# Set the instance attributes so that the instance can know where its associated properties file is
self.properties_path = new_path
self.properties_file = new_file
# Check if the new path already exists. If not, then create it
if not path.exists(new_path):
makedirs(new_path)
# Copy the old properties file to the new location with the new name
shutil.copy(old_path + old_file,
new_path + new_file)
# Open the new properties file and scan it for the ports that need to be changed
with open(new_path + new_file) as fp:
properties = jprops.load_properties(fp)
http = [key for key, value in properties.items() if key.endswith('httpPort')][0]
https = [key for key, value in properties.items() if key.endswith('httpsPort')][0]
openflow = [key for key, value in properties.items() if key.endswith('openFlowPort')][0]
syncmanager = [key for key, value in properties.items() if key.endswith('SyncManager.port')][0]
properties[http] = str(Floodlight.http_port + 10)
properties[https] = str(Floodlight.https_port + 10)
properties[openflow] = str(Floodlight.openflow_port + 10)
properties[syncmanager] = str(Floodlight.sync_manager_port + 10)
# Update the class attributes so that everyone knows what ports are available now
Floodlight.http_port += 10
Floodlight.https_port += 10
Floodlight.openflow_port += 10
Floodlight.sync_manager_port += 10
log.debug('Ports being used in controller ' + self.name + ' property file...\n')
log.debug(http + ' = ' + properties[http] + '\n')
log.debug(https + ' = ' + properties[https] + '\n')
log.debug(openflow + ' = ' + properties[openflow] + '\n')
log.debug(syncmanager + ' = ' + properties[syncmanager] + '\n')
# Write the updated ports to the new properties file
with open(new_path + new_file, 'w') as fp:
# print 'Writing to file ' + new_file
jprops.store_properties(fp, properties)
def setup_properties(self, properties_file):
config_loc = os.path.dirname(docker.apply_base(self.options['config']))
project_loc = docker.apply_base('/')
properties = {
'config_loc': config_loc,
'samedir': config_loc,
'project_loc': project_loc,
'basedir': project_loc,
}
jprops.store_properties(properties_file, properties)
def setup_properties(self, properties_file):
config_loc = os.path.dirname(docker.apply_base(self.options['config']))
project_loc = docker.apply_base('/')
properties = {
'config_loc': config_loc,
'samedir': config_loc,
'project_loc': project_loc,
'basedir': project_loc,
}
jprops.store_properties(properties_file, properties)
def save_properties(self, path=None):
# store the Allure properties
#
if (path is None):
output_path = self.get_property(
'allure.cli.logs.xml') + '\\allure.properties'
else:
output_path = path
with open(output_path, 'w+') as fp:
jprops.store_properties(fp, self.properties, timestamp=False)
fp.close()
return True
def compute_rouge(event):
"""
Compute rouge-1 score for an event
:param event: Name of the event
:return:
"""
summaries_path = Path(LOCAL_DATA_DIR_2, 'data', event, 'summaries')
with open('rouge.properties', 'r+') as fp:
props = jprops.load_properties(fp, collections.OrderedDict)
ngrams = props.get('ngram')
result_path = Path(LOCAL_DATA_DIR_2, 'data', event, 'summaries', 'result_rouge_ngram' + str(ngrams) + '.csv')
props.pop('project.dir')
props.pop('outputFile')
props.update({'project.dir': str(summaries_path.absolute()), 'outputFile': str(result_path.absolute())})
fp.seek(0)
fp.truncate()
jprops.store_properties(fp, props)
subprocess.call(['java', '-jar', 'rouge2.0_0.2.jar', '-Drouge.prop=', 'rouge.properties'])
def modify_properties_file(fold):
with open(CARSKIT_ORIGINAL_CONF_FILE) as read_file:
properties = jprops.load_properties(read_file, collections.OrderedDict)
CARSKIT_CONF_FOLD_FOLDER = Constants.RIVAL_RATINGS_FOLD_FOLDER + 'carskit/'
CARSKIT_MODIFIED_CONF_FILE = CARSKIT_CONF_FOLD_FOLDER + '%s.conf'
recommender = Constants.CARSKIT_RECOMMENDERS
carskit_conf_fold_folder = CARSKIT_CONF_FOLD_FOLDER % fold
ratings_fold_folder = Constants.RIVAL_RATINGS_FOLD_FOLDER % fold
prediction_type_map = {
'user_test': 'rating',
'test_items': 'rating',
'rel_plus_n': 'ranking'
}
prediction_type = prediction_type_map[Constants.RIVAL_EVALUATION_STRATEGY]
if not os.path.exists(carskit_conf_fold_folder):
os.makedirs(carskit_conf_fold_folder)
modified_file = CARSKIT_MODIFIED_CONF_FILE % (fold, recommender)
properties['recommender'] = recommender
train_file = ratings_fold_folder + 'carskit_train.csv'
test_file = \
ratings_fold_folder + 'carskit_predictions_%s.csv' % (prediction_type)
properties['dataset.ratings.lins'] = train_file
properties['evaluation.setup'] = \
'test-set -f %s --rand-seed 1 --test-view all' % test_file
properties['item.ranking'] = 'off'
properties['output.setup'] =\
'-folder %s -verbose on, off --to-file %s%s_summary.txt' % (
ratings_fold_folder, ratings_fold_folder, recommender)
params = process_carskit_parameters(Constants.CARSKIT_PARAMETERS)
for key, value in params.items():
properties[key] = value
with open(modified_file, 'w') as write_file:
jprops.store_properties(write_file, properties)
def modify_properties_file(fold):
with open(CARSKIT_ORIGINAL_CONF_FILE) as read_file:
properties = jprops.load_properties(read_file, collections.OrderedDict)
CARSKIT_CONF_FOLD_FOLDER = Constants.RIVAL_RATINGS_FOLD_FOLDER + 'carskit/'
CARSKIT_MODIFIED_CONF_FILE = CARSKIT_CONF_FOLD_FOLDER + '%s.conf'
recommender = Constants.CARSKIT_RECOMMENDERS
carskit_conf_fold_folder = CARSKIT_CONF_FOLD_FOLDER % fold
ratings_fold_folder = Constants.RIVAL_RATINGS_FOLD_FOLDER % fold
prediction_type_map = {
'user_test': 'rating',
'test_items': 'rating',
'rel_plus_n': 'ranking'
}
prediction_type = prediction_type_map[Constants.RIVAL_EVALUATION_STRATEGY]
if not os.path.exists(carskit_conf_fold_folder):
os.makedirs(carskit_conf_fold_folder)
modified_file = CARSKIT_MODIFIED_CONF_FILE % (fold, recommender)
properties['recommender'] = recommender
train_file = ratings_fold_folder + 'carskit_train.csv'
test_file = \
ratings_fold_folder + 'carskit_predictions_%s.csv' % (prediction_type)
properties['dataset.ratings.lins'] = train_file
properties['evaluation.setup'] = \
'test-set -f %s --rand-seed 1 --test-view all' % test_file
properties['item.ranking'] = 'off'
properties['output.setup'] =\
'-folder %s -verbose on, off --to-file %s%s_summary.txt' % (
ratings_fold_folder, ratings_fold_folder, recommender)
params = process_carskit_parameters(Constants.CARSKIT_PARAMETERS)
for key, value in params.items():
properties[key] = value
with open(modified_file, 'w') as write_file:
jprops.store_properties(write_file, properties)
def test_file_encoding(props, java, encoding):
bytes_fp = io.BytesIO()
text_fp = io.TextIOWrapper(bytes_fp, encoding=encoding)
jprops.store_properties(text_fp, props)
text_fp.flush()
assert java_load(bytes_fp.getvalue(), encoding=encoding) == props
def store(self):
f = open(self.sysprops, 'wb')
jprops.store_properties(f, self.props)
f.close()
FILES = [
'cluster.properties',
'java.properties',
'service.properties',
'utf8.properties',
'wiki.properties',
]
pwd = os.getcwd()
for pfile in FILES:
values = OrderedDict()
# Should autodetect utf8 or latin-1
with open(pwd + "/tests/" + pfile, mode="r") as f:
values = jprops.load_properties(f, OrderedDict)
# This will write out in latin-1 unless utf8 data exists,
# switching mode to "wb" forces latin-1
with open(pwd + "/out/" + pfile, mode="w") as w:
jprops.store_properties(w, values)
# Append this file to the DEFAULTS dict
DEFAULTS.update(values)
# use "wb" so that UTF8 is escaped for java as \uffff and encoded as latin-1
with open(pwd + "/out/all.properties", mode="wb") as a:
jprops.store_properties(a, DEFAULTS)
# force utf8 output
with open(pwd + "/out/all-utf8.properties", mode="w", encoding='utf-8') as a:
jprops.store_properties(a, DEFAULTS)
def jprops_store(fp, props):
jprops.store_properties(fp, props, timestamp=False)
def save_props(f, d: Mapping) -> None:
import jprops
jprops.store_properties(f, d)
def dumps(d):
out = StringIO.StringIO()
stringified_d = { k:str(v) for k, v in d.iteritems() }
jprops.store_properties(out, stringified_d)
return out.getvalue()
def jprops_store(fp, props): jprops.store_properties(fp, props, timestamp=False)
def test_file_encoding(props, java, encoding): bytes_fp = io.BytesIO() text_fp = io.TextIOWrapper(bytes_fp, encoding=encoding) jprops.store_properties(text_fp, props) text_fp.flush() assert java_load(bytes_fp.getvalue(), encoding=encoding) == props
def test(self):
self.props = {'home_lat': '59.1234', 'home_lon': '5.5678'}
f = open(self.sysprops, 'wb')
jprops.store_properties(f, self.props)
f.close()