def get_departments():
# Load properties and check that they are structurally OK
props = utils.load_properties()
try:
assert props['hrMgmtSystem'] != None, 'No HRM properties section'
assert props['hrMgmtSystem'][
'departmentsFile'] != None, 'HRM Departments Excel file not set'
except Exception as ex:
print("Properties file not complete:", repr(ex))
exit()
wb = load_workbook(props['hrMgmtSystem']['departmentsFile'])
departments = []
for row in wb.active.iter_rows(min_row=2):
external_id, fi, sv, en = row
dep = {'externalId': external_id.value, 'names': {}}
if row[1].value:
dep['names']['fi'] = fi.value
if row[2].value:
dep['names']['sv'] = sv.value
if row[3].value:
dep['names']['en'] = en.value
departments.append(dep)
return departments
def get_absences():
# Load properties and check that they are structurally OK
props = utils.load_properties()
try:
assert props['hourTrackingSystem'] != None, 'No HRM properties section'
assert props['hourTrackingSystem'][
'absenceFile'] != None, 'Hour tracking system Absence Excel file not set'
except Exception as ex:
print("Properties file not complete:", repr(ex))
exit()
wb = load_workbook(props['hourTrackingSystem']['absenceFile'])
absences = []
for row in wb.active.iter_rows(min_row=2):
external_id, start_date, end_date, approval_type = row
absence = {
'externalId': external_id.value,
'startDate': start_date.value.strftime('%Y-%m-%d')
}
if end_date.value:
absence['endDate'] = end_date.value.strftime('%Y-%m-%d')
if approval_type.value:
absence['approvalType'] = approval_type.value
absences.append(absence)
return absences
def get_absences():
# Load properties and check that they are structurally OK
props = utils.load_properties()
try:
assert props[
'hourTrackingSystem'] != None, 'No hourTrackingSystem properties section'
assert props['hourTrackingSystem'][
'moduleName'] != None, 'Time tracking module name not set'
assert props['hourTrackingSystem'][
'host'] != None, 'Time tracking host not set'
assert props['hourTrackingSystem'][
'port'] != None, 'Time tracking server port not set'
assert props['hourTrackingSystem'][
'path'] != None, 'Time tracking file path not set'
assert props['hourTrackingSystem'][
'id'] != None, 'Time tracking user ID not set'
assert props['hourTrackingSystem'][
'pw'] != None, 'Time tracking password not set'
assert props['hourTrackingSystem'][
'hostKey'] != None, 'Time tracking hostKey not set'
except Exception as ex:
print("Properties file not complete:", repr(ex))
exit()
absences = []
# For added security, the server's hostkey is verified against the one stored in properties
bHostKey = str.encode(props['hourTrackingSystem']['hostKey'])
hostKey = paramiko.DSSKey(data=decodebytes(bHostKey))
cnopts = pysftp.CnOpts()
cnopts.hostkeys.add(props['hourTrackingSystem']['host'], 'ssh-rsa',
hostKey)
tempfile = '__temp.csv'
with pysftp.Connection(host=props['hourTrackingSystem']['host'],
username=props['hourTrackingSystem']['id'],
password=props['hourTrackingSystem']['pw'],
cnopts=cnopts) as sftp:
sftp.get(props['hourTrackingSystem']['path'], tempfile)
csvfile = open(tempfile, 'r')
reader = csv.reader(csvfile, delimiter=';')
for row in reader:
external_id, start_date, end_date = row
absence = {
'externalId': external_id,
'startDate': parse_date(start_date),
'endDate': parse_date(end_date)
}
absences.append(absence)
os.remove(tempfile)
return absences
def main():
args = get_arguments()
np.random.seed(1380)
if args.model_folder:
model_dir = args.model_folder
params = utils.AttnParams()
params.load(model_dir + "/params.pkl")
model_name = params["model"]
else:
model_name = "{}_{}_d{}_{}".format(mnames[args.bottleneck], args.model_size, args.latent_dim,
"WAE" if args.use_WAE else "VAE")
model_dir = args.models_dir + model_name + "/"
if not os.path.exists(model_dir):
os.mkdir(model_dir)
# Get default attention parameters
params = utils.AttnParams()
# Standard training params
params["epochs"] = args.epochs
params["batch_size"] = args.batch_size
params["kl_pretrain_epochs"] = 2
params["kl_anneal_epochs"] = 5
params["bottleneck"] = args.bottleneck
params["stddev"] = 1
params["decoder"] = "TRANSFORMER"
params["latent_dim"] = args.latent_dim
params["model"] = model_name
if args.use_FILM:
model_name += "_FILM"
params["decoder"] += "_FILM"
# Get training and test data from data file
# Set up model
if not args.model_folder:
if args.model_size == "small":
# AVG1: 47741
# AVG2: 53216
# GRU: 51076
# GRU_ATTN: 50806
# CONV: 51629
# AR_SLIM: 58394
params["d_model"] = 32
params["d_inner_hid"] = 196
params["d_k"] = 6
params["heads"] = 6
params["layers"] = 2
if params["bottleneck"] == "ar_slim":
params["ID_layers"] = 3
params["ID_d_model"] = 6
params["ID_width"] = 6
params["ID_d_inner_hid"] = 30
params["ID_d_k"] = 4
params["ID_d_v"] = 4
params["ID_heads"] = 5
elif "ar" in params["bottleneck"]:
params["ID_layers"] = 2
params["ID_d_model"] = 8
params["ID_d_inner_hid"] = 64
params["ID_width"] = 4
params["ID_d_k"] = 6
params["ID_d_v"] = 6
params["ID_heads"] = 4
elif params["bottleneck"] == "gru":
params["ID_layers"] = 3
params["ID_d_model"] = 48
elif params["bottleneck"] == "gru_attn":
params["ID_layers"] = 3
params["ID_d_model"] = 42
elif params["bottleneck"] == "conv":
params["ID_layers"] = 2 # num layers
params["ID_d_k"] = 5 # min_filt_size/num
params["ID_d_model"] = 64 # dense dim
elif args.model_size == "medium":
# AVG1: 171413
# AVG2: 174488
# GRU: 172664
# GRU_ATTN: 171308
# CONV: 171800
# AR_SLIM: 184968
params["d_model"] = 64
params["d_inner_hid"] = 256
params["d_k"] = 8
params["heads"] = 8
params["layers"] = 3
if params["bottleneck"] == "ar_slim":
params["ID_layers"] = 4
params["ID_d_model"] = 8
params["ID_width"] = 6
params["ID_d_inner_hid"] = 64
params["ID_d_k"] = 4
params["ID_d_v"] = 4
params["ID_heads"] = 4
elif "ar" in params["bottleneck"]:
params["ID_layers"] = 2
params["ID_d_model"] = 32
params["ID_width"] = 4
params["ID_d_inner_hid"] = 196
params["ID_d_k"] = 7
params["ID_d_v"] = 7
params["ID_heads"] = 5
elif params["bottleneck"] == "gru_attn":
params["ID_layers"] = 4
params["ID_d_model"] = 78
elif params["bottleneck"] == "gru":
params["ID_layers"] = 4
params["ID_d_model"] = 82
elif params["bottleneck"] == "conv":
params["ID_layers"] = 4
params["ID_d_k"] = 8
params["ID_d_model"] = 156
elif args.model_size == "big" or args.model_size == "large":
# big avg: 1,131,745
# big ar_log: 1,316,449
# big GRU: 1,029,152
# big CONV: 439,419
params["d_model"] = 128
params["d_inner_hid"] = 768
params["d_k"] = 12
params["heads"] = 12
params["layers"] = 4
if "ar" in params["bottleneck"]:
params["ID_layers"] = 3
params["ID_d_model"] = 40
params["ID_width"] = 4
params["ID_d_inner_hid"] = 256
params["ID_d_k"] = 8
params["ID_d_v"] = 8
params["ID_heads"] = 6
elif "gru" in params["bottleneck"]:
params["ID_layers"] = 5
params["ID_d_model"] = 160
elif params["bottleneck"] == "conv":
params["ID_layers"] = 4
params["ID_d_k"] = 9
params["ID_d_model"] = 512
params["d_v"] = params["d_k"]
if args.use_WAE:
params["WAE_kernel"] = "IMQ_normal"
params["kl_max_weight"] = 10
params["WAE_s"] = 2
# Handle interim decoder parameters
params.setIDparams()
# Create model tracking folder
if not exists(model_dir):
mkdir(model_dir)
# Handle parameters
param_filename = model_dir + "params.pkl"
loaded_params = utils.AttnParams()
if not exists(param_filename):
print("Starting new model {} with params:".format(model_name))
params.dump()
params.save(param_filename)
else:
loaded_params.load(param_filename)
print("Found model also named {} trained for {} epochs with params:".format(model_name,
loaded_params["current_epoch"]))
loaded_params.dump()
# Allow for increasing number of epochs of pre-trained model
if params["epochs"] > loaded_params["epochs"]:
print(
"Number of epochs increased to {} from {}. Autoencoder will be trained more.".format(
params["epochs"], loaded_params["epochs"]))
loaded_params["epochs"] = params["epochs"]
params = loaded_params
model, results = trainTransformer(params=params, data_file=args.data,
model_dir=model_dir)
data_train, data_test, _, _, tokens = utils.load_dataset(args.data,
"cat",
params["pp_weight"])
props_train, props_test, prop_labels = utils.load_properties(args.data)
num_seeds = 500
num_decodings = 10
num_prior_samples = 1000
with supress_stderr():
seed_output = property_distributions(data_test, props_test,
num_seeds=num_seeds,
num_decodings=num_decodings,
model=model,
beam_width=5, data_file='data/zinc12.h5')
rand_output = rand_mols(num_prior_samples, params["latent_dim"], model, 5, data_file='data/zinc12.h5')
# SAVE DATA
val_acc = getBestValAcc(args.models_dir + "/runs.csv", params)
createResultsFile(args.models_dir)
saveResults(params, val_acc, seed_output, rand_output, num_seeds, num_decodings, num_prior_samples,
models_dir=args.models_dir)
print("\tValidation accuracy:\t {:.2f}".format(val_acc))
# TODO(Basil): Add getting results from CSV file...
for (mode, output) in zip(["SAMPLING PRIOR", "SAMPLING WITH SEEDS"], [rand_output, seed_output]):
print("BY", mode)
print("\tGenerated {} molecules, of which {} were valid and {} were novel.".format(output["num_mols"],
output["num_valid"],
output["num_novel"]))
print("\t\tValid mols:\t {:.2f}".format(output["num_valid"] / output["num_mols"]))
if "num_novel" in output: print("\t\tNovel mols:\t{:.2f}".format(output["num_novel"] / output["num_valid"]))
print("\t\tSuccess frac:\t{:.2f}".format(output["success_frac"]))
print("\t\tYield:\t{:.2f}".format(output["yield"]))
for (i, key) in enumerate(utils.rdkit_funcs):
if key in prop_labels:
k = prop_labels.index(key)
print("\t\t{}:".format(key))
dat = props_test[:, k]
print("\t\t\tTest distribution:\t {:.2f} ± {:.2f}".format(np.mean(dat), np.std(dat)))
gen_dat = output["gen_props"][:, i]
print("\t\t\tGenerated distribution:\t {:.2f} ± {:.2f}".format(np.mean(gen_dat), np.std(gen_dat)))
def main():
args = get_arguments()
model_dir = args.model_path
# Get Params
model_params = AttnParams()
model_params.load(model_dir + "params.pkl")
print("Analysing model", model_dir)
model_params.dump()
# Get data
d_file = model_params["data"]
if model_params["bottleneck"] == "conv" or model_params["decoder"] == "VAE":
d_type = "onehot"
else:
d_type = "cat"
data_train, data_test, props_train, props_test, tokens = load_dataset(d_file, d_type, False)
props_train, props_test, prop_labels = load_properties(d_file)
if "TRANSFORMER" in model_params["decoder"]:
# Model is an attention based model
model = TriTransformer(tokens, model_params)
model.build_models()
model.compile_vae(Adam(0.001, 0.9, 0.98, epsilon=1e-9))
else:
# Model is GRU
model = MoleculeVAE(tokens, model_params)
# Assess how close each dimension is to a Gaussian
# Try to load property training data
if not exists(model_dir + "latents.h5") and "dothis" == "nothanks":
print("Generating latent representations from auto-encoder")
z_train = model.encode_sample.predict([data_train], 64)
z_test = model.encode_sample.predict([data_test], 64)
with h5py.File(model_dir + "latents.h5", 'w') as dfile:
dfile.create_dataset('z_test', data=z_test)
dfile.create_dataset('z_train', data=z_train)
print("KURTOSIS:")
# latent_distributions(model_dir + 'latents.h5', plot_kd=True)
# Test random molecule
print("Example decodings with ibruprofen (beam width = 5):")
print("\tIbuprofen smiles:\t{}".format(IBUPROFEN_SMILES))
s = model.decode_from_string(IBUPROFEN_SMILES, beam_width=5)
[print("\t\tDecoding {}:\t\t{}".format(i + 1, seq[0])) for (i, seq) in enumerate(s)]
print("Exploring property distributions of chemicals from {} decoding(s) of {} random seed(s):".format(
args.n_decodings,
args.n_seeds))
with supress_stderr():
if args.prior_sample:
output = rand_mols(args.n_seeds, model_params["latent_dim"], model, args.beam_width)
else:
output = property_distributions(data_test, props_test,
num_seeds=args.n_seeds,
num_decodings=args.n_decodings,
model=model,
beam_width=args.beam_width,
data_file=None) # ,
print("Generated {} molecules, of which {} were valid.".format(output["num_mols"], output["num_valid"]))
print("\tValid mols:\t {:.2f}".format(output["num_valid"] / output["num_mols"]))
if "num_novel" in output: print("\tNovel mols:\t{:.2f}".format(output["num_novel"]))
print("\tSuccess frac:\t{:.2f}".format(output["success_frac"]))
print("\tYield:\t{:.2f}".format(output["yield"]))
for (i, key) in enumerate(rdkit_funcs):
if key in prop_labels:
k = prop_labels.index(key)
print("\t{}:".format(key))
dat = props_test[:, k]
print("\t\tTest distribution:\t {:.2f} ± {:.2f}".format(np.mean(dat), np.std(dat)))
gen_dat = output["gen_props"][:, i]
print("\t\tGenerated distribution:\t {:.2f} ± {:.2f}".format(np.mean(gen_dat), np.std(gen_dat)))
logger.info('检查测试目标主机[%s]的PING连通性。' % properties['host.ip'])
if not utils.is_host_pingable(properties['host.ip']):
logger.error('不能PING通所要测试的目标主机[%s]。测试中断！请检查机器IP地址设置，或者目标服务器是否已经启动。' % properties['host.ip'])
sys.exit()
stats = {}
if args.testfile is not None:
case_id = None
items = args.testfile.split(':')
testfile = locate_test_file(items[0])
if len(items) > 1:
case_id = items[1]
stat = runner.runfile(testfile, properties, case_id=case_id, checktestonly=args.checktestonly,
includemanual=args.includemanual)
stats[testfile] = stat
else:
stats = runner.runfiles(properties, checktestonly=args.checktestonly, includemanual=args.includemanual)
report.console(None,stats,properties['host.ip'])
if __name__ == '__main__':
parser = get_cmd_parser()
args = parser.parse_args(sys.argv[1:])
properties = utils.load_properties('./conf/test.%s.properties' % args.profile)
logger.info('运行参数：%s' % args)
logger.info('测试设置：%s' % properties)
properties['debug'] = True#args.debug
properties['fuzz'] = args.fuzz
properties['failfast'] = args.failfast
main(args, properties)
def load_sympa():
# Load properties and check that they are structurally OK
props = utils.load_properties()
try:
assert props['hrMgmtSystem'] != None, 'No HRM properties section'
assert props['hrMgmtSystem'][
'moduleName'] != None, 'HRM module name not set'
assert props['hrMgmtSystem']['url'] != None, 'HRM URL not set'
assert props['hrMgmtSystem']['pw'] != None, 'HRM user ID not set'
assert props['hrMgmtSystem']['id'] != None, 'HRM password not set'
except Exception as ex:
print("Properties file not complete:", repr(ex))
exit()
# Load all the information for use by the other two functions
response = requests.get(props['hrMgmtSystem']['url'],
auth=(props['hrMgmtSystem']['id'],
props['hrMgmtSystem']['pw']))
if not response:
print("Could not read employee info")
print(response.content)
exit()
errors = []
for e in response.json()["value"]:
# Not all employees are created perfect
try:
assert e["Henkilönumero"] != None
assert e["Henkilötunnus"] != None
assert len(e["Työsuhdetiedot"]) > 0
except Exception as ex:
errors.append(repr(ex))
continue
e["Etunimet"] = ' '.join(
map(lambda x: x.capitalize(), e["Etunimet"].split(' ')))
# Initialize the dictionary with information we trust to
# always be available
employee = {
'externalId': e["Henkilönumero"],
'identifier': e["Henkilönumero"],
'ssn': e["Henkilötunnus"],
'callName': e["Etunimet"].split(' ')[0],
'lastName': e["Sukunimi"],
'emailAddress': e["Työsähköposti"],
'localPhoneNumber': e["Puhelinnumero_työ"],
'startDate': e["Työsuhdetiedot"][0]["Työsuhteen_alkupvm"],
'departments': []
}
# Add employment end date, if set
if e["Työsuhdetiedot"][0]["Työsuhteen_päättymispvm"]:
employee['endDate'] = e["Työsuhdetiedot"][0][
"Työsuhteen_päättymispvm"]
# Add supervisor information, if available
if e["Lähin_esimies"]:
employee['supervisors'] = [{
'externalId':
e["Lähin_esimies"],
'startDate':
e["Työsuhdetiedot"][0]["Työsuhteen_alkupvm"]
}]
# Add information of past and current departments
for d in e["Työsuhdetiedot"]:
try:
assert d["Osasto"] != None
except Exception as ex:
errors.append(repr(ex))
continue
d_id = get_depId(d["Osasto"])
# make sure the department info is in the deps array
deps[d_id] = d["Osasto"]
d_info = {
'externalId': d_id,
'startDate': d["Rivi_voimassa_alkaen_pvm"]
}
if d["Rivi_voimassa_asti_pvm"] != None:
d_info['endDate'] = d["Rivi_voimassa_asti_pvm"]
employee['departments'].append(d_info)
employees.append(employee)
if len(errors) > 0:
print("Found {} errors when loading employee data.".format(
len(errors)))
def get_personnel():
# Load properties and check that they are structurally OK
props = utils.load_properties()
try:
assert props['hrMgmtSystem'] != None, 'No HRM properties section'
assert props['hrMgmtSystem'][
'employeeFile'] != None, 'HRM Employees Excel file not set'
except Exception as ex:
print("Properties file not complete:", repr(ex))
exit()
wb = load_workbook(props['hrMgmtSystem']['employeeFile'])
employees = []
for row in wb.active.iter_rows(min_row=2):
external_id, identifier, ssn, call_name, last_name, email_address, private_email_address, \
job_title, local_phone_number, phone_country_code, start_date, end_date, department, department_start, \
supervisor, supervisor_start = row
employee = {
'externalId':
external_id.value,
'identifier':
identifier.value,
'ssn':
ssn.value,
'callName':
call_name.value,
'lastName':
last_name.value,
'emailAddress':
email_address.value,
'privateEmailAddress':
private_email_address.value,
'jobTitle':
job_title.value,
'localPhoneNumber':
local_phone_number.value,
'phoneCountryCode':
phone_country_code.value,
'startDate':
start_date.value.strftime('%Y-%m-%d'),
'departments': [{
'externalId':
department.value,
'startDate':
department_start.value.strftime('%Y-%m-%d'),
}]
}
if end_date.value:
employee['endDate'] = end_date.value.strftime('%Y-%m-%d')
if supervisor.value and supervisor_start.value:
employee['supervisors'] = [{
'externalId':
supervisor.value,
'startDate':
supervisor_start.value.strftime('%Y-%m-%d')
}]
employees.append(employee)
return employees
def cfg_init(config_dir):
global config
config = load_properties(config_dir + '/' + CONFIG_FILE)