def on_pubmsg(self, connection, event):
msg = event.arguments[0]
if msg.startswith(self.command_delimiter):
parts = msg[1:].split(' ', 1)
cmd = parts[0]
arg = parts[1] if len(parts) == 2 else None
command.call(cmd, arg, self)
def calcCCP4Maps(mtzin,
output_file_prefix,
job_dir,
file_stdout,
file_stderr,
source_key="refmac",
log_parser=None):
# Calculate CCP4 Maps from refinement mtz, given in mtzin. The maps will be
# placed in files output_file_prefix_map.map and output_file_prefix_dmap.map
#
# Sigmaa style 2mfo-dfc map with restored data
scr_file = open(fft_map_script(), "w")
scr_file.write(
"TITLE Sigmaa style 2mfo-dfc map calculated with refmac coefficients\n"
+ "LABI F1=" + _columns[source_key][0] + " PHI=" +
_columns[source_key][1] + "\nEND\n")
scr_file.close()
# Start fft
rc = command.call(
"fft", ["HKLIN", mtzin, "MAPOUT", output_file_prefix + file_map()],
job_dir, fft_map_script(), file_stdout, file_stderr, log_parser)
if rc.msg:
file_stdout.write("Error calling FFT(1): " + rc.msg + "\n")
file_stderr.write("Error calling FFT(1): " + rc.msg + "\n")
# Sigmaa style mfo-dfc map
if source_key.startswith("refmac"):
scr_file = open(fft_dmap_script(), "w")
scr_file.write(
"TITLE Sigmaa style mfo-dfc map calculated with refmac coefficients\n"
+ "LABI F1=" + _columns[source_key][2] + " PHI=" +
_columns[source_key][3] + "\nEND\n")
scr_file.close()
# Start fft
rc = command.call(
"fft",
["HKLIN", mtzin, "MAPOUT", output_file_prefix + file_dmap()],
job_dir, fft_dmap_script(), file_stdout, file_stderr, log_parser)
if rc.msg:
file_stdout.write("Error calling FFT(2): " + rc.msg + "\n")
file_stderr.write("Error calling FFT(2): " + rc.msg + "\n")
return
def runApp(self, appName, cmd, fpath_stdout=None, fpath_stderr=None):
input_script = None
if self.script_file:
input_script = self.script_path
fstdout = self.file_stdout
fstderr = self.file_stderr
if fpath_stdout:
fstdout = open(fpath_stdout, 'w')
if fpath_stderr:
fstderr = open(fpath_stderr, 'w')
rc = command.call(appName, cmd, self.workdir, input_script, fstdout,
fstderr, self.log_parser)
os.chdir(self.workdir)
self.script_file = None
if fpath_stdout:
fstdout.close()
if fpath_stderr:
fstderr.close()
return rc
def switch_to_certsbridge_conf():
configurations, success = command.call_get_out(
"gcloud config configurations list --filter=name=certsbridge")
configuration_exists = len(configurations) > 0
if not configuration_exists:
command.call("gcloud config configurations create certsbridge",
"Create gcloud conf certsbridge")
command.call("gcloud config set compute/zone europe-west1-b")
command.call("gcloud config set project certsbridge-dev")
if os.path.isfile("/etc/service-account/service-account.json"):
command.call(
"gcloud auth activate-service-account --key-file=/etc/service-account/service-account.json"
)
else:
print("### Gcloud conf certsbridge exists")
command.call("gcloud config configurations activate certsbridge",
"Switch gcloud conf to certsbridge")
def clear_dns_zone(zone_name):
"""
Removes A sub-records of com.certsbridge from a given DNS zone.
"""
print(
"### Removing all sub-records of com.certsbridge from zone {0}".format(
zone_name))
switch_to_certsbridge_conf()
output = command.call_get_out(
"gcloud dns record-sets list --zone {0} --filter=type=A | grep certsbridge.com | tr -s ' ' | cut -d ' ' -f 1,4"
.format(zone_name))[0]
command.call("gcloud dns record-sets transaction start --zone {0}".format(
zone_name))
for line in output:
domain, ip = line.split(" ")
command.call(
"gcloud dns record-sets transaction remove --zone {0} --name='{1}' --type=A --ttl=300 {2}"
.format(zone_name, domain,
ip), "Remove DNS record for domain {0}".format(domain))
command.call(
"gcloud dns record-sets transaction execute --zone {0}".format(
zone_name))
switch_to_default_conf()
def calcAnomEDMap(xyzin,
hklin,
hkl_dataset,
anom_form,
output_file_prefix,
job_dir,
file_stdout,
file_stderr,
log_parser=None):
# prepare refmac input script
scr_file = open(refmac_script(), "w")
scr_file.write(anom_form + "solv NO\n" + "refi -\n" +
" type UNREST -\n" + " resi MLKF -\n" +
" meth CGMAT -\n" + " bref ISOT\n" + "ncyc 0\n" +
"labin FREE=" + hkl_dataset.FREE + " F+=" +
hkl_dataset.Fpm.plus.value + " SIGF+=" +
hkl_dataset.Fpm.plus.sigma + " F-=" +
hkl_dataset.Fpm.minus.value + " SIGF-=" +
hkl_dataset.Fpm.minus.sigma + "\n" + "end\n")
scr_file.close()
# prepare refmac command line
cmd = [
"XYZIN",
xyzin,
"XYZOUT",
output_file_prefix + file_pdb(),
"HKLIN",
hklin,
"HKLOUT",
output_file_prefix + file_mtz(),
"LIBOUT",
output_file_prefix + file_cif(),
]
# Start refmac
rc = command.call("refmac5", cmd, job_dir, refmac_script(), file_stdout,
file_stderr, log_parser)
if rc.msg:
file_stdout.write("Error calling refmac5: " + rc.msg)
file_stderr.write("Error calling refmac5: " + rc.msg)
else:
# Generate maps
calcCCP4Maps(output_file_prefix + file_mtz(), output_file_prefix,
job_dir, file_stdout, file_stderr, "refmac_anom",
log_parser)
return
def create_random_domains(zone_name):
"""
Generates 2 random domain names in zone_name, under top-level domain com.certsbridge
"""
output, _ = command.call_get_out("gcloud compute addresses describe test-ip-address --global | grep address: | cut -d ' ' -f 2")
ip = output[0]
utils.printf("Creating random domains pointing at ip {0}".format(ip))
command.call("gcloud dns record-sets transaction start --zone {0} --project {1}".format(zone_name, PROJECT))
result = []
for i in range(2):
record = ''.join(random.choice(string.ascii_lowercase) for _ in range(RECORD_LENGTH))
domain = "{0}.{1}.certsbridge.com".format(record, zone_name)
result.append(domain)
command.call("gcloud dns record-sets transaction add --zone {0} --project {1} --name='{2}' --type=A --ttl=300 {3}".format(zone_name, PROJECT, domain, ip), "Add DNS record for domain {0} to ip {1}".format(domain, ip))
command.call("gcloud dns record-sets transaction execute --zone {0} --project {1}".format(zone_name, PROJECT))
return result
def generate(self, access_token, refresh_token, user_project_id,
project_id):
self.info("GenIdentity User Project=%s" % (user_project_id))
pki_role = "pki-backend-role-%s" % project_id
jwt_role = "sae-issue-cert-%s" % project_id
vault_token = self.vault_login(os.environ['VAULT_ADDR'], jwt_role,
access_token, refresh_token, 1)
url = "%s/v1/pki_int/issue/%s" % (os.environ['VAULT_ADDR'], pki_role)
self.info("issue url %s" % url)
payload = {'common_name': user_project_id}
headers = {'X-Vault-Token': vault_token}
x = requests.post(url, data=json.dumps(payload), headers=headers)
self.info("status_code %s" % x.status_code)
if x.status_code != 200:
self.info("text %s" % x.text)
raise Exception("Failed to issue certificate")
j = x.json()
self.info("issue_cert " + json.dumps(j))
call('curl -f -k -O ' + os.environ['CA_CHAIN_URI'])
call(
'openssl pkcs12 -in ca_chain.pfx -out ca_chain.pem -password pass:password'
)
f = open("crt", "w")
f.write(j['data']['certificate'])
f.close()
f = open("key", "w")
f.write(j['data']['private_key'])
f.close()
call(
'openssl pkcs12 -export -out private.pfx -inkey key -in crt -password pass:password'
)
call('rm -rf nssdb')
call('mkdir nssdb')
call('certutil -d nssdb -N --empty-password')
call('pk12util -v -d sql:nssdb -K password -W password -i private.pfx')
call('echo "password" > pass')
call(
'certutil -A -n "ca-vaultpki-root" -t TC -i /cacerts/ca-vaultpki-root.crt -d sql:nssdb'
)
call(
'certutil -A -n "ca-vaultpki-inter" -t TC -i /cacerts/ca-vaultpki-inter.crt -d sql:nssdb'
)
call('certutil -L -d sql:nssdb')
# Install the Root CA into the JAVA keystore
call('rm -f jre_cacerts')
call(
'keytool -trustcacerts -noprompt -keystore jre_cacerts -storepass changeit -alias root -import -file /cacerts/ca-vaultpki-root.crt'
)
secret_data = {}
secret_data['postgresql.crt'] = base64.b64encode(
j['data']['certificate'].encode('utf-8')).decode('utf-8')
secret_data['postgresql.key'] = base64.b64encode(
j['data']['private_key'].encode('utf-8')).decode('utf-8')
s3_config = {
"aws_access_key_id": '',
"aws_secret_access_key": '',
"endpoint_url": os.environ['MINIO_ADDR']
}
creds = assume_role_with_web_identity(s3_config, access_token)
aws_credentials = """
[default]
aws_access_key_id={key}
aws_secret_access_key={secret}
aws_session_token={token}
""".format(key=creds['AccessKeyId'],
secret=creds['SecretAccessKey'],
token=creds['SessionToken'])
aws_credentials_r = """
aws_access_key_id <- "{key}"
aws_secret_access_key <- "{secret}"
aws_session_token <- "{token}"
""".format(key=creds['AccessKeyId'],
secret=creds['SecretAccessKey'],
token=creds['SessionToken'])
self.info("minio/s3 expiration " + str(creds['Expiration']))
self.info("minio/s3 access key " + creds['AccessKeyId'])
def diff_dates(date1, date2):
return abs(date2 - date1)
minutes = divmod(
diff_dates(datetime.now(timezone.utc),
creds['Expiration']).seconds, 60)
self.info("minio/s3 expires in %d mins %d secs" %
(minutes[0], minutes[1]))
secret_data['aws-credentials'] = base64.b64encode(
aws_credentials.encode('utf-8')).decode('utf-8')
secret_data['aws-credentials-r'] = base64.b64encode(
aws_credentials_r.encode('utf-8')).decode('utf-8')
for f in listdir('nssdb'):
data = open("nssdb/%s" % f, "rb").read()
b64 = base64.b64encode(data)
secret_data[f] = b64.decode('utf-8')
data = open("jre_cacerts", "rb").read()
b64 = base64.b64encode(data)
secret_data["jre_cacerts"] = b64.decode('utf-8')
namespace = 'vdi'
metadata = {
'name': "%s-cert" % user_project_id,
'namespace': namespace
}
self.info(secret_data.keys())
return secret_data
def call(cmdline):
log.info("Running external command: " + cmdline)
command.call(cmdline)
def __init__(self, addr, access_key, secret_key):
self.addr = addr
self.access_key = access_key
self.secret_key = secret_key
call("mc config host add s3 %s %s %s" % (addr, access_key, secret_key))
def clear_conf():
command.call("echo y | gcloud config configurations delete certsbridge",
"Remove gcloud conf certsbridge")
def switch_to_default_conf():
command.call("gcloud config configurations activate default",
"Switch gcloud conf to default")
def calcEDMap(xyzin,
hklin,
libin,
hkl_dataset,
output_file_prefix,
job_dir,
file_stdout,
file_stderr,
log_parser=None):
# prepare refmac input script
scr_file = open(refmac_script(), "w")
scr_file.write(
"make check NONE\n" + "make -\n" + " hydrogen ALL -\n" +
" hout NO -\n" + " peptide NO -\n" + " cispeptide YES -\n" +
" ssbridge YES -\n" + " symmetry YES -\n" + " sugar YES -\n" +
" connectivity NO -\n" + " link NO\n" + "refi -\n" +
" type UNREST -\n" + " resi MLKF -\n" + " meth CGMAT -\n" +
" bref ISOT\n" + "ncyc 0\n" + "scal -\n" + " type SIMP -\n" +
" LSSC -\n" + " ANISO -\n" + " EXPE\n" + "solvent YES\n" +
"weight -\n" + " AUTO\n" + "monitor MEDIUM -\n" +
" torsion 10.0 -\n" + " distance 10.0 -\n" +
" angle 10.0 -\n" + " plane 10.0 -\n" + " chiral 10.0 -\n" +
" bfactor 10.0 -\n" + " bsphere 10.0 -\n" +
" rbond 10.0 -\n" + " ncsr 10.0\n" + " labin FP=" +
hkl_dataset.Fmean.value + " SIGFP=" + hkl_dataset.Fmean.sigma +
" -\n" + " FREE=" + hkl_dataset.FREE + "\n" +
"labout FC=FC FWT=FWT PHIC=PHIC PHWT=PHWT DELFWT=DELFWT PHDELWT=PHDELWT FOM=FOM\n"
+ "PNAME CoFE\n" + "DNAME\n" + "RSIZE 80\n" +
"EXTERNAL WEIGHT SCALE 10.0\n" + "EXTERNAL USE MAIN\n" +
"EXTERNAL DMAX 4.2\n" + "END\n")
scr_file.close()
# prepare refmac command line
cmd = [
"XYZIN",
xyzin,
"XYZOUT",
output_file_prefix + file_pdb(),
"HKLIN",
hklin,
"HKLOUT",
output_file_prefix + file_mtz(),
"LIBOUT",
output_file_prefix + file_cif(),
]
if libin:
cmd += ["LIBIN", libin]
# Start refmac
rc = command.call("refmac5", cmd, job_dir, refmac_script(), file_stdout,
file_stderr, log_parser)
if rc.msg:
file_stdout.write("Error calling refmac5: " + rc.msg)
file_stderr.write("Error calling refmac5: " + rc.msg)
else: # Generate maps
calcCCP4Maps(output_file_prefix + file_mtz(), output_file_prefix,
job_dir, file_stdout, file_stderr, "refmac", log_parser)
return