def changed_dict(properties, block, key, input):
if get_dict_value(properties, block, key) != input:
print "%s.%s = %s != %s" % (
block, key, get_dict_value(properties, block, key), input)
return True
else:
return False
def changed_dict(properties, block, key, input):
if get_dict_value(properties, block, key) != input:
print "%s.%s = %s != %s" % (block, key,
get_dict_value(properties, block, key),
input)
return True
else:
return False
def verify_ope_account_in_smc(student_user, smc_url, smc_admin_user, smc_admin_pw):
# Bounce off the SMC server to see if the student account exists in SMC
# NOTE - this one does NOT check canvas for the user
laptop_admin_user = ""
laptop_admin_password = ""
student_full_name = ""
smc_version = ""
p("\n}}gnChecking user status in SMC tool...}}xx")
json_response = RestClient.send_rest_call(server=smc_url,
api_endpoint="lms/verify_ope_account_in_smc.json/" + student_user,
auth_user=smc_admin_user, auth_password=smc_admin_pw)
if json_response is None:
# If None - fatal error!
return None
# Interpret response from SMC
try:
msg = util.get_dict_value(json_response, "msg")
if msg == "":
p("}}rbUnable to interpret response from SMC - no msg parameter returned}}xx")
p(json_response)
return None
if msg.startswith("Invalid User!"):
p("\n}}rbInvalid User!}}xx")
p("}}ynUser doesn't exit in system, please import this student in the SMC first!}}xx")
return None
if msg == "No username specified!":
p("\n}}rbInvalid User!}}xx")
p("}}mnNo user with this name exists, please import this student in the SMC first!}}xx")
return None
student_full_name = util.get_dict_value(json_response, "student_full_name")
laptop_admin_user = util.get_dict_value(json_response, "laptop_admin_user")
smc_version = util.get_dict_value(json_response, "smc_version")
# Password moved to credential step
#laptop_admin_password = util.get_dict_value(json_response, "laptop_admin_password")
except Exception as ex:
p("}}rbUnable to interpret response from SMC - no msg parameter returned}}xx")
p("}}mn" + str(ex) + "}}xx")
# p(str(smc_response))
return None
if laptop_admin_user == "": # or laptop_admin_password == "":
p("}}rbERR - Please set the laptop admin credentials in the SMC before continuing (Admin -> Configure App -> Laptop Admin Credentials) }}xx")
return None # sys.exit(-1)
if student_full_name == "":
p("}}rbERR - Unable to find student user in the SMC? Make sure it is imported.}}xx")
return None # sys.exit(-1)
#return (laptop_admin_user, laptop_admin_password, student_full_name)
return (laptop_admin_user, student_full_name, smc_version)
def ping_smc(smc_url):
json_response = RestClient.send_rest_call(server=smc_url,
api_endpoint="lms/ping.json",
)
if json_response is None:
# If None - unable to communicate - likely offline
p("}}ynNo response from smc server}}xx", log_level=4)
return False
server_time = util.get_dict_value(json_response, "server_time")
p("}}mnPING - Got SMC Server time: " + str(server_time) + "}}xx")
return True
def credential_student_in_smc(student_user, smc_url, smc_admin_user, smc_admin_pw, ex_info):
# Send the info to the SMC server to
json_response = RestClient.send_rest_call(server=smc_url,
api_endpoint="lms/credential_student.json/" + student_user,
method="POST", json_params=ex_info,
auth_user=smc_admin_user, auth_password=smc_admin_pw)
if json_response is None:
# If None - fatal error!
return None
# Interpret response from SMC
try:
#p("RESP: " + str(smc_response))
msg = util.get_dict_value(json_response, "msg", default="missing")
if msg == "missing":
p("}}rbUnable to interpret response from SMC - no msg parameter returned}}xx")
return None
if msg == "Invalid User!":
p("\n}}rbInvalid User!}}xx")
p("}}mnUser doesn't exit in system, please import this student in the SMC first!}}xx")
return None
if msg == "No username specified!":
p("\n}}rbInvalid User!}}xx")
p("}}mnNo user with this name exists, please import this student in the SMC first!}}xx")
return None
if "unable to connect to canvas db" in msg:
p("\n}}rbSMC Unable to connect to Canvas DB - make sure canvas app is running and\n" +
"the SMC tool is configured to talk to Canvas}}xx")
p("}}yn" + str(msg) + "}}xx")
return None
if "Unable to find user in canvas:" in msg:
p("\n}}rbInvalid User!}}xx")
p("}}mnUser exists in SMC but not in Canvas, please rerun import this student in the SMC to correct the issue!}}xx")
return None
#full_name = util.get_dict_value(json_response, "full_name")
canvas_access_token = util.get_dict_value(json_response, "key")
student_hash = util.get_dict_value(json_response, "hash")
admin_hash = util.get_dict_value(json_response, "admin_hash")
student_full_name = util.get_dict_value(json_response, "full_name")
canvas_url = util.get_dict_value(json_response, "canvas_url")
except Exception as ex:
p("}}rbUnable to interpret response from SMC - no msg parameter returned}}xx")
p("}}mn" + str(ex) + "}}xx")
return None
# Decrypt scrambled parts
student_password = Encryption.decrypt(student_hash, canvas_access_token)
laptop_admin_password = Encryption.decrypt(admin_hash, canvas_access_token)
# TODO - DEBUG - Disable this line!
adm_pw_masked = (len(laptop_admin_password)-1) * "*" + laptop_admin_password[-1:]
#p(student_password + "/" + adm_pw_masked, log_level=5)
return (student_full_name, canvas_url, canvas_access_token,
student_password, laptop_admin_password)
# Only show commands if UAC active
if is_admin[1]:
commands = sorted(valid_commands.keys())
p("}}yn Valid Commands: " + str(commands) + "}}xx")
p("}}ybFor help - type mgmt.exe help (command)}}xx")
sys.exit(1)
# Run the function associated w the command
cmd_parts = valid_commands[cmd]
if cmd_parts is None:
p("}}rnERROR - Command not avaialable " + cmd +
" - coming soon...}}xx",
log_level=1)
sys.exit(1)
cmd_requires_admin = util.get_dict_value(cmd_parts, "require_admin", True)
if cmd_requires_admin is True and is_admin[1] is not True:
# Command requires elevation and this user doesn't have it!
if is_admin[0] is not True:
# User is NOT in the administrators group
p("}}rbINVALID USER - Must be in the administrators group to use this utility!\n"
+ "Attempt logged for user " + is_admin[2] + ".}}xx",
log_level=1)
sys.exit(2)
if is_admin[1] is not True:
# User is NOT running with UAC enabled
p("}}rbINVALID USER - Must be in UAC prompt to use this utility!\n"
+ "Attempt logged for user " + is_admin[2] + ".}}xx",
def update_fields(service_id):
properties = get_properties(service_id)
if not properties.keys():
return
if get_value(properties, "AutoConnect") == "true":
autoconn = "Yes"
else:
autoconn = "No"
form.get('autoconnect').value = autoconn
form.get('nameservers').value = get_str_value(properties,
"Nameservers.Configuration")
form.get('timeservers').value = get_str_value(properties,
"Timeservers.Configuration")
form.get('domains').value = get_str_value(properties,
"Domains.Configuration")
value = get_dict_value(properties, "Proxy.Configuration", "Method")
if value == "":
value = "auto"
form.get('proxymethod').value = value
form.get('proxyurl').value = get_dict_value(properties,
"Proxy.Configuration", "URL")
form.get('proxyservers').value = get_dict_value(properties,
"Proxy.Configuration",
"Servers")
form.get('proxyexcludes').value = get_dict_value(properties,
"Proxy.Configuration",
"Excludes")
form.get('ipv4method').value = get_dict_value(properties,
"IPv4.Configuration",
"Method")
form.get('ipv4address').value = get_dict_value(properties,
"IPv4.Configuration",
"Address")
form.get('ipv4netmask').value = get_dict_value(properties,
"IPv4.Configuration",
"Netmask")
form.get('ipv4gateway').value = get_dict_value(properties,
"IPv4.Configuration",
"Gateway")
form.get('ipv6method').value = get_dict_value(properties,
"IPv6.Configuration",
"Method")
form.get('ipv6address').value = get_dict_value(properties,
"IPv6.Configuration",
"Address")
value = get_dict_value(properties, "IPv6.Configuration",
"PrefixLength")
if value != "":
value = "%d" % value
form.get('ipv6prefixlen').value = value
form.get('ipv6gateway').value = get_dict_value(properties,
"IPv6.Configuration",
"Gateway")
form.get('ipv6privacy').value = get_dict_value(properties,
"IPv6.Configuration",
"Privacy")
def changed_dict_byte(properties, block, key, input):
value = "%d" % get_dict_value(properties, block, key)
if value != input:
return True
else:
return False
def build_donor(app_ctx):
click.echo('in build_donor...')
schema = app_ctx['input_schemas']['donor']
fname = os.path.join(schema['data_path'], schema['main_file_name'])
annotations = {}
util.read_annotations(annotations, 'pcawg_donorlist', schema['annotations_path']+'pc_annotation-pcawg_final_list.tsv')
util.read_annotations(annotations, 'blacklist', schema['annotations_path']+'blacklist/pc_annotation-donor_blacklist.tsv')
util.read_annotations(annotations, 'graylist', schema['annotations_path']+'graylist/pc_annotation-donor_graylist.tsv')
util.read_annotations(annotations, 'uuid_to_barcode', schema['annotations_path']+'pc_annotation-tcga_uuid2barcode.tsv')
util.read_annotations(annotations, 'icgc_donor_id', schema['annotations_path']+'icgc_bioentity_ids/pc_annotation-icgc_donor_ids.csv')
util.read_annotations(annotations, 'icgc_specimen_id', schema['annotations_path']+'icgc_bioentity_ids/pc_annotation-icgc_specimen_ids.csv')
util.read_annotations(annotations, 'icgc_sample_id', schema['annotations_path']+'icgc_bioentity_ids/pc_annotation-icgc_sample_ids.csv')
util.read_annotations(annotations, 'specimen_type_to_cv', schema['annotations_path']+'pc_annotation-tcga_specimen_type2icgc_cv.tsv')
util.read_annotations(annotations, 'specimen_uuid_to_type', schema['annotations_path']+'pc_annotation-tcga_specimen_uuid2type.tsv')
donor = {}
specimen = {}
# build the donor and specimen objects
with open(fname, 'r') as f:
reader = csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
for r in reader:
donor_unique_id = r.get('disease.x')+'-US::'+r.get('participant_id')
if not donor.get(donor_unique_id):
#new donor: initial the donor
donor[donor_unique_id] = create_obj(donor_unique_id, schema, r, annotations, 'donor')
if not specimen.get('mirna'):
specimen['mirna'] = {}
specimen_id = r.get('sample_id')
if not specimen['mirna'].get(specimen_id):
specimen['mirna'][specimen_id] = create_obj(donor_unique_id, schema, r, annotations, 'mirna')
aliquot = create_obj(donor_unique_id, schema, r, annotations, 'aliquot')
specimen['mirna'][specimen_id]['aliquots'].append(copy.deepcopy(aliquot))
specimen['mirna'][specimen_id]['donor_unique_id'] = donor_unique_id
# build the specimen object from help file
fname = os.path.join(schema['data_path'], schema['help_file_name'])
with open(fname, 'r') as f:
reader = csv.DictReader(f, delimiter='\t', quoting=csv.QUOTE_NONE)
for r in reader:
donor_unique_id = r.get('donor_unique_id')
specimen_id = r.get('submitter_specimen_id')
dtype = 'rna_seq' if r.get('library_strategy') == 'RNA-Seq' else 'wgs'
if not specimen.get(dtype):
specimen[dtype] = {}
if not specimen[dtype].get(specimen_id):
specimen[dtype][specimen_id] = create_obj(donor_unique_id, schema, r, annotations, dtype)
specimen[dtype][specimen_id]['donor_unique_id'] = donor_unique_id
if os.path.exists(app_ctx['output_dir']):
shutil.rmtree(app_ctx['output_dir'])
os.makedirs(app_ctx['output_dir'])
# add specimen to donor
crossref_specimen_ids = {}
crossref_specimen_ids['rna_seq'] = specimen['rna_seq'].keys()
crossref_specimen_ids['wgs'] = specimen['wgs'].keys()
json2tsv_fields_map = schema['json2tsv_fields_map']
sample_sheet_fields_donor = schema['target_tsv_file']['mirna_sample_sheet']['donor']
sample_sheet_fields_specimen = schema['target_tsv_file']['mirna_sample_sheet']['specimen']
sample_sheet_fields_sample = schema['target_tsv_file']['mirna_sample_sheet']['sample']
with open(os.path.join(app_ctx['output_dir'], 'mirna_sample_sheet.tsv'), 'a') as f:
f.write('\t'.join(sample_sheet_fields_donor+sample_sheet_fields_specimen+sample_sheet_fields_sample) + '\n')
for dtype in ['mirna', 'rna_seq', 'wgs']:
#loop on the specimen to add it to donor
for k, v in specimen[dtype].iteritems():
donor_unique_id = v.get('donor_unique_id')
if not donor.get(donor_unique_id):
continue
if (not donor[donor_unique_id]['pcawg_donor']) and (not 'normal' in v.get('dcc_specimen_type').lower()):
continue
sample_sheet = OrderedDict()
sample_sheet.update(util.get_dict_value(sample_sheet_fields_donor, donor[donor_unique_id], json2tsv_fields_map))
# check whether wgs and rna_seq has the mirna specimen
if dtype == 'mirna':
v['has_matched_wgs_specimen'] = True if k in crossref_specimen_ids.get('wgs') else False
v['has_matched_rna_seq_specimen'] = True if k in crossref_specimen_ids.get('rna_seq') else False
sample_sheet.update(util.get_dict_value(sample_sheet_fields_specimen, v, json2tsv_fields_map))
for aliquot in v.get('aliquots'):
sample_sheet.update(util.get_dict_value(sample_sheet_fields_sample, aliquot, json2tsv_fields_map))
line = util.get_line(sample_sheet)
# # generate mirna_sample_sheet
with open(os.path.join(app_ctx['output_dir'], 'mirna_sample_sheet.tsv'), 'a') as f:
f.write('\t'.join(line) + '\n')
v.pop('donor_unique_id')
if 'normal' in v.get('dcc_specimen_type').lower():
donor[donor_unique_id]['normal'][dtype] = v
else:
donor[donor_unique_id]['tumor'][dtype].append(v)
# get release_tsv fields
release_fields = schema['target_tsv_file']['mirna_release_tsv']
with open(os.path.join(app_ctx['output_dir'], 'mirna_release.tsv'), 'a') as f:
f.write('\t'.join(release_fields) + '\n')
# remove the donors only have tumor mirna specimen
for k, v in donor.iteritems():
if (not v['pcawg_donor']) and (not v.get('normal').get('mirna')):
continue
for dtype in ['mirna', 'rna_seq', 'wgs']:
v['tumor_'+dtype+'_specimen_count'] = len(v['tumor'][dtype]) if v['tumor'].get(dtype) else 0
# generate the jsonl dump
with open(os.path.join(app_ctx['output_dir'], 'mirna_release.jsonl'), 'a') as f:
f.write(json.dumps(v) + '\n')
# # generate mirna_release_tsv list
tsv_obj = util.get_dict_value(release_fields, v, json2tsv_fields_map)
line = util.get_line(tsv_obj)
with open(os.path.join(app_ctx['output_dir'], 'mirna_release.tsv'), 'a') as f:
f.write('\t'.join(line) + '\n')
click.echo('complete in build_donor...')
def update_fields(service_id):
properties = get_properties(service_id)
if not properties.keys():
return
if get_value(properties, "AutoConnect") == "true":
autoconn = "Yes"
else:
autoconn = "No"
form.get('autoconnect').value = autoconn
form.get('nameservers').value = get_str_value(properties,
"Nameservers.Configuration")
form.get('timeservers').value = get_str_value(properties,
"Timeservers.Configuration")
form.get('domains').value = get_str_value(properties,
"Domains.Configuration")
value = get_dict_value(properties, "Proxy.Configuration", "Method")
if value == "":
value = "auto"
form.get('proxymethod').value = value
form.get('proxyurl').value = get_dict_value(properties,
"Proxy.Configuration", "URL")
form.get('proxyservers').value = get_dict_value(properties,
"Proxy.Configuration",
"Servers")
form.get('proxyexcludes').value = get_dict_value(properties,
"Proxy.Configuration",
"Excludes")
form.get('ipv4method').value = get_dict_value(properties,
"IPv4.Configuration",
"Method")
form.get('ipv4address').value = get_dict_value(properties,
"IPv4.Configuration",
"Address")
form.get('ipv4netmask').value = get_dict_value(properties,
"IPv4.Configuration",
"Netmask")
form.get('ipv4gateway').value = get_dict_value(properties,
"IPv4.Configuration",
"Gateway")
form.get('ipv6method').value = get_dict_value(properties,
"IPv6.Configuration",
"Method")
form.get('ipv6address').value = get_dict_value(properties,
"IPv6.Configuration",
"Address")
value = get_dict_value(properties, "IPv6.Configuration", "PrefixLength")
if value != "":
value = "%d" % value
form.get('ipv6prefixlen').value = value
form.get('ipv6gateway').value = get_dict_value(properties,
"IPv6.Configuration",
"Gateway")
form.get('ipv6privacy').value = get_dict_value(properties,
"IPv6.Configuration",
"Privacy")
def changed_dict_byte(properties, block, key, input):
value = "%d" % get_dict_value(properties, block, key)
if value != input:
return True
else:
return False
