def generate_sesskey(user_id, node_id=None):
if node_id:
node_id_part = '%08x' % node_id
else:
node_id_part = 'XXXXXXXX'
return '%08x%s%s' % (user_id, random_string(8), node_id_part)
def generate_report_2( batchcode, querytext, marker_ids,
allele_absolute_threshold, allele_relative_threshold,
sample_quality_threshold, marker_quality_threshold,
location_level, spatial_differentiation, temporal_differentiation,
template_file, fmt = 'PDF' ):
report = ReportResult()
# prepare the query
fullquery = ''
if batchcode:
fullquery += ' | '.join( '%s[batch]' % batch for batch in batchcode.split() )
if querytext:
fullquery += querytext
if not fullquery:
raise RuntimeError( 'Please provide batch code or querytext' )
sample_ids = parse_querycmd( fullquery )
tmp_dir = get_temp_path( random_string(8) )
os.mkdir( tmp_dir )
tmp_dir += '/'
result = generate_comprehensive_summary_2(
sample_ids = sample_ids, querysets = None, marker_ids = marker_ids,
location_level = location_level,
spatial_differentiation = spatial_differentiation,
temporal_differentiation = temporal_differentiation,
allele_absolute_threshold = allele_absolute_threshold,
allele_relative_threshold = allele_relative_threshold,
sample_quality_threshold = sample_quality_threshold,
marker_quality_threshold = marker_quality_threshold,
tmp_dir = tmp_dir )
# format the TeX file
tex_filename = tmp_dir + 'report.tex'
#template = open( template_file ).read()
template = format_tex_report( template_file, result, tmp_dir )
with open(tex_filename, 'w') as f:
f.write( template )
if fmt == 'PDF':
# run pdflatex here
ok = call([ 'pdflatex',
'-halt-on-error',
'-output-directory', tmp_dir,
tex_filename
])
if ok != 0:
raise SysError('pdflatex returned error code: %d, produced with TeX source at %s'
% (ok, tex_filename))
report.fullpath = tmp_dir + 'report.pdf'
return report
raise SysError('should not be here!')
def nj_distance( names, m, tempdir ):
""" run R's ape to create NJ and display tree """
# prepare a file for the matrix
file_id = random_string(3)
matrix_file = tempdir + 'matrix-distance-%s.txt' % file_id
script_file = tempdir + 'njtree-%s.r' % file_id
njtree_file = tempdir + 'njtree-%s.pdf' % file_id
with open(matrix_file, 'w') as out:
out.write( '\t'.join( names ) )
out.write( '\n')
for name, vals in zip( names, m ):
out.write( '%s\t%s\n' % (name, '\t'.join( ['%2.3f' % x for x in vals] ) ))
with open(script_file, 'w') as scriptout:
scriptout.write("""
library(ape)
M <- as.matrix( read.table("%s", sep='\t', header=T) )
tree <- nj( M )
pdf("%s")
plot(tree, "fan", font=1, cex=0.5)
""" % (matrix_file, njtree_file) )
ok = call( ['Rscript', script_file] )
if ok != 0:
raise RuntimeError("Rscript run unsucessfully")
return njtree_file
def submit(self, uid, wd, func, *args, **kwargs):
# submit a process
# func at least has ns and cerr, ie: f(ns=None, cerr=None)
with self.lock: # can use with self.lock.acquire() ???
if self.queue >= self.max_queue:
raise MaxQueueError(
"Queue reached maximum number. Please try again in few moments"
)
while True:
procid = random_string(16)
if procid not in self.procs:
break
procunit = ProcUnit(taskid=procid, uid=uid, wd=wd)
procunit.ns = self.prepare_namespace()
procunit.cerr = self.manager().list()
kwargs['ns'] = procunit.ns
kwargs['cerr'] = procunit.cerr
proc = self.pool.submit(func, *args, **kwargs)
procunit.proc = proc
self.procs[procid] = procunit
self.queue += 1
proc.add_done_callback(lambda x: self.callback(x, procid))
return procunit
def run_arlequin( genotype_sets, recode = False, tmp_dir = None ):
""" genotype_sets: ( AnalyticalSet, ...) """
# write to a file, similar to how Lian works
if len(genotype_sets) == 0:
return None
buf, used_sets = export_arlequin( genotype_sets, recode = recode )
tmp_dir = tmp_dir or get_temp_path( random_string(8) + '-arlequin')
os.mkdir( tmp_dir )
tmp_dir += '/'
arp_file = tmp_dir + 'msaf.arp'
ars_file = tmp_dir + 'msaf.ars'
with open(arp_file, 'wt') as f:
f.write( buf.read() )
with open(ars_file, 'wt') as f:
f.write( ars )
print('Running arlequin in: %s' % tmp_dir)
call([get_libexec_path('arlecore'), arp_file, ars_file],
cwd = tmp_dir)
result = ArlequinResult( output = open( tmp_dir + 'msaf.res/msaf.htm' ).read(),
sets = used_sets )
return result
def verify(request, dictfmt = msaf_dictfmt):
# perform consistency check, sanity check
# dictfmt -> module for dictionary format functions
if not request.POST:
return Response("ERROR: not a valid operation [upload:100]")
input_file = request.POST.get('input_file', None)
#opt_replace_existing = request.POST.get('replace_existing', None)
name, ext = os.path.splitext( input_file.filename )
if ext in [ '.csv', '.tab', '.tsv' ]:
# convert to JSON first
# consistency checks
if ext == '.csv':
delim = ','
else:
delim = '\t'
try:
dict_samples, report_log = dictfmt.csv2dict(
StringIO(input_file.file.read().decode('UTF-8')),
with_report=True,
delimiter = delim )
except ValueError as err:
return error_page( 'ValueError: {0}'.format(err) )
if dict_samples is None:
return render_to_response( "msaf:templates/upload/error.mako",
{ 'report_log': report_log.getvalue() }, request = request )
dict_text = yaml.dump( dict_samples )
ext = '.yaml'
elif ext in ['.json', '.yaml']:
dict_text = input_file.file.read().decode('UTF-8')
report_log = StringIO()
else:
return error_page('Unrecognized format')
# sanity check
# data integrity check (flush to database, but no commit yet)
temppath = get_temp_path( random_string(8) + ext )
with open(temppath, 'w') as f:
f.write( dict_text )
ticket = request.get_ticket( { 'dict.path': temppath } )
# 'replace_allele': opt_replace_existing } )
return render_to_response( "msaf:templates/upload/verify.mako",
{ 'report_log': report_log.getvalue(), 'ticket': ticket,
'filename': input_file.filename },
request = request )
def add_assay_info(assayinfo_file, assaydata_file):
assayinfo_name, assayinfo_ext = os.path.splitext( ifile.filename )
assaydata_name, assaydata_ext = os.path.splitext( assaydata_file.filename )
pathname = random_string(16) + '.yaml'
temppath = get_temp_path( pathname )
os.mkdir( temppath )
with open( '%s/%s.%s' % (temppath, assaydata_name, assaydata_ext)) as f:
f.write()
def add_assay_info(assayinfo_file, assaydata_file):
assayinfo_name, assayinfo_ext = os.path.splitext( ifile.filename )
assaydata_name, assaydata_ext = os.path.splitext( assaydata_file.filename )
pathname = random_string(16) + '.yaml'
temppath = get_temp_path( pathname )
os.mkdir( temppath )
with open( '%s/%s.%s' % (temppath, assaydata_name, assaydata_ext)) as f:
f.write()
def generate_report_3( baseparams, location_level,
spatial_differentiation, temporal_differentiation,
detection_differentiation,
template_file, fmt = 'PDF' ):
report = ReportResult()
# prepare the query
sample_sets = parse_advquerycmd( baseparams.queryset )
tmp_dir = get_temp_path( random_string(8) )
os.mkdir( tmp_dir )
tmp_dir += '/'
result = generate_comprehensive_summary_3(
sample_sets = sample_sets,
baseparams = baseparams,
location_level = location_level,
spatial_differentiation = spatial_differentiation,
temporal_differentiation = temporal_differentiation,
detection_differentiation = detection_differentiation,
tmp_dir = tmp_dir )
# format the TeX file
tex_filename = tmp_dir + 'report.tex'
#template = open( template_file ).read()
template = format_tex_report( template_file, result, tmp_dir )
with open(tex_filename, 'w') as f:
f.write( template )
if fmt == 'PDF':
# run pdflatex here
ok = call([ 'pdflatex',
'-halt-on-error',
'-output-directory', tmp_dir,
tex_filename
])
if ok != 0:
raise SysError('pdflatex returned error code: %d, produced with TeX source at %s'
% (ok, tex_filename))
report.fullpath = tmp_dir + 'report.pdf'
return report
raise SysError('should not be here!')
def xxx_generate_report(batchcode, querytext, location_level = 4,
threshold = '100 | 33%', fmt='PDF',
template_file = None):
report = ReportResult()
# prepare the query
fullquery = ''
if batchcode:
fullquery += '%s[batch]' % batchcode
if querytext:
fullquery += querytext
if not fullquery:
raise RuntimeError( 'Please provide batch code or querytext' )
sample_ids = parse_querycmd( fullquery )
result = generate_comprehensive_summary( sample_ids, location_level )
# format the TeX file
tmp_dir = get_temp_path( random_string(8) )
os.mkdir( tmp_dir )
tmp_dir += '/'
tex_filename = tmp_dir + 'report.tex'
template = open( template_file ).read()
template = format_report( template, result, tmp_dir )
with open(tex_filename, 'w') as f:
f.write( template )
if fmt == 'PDF':
# run pdflatex here
ok = call([ 'pdflatex',
'-halt-on-error',
'-output-directory', tmp_dir,
tex_filename
])
if ok != 0:
raise SysError('pdflatex returned error code: %d' % ok)
report.fullpath = tmp_dir + 'report.pdf'
return report
raise SysError('should not be here!')
def add_sample_info( batch, ifile, request):
""" parse file and save it to temporary file as YAML file """
name, ext = os.path.splitext( ifile.filename )
if ext in [ '.csv', '.tab', '.tsv', '.txt' ]:
# convert to JSON first
# consistency checks
#if ext == '.csv':
# delim = ','
#else:
# delim = '\t'
buf, delim = detect_buffer( ifile.file.read().decode('UTF-8') )
try:
## the csv2dict function has to be sample-specific method
## use batch.Sample.csv2dict() ??
dict_samples, errlog, sample_codes = batch.get_sample_class().csv2dict(
StringIO(buf),
with_report=True,
delimiter = delim )
except ValueError as err:
return error_page(request, 'ValueError: {0}'.format(err) )
if dict_samples is None:
return render_to_response( "msaf:templates/upload/error.mako",
{ 'report_log': '<br/>'.join(errlog) }, request = request )
dict_text = yaml.dump( dict(codes = sample_codes, samples = dict_samples) )
elif ext in ['.json', '.yaml']:
dict_text = yaml.dump( yaml.load( ifile.file.read().decode('UTF-8') ) )
errlog = ''
else:
return error_page(request, 'Unrecognized format')
pathname = random_string(16) + '.yaml'
temppath = get_temp_path( pathname )
with open(temppath, 'w') as f:
f.write( dict_text )
return (pathname, errlog)
def add_sample_info( batch, ifile, request):
""" parse file and save it to temporary file as YAML file """
name, ext = os.path.splitext( ifile.filename )
if ext in [ '.csv', '.tab', '.tsv', '.txt' ]:
# convert to JSON first
# consistency checks
if ext == '.csv':
delim = ','
else:
delim = '\t'
try:
## the csv2dict function has to be sample-specific method
## use batch.Sample.csv2dict() ??
dict_samples, errlog, sample_codes = batch.get_sample_class().csv2dict(
StringIO(ifile.file.read().decode('UTF-8')),
with_report=True,
delimiter = delim )
except ValueError as err:
return error_page(request, 'ValueError: {0}'.format(err) )
if dict_samples is None:
return render_to_response( "msaf:templates/upload/error.mako",
{ 'report_log': '<br/>'.join(errlog) }, request = request )
dict_text = yaml.dump( dict(codes = sample_codes, samples = dict_samples) )
elif ext in ['.json', '.yaml']:
dict_text = yaml.dump( yaml.load( ifile.file.read().decode('UTF-8') ) )
errlog = ''
else:
return error_page(request, 'Unrecognized format')
pathname = random_string(16) + '.yaml'
temppath = get_temp_path( pathname )
with open(temppath, 'w') as f:
f.write( dict_text )
return (pathname, errlog)
def nj_tree( aDistanceMatrix, tempdir = None, fmt='pdf' ):
file_id = random_string(3)
matrix_file = tempdir + 'matrix-distance-%s.txt' % file_id
colors_file = tempdir + 'colors-distance-%s.txt' % file_id
script_file = tempdir + 'njtree-%s.r' % file_id
njtree_file = tempdir + 'njtree-%s.%s' % (file_id, fmt)
with open(matrix_file, 'w') as out_m, open(colors_file, 'w') as out_c:
out_m.write( '\t'.join( aDistanceMatrix.L ) )
out_m.write( '\n')
for name, vals in zip( aDistanceMatrix.L, aDistanceMatrix.M ):
out_m.write( '%s\t%s\n' % (name, '\t'.join( ['%2.3f' % x for x in vals] ) ))
out_c.write('\n'.join( aDistanceMatrix.C ) )
with open(script_file, 'w') as scriptout:
if fmt == 'pdf':
cmd = 'pdf("%s", width = 11.2, height=7)' % njtree_file
elif fmt == 'png':
cmd = 'png("%s", width = 1024, height = 640)' % njtree_file
scriptout.write("""
library(ape)
M <- as.matrix( read.table("%s", sep='\t', header=T) )
C <- as.vector( read.table("%s", sep='\t', header=F)[,1] )
tree <- nj( M )
%s
plot(tree, "fan", tip.color = C, font=1, cex=0.7, label.offset = 0.009)
legend('topright', inset=c(0,0), c(%s), col = c(%s), lty=1, cex=0.85, xpd=T)
""" % (matrix_file, colors_file, cmd,
",".join( '"%s"' % e.get_label() for e in aDistanceMatrix.genotype_sets),
",".join( '"%s"' % e.get_colour() for e in aDistanceMatrix.genotype_sets) )
)
ok = call( ['Rscript', script_file] )
if ok != 0:
raise RuntimeError("Rscript run unsucessfully")
return njtree_file
def submit(self, uid, wd, func, *args, **kwargs):
with self.lock: # can use with self.lock.acquire() ???
if self.queue >= self.max_queue:
raise MaxQueueError(
"Queue reached maximum number. Please try again in few moments")
while True:
procid = random_string(16)
if procid not in self.procs:
break
procunit = ProcUnit(None, uid, wd)
procunit.ns = self.prepare_namespace()
kwargs['ns'] = procunit.ns
proc = self.pool.submit( func, *args, **kwargs )
procunit.proc = proc
self.procs[procid] = procunit
self.queue += 1
proc.add_done_callback( lambda x: self.callback(x, procid) )
return (procid, "Task submitted to queue")
def get_proc_path( path = None ):
if path is None:
path = random_string(8)
return "%s/%s" % (PROC_DIR, path)
def update(self, obj):
if type(obj) == dict:
if obj.get('storage') is not None:
self.storage = obj.get('storage') # blood storage
if obj.get('method') is not None:
self.method = obj.get('method') # blood withdrawal method
if obj.get('pcr_method') is not None:
self.pcr_method_id = EK._id(obj.get('pcr_method'), grp = '@PCR_METHOD',
dbsession = object_session(self), auto=True)
if obj.get('pcr') is not None:
self.pcr = obj.get('pcr')
if obj.get('microscopy') is not None:
self.microscopy = obj.get('microscopy')
if obj.get('type') is not None:
self.type = obj.get('type')
if obj.get('day') is not None:
self.day = int(obj.get('day'))
if self.day == 0 and self.type == '':
self.type = 'P'
if obj.get('case_detection') is not None:
self.passive_case_detection = obj.get('case_detection').lower().startswith('y')
if obj.get('symptomatic_status') is not None:
self.symptomatic_status = obj.get('symptomatic_status').lower().startswith('y')
if obj.get('nationality_status') is not None:
self.nationality_status = obj.get('nationality_status').lower().startswith('y')
if obj.get('imported_case') is not None:
self.imported_case = obj.get('imported_case').lower().startswith('y')
if obj.get('parasite_density') is not None:
self.parasitemia = int(obj.get('parasite_density', -1))
# deals with subject
subject_code = obj.get('subject_code', None)
related_sample = obj.get('related_sample', None)
session = object_session(self)
if subject_code:
# find subject
subject = Subject.search(code=subject_code, dbsession = session)
self.subject_id = subject.id
elif related_sample:
# find sample
sample = self.batch.search_sample(related_sample)
if sample is None:
raise RuntimeError('ERROR for sample code %s: related sample code %s does not exist' % (self.code, related_sample))
self.subject_id = sample.subject_id
elif not self.subject:
# create new subject
while True:
code = '#' + random_string(8)
# we need to check subject code first !
if Subject.query(session).filter(Subject.code == code).count() == 0:
break
subject = Subject( code = code, gender = obj['gender'],
yearofbirth = obj['yearofbirth'] )
session.add(subject)
subject.update( obj )
session.flush([subject])
cerr('subject %s nationality_id %d' % (subject.code, subject.nationality_id))
self.subject = subject
#self.subject_id = 0
# now if subject_code &
else:
raise NotImplementedError('PROG/ERR - not implemented yet')
super().update( obj )
return
if obj.passive_case_detection is not None:
self.passive_case_detection = obj.passive_case_detection
if obj.storage_id is not None:
self.storage_id = 0 #obj.storage_id
if obj.method_id is not None:
self.method_id = 0 #obj.method_id
if obj.pcr_method_id is not None:
self.pcr_method_id = 0 #obj.pcr_method_id
if obj.pcr_id is not None:
self.pcr_id = 0 #obj.pcr_id
if obj.microscopy_id is not None:
self.microscopy_id = 0 #obj.microscopy_id
if obj.recurent is not None:
self.recurrent = obj.recurrent
if obj.subject_is:
self.subject_id = obj.subject_id
def mkranddir(parent_directory, userid):
d = FileOverlay(abspath="%s/%s/" % (parent_directory, random_string(12)), type="dir")
d.add_permission(userid)
d.create_directory()
return d
dbsession = dbh.session()
EK.bulk_update( ek_initlist, dbsession=dbsession )
Group.bulk_insert( essential_groups, dbsession=dbsession )
UserClass.bulk_insert( system_userclass, dbsession=dbsession )
group_id = Group._id('_SysAdm_', dbsession)
file = File( path='/', group_id = group_id, permanent = True )
dbsession.add( file )
file.type = 'file/folder'
file.mimetype = 'application/x-directory'
cerr('INFO: root password is %s\n' % root_password)
root_password = random_string(16)
system_userclass = ( '_SYSTEM_', 'Rhombus System', None, {},
[ ('system', '', '', 'root@localhost', '_SysAdm_', root_password, [] ),
('dataadm', '', '', 'root@localhost', '_DataAdm_', '{X}', [] ),
] )
essential_groups = [
( '__default__', [ PUBLIC ] ),
( '_SysAdm_', [SYSADM, SYSVIEW] ),
( '_EKMgr_', [ EK_CREATE, EK_MODIFY, EK_DELETE ] ),
( '_UserClassMgr_', [ USERCLASS_CREATE, USERCLASS_MODIFY, USERCLASS_DELETE ] ),
( '_UserMgr_', [ USER_CREATE, USER_MODIFY, USER_DELETE ] ),
( '_GroupMgr_', [ GROUP_CREATE, GROUP_MODIFY, GROUP_DELETE,
GROUP_ADDUSER, GROUP_DELUSER ] ),
( '_DataAdm_', [DATAADM, DATAVIEW] ),
def update(self, obj):
if type(obj) == dict:
if obj.get('storage') is not None:
self.storage = obj.get('storage') # blood storage
if obj.get('method') is not None:
self.method = obj.get('method') # blood withdrawal method
if obj.get('pcr_method') is not None:
self.pcr_method_id = EK._id(obj.get('pcr_method'), grp = '@PCR_METHOD',
dbsession = object_session(self), auto=True)
if obj.get('pcr') is not None:
self.pcr = obj.get('pcr')
if obj.get('microscopy') is not None:
self.microscopy = obj.get('microscopy')
if obj.get('type') is not None:
self.type = obj.get('type')
if obj.get('day') is not None:
self.day = int(obj.get('day'))
if self.day == 0 and self.type == '':
self.type = 'P'
if obj.get('case_detection') is not None:
self.passive_case_detection = obj.get('case_detection').lower().startswith('y')
if obj.get('symptomatic_status') is not None:
self.symptomatic_status = obj.get('symptomatic_status').lower().startswith('y')
if obj.get('nationality_status') is not None:
self.nationality_status = obj.get('nationality_status').lower().startswith('y')
if obj.get('imported_case') is not None:
self.imported_case = obj.get('imported_case').lower().startswith('y')
if obj.get('parasite_density') is not None:
self.parasitemia = int(obj.get('parasite_density', -1))
# deals with subject
subject_code = obj.get('subject_code', None)
related_sample = obj.get('related_sample', None)
session = object_session(self)
if subject_code:
# find subject
subject = Subject.search(code=subject_code, dbsession = session)
self.subject_id = subject.id
elif related_sample:
# find sample
sample = self.batch.search_sample(related_sample)
if sample is None:
raise RuntimeError('ERROR for sample code %s: related sample code %s does not exist' % (self.code, related_sample))
self.subject_id = sample.subject_id
elif not self.subject:
# create new subject
while True:
code = '#' + random_string(8)
# we need to check subject code first !
if Subject.query(session).filter(Subject.code == code).count() == 0:
break
subject = Subject( code = code, gender = obj['gender'],
yearofbirth = obj['yearofbirth'] )
session.add(subject)
subject.update( obj )
session.flush([subject])
cerr('subject %s nationality_id %d' % (subject.code, subject.nationality_id))
self.subject = subject
#self.subject_id = 0
# now if subject_code &
else:
raise NotImplementedError('PROG/ERR - not implemented yet')
super().update( obj )
return
def get_proc_path( path = None ):
if path is None:
path = random_string(8)
return "%s/%s" % (PROC_DIR, path)
