def do_listenumkey(args, dbh, settings):
cerr('List enumkey')
ekeys = dbh.list_ekeys(group = args.ekeygroup)
for ek in ekeys:
cout('%s' % ek.key)
def setup(session):
""" populate database with essential GenAF data, mostly taken from FATools constants """
EK.bulk_insert(ek_initlist, dbsession=session)
# get default group, which is system
adm_group = Group.search("_DataAdm_", dbsession=session)
# create undefined marker
marker = Marker(code="undefined", species="X")
cerr("INFO - marker 'undefined' created.")
session.add(marker)
# create ladder marker
marker = Marker(code="ladder", species="X")
cerr("INFO - marker 'ladder' created.")
session.add(marker)
# create combined marker
marker = Marker(code="combined", species="X")
cerr("INFO - marker 'combined' created.")
session.add(marker)
# create default panel
panel = Panel(code="undefined", group_id=adm_group.id)
cerr("INFO - panel 'undefined' created.")
session.add(panel)
# create default batch
batch = Batch(code="default", group_id=adm_group.id, assay_provider_id=adm_group.id, species="X")
cerr("INFO - default batch created.")
session.add(batch)
def main(args):
cerr('genaf analyze main()')
settings = setup_settings( args )
do_analyze( args, settings )
def initdb(self, create_table=True, init_data=True, rootpasswd=None):
""" initialize database """
super().initdb(create_table, init_data, rootpasswd)
if init_data:
from .setup import setup
setup(self)
cerr('[genaf-base] Database has been initialized')
def main(global_config, **settings):
cerr('CMSFix main() is running...')
config = init_app(global_config, settings, prefix='/mgr')
config.include(includeme)
set_workflow(GroupwareWorkflow())
return config.make_wsgi_app()
def includeme( config ):
cerr('rhombus configuration with prefix: %s' % config.route_prefix)
config.include('pyramid_mako')
config.add_static_view(name='rhombus_static', path="rhombus:static/")
session_factory = SignedCookieSessionFactory('Rh0S35s1On')
config.set_session_factory(session_factory)
# configure RbRequest
# configure exception views if debugtoolbar is not enabled
settings = config.get_settings()
if 'debugtoolbar.includes' not in settings:
config.add_view('rhombus.views.generics.usererror_page', context=RuntimeError)
# configure routes & views
if config.route_prefix:
config.add_route('rhombus.dashboard', '/')
config.add_view('rhombus.views.dashboard.index', route_name = 'rhombus.dashboard')
add_route_view( config, 'rhombus.views.group', 'rhombus.group',
'/group',
'/group/@@action',
'/group/@@user_action',
'/group/@@role_action',
'/group/@@lookup',
'/group/{id}@@edit',
'/group/{id}@@save',
('/group/{id}', 'view'),
)
add_route_view( config, 'rhombus.views.ek', 'rhombus.ek',
'/ek',
'/ek/@@action',
('/ek/@@lookup', 'lookup', 'json'),
'/ek/{id}@@edit',
'/ek/{id}@@save',
('/ek/{id}', 'view'),
)
add_route_view( config, 'rhombus.views.userclass', 'rhombus.userclass',
'/userclass',
'/userclass/@@action',
'/userclass/{id}@@edit',
('/userclass/{id}', 'view'),
)
add_route_view( config, 'rhombus.views.user', 'rhombus.user',
'/user',
'/user/@@action',
'/user/@@passwd',
('/user/@@lookup', 'lookup', 'json'),
'/user/{id}@@edit',
#'/user/{id}@@passwd',
('/user/{id}', 'view'),
)
def do_importgroup(args, dbh, settings):
cerr('Importing group')
groups = yaml.load(open(args.importgroup))
from rhombus.models.user import UserClass, Group
Group.bulk_insert(groups, dbsession=dbh.session())
def validate_by_LDAP(username, passwd, scheme):
import ldap3
s = ldap3.Server(host = scheme['host'], get_info=ldap3.ALL)
c = ldap3.Connection(s, user=scheme['DN'] % username, password = passwd)
if not c.bind():
cerr('Failed LDAP authentication for user: %s' % username)
return False
return True
def do_dbmgr(args, settings, dbh=None):
if dbh is None:
dbh = get_dbhandler(settings, initial = args.initdb)
print(dbh)
if not rhombus_do_dbmgr(args, settings, dbh):
cerr('ERR - command unrecognized!')
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
cerr('genaf-base main() is running...')
# attach rhombus to /mgr url, include custom configuration
config = init_app(global_config, settings, prefix='/mgr')
return config.make_wsgi_app()
def update(self, d):
if type(d) == dict:
if not self.nationality_id:
self.nationality = ''
if d.get('nationality', None) is not None:
self.nationality = d['nationality']
cerr('set nationality to %s' % d['nationality'])
else:
raise NotImplementedError('can only update from dict')
def index_all():
dbh = get_dbhandler()
index_service = get_index_service()
with index_service.ix.writer() as writer:
for n in dbh.get_nodes():
writer.delete_by_term('nodeid', n.id)
writer.add_document(nodeid=n.id, mtime=n.stamp, text=n.search_text(), keywords=n.search_keywords())
cerr('indexing node: %d' % n.id)
def update(self, d):
if type(d) == dict:
if not self.nationality_id:
self.nationality = ''
if d.get('nationality', None) is not None:
self.nationality = d['nationality']
cerr('set nationality to %s' % d['nationality'])
else:
raise NotImplementedError('can only update from dict')
def initdb(self, create_table=True, init_data=True):
""" prepare the database for the first time by initializing it with
necessary, basic, default data set """
# WARN! if possible, use alembic to create tables
if create_table:
core.Base.metadata.create_all(self.engine)
cerr('[rhombus] Database tables created.')
if init_data:
from rhombus.models.setup import setup
setup( self )
cerr('[rhombus] Database has been initialized.')
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
cerr('PlasmoGEN main() in running...')
config = init_app(global_config, settings, prefix='/mgr')
config.include(includeme)
cover_template = settings.get('override.cover', None)
if cover_template:
config.override_asset( 'plasmogen:templates/cover.mako', cover_template)
set_form_factory(plasmogen_form_factory)
return config.make_wsgi_app()
def do_addenumkey(args, dbh, settings):
cerr('Add enumkey')
key = args.addenumkey
key = key.upper() if key.startswith('@') else key.lower()
key_group = args.ekeygroup
if key_group is None and not key.startswith('@'):
cexit('ekey must under a group, or a group key which starts with @')
ekey = dbh.add_ekey(key, key_group)
cerr('I - enumkey: %s / %s has been added.' %
(ekey.key, ekey.group.key if ekey.group else '*'))
def setup(dbh):
dbh.EK.bulk_update(ek_initlist, dbsession=dbh.session())
adm_group = dbh.Group.search('_DataAdm_', dbsession=dbh.session())
# create default batch
batch = dbh.Batch(code='default',
group_id=adm_group.id,
assay_provider_id=adm_group.id,
species='X',
lastuser_id=1,
public=0)
cerr("INFO - default batch created.")
dbh.session().add(batch)
def get_userobj(request):
user_id = request.unauthenticated_userid
db_session = get_dbhandler().session()
if user_id is None:
db_session.user = None
return None
user_id = user_id.encode('ASCII')
userinstance = auth_cache.get(user_id, None)
if db_session.user and userinstance and db_session.user.id != userinstance.id:
cerr('WARNING PROGRAMMING ERROR: reuse of db_session.user -> %d >> %d'
% (db_session.user.id, userinstance.id))
db_session.user = userinstance
return userinstance
def main(global_config, **settings):
""" This function returns a Pyramid WSGI application.
"""
cerr('plasmogen_snp main() in running...')
config = init_app(global_config, settings, prefix='/mgr')
config.include(includeme)
cover_template = settings.get('override.cover', None)
if cover_template:
config.override_asset('plasmogen_snp:templates/cover.mako',
cover_template)
set_form_factory(plasmogen_snp_form_factory)
return config.make_wsgi_app()
def main(args):
settings = setup_settings(args)
if not args.test and args.commit:
with transaction.manager:
do_dbmgr(args, settings)
cerr('** COMMIT database **')
else:
cerr('** WARNING -- running without database COMMIT **')
if not args.test:
keys = input('Do you want to continue [y/n]: ')
if keys.lower()[0] != 'y':
sys.exit(1)
do_dbmgr(args, settings)
def do_setcred(args, dbh, settings):
if not args.userclass:
cexit('ERR - please provide userclass')
if not args.login:
cexit('ERR - please provide login name')
domain = dbh.get_userclass(args.userclass)
user = domain.search_user(args.login)
if not user:
cexit('WARN - user does not exist')
if args.credential:
user.set_credential(args.credential)
cerr('Credential for user %s has been modified sucessfully' % user.login)
def update(self, obj):
self._update(obj)
if type(obj) == dict:
if 'type' in obj:
self.type = obj['type']
if 'category' in obj:
self.category = obj['category']
if 'collection_date' in obj:
collection_date = obj['collection_date']
if type(collection_date) is str:
collection_date = date(*[ int(x) for x in collection_date.split('/') ])
self.collection_date = collection_date
if 'location' in obj:
location_code = obj['location']
location = Location.search(
location_code[0], location_code[1],
location_code[2], location_code[3],
location_code[4], auto=True, dbsession = object_session(self) )
cerr('LOCATION: %s' % location)
self.location = location
if 'latitude' in obj:
self.latitute = obj['latitude']
if 'longitude' in obj:
self.longitude = obj['longitude']
if 'altitute' in obj:
self.altitute = obj['altitude']
if 'remark' in obj:
self.remark = obj['remark']
if 'int1' in obj:
self.int1 = obj['int1']
if 'int2' in obj:
self.int2 = obj['int2']
if 'int3' in obj:
self.int3 = obj['int3']
if 'string1' in obj:
self.string1 = obj['string1']
if 'string2' in obj:
self.string2 = obj['string2']
if 'string3' in obj:
self.string3 = obj['string3']
else:
raise NotImplementedError('PROG/ERR - not implemented yet')
def main(args):
settings = setup_settings( args )
if any( (args.exportuserclass, args.exportgroup, args.exportenumkey) ):
do_rbmgr( args, settings )
elif not args.rollback and (args.commit or args.initdb):
with transaction.manager:
do_rbmgr( args, settings )
cerr('** COMMIT database **')
else:
cerr('** WARNING -- running without database COMMIT **')
if not args.rollback:
keys = input('Do you want to continue? ')
if keys.lower()[0] != 'y':
sys.exit(1)
do_rbmgr( args, settings )
def estimate_time(start_time, current_time, processed, unprocessed):
""" return string of '2d 3h 23m' """
used_time = current_time - start_time # in seconds
if processed == 0:
return 'undetermined'
average_time = used_time / processed
estimated = average_time * unprocessed
cerr('Average time: %3.6f sec' % average_time)
cerr('Estimated remaining time: %3.6f sec' % estimated)
text = ''
if estimated > 86400:
text += '%dd ' % int(estimated / 86400)
estimated = estimated % 86400
if estimated > 3600:
text += '%dh ' % int(estimated / 3600)
estimated = estimated % 3600
text += '%dm' % (int(estimated / 60) + 1)
return text
def estimate_time(start_time, current_time, processed, unprocessed):
""" return string of '2d 3h 23m' """
used_time = current_time - start_time # in seconds
if processed == 0:
return 'undetermined'
average_time = used_time / processed
estimated = average_time * unprocessed
cerr('Average time: %3.6f sec' % average_time)
cerr('Estimated remaining time: %3.6f sec' % estimated)
text = ''
if estimated > 86400:
text += '%dd ' % int( estimated / 86400 )
estimated = estimated % 86400
if estimated > 3600:
text += '%dh ' % int( estimated / 3600 )
estimated = estimated % 3600
text += '%dm' % (int( estimated / 60) + 1)
return text
def setup( dbh ):
""" populate the database with basic, essential data """
if get_datalogger():
get_clsreg().sync()
session.commit()
dbsession = dbh.session()
EK.bulk_update( ek_initlist, dbsession=dbsession )
Group.bulk_insert( essential_groups, dbsession=dbsession )
UserClass.bulk_insert( system_userclass, dbsession=dbsession )
group_id = Group._id('_SysAdm_', dbsession)
file = File( path='/', group_id = group_id, permanent = True )
dbsession.add( file )
file.type = 'file/folder'
file.mimetype = 'application/x-directory'
cerr('INFO: root password is %s\n' % root_password)
def execute(command, run_args):
print('paths:', PATHS)
M = None
for module_path in PATHS:
try:
print(module_path)
M = importlib.import_module(module_path + command)
break
except ImportError as exc:
pass
if M is None:
cexit('Cannot locate script name: %s' % command)
print(M)
if hasattr(M, 'init_argparser'):
parser = M.init_argparser()
assert parser, "FATAL ERROR - init_argparser() does not return an instance"
else:
print('Use default arg parser!')
parser = arg_parser()
args = parser.parse_args(run_args)
#configfile = args.config or os.environ.get('RHOMBUS_CONFIG')
#cinfo('Using config file: %s' % configfile)
#if not configfile:
# print('need -c or --config option, or set RHOMBUS_CONFIG environment')
# sys.exit(1)
#print('Connecting to database...')
#setup_logging( configfile )
#settings = get_appsettings( configfile )
#init_db( settings )
cerr('Running module: %s' % command)
M.main( args )
def main(args):
cerr('genaf facmd main()')
settings = setup_settings( args )
if args.commit:
with transaction.manager:
do_facmd( args, settings )
cerr('** COMMIT database **')
else:
cerr('** WARNING -- running without database COMMIT **')
if not args.test:
keys = input('Do you want to continue? ')
if keys.lower()[0] != 'y':
sys.exit(1)
do_facmd( args, settings )
def main(args):
# initialize database
cerr('initialize ')
settings = setup_settings(args)
get_dbhandler(settings)
# init taskqueue here
cerr('initialize taskcache')
init_taskcache(settings)
# running working
with Connection():
cerr('running worker')
w = Worker(settings['genaf.taskqueue.name'])
w.work()
def initdb(self, create_table=True, init_data=True):
super().initdb(create_table, init_data)
if init_data:
from plasmogen.models.setup import setup
setup( self.session() )
cerr('[plasmogen] Database has been initialized.')
def greet():
cerr('command line utility for genaf-base')
def greet():
cerr('genaf-run')
def initdb(self, create_table=True, init_data=True):
super().initdb(create_table, init_data)
if init_data:
from cmsfix.models.setup import setup
setup( self )
cerr('[cmsfix] Database has been initialized.')
def usage():
cerr('genaf_base-run - command line utility for genaf-base')
cerr('usage:')
cerr('\t%s scriptname [options]' % sys.argv[0])
sys.exit(0)
def greet():
cerr('cmsfix-shell - shell for CMSFix')
def update(self, obj):
if type(obj) == dict:
if obj.get('storage') is not None:
self.storage = obj.get('storage') # blood storage
if obj.get('method') is not None:
self.method = obj.get('method') # blood withdrawal method
if obj.get('pcr_method') is not None:
self.pcr_method_id = EK._id(obj.get('pcr_method'), grp = '@PCR_METHOD',
dbsession = object_session(self), auto=True)
if obj.get('pcr') is not None:
self.pcr = obj.get('pcr')
if obj.get('microscopy') is not None:
self.microscopy = obj.get('microscopy')
if obj.get('type') is not None:
self.type = obj.get('type')
if obj.get('day') is not None:
self.day = int(obj.get('day'))
if self.day == 0 and self.type == '':
self.type = 'P'
if obj.get('case_detection') is not None:
self.passive_case_detection = obj.get('case_detection').lower().startswith('y')
if obj.get('symptomatic_status') is not None:
self.symptomatic_status = obj.get('symptomatic_status').lower().startswith('y')
if obj.get('nationality_status') is not None:
self.nationality_status = obj.get('nationality_status').lower().startswith('y')
if obj.get('imported_case') is not None:
self.imported_case = obj.get('imported_case').lower().startswith('y')
if obj.get('parasite_density') is not None:
self.parasitemia = int(obj.get('parasite_density', -1))
# deals with subject
subject_code = obj.get('subject_code', None)
related_sample = obj.get('related_sample', None)
session = object_session(self)
if subject_code:
# find subject
subject = Subject.search(code=subject_code, dbsession = session)
self.subject_id = subject.id
elif related_sample:
# find sample
sample = self.batch.search_sample(related_sample)
if sample is None:
raise RuntimeError('ERROR for sample code %s: related sample code %s does not exist' % (self.code, related_sample))
self.subject_id = sample.subject_id
elif not self.subject:
# create new subject
while True:
code = '#' + random_string(8)
# we need to check subject code first !
if Subject.query(session).filter(Subject.code == code).count() == 0:
break
subject = Subject( code = code, gender = obj['gender'],
yearofbirth = obj['yearofbirth'] )
session.add(subject)
subject.update( obj )
session.flush([subject])
cerr('subject %s nationality_id %d' % (subject.code, subject.nationality_id))
self.subject = subject
#self.subject_id = 0
# now if subject_code &
else:
raise NotImplementedError('PROG/ERR - not implemented yet')
super().update( obj )
return
def usage():
cerr('cmsfix-shell - shell for CMSFix')
cerr('usage:')
cerr('\t%s scriptname [options]' % sys.argv[0])
sys.exit(0)
def greet():
cerr('command line utility for genaf-snp')
def greet():
cerr('cmsfix-run - command line utility for CMSFix/rhombus')
def initdb(self, create_table=True, init_data=True, rootpasswd=None):
super().initdb(create_table, init_data, rootpasswd=rootpasswd)
if init_data:
from plasmogen_snp.models.setup import setup
setup( self.session() )
cerr('[plasmogen] Database has been initialized.')
def usage():
cerr('cmsfix-run - command line utility for CMSFix/rhombus')
cerr('usage:')
cerr('\t%s scriptname [options]' % sys.argv[0])
sys.exit(0)
def greet():
cerr('genaf-base-shell - shell for genaf-base')
def usage():
cerr('genaf-run usage')
def usage():
cerr('genaf-base-shell - shell for genaf-base')
cerr('usage:')
cerr('\t%s scriptname [options]' % sys.argv[0])
sys.exit(0)
def greet():
cerr(
'plasmogen-snp-run - command line utility for plasmogen/genaf/rhombus')
def usage():
cerr(
'plasmogen-snp-run - command line utility for plasmogen/genaf/rhombus')
cerr('usage:')
cerr('\t%s scriptname [options]' % sys.argv[0])
sys.exit(0)
def cerr(self, text):
cerr(text)
def update(self, obj):
if type(obj) == dict:
if obj.get('storage') is not None:
self.storage = obj.get('storage') # blood storage
if obj.get('method') is not None:
self.method = obj.get('method') # blood withdrawal method
if obj.get('pcr_method') is not None:
self.pcr_method_id = EK._id(obj.get('pcr_method'), grp = '@PCR_METHOD',
dbsession = object_session(self), auto=True)
if obj.get('pcr') is not None:
self.pcr = obj.get('pcr')
if obj.get('microscopy') is not None:
self.microscopy = obj.get('microscopy')
if obj.get('type') is not None:
self.type = obj.get('type')
if obj.get('day') is not None:
self.day = int(obj.get('day'))
if self.day == 0 and self.type == '':
self.type = 'P'
if obj.get('case_detection') is not None:
self.passive_case_detection = obj.get('case_detection').lower().startswith('y')
if obj.get('symptomatic_status') is not None:
self.symptomatic_status = obj.get('symptomatic_status').lower().startswith('y')
if obj.get('nationality_status') is not None:
self.nationality_status = obj.get('nationality_status').lower().startswith('y')
if obj.get('imported_case') is not None:
self.imported_case = obj.get('imported_case').lower().startswith('y')
if obj.get('parasite_density') is not None:
self.parasitemia = int(obj.get('parasite_density', -1))
# deals with subject
subject_code = obj.get('subject_code', None)
related_sample = obj.get('related_sample', None)
session = object_session(self)
if subject_code:
# find subject
subject = Subject.search(code=subject_code, dbsession = session)
self.subject_id = subject.id
elif related_sample:
# find sample
sample = self.batch.search_sample(related_sample)
if sample is None:
raise RuntimeError('ERROR for sample code %s: related sample code %s does not exist' % (self.code, related_sample))
self.subject_id = sample.subject_id
elif not self.subject:
# create new subject
while True:
code = '#' + random_string(8)
# we need to check subject code first !
if Subject.query(session).filter(Subject.code == code).count() == 0:
break
subject = Subject( code = code, gender = obj['gender'],
yearofbirth = obj['yearofbirth'] )
session.add(subject)
subject.update( obj )
session.flush([subject])
cerr('subject %s nationality_id %d' % (subject.code, subject.nationality_id))
self.subject = subject
#self.subject_id = 0
# now if subject_code &
else:
raise NotImplementedError('PROG/ERR - not implemented yet')
super().update( obj )
return
if obj.passive_case_detection is not None:
self.passive_case_detection = obj.passive_case_detection
if obj.storage_id is not None:
self.storage_id = 0 #obj.storage_id
if obj.method_id is not None:
self.method_id = 0 #obj.method_id
if obj.pcr_method_id is not None:
self.pcr_method_id = 0 #obj.pcr_method_id
if obj.pcr_id is not None:
self.pcr_id = 0 #obj.pcr_id
if obj.microscopy_id is not None:
self.microscopy_id = 0 #obj.microscopy_id
if obj.recurent is not None:
self.recurrent = obj.recurrent
if obj.subject_is:
self.subject_id = obj.subject_id
def get_locus_by_pos(self, ref, pos):
""" return locus """
cerr('[%s %d]' % (ref, pos))
return self.Locus.query(self.session()).filter(self.Locus.refseq == ref, self.Locus.position == pos).one()