def _build_study_info(user, search_type, study_proc=None, proc_samples=None): """Builds list of dicts for studies table, with all HTML formatted Parameters ---------- user : User object logged in user search_type : choice, ['user', 'public'] what kind of search to perform study_proc : dict of lists, optional Dictionary keyed on study_id that lists all processed data associated with that study. Required if proc_samples given. proc_samples : dict of lists, optional Dictionary keyed on proc_data_id that lists all samples associated with that processed data. Required if study_proc given. Returns ------- infolist: list of dict of lists and dicts study and processed data info for JSON serialiation for datatables Each dict in the list is a single study, and contains the text Notes ----- Both study_proc and proc_samples must be passed, or neither passed. """ build_samples = False # Logic check to make sure both needed parts passed if study_proc is not None and proc_samples is None: raise IncompetentQiitaDeveloperError( 'Must pass proc_samples when study_proc given') elif proc_samples is not None and study_proc is None: raise IncompetentQiitaDeveloperError( 'Must pass study_proc when proc_samples given') elif study_proc is None: build_samples = True # get list of studies for table if search_type == 'user': user_study_set = user.user_studies.union(user.shared_studies) if user.level == 'admin': user_study_set = (user_study_set | Study.get_by_status('sandbox') | Study.get_by_status('private')) study_set = user_study_set - Study.get_by_status('public') elif search_type == 'public': study_set = Study.get_by_status('public') else: raise ValueError('Not a valid search type') if study_proc is not None: study_set = study_set.intersection(study_proc) if not study_set: # No studies left so no need to continue return [] return generate_study_list([s.id for s in study_set], build_samples, public_only=(search_type == 'public'))
def _build_study_info(user, search_type, study_proc=None, proc_samples=None): """Builds list of dicts for studies table, with all HTML formatted Parameters ---------- user : User object logged in user search_type : choice, ['user', 'public'] what kind of search to perform study_proc : dict of lists, optional Dictionary keyed on study_id that lists all processed data associated with that study. Required if proc_samples given. proc_samples : dict of lists, optional Dictionary keyed on proc_data_id that lists all samples associated with that processed data. Required if study_proc given. Returns ------- infolist: list of dict of lists and dicts study and processed data info for JSON serialiation for datatables Each dict in the list is a single study, and contains the text Notes ----- Both study_proc and proc_samples must be passed, or neither passed. """ build_samples = False # Logic check to make sure both needed parts passed if study_proc is not None and proc_samples is None: raise IncompetentQiitaDeveloperError( 'Must pass proc_samples when study_proc given') elif proc_samples is not None and study_proc is None: raise IncompetentQiitaDeveloperError( 'Must pass study_proc when proc_samples given') elif study_proc is None: build_samples = True # get list of studies for table if search_type == 'user': user_study_set = user.user_studies.union(user.shared_studies) if user.level == 'admin': user_study_set = (user_study_set | Study.get_by_status('sandbox') | Study.get_by_status('private')) study_set = user_study_set - Study.get_by_status('public') elif search_type == 'public': study_set = Study.get_by_status('public') else: raise ValueError('Not a valid search type') if study_proc is not None: study_set = study_set.intersection(study_proc) if not study_set: # No studies left so no need to continue return [] return generate_study_list([s.id for s in study_set], build_samples, public_only=(search_type == 'public'))
def _build_study_info(studytype, user=None): """builds list of namedtuples for study listings""" if studytype == "private": studylist = user.user_studies elif studytype == "shared": studylist = user.shared_studies elif studytype == "public": studylist = Study.get_by_status('public') else: raise IncompetentQiitaDeveloperError("Must use private, shared, " "or public!") StudyTuple = namedtuple( 'StudyInfo', 'id title meta_complete ' 'num_samples_collected shared num_raw_data pi ' 'pmids owner status') infolist = [] for s_id in studylist: study = Study(s_id) status = study.status # Just passing the email address as the name here, since # name is not a required field in qiita.qiita_user owner = study_person_linkifier((study.owner, study.owner)) info = study.info PI = StudyPerson(info['principal_investigator_id']) PI = study_person_linkifier((PI.email, PI.name)) pmids = ", ".join([pubmed_linkifier([pmid]) for pmid in study.pmids]) shared = _get_shared_links_for_study(study) infolist.append( StudyTuple(study.id, study.title, info["metadata_complete"], info["number_samples_collected"], shared, len(study.raw_data()), PI, pmids, owner, status)) return infolist
def _build_study_info(studytype, user=None): """builds list of namedtuples for study listings""" if studytype == "private": studylist = user.user_studies elif studytype == "shared": studylist = user.shared_studies elif studytype == "public": studylist = Study.get_by_status('public') else: raise IncompetentQiitaDeveloperError("Must use private, shared, " "or public!") StudyTuple = namedtuple('StudyInfo', 'id title meta_complete ' 'num_samples_collected shared num_raw_data pi ' 'pmids owner status') infolist = [] for s_id in studylist: study = Study(s_id) status = study.status # Just passing the email address as the name here, since # name is not a required field in qiita.qiita_user owner = study_person_linkifier((study.owner, study.owner)) info = study.info PI = StudyPerson(info['principal_investigator_id']) PI = study_person_linkifier((PI.email, PI.name)) pmids = ", ".join([pubmed_linkifier([pmid]) for pmid in study.pmids]) shared = _get_shared_links_for_study(study) infolist.append(StudyTuple(study.id, study.title, info["metadata_complete"], info["number_samples_collected"], shared, len(study.raw_data()), PI, pmids, owner, status)) return infolist
def get(self): stats = yield Task(self._get_stats) # Pull a random public study from the database public_studies = Study.get_by_status('public') study = choice(list(public_studies)) if public_studies else None if study is None: random_study_info = None random_study_title = None random_study_id = None else: random_study_info = study.info random_study_title = study.title random_study_id = study.id self.render('stats.html', number_studies=stats['number_studies'], number_of_samples=stats['number_of_samples'], num_users=stats['num_users'], lat_longs=eval( stats['lat_longs']) if stats['lat_longs'] else [], num_studies_ebi=stats['num_studies_ebi'], num_samples_ebi=stats['num_samples_ebi'], number_samples_ebi_prep=stats['number_samples_ebi_prep'], img=stats['img'], time=stats['time'], num_processing_jobs=stats['num_processing_jobs'], random_study_info=random_study_info, random_study_title=random_study_title, random_study_id=random_study_id)
def get(self): stats = yield Task(self._get_stats) # Pull a random public study from the database public_studies = Study.get_by_status('public') study = choice(list(public_studies)) if public_studies else None if study is None: random_study_info = None random_study_title = None random_study_id = None else: random_study_info = study.info random_study_title = study.title random_study_id = study.id self.render('stats.html', number_studies=stats['number_studies'], number_of_samples=stats['number_of_samples'], num_users=stats['num_users'], lat_longs=eval( stats['lat_longs']) if stats['lat_longs'] else [], num_studies_ebi=stats['num_studies_ebi'], num_samples_ebi=stats['num_samples_ebi'], number_samples_ebi_prep=stats['number_samples_ebi_prep'], img=stats['img'], time=stats['time'], random_study_info=random_study_info, random_study_title=random_study_title, random_study_id=random_study_id)
def get(self): num_studies, num_samples, num_users, lat_longs = \ yield Task(self._get_stats) # Pull a random public study from the database public_studies = Study.get_by_status('public') study = choice(list(public_studies)) if public_studies else None if study is None: random_study_info = None random_study_title = None random_study_id = None else: random_study_info = study.info random_study_title = study.title random_study_id = study.id self.render('stats.html', num_studies=num_studies, num_samples=num_samples, num_users=num_users, lat_longs=lat_longs, random_study_info=random_study_info, random_study_title=random_study_title, random_study_id=random_study_id)
def test_get_by_status(self): obs = Study.get_by_status('sandbox') self.assertEqual(obs, set()) Study.create(User('*****@*****.**'), 'NOT Identification of the ' 'Microbiomes for Cannabis Soils', [1], self.info) obs = Study.get_by_status('private') self.assertEqual(obs, {1}) obs = Study.get_by_status('sandbox') self.assertEqual(obs, {2}) obs = Study.get_by_status('public') self.assertEqual(obs, set()) obs = Study.get_by_status('awaiting_approval') self.assertEqual(obs, set())
def test_get_by_status(self): obs = Study.get_by_status("sandbox") self.assertEqual(obs, set()) Study.create( User("*****@*****.**"), "NOT Identification of the " "Microbiomes for Cannabis Soils", [1], self.info ) obs = Study.get_by_status("private") self.assertEqual(obs, {1}) obs = Study.get_by_status("sandbox") self.assertEqual(obs, {2}) obs = Study.get_by_status("public") self.assertEqual(obs, set()) obs = Study.get_by_status("awaiting_approval") self.assertEqual(obs, set())
def get(self): user = self.current_user if user.level != 'admin': raise HTTPError(403, 'User %s is not admin' % self.current_user) parsed_studies = [] for sid in Study.get_by_status('awaiting_approval'): study = Study(sid) parsed_studies.append((study.id, study.title, study.owner)) self.render('admin_approval.html', study_info=parsed_studies)
def get(self): user = self.current_user if user.level != 'admin': raise HTTPError(403, 'User %s is not admin' % self.current_user) parsed_studies = [] for sid in Study.get_by_status('awaiting_approval'): study = Study(sid) parsed_studies.append((study.id, study.title, study.owner)) self.render('admin_approval.html', study_info=parsed_studies)
def __call__(self, searchstr, user): """Runs a Study query and returns matching studies and samples Parameters ---------- searchstr : str Search string to use user : User object User making the search. Needed for permissions checks. Returns ------- dict Found samples in format {study_id: [[samp_id1, meta1, meta2, ...], [samp_id2, meta1, meta2, ...], ...} list metadata column names searched for Notes ----- Metadata information for each sample is in the same order as the metadata columns list returned Metadata column names and string searches are case-sensitive """ with TRN: study_sql, sample_sql, meta_headers = \ self._parse_study_search_string(searchstr, True) # get all studies containing the metadata headers requested TRN.add(study_sql) study_ids = set(TRN.execute_fetchflatten()) # strip to only studies user has access to if user.level not in {'admin', 'dev', 'superuser'}: study_ids = study_ids.intersection( Study.get_by_status('public') | user.user_studies | user.shared_studies) results = {} # run search on each study to get out the matching samples for sid in study_ids: TRN.add(sample_sql.format(sid)) study_res = TRN.execute_fetchindex() if study_res: # only add study to results if actually has samples # in results results[sid] = study_res self.results = results self.meta_headers = meta_headers return results, meta_headers
def __call__(self, searchstr, user): """Runs a Study query and returns matching studies and samples Parameters ---------- searchstr : str Search string to use user : User object User making the search. Needed for permissions checks. Returns ------- dict Found samples in format {study_id: [[samp_id1, meta1, meta2, ...], [samp_id2, meta1, meta2, ...], ...} list metadata column names searched for Notes ----- Metadata information for each sample is in the same order as the metadata columns list returned Metadata column names and string searches are case-sensitive """ study_sql, sample_sql, meta_headers = \ self._parse_study_search_string(searchstr, True) conn_handler = SQLConnectionHandler() # get all studies containing the metadata headers requested study_ids = {x[0] for x in conn_handler.execute_fetchall(study_sql)} # strip to only studies user has access to if user.level not in {'admin', 'dev', 'superuser'}: study_ids = study_ids.intersection( Study.get_by_status('public') + user.user_studies + user.shared_studies) results = {} # run search on each study to get out the matching samples for sid in study_ids: study_res = conn_handler.execute_fetchall(sample_sql.format(sid)) if study_res: # only add study to results if actually has samples in results results[sid] = study_res return results, meta_headers
def __call__(self, searchstr, user): """Runs a Study query and returns matching studies and samples Parameters ---------- searchstr : str Search string to use user : str User making the search. Needed for permissions checks. Returns ------- dict Found samples in format {study_id: [[samp_id1, meta1, meta2, ...], [samp_id2, meta1, meta2, ...], ...} list metadata column names searched for Notes ----- Metadata information for each sample is in the same order as the metadata columns list returned Metadata column names and string searches are case-sensitive """ study_sql, sample_sql, meta_headers = \ self._parse_study_search_string(searchstr, True) conn_handler = SQLConnectionHandler() # get all studies containing the metadata headers requested study_ids = {x[0] for x in conn_handler.execute_fetchall(study_sql)} # strip to only studies user has access to userobj = User(user) study_ids = study_ids.intersection(Study.get_by_status('public') + userobj.user_studies + userobj.shared_studies) results = {} # run search on each study to get out the matching samples for sid in study_ids: study_res = conn_handler.execute_fetchall(sample_sql.format(sid)) if study_res: # only add study to results if actually has samples in results results[sid] = study_res return results, meta_headers
def get(self): num_studies, num_samples, num_users, lat_longs = \ yield Task(self._get_stats) # Pull a random public study from the database public_studies = Study.get_by_status('public') study = Study(choice(list(public_studies))) if public_studies else None if study is None: random_study_info = None random_study_title = None random_study_id = None else: random_study_info = study.info random_study_title = study.title random_study_id = study.id self.render('stats.html', num_studies=num_studies, num_samples=num_samples, num_users=num_users, lat_longs=lat_longs, random_study_info=random_study_info, random_study_title=random_study_title, random_study_id=random_study_id)
def _build_study_info(user, study_proc=None, proc_samples=None): """Builds list of dicts for studies table, with all HTML formatted Parameters ---------- user : User object logged in user study_proc : dict of lists, optional Dictionary keyed on study_id that lists all processed data associated with that study. Required if proc_samples given. proc_samples : dict of lists, optional Dictionary keyed on proc_data_id that lists all samples associated with that processed data. Required if study_proc given. Returns ------- infolist: list of dict of lists and dicts study and processed data info for JSON serialiation for datatables Each dict in the list is a single study, and contains the text Notes ----- Both study_proc and proc_samples must be passed, or neither passed. """ build_samples = False # Logic check to make sure both needed parts passed if study_proc is not None and proc_samples is None: raise IncompetentQiitaDeveloperError( 'Must pass proc_samples when study_proc given') elif proc_samples is not None and study_proc is None: raise IncompetentQiitaDeveloperError( 'Must pass study_proc when proc_samples given') elif study_proc is None: build_samples = True # get list of studies for table study_set = user.user_studies.union( Study.get_by_status('public')).union(user.shared_studies) if study_proc is not None: study_set = study_set.intersection(study_proc) if not study_set: # No studies left so no need to continue return [] cols = ['study_id', 'email', 'principal_investigator_id', 'publication_doi', 'study_title', 'metadata_complete', 'number_samples_collected', 'study_abstract'] study_info = Study.get_info([s.id for s in study_set], cols) # get info for the studies infolist = [] for info in study_info: # Convert DictCursor to proper dict info = dict(info) study = Study(info['study_id']) # Build the processed data info for the study if none passed if build_samples: proc_data_list = [ar for ar in study.artifacts() if ar.artifact_type == 'BIOM'] proc_samples = {} study_proc = {study.id: defaultdict(list)} for proc_data in proc_data_list: study_proc[study.id][proc_data.data_type].append(proc_data.id) # there is only one prep template for each processed data proc_samples[proc_data.id] = proc_data.prep_templates[0].keys() study_info = _build_single_study_info(study, info, study_proc, proc_samples) infolist.append(study_info) return infolist
parameters = {'url': ['string', None], 'private_key': ['string', None], 'study_id': ['integer', None]} create_command(qiita_plugin, "list_remote_files", "retrieves list of valid study files from remote dir", parameters) # Create the 'download_remote_files' command parameters = {'url': ['string', None], 'destination': ['string', None], 'private_key': ['string', None]} create_command(qiita_plugin, "download_remote_files", "downloads valid study files from remote dir", parameters) # August 31, 2018 # Strip any UTF-8 characters that are not also printable ASCII characters # from study titles. As some analysis packages cannot interpret UTF-8 # characters, it becomes important to remove them from study titles, as # they are used as metadata/identifiers when creating new analyses. # insert new status_types into list, or replace w/a call to an appropriate # method. status_types = ['awaiting_approval', 'sandbox', 'private', 'public'] for status_type in status_types: for study in Study.get_by_status(status_type): new_title = sub(r'[^\x20-\x7E]+', '', study.title) if new_title != study.title: study.title = new_title
def _build_study_info(user, results=None): """builds list of dicts for studies table, with all html formatted""" # get list of studies for table study_list = user.user_studies.union( Study.get_by_status('public')).union(user.shared_studies) if results is not None: study_list = study_list.intersection(results) if not study_list: # No studies left so no need to continue return [] # get info for the studies cols = ['study_id', 'email', 'principal_investigator_id', 'pmid', 'study_title', 'metadata_complete', 'number_samples_collected', 'study_abstract'] study_info = Study.get_info(study_list, cols) infolist = [] for row, info in enumerate(study_info): study = Study(info['study_id']) status = study.status # Just passing the email address as the name here, since # name is not a required field in qiita.qiita_user PI = StudyPerson(info['principal_investigator_id']) PI = study_person_linkifier((PI.email, PI.name)) if info['pmid'] is not None: pmids = ", ".join([pubmed_linkifier([p]) for p in info['pmid']]) else: pmids = "" if info["number_samples_collected"] is None: info["number_samples_collected"] = "0" shared = _get_shared_links_for_study(study) meta_complete_glyph = "ok" if info["metadata_complete"] else "remove" # build the HTML elements needed for table cell title = ("<a href='#' data-toggle='modal' " "data-target='#study-abstract-modal' " "onclick='fillAbstract(\"studies-table\", {0})'>" "<span class='glyphicon glyphicon-file' " "aria-hidden='true'></span></a> | " "<a href='/study/description/{1}' " "id='study{0}-title'>{2}</a>").format( str(row), str(study.id), info["study_title"]) meta_complete = "<span class='glyphicon glyphicon-%s'></span>" % \ meta_complete_glyph if status == 'public': shared = "Not Available" else: shared = ("<span id='shared_html_{0}'>{1}</span><br/>" "<a class='btn btn-primary btn-xs' data-toggle='modal' " "data-target='#share-study-modal-view' " "onclick='modify_sharing({0});'>Modify</a>".format( study.id, shared)) infolist.append({ "checkbox": "<input type='checkbox' value='%d' />" % study.id, "id": study.id, "title": title, "meta_complete": meta_complete, "num_samples": info["number_samples_collected"], "shared": shared, "num_raw_data": len(study.raw_data()), "pi": PI, "pmid": pmids, "status": status, "abstract": info["study_abstract"] }) return infolist
def _build_study_info(user, study_proc=None, proc_samples=None): """Builds list of dicts for studies table, with all HTML formatted Parameters ---------- user : User object logged in user study_proc : dict of lists, optional Dictionary keyed on study_id that lists all processed data associated with that study. Required if proc_samples given. proc_samples : dict of lists, optional Dictionary keyed on proc_data_id that lists all samples associated with that processed data. Required if study_proc given. Returns ------- infolist: list of dict of lists and dicts study and processed data info for JSON serialiation for datatables Each dict in the list is a single study, and contains the text Notes ----- Both study_proc and proc_samples must be passed, or neither passed. """ build_samples = False # Logic check to make sure both needed parts passed if study_proc is not None and proc_samples is None: raise IncompetentQiitaDeveloperError( 'Must pass proc_samples when study_proc given') elif proc_samples is not None and study_proc is None: raise IncompetentQiitaDeveloperError( 'Must pass study_proc when proc_samples given') elif study_proc is None: build_samples = True # get list of studies for table study_set = user.user_studies.union( Study.get_by_status('public')).union(user.shared_studies) if study_proc is not None: study_set = study_set.intersection(study_proc) if not study_set: # No studies left so no need to continue return [] # get info for the studies cols = ['study_id', 'email', 'principal_investigator_id', 'pmid', 'study_title', 'metadata_complete', 'number_samples_collected', 'study_abstract'] study_info = Study.get_info(study_set, cols) infolist = [] for info in study_info: # Convert DictCursor to proper dict info = dict(info) study = Study(info['study_id']) # Build the processed data info for the study if none passed if build_samples: proc_data_list = study.processed_data() proc_samples = {} study_proc = {study.id: defaultdict(list)} for pid in proc_data_list: proc_data = ProcessedData(pid) study_proc[study.id][proc_data.data_type()].append(pid) proc_samples[pid] = proc_data.samples study_info = _build_single_study_info(study, info, study_proc, proc_samples) infolist.append(study_info) return infolist
# ----------------------------------------------------------------------------- # Copyright (c) 2014--, The Qiita Development Team. # # Distributed under the terms of the BSD 3-clause License. # # The full license is in the file LICENSE, distributed with this software. # ----------------------------------------------------------------------------- from qiita_db.study import Study studies = Study.get_by_status('private').union( Study.get_by_status('public')).union(Study.get_by_status('sandbox')) raw_data = [pt.artifact for s in studies for pt in s.prep_templates() if pt.artifact is not None] for rd in raw_data: # getting the most open visibility of all the children in the pipeline children = rd.descendants.nodes() vis = [a.visibility for a in children] vis.append(rd.visibility) new_vis = 'sandbox' if 'public' in vis: new_vis = 'public' elif 'private' in vis: new_vis = 'private' rd.visibility = new_vis
# replacing all \t and \n for space as those chars brake QIIME from qiita_db.study import Study from qiita_db.sql_connection import TRN def searcher(df): search = r"\t|\n" return [ col for col in df if df[col].str.contains(search, na=False, regex=True).any() ] studies = Study.get_by_status('private').union( Study.get_by_status('public')).union(Study.get_by_status('sandbox')) # we will start search using pandas as is much easier and faster # than using pgsql. remember that to_dataframe actually transforms what's # in the db to_fix = [] for s in studies: st = s.sample_template if st is None: continue cols = searcher(st.to_dataframe()) if cols: to_fix.append((st, cols)) for pt in s.prep_templates(): if pt is None:
Notes ----- Both study_proc and proc_samples must be passed, or neither passed. """ # Logic check to make sure both needed parts passed if study_proc is not None and proc_samples is None: raise IncompetentQiitaDeveloperError( 'Must pass proc_samples when study_proc given') elif proc_samples is not None and study_proc is None: raise IncompetentQiitaDeveloperError( 'Must pass study_proc when proc_samples given') # get list of studies for table <<<<<<< HEAD study_set = user.user_studies.union( Study.get_by_status('public')).union(user.shared_studies) ======= user_study_set = user.user_studies.union(user.shared_studies) if search_type == 'user': if user.level == 'admin': user_study_set = (user_study_set | Study.get_by_status('sandbox') | Study.get_by_status('private') | Study.get_by_status('awaiting_approval') - Study.get_by_status('public')) study_set = user_study_set elif search_type == 'public': study_set = Study.get_by_status('public') - user_study_set else: raise ValueError('Not a valid search type') >>>>>>> 405cbef0c9f71c620da95a0c1ba6c7d3d588b3ed
def test_get_by_status(self): Study.create( User('*****@*****.**'), 'NOT Identification of the ' 'Microbiomes for Cannabis Soils', [1], self.info) obs = Study.get_by_status('private') self.assertEqual(obs, [1])
def test_get_by_status(self): Study.create(User('*****@*****.**'), 'NOT Identification of the ' 'Microbiomes for Cannabis Soils', [1], self.info) obs = Study.get_by_status('private') self.assertEqual(obs, [1])