示例#1
0
 def test_Document(self):
     """Instantiate and authenticate a Document object"""
     credentials = get_credentials(self.credentials_file)
     obs_type = type(gdocs.Document(credentials))
     exp_type = gdocs.Document
     self.assertIs(obs_type, exp_type,
                   "Did not instantiate an object of the correct type")
def get(project_ID, proj):
    CREDENTIALS_FILE = os.path.join(os.environ['HOME'],
                                    'opt/config/gdocs_credentials')
    credentials = get_credentials(CREDENTIALS_FILE)
    info = get_20158_info(credentials, project_ID)
    if info:
        if proj.has_key('samples'):
            for sample in proj['samples']:
                if sample in info.keys():
                    proj['samples'][sample]['status'] = info[sample][0]
                    proj['samples'][sample]['m_reads_sequenced'] = info[
                        sample][1]
                    if not proj['samples'][sample].has_key('details'):
                        proj['samples'][sample]['details'] = {
                            'status_(manual)': info[sample][0],
                            'total_reads_(m)': info[sample][1]
                        }
                    else:
                        if not proj['samples'][sample]['details'].has_key(
                                'status_(manual)'):
                            proj['samples'][sample]['details'][
                                'status_(manual)'] = info[sample][0]
                        if not proj['samples'][sample]['details'].has_key(
                                'total_reads_(m)'):
                            proj['samples'][sample]['details'][
                                'total_reads_(m)'] = info[sample][1]
    return proj
示例#3
0
 def __init__(self, config):
     super(GDocsUpdater, self).__init__(config)
         
     # Connect to the Google Docs api
     gdconf = self.config.get("gdocs",{})
     creds = os.path.expanduser(gdconf.get("credentials_file",""))
     assert os.path.exists(creds), "Supplied GDocs credentials file does not exist"
     self.gdcon = SpreadSheet(get_credentials(creds))
     assert self.gdcon, "Could not get a SpreadSheet object, please verify gdocs credentials"
     doc = gdconf.get("qc_checklist",None)
     assert doc, "No QC checklist specified in configuration, please specify"
     ssheet = self.gdcon.get_spreadsheet(doc)
     assert ssheet, "Could not locate QC checklist '{}' on Google Docs. Please make sure it exists".format(doc)
     self.gdcon.ssheet = ssheet
     
     # Get the Ongoing, Finished and Coming worksheets
     self.ongoing = self.gdcon.get_worksheet("Ongoing")
     self.coming = self.gdcon.get_worksheet("Coming")
     self.finished = self.gdcon.get_worksheet("Finished")
     assert self.ongoing and self.coming and self.finished, "Could not get 'Ongoing', 'Finished' and 'Coming' worksheets from '{}'. Please make sure that they exist".format(doc)
     
     # Get a connection to the StatusDB project database
     dbconf = self.config.get("statusdb",{})
     try:
         self.pcon = ProjectSummaryConnection(url=dbconf.get("url","localhost"), 
                                              username=dbconf.get("user","user"), 
                                              password=dbconf.get("password","pass"))
     except ConnectionError:
         self.pcon = None
示例#4
0
 def test_Document(self):
     """Instantiate and authenticate a Document object"""
     credentials = get_credentials(self.credentials_file)
     obs_type = type(gdocs.Document(credentials))
     exp_type = gdocs.Document
     self.assertIs(obs_type,
                   exp_type,
                   "Did not instantiate an object of the correct type")
示例#5
0
def upload_to_gdocs(fcdir, credentials_file=None, gdocs_folder=None):

    output_data = {
        'stdout': StringIO(),
        'stderr': StringIO(),
        'debug': StringIO()
    }

    if not os.path.exists(fcdir):
        LOG.error("The run folder, {} does not exist!".format(
            os.path.basename(fcdir)))
        return output_data

    credentials = google.get_credentials(credentials_file)
    if credentials is None:
        LOG.error("Could not parse the Google Docs credentials")
        return output_data

    metrics = collect_metrics(fcdir)
    samples = _format_samples(metrics)

    ssheet_name = _demultiplex_spreadsheet(metrics['RunInfo'].get(
        'Date', None))
    ssheet = SpreadSheet(credentials, ssheet_name)
    ssheet.move_to_folder(gdocs_folder)

    run_id = metrics['RunInfo']['Id'].split("_")
    wsheet_name = "_".join([run_id[0], run_id[-1]])

    # Write the metrics for the entire flowcell
    write_flowcell_metrics(samples, ssheet, wsheet_name)

    # Write project-centered metrics
    projects = list(set([sample.get('Project name', '')
                         for sample in samples]))
    for project in projects:
        if project in ['Undetermined_indices', '']:
            continue
        project_samples = [
            sample for sample in samples
            if sample.get('Project name', '') == project
        ]
        # Insert the run name as description
        for sample in project_samples:
            sample['Description'] = wsheet_name

        ssheet_name = "{}_sequencing_results".format(project)
        ssheet = SpreadSheet(credentials, ssheet_name)
        ssheet.move_to_folder(gdocs_folder)
        # Truncate the summary worksheet so that it won't show the wrong information in case upload fails
        write_flowcell_metrics([], ssheet, "Summary")
        write_flowcell_metrics(project_samples, ssheet, wsheet_name)

        # Create the summary over all worksheets in the project
        summary_samples = summarize_project(ssheet)
        write_flowcell_metrics(summary_samples, ssheet, "Summary")

    return output_data
示例#6
0
 def test_get_credentials(self):
     """Parse credentials file"""
     cfile = "credentials.tmp"
     exp_credentials = "credentials string"
     with open(cfile,"w") as fh:
         fh.write(exp_credentials)
     
     obs_credentials = get_credentials(cfile)
     self.assertEqual(exp_credentials,
                      obs_credentials,
                      "Expected and observed credentials are not identical")
示例#7
0
    def test_get_credentials(self):
        """Parse credentials file"""
        cfile = "credentials.tmp"
        exp_credentials = "credentials string"
        with open(cfile, "w") as fh:
            fh.write(exp_credentials)

        obs_credentials = get_credentials(cfile)
        self.assertEqual(
            exp_credentials, obs_credentials,
            "Expected and observed credentials are not identical")
def main(project_name, conf, cred):
    credentials = get_credentials(cred)
    client = SpreadSheet(credentials)
    config = cl.load_config(conf)
    couch = load_couch_server(conf)
    analysis_db = couch['analysis']
    #proj_db = couch['projects']
    BP_RNA = DB.BP_RNA(project_name)
    key = find_proj_from_view(analysis_db, project_name)
    BP_RNA.obj['_id'] = find_or_make_key(key)
    info = save_couchdb_obj(analysis_db, BP_RNA.obj)
    LOG.info('project %s %s : _id = %s' % (project_name, info, BP_RNA.obj['_id']))
示例#9
0
def upload_to_gdocs(log, fcdir, credentials_file=None, gdocs_folder=None):

    output_data = {'stdout':StringIO(), 'stderr':StringIO(), 'debug':StringIO()}

    if not os.path.exists(fcdir):
        log.error("The run folder, {} does not exist!".format(os.path.basename(fcdir)))
        return output_data

    credentials = google.get_credentials(credentials_file)
    if credentials is None:
        log.error("Could not parse the Google Docs credentials")
        return output_data

    metrics = collect_metrics(fcdir, log)
    samples = _format_samples(metrics)

    ssheet_name = _demultiplex_spreadsheet(metrics['RunInfo'].get('Date',None))
    ssheet = SpreadSheet(credentials,ssheet_name)
    if ssheet.new_doc:
        ssheet.move_to_folder(gdocs_folder)

    run_id = metrics['RunInfo']['Id'].split("_")
    wsheet_name = "_".join([run_id[0],run_id[-1]])

    # Write the metrics for the entire flowcell
    write_flowcell_metrics(samples, ssheet, wsheet_name)

    # Write project-centered metrics
    projects = list(set([sample.get('Project name','') for sample in samples]))
    for project in projects:
        if project in ['Undetermined_indices','']:
            continue
        project_samples = [sample for sample in samples if sample.get('Project name','') == project]
        # Insert the run name as description
        for sample in project_samples:
            sample['Description'] = wsheet_name

        ssheet_name = "{}_sequencing_results".format(project)
        ssheet = SpreadSheet(credentials,ssheet_name)
        if ssheet.new_doc:
            ssheet.move_to_folder(gdocs_folder)
        # Truncate the summary worksheet so that it won't show the wrong information in case upload fails
        write_flowcell_metrics([], ssheet, "Summary")
        project_samples = summarize_project(log, ssheet,{wsheet_name: project_samples})
        write_flowcell_metrics(project_samples, ssheet, wsheet_name)

        # Create the summary over all worksheets in the project
        summary_samples = summarize_project(log, ssheet)
        write_flowcell_metrics(summary_samples, ssheet, "Summary")

    return output_data
示例#10
0
    def __init__(self, project_name, config):
        """Initialize the object"""

        # Map internal attribute names to the GPL column headers
        col_mapping = self.column_mapping()
        for attr in col_mapping.keys():
            setattr(self, attr, None)

        # Get the name of the spreadsheet where uppnex ids can be found
        gdocs_config = config.get("gdocs", config.get("gdocs_upload",{}))
        cred_file = gdocs_config.get("credentials_file",gdocs_config.get("gdocs_credentials"))
        ssheet_title = gdocs_config.get("projects_spreadsheet")
        wsheet_title = gdocs_config.get("projects_worksheet")

        # Get the credentials
        credentials = get_credentials(cred_file)
        assert credentials is not None, \
        "The Google Docs credentials could not be found."
        assert ssheet_title is not None and wsheet_title is not None, \
            "The names of the projects spreadsheet and worksheet on Google \
            Docs could not be found."

        # Connect to the spread- and worksheet
        ssheet = SpreadSheet(credentials, ssheet_title)
        assert ssheet is not None, \
            "Could not fetch '{}' from Google Docs.".format(ssheet_title)

        # We allow multiple, comma-separated worksheets to be searched
        for wtitle in wsheet_title.split(','):
            wsheet = ssheet.get_worksheet(wtitle.strip())
            if not wsheet:
                print("WARNING: Could not locate {} in {}".format(wsheet_title, ssheet_title))
                continue

            # Get the rows for the project
            rows = ssheet.get_cell_content(wsheet)
            header = ssheet.get_header(wsheet)
            column_indexes = {attr: ssheet.get_column_index(wsheet,col)-1 for attr, col in col_mapping.items()}
            for row in rows:
                # skip if this is not the project we're interested in
                if row[column_indexes["project_name"]] != project_name:
                    continue
                
                # Will only use the first result found to set each attribute
                for attr, index in column_indexes.items():
                    setattr(self, attr, row[index])

                # We have found the project data so stop iterating
                return
def main(project_name, conf):
    couch = load_couch_server(conf)
    proj_db = couch['projects']
    key = find_proj_from_view(proj_db, project_name)
    print key
    proj = proj_db.get(key)
    if not proj:
        LOG.warning('No project named {man_name} in Lims'.format(
                        man_name = project_name))
    else:
        CREDENTIALS_FILE = os.path.join(os.environ['HOME'], 'opt/config/gdocs_credentials')
        credentials = get_credentials(CREDENTIALS_FILE)
        info = get_20158_info(credentials, project_name)
        proj = set_db_fields(info, proj)
        save_couchdb_obj(proj_db, proj)
def get(project_ID, proj):
    CREDENTIALS_FILE = os.path.join(os.environ['HOME'], 'opt/config/gdocs_credentials')
    credentials = get_credentials(CREDENTIALS_FILE)
    info = get_20158_info(credentials, project_ID)
    if info:
        if proj.has_key('samples'):
            for sample in proj['samples']:
                if sample in info.keys():
                    proj['samples'][sample]['status'] = info[sample][0]
                    proj['samples'][sample]['m_reads_sequenced'] = info[sample][1]
                    if not proj['samples'][sample].has_key('details'):
                        proj['samples'][sample]['details'] = {
                                            'status_(manual)' : info[sample][0],
                                            'total_reads_(m)' : info[sample][1]}
                    else:
                        if not proj['samples'][sample]['details'].has_key('status_(manual)'):
                            proj['samples'][sample]['details']['status_(manual)'] = info[sample][0]
                        if not proj['samples'][sample]['details'].has_key('total_reads_(m)'):
                            proj['samples'][sample]['details']['total_reads_(m)'] = info[sample][1]
    return proj
def get(project_ID):
    CREDENTIALS_FILE = os.path.join(os.environ['HOME'], 'opt/config/gdocs_credentials')
    credentials = get_credentials(CREDENTIALS_FILE)
    info = get_20158_info(credentials, project_ID)
    return info
	sample_dir = [h for h in os.listdir(p_path) if not os.path.isfile("{}/{}".format(p_path,h))]
	for sample in sample_dir:
		flag = os.path.join(p_path,sample,'FINISHED_AND_DELIVERED')
		if not os.path.exists(flag):
			sam_cnt = sam_cnt+1
			sam_nm.append(sample)
	if sam_cnt == 0:
		status = 'full_marked'
	elif sam_cnt == len(sample_dir):
		status = 'none_marked'
	else:
		status = 'some_marked'
	return status,sam_nm;

## getting the spreadsheet from given name and appropriate worksheet
credentials = get_credentials(config['credentials_file'])
ssheet = SpreadSheet(credentials,config['gpl_spreadsheet'])
wksheet = ssheet.get_worksheet(config['gpl_worksheet'])
cell_feed = ssheet.get_cell_feed(wksheet)

print "**** DONE ****\n" ##log
print "Parsing worksheet and obtaining information..." ##log

## iterating through cell's content and gives list of prject signed delivered
col_tot = int(cell_feed.col_count.text)
projects_done, cell_content, row, ck, rNum = ([], [], [], 0, 1)
for cell in cell_feed.entry:
	row.append(cell.content.text or "")
	ck = ck+1
	if ck == col_tot:
		if rNum == 1: