def test_Document(self):
"""Instantiate and authenticate a Document object"""
credentials = get_credentials(self.credentials_file)
obs_type = type(gdocs.Document(credentials))
exp_type = gdocs.Document
self.assertIs(obs_type, exp_type,
"Did not instantiate an object of the correct type")
def get(project_ID, proj):
CREDENTIALS_FILE = os.path.join(os.environ['HOME'],
'opt/config/gdocs_credentials')
credentials = get_credentials(CREDENTIALS_FILE)
info = get_20158_info(credentials, project_ID)
if info:
if proj.has_key('samples'):
for sample in proj['samples']:
if sample in info.keys():
proj['samples'][sample]['status'] = info[sample][0]
proj['samples'][sample]['m_reads_sequenced'] = info[
sample][1]
if not proj['samples'][sample].has_key('details'):
proj['samples'][sample]['details'] = {
'status_(manual)': info[sample][0],
'total_reads_(m)': info[sample][1]
}
else:
if not proj['samples'][sample]['details'].has_key(
'status_(manual)'):
proj['samples'][sample]['details'][
'status_(manual)'] = info[sample][0]
if not proj['samples'][sample]['details'].has_key(
'total_reads_(m)'):
proj['samples'][sample]['details'][
'total_reads_(m)'] = info[sample][1]
return proj
def __init__(self, config):
super(GDocsUpdater, self).__init__(config)
# Connect to the Google Docs api
gdconf = self.config.get("gdocs",{})
creds = os.path.expanduser(gdconf.get("credentials_file",""))
assert os.path.exists(creds), "Supplied GDocs credentials file does not exist"
self.gdcon = SpreadSheet(get_credentials(creds))
assert self.gdcon, "Could not get a SpreadSheet object, please verify gdocs credentials"
doc = gdconf.get("qc_checklist",None)
assert doc, "No QC checklist specified in configuration, please specify"
ssheet = self.gdcon.get_spreadsheet(doc)
assert ssheet, "Could not locate QC checklist '{}' on Google Docs. Please make sure it exists".format(doc)
self.gdcon.ssheet = ssheet
# Get the Ongoing, Finished and Coming worksheets
self.ongoing = self.gdcon.get_worksheet("Ongoing")
self.coming = self.gdcon.get_worksheet("Coming")
self.finished = self.gdcon.get_worksheet("Finished")
assert self.ongoing and self.coming and self.finished, "Could not get 'Ongoing', 'Finished' and 'Coming' worksheets from '{}'. Please make sure that they exist".format(doc)
# Get a connection to the StatusDB project database
dbconf = self.config.get("statusdb",{})
try:
self.pcon = ProjectSummaryConnection(url=dbconf.get("url","localhost"),
username=dbconf.get("user","user"),
password=dbconf.get("password","pass"))
except ConnectionError:
self.pcon = None
def test_Document(self):
"""Instantiate and authenticate a Document object"""
credentials = get_credentials(self.credentials_file)
obs_type = type(gdocs.Document(credentials))
exp_type = gdocs.Document
self.assertIs(obs_type,
exp_type,
"Did not instantiate an object of the correct type")
def upload_to_gdocs(fcdir, credentials_file=None, gdocs_folder=None):
output_data = {
'stdout': StringIO(),
'stderr': StringIO(),
'debug': StringIO()
}
if not os.path.exists(fcdir):
LOG.error("The run folder, {} does not exist!".format(
os.path.basename(fcdir)))
return output_data
credentials = google.get_credentials(credentials_file)
if credentials is None:
LOG.error("Could not parse the Google Docs credentials")
return output_data
metrics = collect_metrics(fcdir)
samples = _format_samples(metrics)
ssheet_name = _demultiplex_spreadsheet(metrics['RunInfo'].get(
'Date', None))
ssheet = SpreadSheet(credentials, ssheet_name)
ssheet.move_to_folder(gdocs_folder)
run_id = metrics['RunInfo']['Id'].split("_")
wsheet_name = "_".join([run_id[0], run_id[-1]])
# Write the metrics for the entire flowcell
write_flowcell_metrics(samples, ssheet, wsheet_name)
# Write project-centered metrics
projects = list(set([sample.get('Project name', '')
for sample in samples]))
for project in projects:
if project in ['Undetermined_indices', '']:
continue
project_samples = [
sample for sample in samples
if sample.get('Project name', '') == project
]
# Insert the run name as description
for sample in project_samples:
sample['Description'] = wsheet_name
ssheet_name = "{}_sequencing_results".format(project)
ssheet = SpreadSheet(credentials, ssheet_name)
ssheet.move_to_folder(gdocs_folder)
# Truncate the summary worksheet so that it won't show the wrong information in case upload fails
write_flowcell_metrics([], ssheet, "Summary")
write_flowcell_metrics(project_samples, ssheet, wsheet_name)
# Create the summary over all worksheets in the project
summary_samples = summarize_project(ssheet)
write_flowcell_metrics(summary_samples, ssheet, "Summary")
return output_data
def test_get_credentials(self):
"""Parse credentials file"""
cfile = "credentials.tmp"
exp_credentials = "credentials string"
with open(cfile,"w") as fh:
fh.write(exp_credentials)
obs_credentials = get_credentials(cfile)
self.assertEqual(exp_credentials,
obs_credentials,
"Expected and observed credentials are not identical")
def test_get_credentials(self):
"""Parse credentials file"""
cfile = "credentials.tmp"
exp_credentials = "credentials string"
with open(cfile, "w") as fh:
fh.write(exp_credentials)
obs_credentials = get_credentials(cfile)
self.assertEqual(
exp_credentials, obs_credentials,
"Expected and observed credentials are not identical")
def main(project_name, conf, cred):
credentials = get_credentials(cred)
client = SpreadSheet(credentials)
config = cl.load_config(conf)
couch = load_couch_server(conf)
analysis_db = couch['analysis']
#proj_db = couch['projects']
BP_RNA = DB.BP_RNA(project_name)
key = find_proj_from_view(analysis_db, project_name)
BP_RNA.obj['_id'] = find_or_make_key(key)
info = save_couchdb_obj(analysis_db, BP_RNA.obj)
LOG.info('project %s %s : _id = %s' % (project_name, info, BP_RNA.obj['_id']))
def upload_to_gdocs(log, fcdir, credentials_file=None, gdocs_folder=None):
output_data = {'stdout':StringIO(), 'stderr':StringIO(), 'debug':StringIO()}
if not os.path.exists(fcdir):
log.error("The run folder, {} does not exist!".format(os.path.basename(fcdir)))
return output_data
credentials = google.get_credentials(credentials_file)
if credentials is None:
log.error("Could not parse the Google Docs credentials")
return output_data
metrics = collect_metrics(fcdir, log)
samples = _format_samples(metrics)
ssheet_name = _demultiplex_spreadsheet(metrics['RunInfo'].get('Date',None))
ssheet = SpreadSheet(credentials,ssheet_name)
if ssheet.new_doc:
ssheet.move_to_folder(gdocs_folder)
run_id = metrics['RunInfo']['Id'].split("_")
wsheet_name = "_".join([run_id[0],run_id[-1]])
# Write the metrics for the entire flowcell
write_flowcell_metrics(samples, ssheet, wsheet_name)
# Write project-centered metrics
projects = list(set([sample.get('Project name','') for sample in samples]))
for project in projects:
if project in ['Undetermined_indices','']:
continue
project_samples = [sample for sample in samples if sample.get('Project name','') == project]
# Insert the run name as description
for sample in project_samples:
sample['Description'] = wsheet_name
ssheet_name = "{}_sequencing_results".format(project)
ssheet = SpreadSheet(credentials,ssheet_name)
if ssheet.new_doc:
ssheet.move_to_folder(gdocs_folder)
# Truncate the summary worksheet so that it won't show the wrong information in case upload fails
write_flowcell_metrics([], ssheet, "Summary")
project_samples = summarize_project(log, ssheet,{wsheet_name: project_samples})
write_flowcell_metrics(project_samples, ssheet, wsheet_name)
# Create the summary over all worksheets in the project
summary_samples = summarize_project(log, ssheet)
write_flowcell_metrics(summary_samples, ssheet, "Summary")
return output_data
def __init__(self, project_name, config):
"""Initialize the object"""
# Map internal attribute names to the GPL column headers
col_mapping = self.column_mapping()
for attr in col_mapping.keys():
setattr(self, attr, None)
# Get the name of the spreadsheet where uppnex ids can be found
gdocs_config = config.get("gdocs", config.get("gdocs_upload",{}))
cred_file = gdocs_config.get("credentials_file",gdocs_config.get("gdocs_credentials"))
ssheet_title = gdocs_config.get("projects_spreadsheet")
wsheet_title = gdocs_config.get("projects_worksheet")
# Get the credentials
credentials = get_credentials(cred_file)
assert credentials is not None, \
"The Google Docs credentials could not be found."
assert ssheet_title is not None and wsheet_title is not None, \
"The names of the projects spreadsheet and worksheet on Google \
Docs could not be found."
# Connect to the spread- and worksheet
ssheet = SpreadSheet(credentials, ssheet_title)
assert ssheet is not None, \
"Could not fetch '{}' from Google Docs.".format(ssheet_title)
# We allow multiple, comma-separated worksheets to be searched
for wtitle in wsheet_title.split(','):
wsheet = ssheet.get_worksheet(wtitle.strip())
if not wsheet:
print("WARNING: Could not locate {} in {}".format(wsheet_title, ssheet_title))
continue
# Get the rows for the project
rows = ssheet.get_cell_content(wsheet)
header = ssheet.get_header(wsheet)
column_indexes = {attr: ssheet.get_column_index(wsheet,col)-1 for attr, col in col_mapping.items()}
for row in rows:
# skip if this is not the project we're interested in
if row[column_indexes["project_name"]] != project_name:
continue
# Will only use the first result found to set each attribute
for attr, index in column_indexes.items():
setattr(self, attr, row[index])
# We have found the project data so stop iterating
return
def main(project_name, conf):
couch = load_couch_server(conf)
proj_db = couch['projects']
key = find_proj_from_view(proj_db, project_name)
print key
proj = proj_db.get(key)
if not proj:
LOG.warning('No project named {man_name} in Lims'.format(
man_name = project_name))
else:
CREDENTIALS_FILE = os.path.join(os.environ['HOME'], 'opt/config/gdocs_credentials')
credentials = get_credentials(CREDENTIALS_FILE)
info = get_20158_info(credentials, project_name)
proj = set_db_fields(info, proj)
save_couchdb_obj(proj_db, proj)
def get(project_ID, proj):
CREDENTIALS_FILE = os.path.join(os.environ['HOME'], 'opt/config/gdocs_credentials')
credentials = get_credentials(CREDENTIALS_FILE)
info = get_20158_info(credentials, project_ID)
if info:
if proj.has_key('samples'):
for sample in proj['samples']:
if sample in info.keys():
proj['samples'][sample]['status'] = info[sample][0]
proj['samples'][sample]['m_reads_sequenced'] = info[sample][1]
if not proj['samples'][sample].has_key('details'):
proj['samples'][sample]['details'] = {
'status_(manual)' : info[sample][0],
'total_reads_(m)' : info[sample][1]}
else:
if not proj['samples'][sample]['details'].has_key('status_(manual)'):
proj['samples'][sample]['details']['status_(manual)'] = info[sample][0]
if not proj['samples'][sample]['details'].has_key('total_reads_(m)'):
proj['samples'][sample]['details']['total_reads_(m)'] = info[sample][1]
return proj
def get(project_ID):
CREDENTIALS_FILE = os.path.join(os.environ['HOME'], 'opt/config/gdocs_credentials')
credentials = get_credentials(CREDENTIALS_FILE)
info = get_20158_info(credentials, project_ID)
return info
sample_dir = [h for h in os.listdir(p_path) if not os.path.isfile("{}/{}".format(p_path,h))]
for sample in sample_dir:
flag = os.path.join(p_path,sample,'FINISHED_AND_DELIVERED')
if not os.path.exists(flag):
sam_cnt = sam_cnt+1
sam_nm.append(sample)
if sam_cnt == 0:
status = 'full_marked'
elif sam_cnt == len(sample_dir):
status = 'none_marked'
else:
status = 'some_marked'
return status,sam_nm;
## getting the spreadsheet from given name and appropriate worksheet
credentials = get_credentials(config['credentials_file'])
ssheet = SpreadSheet(credentials,config['gpl_spreadsheet'])
wksheet = ssheet.get_worksheet(config['gpl_worksheet'])
cell_feed = ssheet.get_cell_feed(wksheet)
print "**** DONE ****\n" ##log
print "Parsing worksheet and obtaining information..." ##log
## iterating through cell's content and gives list of prject signed delivered
col_tot = int(cell_feed.col_count.text)
projects_done, cell_content, row, ck, rNum = ([], [], [], 0, 1)
for cell in cell_feed.entry:
row.append(cell.content.text or "")
ck = ck+1
if ck == col_tot:
if rNum == 1: