def index_database(database_connection, channel):
try:
channel.send(Message("Analyzing Database", 'update'))
handle = DatabaseBoundOperation(database_connection)
for (table, ix_name) in index_updates:
session = handle.session()
connection = session.connection()
index = find_index_by_name(table, ix_name)
if index is None:
continue
try:
index.create(connection)
except:
session.rollback()
session.commit()
handle._analyze_database()
session = handle.session()
for query in target_queries:
result = session.execute("EXPLAIN QUERY PLAN " + query)
channel.log("%s:\n\t%r" % (query, ' '.join(map(str, result))))
channel.send(Message("Indexing Complete", 'update'))
except:
channel.send(Message.traceback())
def glycopeptide_mzidentml(database_connection,
analysis_identifier,
output_path=None,
mzml_path=None,
embed_protein_sequences=True):
'''Write identified glycopeptides as mzIdentML file, and associated MSn spectra
to a paired mzML file if the matched data are available. If an mzML file is written
it will also contain the extracted ion chromatograms for each glycopeptide with an
extracted elution profile.
'''
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session() # pylint: disable=not-callable
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycopeptide_lc_msms:
click.secho("Analysis %r is of type %r." %
(str(analysis.name), str(analysis.analysis_type)),
fg='red',
err=True)
raise click.Abort()
loader = AnalysisDeserializer(database_connection._original_connection,
analysis_id=analysis.id)
click.echo("Loading Identifications")
# glycopeptides = loader.load_identified_glycopeptides()
glycopeptides = loader.query(IdentifiedGlycopeptide).filter(
IdentifiedGlycopeptide.analysis_id == analysis_identifier).all()
with open(output_path, 'wb') as outfile:
writer = MzIdentMLSerializer(
outfile,
glycopeptides,
analysis,
loader,
source_mzml_path=mzml_path,
embed_protein_sequences=embed_protein_sequences)
writer.run()
def glycopeptide_hypothesis(database_connection, hypothesis_identifier, output_path, multifasta=False):
'''Write each theoretical glycopeptide in CSV format
'''
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session()
hypothesis = get_by_name_or_id(session, GlycopeptideHypothesis, hypothesis_identifier)
def generate():
interval = 100000
i = 0
while True:
session.expire_all()
chunk = hypothesis.glycopeptides.slice(i, i + interval).all()
if len(chunk) == 0:
break
for glycopeptide in chunk:
yield glycopeptide
i += interval
if output_path is None:
output_stream = ctxstream(sys.stdout)
else:
output_stream = open(output_path, 'wb')
with output_stream:
job = GlycopeptideHypothesisCSVSerializer(output_stream, generate())
job.run()
def export_identified_glycans_from_glycopeptides(database_connection,
analysis_identifier,
output_path):
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session() # pylint: disable=not-callable
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycopeptide_lc_msms:
click.secho("Analysis %r is of type %r." %
(str(analysis.name), str(analysis.analysis_type)),
fg='red',
err=True)
raise click.Abort()
glycans = session.query(GlycanComposition).join(
GlycanCombinationGlycanComposition).join(GlycanCombination).join(
Glycopeptide,
Glycopeptide.glycan_combination_id == GlycanCombination.id).join(
IdentifiedGlycopeptide,
IdentifiedGlycopeptide.structure_id == Glycopeptide.id).filter(
IdentifiedGlycopeptide.analysis_id == analysis.id).all()
if output_path is None:
output_stream = ctxstream(click.get_binary_stream('stdout'))
else:
output_stream = open(output_path, 'wb')
with output_stream:
job = ImportableGlycanHypothesisCSVSerializer(output_stream, glycans)
job.run()
def glycopeptide_hypothesis(database_connection, hypothesis_identifier, output_path, multifasta=False):
'''Write each theoretical glycopeptide in CSV format
'''
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session()
hypothesis = get_by_name_or_id(session, GlycopeptideHypothesis, hypothesis_identifier)
def generate():
interval = 100000
i = 0
while True:
session.expire_all()
chunk = hypothesis.glycopeptides.slice(i, i + interval).all()
if len(chunk) == 0:
break
for glycopeptide in chunk:
yield glycopeptide
i += interval
if output_path is None:
output_stream = ctxstream(sys.stdout)
else:
output_stream = open(output_path, 'wb')
with output_stream:
job = GlycopeptideHypothesisCSVSerializer(output_stream, generate())
job.run()
def sql_shell(database_connection, script=None):
db = DatabaseBoundOperation(database_connection)
session = db.session()
interpreter = SQLShellInterpreter(session)
if script is None:
interpreter.cmdloop()
else:
result = session.execute(script)
interpreter._to_csv(list(result), sys.stdout)
def glycan_composition_identification(database_connection, analysis_identifier, output_path=None,
threshold=0, report=False):
'''Write each glycan chromatogram in CSV format
'''
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session()
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycan_lc_ms:
click.secho("Analysis %r is of type %r." % (
str(analysis.name), str(analysis.analysis_type)), fg='red', err=True)
raise click.Abort()
analysis_id = analysis.id
if output_path is None:
output_stream = ctxstream(sys.stdout)
else:
output_stream = open(output_path, 'wb')
if report:
with output_stream:
job = GlycanChromatogramReportCreator(
database_connection._original_connection,
analysis_id, output_stream, threshold=threshold)
job.run()
else:
def generate():
i = 0
interval = 100
query = session.query(GlycanCompositionChromatogram).filter(
GlycanCompositionChromatogram.analysis_id == analysis_id,
GlycanCompositionChromatogram.score > threshold)
while True:
session.expire_all()
chunk = query.slice(i, i + interval).all()
if len(chunk) == 0:
break
for gcs in chunk:
yield gcs.convert()
i += interval
i = 0
query = session.query(UnidentifiedChromatogram).filter(
UnidentifiedChromatogram.analysis_id == analysis_id,
UnidentifiedChromatogram.score > threshold)
while True:
session.expire_all()
chunk = query.slice(i, i + interval).all()
if len(chunk) == 0:
break
for gcs in chunk:
yield gcs.convert()
i += interval
with output_stream:
job = GlycanLCMSAnalysisCSVSerializer(output_stream, generate())
job.run()
def sql_shell(database_connection, script=None):
db = DatabaseBoundOperation(database_connection)
session = db.session() # pylint: disable=not-callable
interpreter = SQLShellInterpreter(session)
if script is None:
interpreter.cmdloop()
else:
result = session.execute(script)
interpreter._to_csv(list(result), sys.stdout)
def glycan_composition_identification(database_connection, analysis_identifier, output_path=None,
threshold=0, report=False):
'''Write each glycan chromatogram in CSV format
'''
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session()
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycan_lc_ms:
click.secho("Analysis %r is of type %r." % (
str(analysis.name), str(analysis.analysis_type)), fg='red', err=True)
raise click.Abort()
analysis_id = analysis.id
if output_path is None:
output_stream = ctxstream(sys.stdout)
else:
output_stream = open(output_path, 'wb')
if report:
with output_stream:
job = GlycanChromatogramReportCreator(
database_connection._original_connection,
analysis_id, output_stream, threshold=threshold)
job.run()
else:
def generate():
i = 0
interval = 100
query = session.query(GlycanCompositionChromatogram).filter(
GlycanCompositionChromatogram.analysis_id == analysis_id,
GlycanCompositionChromatogram.score > threshold)
while True:
session.expire_all()
chunk = query.slice(i, i + interval).all()
if len(chunk) == 0:
break
for gcs in chunk:
yield gcs.convert()
i += interval
i = 0
query = session.query(UnidentifiedChromatogram).filter(
UnidentifiedChromatogram.analysis_id == analysis_id,
UnidentifiedChromatogram.score > threshold)
while True:
session.expire_all()
chunk = query.slice(i, i + interval).all()
if len(chunk) == 0:
break
for gcs in chunk:
yield gcs.convert()
i += interval
with output_stream:
job = GlycanLCMSAnalysisCSVSerializer(output_stream, generate())
job.run()
def stream_from_hypotheses(self, connection, hypothesis_id):
self.log("Streaming from %s for hypothesis %d" % (connection, hypothesis_id))
connection = DatabaseBoundOperation(connection)
session = connection.session()
for db_composition in session.query(DBGlycanComposition).filter(
DBGlycanComposition.hypothesis_id == hypothesis_id):
structure_classes = list(db_composition.structure_classes)
if len(structure_classes) > 0:
yield db_composition, [sc.name for sc in db_composition.structure_classes]
else:
yield db_composition, [None]
def stream_from_hypotheses(self, connection, hypothesis_id):
self.log("Streaming from %s for hypothesis %d" % (connection, hypothesis_id))
connection = DatabaseBoundOperation(connection)
session = connection.session()
for db_composition in session.query(DBGlycanComposition).filter(
DBGlycanComposition.hypothesis_id == hypothesis_id):
structure_classes = list(db_composition.structure_classes)
if len(structure_classes) > 0:
yield db_composition, [sc.name for sc in db_composition.structure_classes]
else:
yield db_composition, [None]
def glycopeptide_identification(database_connection, analysis_identifier, output_path=None,
report=False, mzml_path=None, threshold=0):
'''Write each distinct identified glycopeptide in CSV format
'''
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session()
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycopeptide_lc_msms:
click.secho("Analysis %r is of type %r." % (
str(analysis.name), str(analysis.analysis_type)), fg='red', err=True)
raise click.Abort()
analysis_id = analysis.id
if output_path is None:
output_stream = ctxstream(sys.stdout)
else:
output_stream = open(output_path, 'wb')
if report:
with output_stream:
if mzml_path is None:
mzml_path = analysis.parameters['sample_path']
if not os.path.exists(mzml_path):
raise click.ClickException(
("Sample path {} not found. Pass the path to"
" this file as `-m/--mzml-path` for this command.").format(
mzml_path))
GlycopeptideDatabaseSearchReportCreator(
database_connection._original_connection, analysis_id,
stream=output_stream, threshold=threshold,
mzml_path=mzml_path).run()
else:
query = session.query(Protein.id, Protein.name).join(Protein.glycopeptides).join(
IdentifiedGlycopeptide).filter(
IdentifiedGlycopeptide.analysis_id == analysis.id)
protein_index = dict(query)
def generate():
i = 0
interval = 100
query = session.query(IdentifiedGlycopeptide).filter(
IdentifiedGlycopeptide.analysis_id == analysis_id)
while True:
session.expire_all()
chunk = query.slice(i, i + interval).all()
if len(chunk) == 0:
break
for glycopeptide in chunk:
yield glycopeptide.convert()
i += interval
with output_stream:
job = GlycopeptideLCMSMSAnalysisCSVSerializer(output_stream, generate(), protein_index)
job.run()
def glycopeptide_identification(database_connection, analysis_identifier, output_path=None,
report=False, mzml_path=None, threshold=0):
'''Write each distinct identified glycopeptide in CSV format
'''
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session()
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycopeptide_lc_msms:
click.secho("Analysis %r is of type %r." % (
str(analysis.name), str(analysis.analysis_type)), fg='red', err=True)
raise click.Abort()
analysis_id = analysis.id
if output_path is None:
output_stream = ctxstream(sys.stdout)
else:
output_stream = open(output_path, 'wb')
if report:
with output_stream:
if mzml_path is None:
mzml_path = analysis.parameters['sample_path']
if not os.path.exists(mzml_path):
raise click.ClickException(
("Sample path {} not found. Pass the path to"
" this file as `-m/--mzml-path` for this command.").format(
mzml_path))
GlycopeptideDatabaseSearchReportCreator(
database_connection._original_connection, analysis_id,
stream=output_stream, threshold=threshold,
mzml_path=mzml_path).run()
else:
query = session.query(Protein.id, Protein.name).join(Protein.glycopeptides).join(
IdentifiedGlycopeptide).filter(
IdentifiedGlycopeptide.analysis_id == analysis.id)
protein_index = dict(query)
def generate():
i = 0
interval = 100
query = session.query(IdentifiedGlycopeptide).filter(
IdentifiedGlycopeptide.analysis_id == analysis_id)
while True:
session.expire_all()
chunk = query.slice(i, i + interval).all()
if len(chunk) == 0:
break
for glycopeptide in chunk:
yield glycopeptide.convert()
i += interval
with output_stream:
job = GlycopeptideLCMSMSAnalysisCSVSerializer(output_stream, generate(), protein_index)
job.run()
def glycopeptide_training_mgf(database_connection, analysis_identifier, output_path=None,
mzml_path=None, threshold=None):
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session()
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycopeptide_lc_msms:
click.secho("Analysis %r is of type %r." % (
str(analysis.name), str(analysis.analysis_type)), fg='red', err=True)
raise click.Abort()
if output_path is None:
output_stream = ctxstream(sys.stdout)
else:
output_stream = open(output_path, 'wb')
with output_stream:
TrainingMGFExporter.from_analysis(
database_connection, analysis.id, output_stream, mzml_path, threshold).run()
def glycopeptide_training_mgf(database_connection, analysis_identifier, output_path=None,
mzml_path=None, threshold=None):
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session()
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycopeptide_lc_msms:
click.secho("Analysis %r is of type %r." % (
str(analysis.name), str(analysis.analysis_type)), fg='red', err=True)
raise click.Abort()
if output_path is None:
output_stream = ctxstream(sys.stdout)
else:
output_stream = open(output_path, 'wb')
with output_stream:
TrainingMGFExporter.from_analysis(
database_connection, analysis.id, output_stream, mzml_path, threshold).run()
def annotate_matched_spectra(database_connection, analysis_identifier, output_path, mzml_path=None):
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session()
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycopeptide_lc_msms:
click.secho("Analysis %r is of type %r." % (
str(analysis.name), str(analysis.analysis_type)), fg='red', err=True)
raise click.Abort()
if output_path is None:
output_path = os.path.dirname(database_connection._original_connection)
task = SpectrumAnnotatorExport(
database_connection._original_connection, analysis.id, output_path,
mzml_path)
task.display_header()
task.start()
def glycopeptide_chromatogram_records(database_connection,
analysis_identifier,
output_path,
apex_time_range=None):
if apex_time_range is None:
apex_time_range = (0, float('inf'))
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session() # pylint: disable=not-callable
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycopeptide_lc_msms:
click.secho("Analysis %r is of type %r." %
(str(analysis.name), str(analysis.analysis_type)),
fg='red',
err=True)
raise click.Abort()
if output_path is None:
fh = click.get_binary_stream('stdout')
else:
fh = open(output_path, 'wb')
idgps = session.query(IdentifiedGlycopeptide).filter(
IdentifiedGlycopeptide.analysis_id == analysis.id).all()
n = len(idgps)
from glycan_profiling.scoring.elution_time_grouping import GlycopeptideChromatogramProxy
cases = []
analysis_name = analysis.name
start_time, stop_time = apex_time_range
for i, idgp in enumerate(idgps):
if i % 50 == 0:
click.echo("%d/%d Records Processed" % (i, n), err=True)
if idgp.chromatogram is None:
continue
if idgp.ms1_score < 0:
continue
obj = GlycopeptideChromatogramProxy.from_obj(
idgp,
ms1_score=idgp.ms1_score,
ms2_score=idgp.ms2_score,
q_value=idgp.q_value,
analysis_name=analysis_name,
mass_shifts=';'.join(
[m.name for m in idgp.chromatogram.mass_shifts]))
if obj.apex_time < start_time or obj.apex_time > stop_time:
continue
cases.append(obj)
click.echo("Writing %d Records" % (len(cases), ), err=True)
with fh:
GlycopeptideChromatogramProxy.to_csv(cases, csv_stream(fh))
def glycopeptide_spectrum_matches(database_connection,
analysis_identifier,
output_path=None):
'''Write each matched glycopeptide spectrum in CSV format
'''
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session() # pylint: disable=not-callable
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycopeptide_lc_msms:
click.secho("Analysis %r is of type %r." %
(str(analysis.name), str(analysis.analysis_type)),
fg='red',
err=True)
raise click.Abort()
analysis_id = analysis.id
query = session.query(Protein.id, Protein.name).join(
Protein.glycopeptides).join(GlycopeptideSpectrumMatch).filter(
GlycopeptideSpectrumMatch.analysis_id == analysis.id)
protein_index = dict(query)
def generate():
i = 0
interval = 100000
query = session.query(GlycopeptideSpectrumMatch).filter(
GlycopeptideSpectrumMatch.analysis_id == analysis_id).order_by(
GlycopeptideSpectrumMatch.scan_id)
while True:
session.expire_all()
chunk = query.slice(i, i + interval).all()
if len(chunk) == 0:
break
for glycopeptide in chunk:
yield glycopeptide.convert()
i += interval
if output_path is None:
output_stream = ctxstream(click.get_binary_stream('stdout'))
else:
output_stream = open(output_path, 'wb')
with output_stream:
job = GlycopeptideSpectrumMatchAnalysisCSVSerializer(
output_stream, generate(), protein_index)
job.run()
def annotate_matched_spectra(database_connection,
analysis_identifier,
output_path,
mzml_path=None):
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session() # pylint: disable=not-callable
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycopeptide_lc_msms:
click.secho("Analysis %r is of type %r." %
(str(analysis.name), str(analysis.analysis_type)),
fg='red',
err=True)
raise click.Abort()
if output_path is None:
output_path = os.path.dirname(database_connection._original_connection)
task = SpectrumAnnotatorExport(database_connection._original_connection,
analysis.id, output_path, mzml_path)
task.display_header()
task.start()
def glycopeptide_mzidentml(database_connection, analysis_identifier, output_path=None,
mzml_path=None):
'''Write identified glycopeptides as mzIdentML file, and associated MSn spectra
to a paired mzML file if the matched data are available. If an mzML file is written
it will also contain the extracted ion chromatograms for each glycopeptide with an
extracted elution profile.
'''
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session()
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycopeptide_lc_msms:
click.secho("Analysis %r is of type %r." % (
str(analysis.name), str(analysis.analysis_type)), fg='red', err=True)
raise click.Abort()
loader = AnalysisDeserializer(
database_connection._original_connection, analysis_id=analysis.id)
glycopeptides = loader.load_identified_glycopeptides()
with open(output_path, 'wb') as outfile:
writer = MzIdentMLSerializer(
outfile, glycopeptides, analysis, loader, source_mzml_path=mzml_path)
writer.run()
def export_identified_glycans_from_glycopeptides(database_connection, analysis_identifier, output_path):
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session()
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycopeptide_lc_msms:
click.secho("Analysis %r is of type %r." % (
str(analysis.name), str(analysis.analysis_type)), fg='red', err=True)
raise click.Abort()
glycans = session.query(GlycanComposition).join(
GlycanCombinationGlycanComposition).join(GlycanCombination).join(
Glycopeptide,
Glycopeptide.glycan_combination_id == GlycanCombination.id).join(
IdentifiedGlycopeptide,
IdentifiedGlycopeptide.structure_id == Glycopeptide.id).filter(
IdentifiedGlycopeptide.analysis_id == analysis.id).all()
if output_path is None:
output_stream = ctxstream(sys.stdout)
else:
output_stream = open(output_path, 'wb')
with output_stream:
job = ImportableGlycanHypothesisCSVSerializer(output_stream, glycans)
job.run()
def glycopeptide_spectrum_matches(database_connection, analysis_identifier, output_path=None):
'''Write each matched glycopeptide spectrum in CSV format
'''
database_connection = DatabaseBoundOperation(database_connection)
session = database_connection.session()
analysis = get_by_name_or_id(session, Analysis, analysis_identifier)
if not analysis.analysis_type == AnalysisTypeEnum.glycopeptide_lc_msms:
click.secho("Analysis %r is of type %r." % (
str(analysis.name), str(analysis.analysis_type)), fg='red', err=True)
raise click.Abort()
analysis_id = analysis.id
query = session.query(Protein.id, Protein.name).join(Protein.glycopeptides).join(
GlycopeptideSpectrumMatch).filter(
GlycopeptideSpectrumMatch.analysis_id == analysis.id)
protein_index = dict(query)
def generate():
i = 0
interval = 100000
query = session.query(GlycopeptideSpectrumMatch).filter(
GlycopeptideSpectrumMatch.analysis_id == analysis_id).order_by(
GlycopeptideSpectrumMatch.scan_id)
while True:
session.expire_all()
chunk = query.slice(i, i + interval).all()
if len(chunk) == 0:
break
for glycopeptide in chunk:
yield glycopeptide.convert()
i += interval
if output_path is None:
output_stream = ctxstream(sys.stdout)
else:
output_stream = open(output_path, 'wb')
with output_stream:
job = GlycopeptideSpectrumMatchAnalysisCSVSerializer(output_stream, generate(), protein_index)
job.run()