pipeline_gwasenrichment.py

##############################################################################
#
#   MRC FGU CGAT
#
#   $Id$
#
#   Copyright (C) 2009 Andreas Heger
#
#   This program is free software; you can redistribute it and/or
#   modify it under the terms of the GNU General Public License
#   as published by the Free Software Foundation; either version 2
#   of the License, or (at your option) any later version.
#
#   This program is distributed in the hope that it will be useful,
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#   GNU General Public License for more details.
#
#   You should have received a copy of the GNU General Public License
#   along with this program; if not, write to the Free Software
#   Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
###############################################################################
"""===========================
Pipeline template
===========================

:Author: Mike Morgan
:Release: $Id$
:Date: |today|
:Tags: Python

Overview
========

This pipeline pulls together different genomic annotations, provided by
the user, over different cell types, and test for enrichment of overlap
for input SNP set(s).  It then finds the SNPs overlapping the enriched
annotations and looks for both enriched motifs in those intervals,
and finds SNPs overlapping the motifs.  These are tested for their
disruptive ability using motifbreakR.

Usage
=====

See :ref:`PipelineSettingUp` and :ref:`PipelineRunning` on general
information how to use CGAT pipelines.

Configuration
-------------

The pipeline requires a configured :file:`pipeline.ini` file.
CGATReport report requires a :file:`conf.py` and optionally a
:file:`cgatreport.ini` file (see :ref:`PipelineReporting`).

Default configuration files can be generated by executing:

   python <srcdir>/pipeline_gwasenrichment.py config

Input files
-----------

None required except the pipeline configuration files.

Requirements
------------

The pipeline requires the results from
:doc:`pipeline_annotations`. Set the configuration variable
:py:data:`annotations_database` and :py:data:`annotations_dir`.

On top of the default CGAT setup, the pipeline requires the following
software to be in the path:

.. Add any additional external requirements such as 3rd party software
   or R modules below:

Requirements:

* samtools >= 1.1

Pipeline output
===============

.. Describe output files of the pipeline here

Glossary
========

.. glossary::


Code
====

"""
from ruffus import *
from ruffus.combinatorics import *

import sys
import os
import sqlite3
import CGAT.Experiment as E
import CGATPipelines.Pipeline as P

# load options from the config file
PARAMS = P.getParameters(
    ["%s/pipeline.ini" % os.path.splitext(__file__)[0],
     "../pipeline.ini",
     "pipeline.ini"])

# add configuration values from associated pipelines
#
# 1. pipeline_annotations: any parameters will be added with the
#    prefix "annotations_". The interface will be updated with
#    "annotations_dir" to point to the absolute path names.
PARAMS.update(P.peekParameters(
    PARAMS["annotations_dir"],
    "pipeline_annotations.py",
    on_error_raise=__name__ == "__main__",
    prefix="annotations_",
    update_interface=True))


# if necessary, update the PARAMS dictionary in any modules file.
# e.g.:
#
# import CGATPipelines.PipelineGeneset as PipelineGeneset
# PipelineGeneset.PARAMS = PARAMS
#
# Note that this is a hack and deprecated, better pass all
# parameters that are needed by a function explicitely.

# -----------------------------------------------
# Utility functions
def connect():
    '''utility function to connect to database.

    Use this method to connect to the pipeline database.
    Additional databases can be attached here as well.

    Returns an sqlite3 database handle.
    '''

    dbh = sqlite3.connect(PARAMS["database"])
    statement = '''ATTACH DATABASE '%s' as annotations''' % (
        PARAMS["annotations_database"])
    cc = dbh.cursor()
    cc.execute(statement)
    cc.close()

    return dbh


# ---------------------------------------------------#
# ---------------------------------------------------#
# ---------------------------------------------------#
# Get the relevant annotations first for enrichments
# Merge enrichments into a single file if necessary
@follows(mkdir("cell_types.dir"))
@collate("%s/*.bed.gz" % PARAMS['bed_files'],
         regex("%s/(.+)-(.+)-(.+).bed.gz" % PARAMS['bed_files']),
         r"cell_types.dir/\1-\2.bed.gz")
def mergeCellTypeAnnotations(infiles, outfile):
    '''
    Use bedtools intersect to get common features
    for each annotation for a given cell/tissue type

    Expected filename format: <Tissue>-<annotation>-<replicate>.bed.gz
    Files must be chrom:start sorted
    '''

    cell_files = " ".join(infiles)
    job_memory = "8G"

    if len(infiles) == 2:
        file1 = infiles[0]
        file2 = infiles[1]
        statement = '''
        bedtools intersect -a %(file1)s -b %(file2)s
        | gzip > %(outfile)s
        '''

    elif len(infiles) > 2:
        statement = '''
        multiIntersectBed -i %(cell_files)s |
        awk '{if($4 == 3) {print $0}' | gzip
        > %(outfile)s
        '''

    else:
        statement = '''
        ln -s %(cell_files)s %(outfile)s
        '''

    P.run()


@follows(mkdir("bed.dir"),
         mergeCellTypeAnnotations)
@collate("cell_types.dir/*.bed.gz",
         regex("cell_types.dir/(.+)-(.+).bed.gz"),
         r"bed.dir/\1.bed.gz")
def mergeBedAnnotations(infiles, outfile):
    '''
    Merge together multiple BED format annotations
    into a single file.  Give the annotation type,
    assumed to be element 2 of the file name,
    as the annotation name in the output file.
    '''

    all_files = ",".join(infiles)

    job_memory = "1G"

    if len(infiles) > 1:
        statement = '''
        python /ifs/devel/projects/proj045/enrichment_pipeline/merge_beds.py
        --regex-filename="-(.+).bed.gz"
        --log=%(outfile)s.log
        %(all_files)s
        | gzip > %(outfile)s
        '''

        P.run()
    else:
        statement = '''
        cp %(all_files)s %(outfile)s
        '''

        P.run()


# assign SNPs to linkage/haplotype blocks in bed format
# process SNP list for GoShifter

@follows(mkdir("snpsets.dir"))
@transform("%s/*.tsv" % PARAMS['snpset_dir'],
           regex("(.+)/(.+)-goshifter.tsv"),
           add_inputs("%s/*.bim" % PARAMS['snpset_bims']),
           r"snpsets.dir/\2.snpmap")
def convertSNPstoSNPset(infiles, outfile):
    '''
    Convert a list of SNP IDs to a GoShifter
    compatible SNP file using position information
    from Plink .bim files
    '''

    snp_file = infiles[0]
    bim_files = ",".join(infiles[1:])
    job_memory = "12G"

    statement = '''
    python /ifs/devel/projects/proj045/enrichment_pipeline/snps2snpset.py
    --bim-file=%(bim_files)s
    --log=%(outfile)s.log
    %(snp_file)s
    > %(outfile)s
    '''

    P.run()


@follows(convertSNPstoSNPset)
@transform("%s/*.tsv" % PARAMS['snpset_dir'],
           regex("(.+)/(.+)-gat.tsv"),
           add_inputs("%s/*.bim" % PARAMS['snpset_bims']),
           r"snpsets.dir/\2.snpset.bed.gz")
def convertSnpsetToBed(infiles, outfile):
    '''
    Convert a SNP set file into a BED4 format
    file
    '''


    snp_file = infiles[0]
    bim_files = ",".join(infiles[1:])
    job_memory = "12G"

    statement = '''
    python /ifs/devel/projects/proj045/enrichment_pipeline/snps2snpset.py
    --bim-file=%(bim_files)s
    --log=%(outfile)s.log
    %(snp_file)s
    | awk '{if(NR > 1) {printf("%%s\\t%%s\\t%%s\\t%%s\\n", $2, $3, $3+1, $1)}}'
    | gzip > %(outfile)s
    '''

    P.run()


@follows(mkdir("haplotypes.dir"))
@transform("%s/*.blocks.det" % PARAMS['snpset_haplotypes'],
           regex("%s/(.+).blocks.det" % PARAMS['snpset_haplotypes']),
           r"haplotypes.dir/\1.bed.gz")
def convertHaplotypes2Bed(infile, outfile):
    '''
    Convert Plink1.9 haplotype blocks to bed interval
    format.
    '''

    job_memory = "1G"

    statement = '''
    python /ifs/devel/projects/proj045/enrichment_pipeline/haplotypes2bed.py
    --header
    --contig-column=1
    --start-column=2
    --end-column=3
    --snp-column=6
    --log=%(outfile)s.log
    <( cat %(infile)s | tr -s ' ' '\\t' | sed 's/^[[:space:]]*//g' | sed 's/*[[:space:]]$//g')
    | gzip > %(outfile)s
    '''

    P.run()

@follows(convertHaplotypes2Bed)
@collate(convertHaplotypes2Bed,
         regex("haplotypes.dir/(.+).bed.gz"),
         r"haplotypes.dir/WholeGenome.bed.gz")
def mergeHaplotypeBeds(infiles, outfile):
    '''
    Merge individual chromosome haplotype
    bed files
    '''

    infiles = " ".join(infiles)

    statement = '''
    zcat %(infiles)s | gzip >> %(outfile)s
    '''

    P.run()

@follows(mergeHaplotypeBeds,
         mkdir("snpsets.dir"))
@transform(convertSnpsetToBed,
           regex("(.+)/(.+).snpset.bed.gz"),
           add_inputs(mergeHaplotypeBeds),
           r"snpsets.dir/\2.haplotype.bed.gz")
def convertSet2Block(infiles, outfile):
    '''
    Convert list of SNP BED file into
    interval haplotype or linkage blocks

    Use bedtools intersect, left outer join.

    Requires bedtools is in your $PATH variable
    '''

    snp_file = infiles[0]
    haplotype_file = infiles[1]

    job_memory = "1G"

    statement = '''
    bedtools
    intersect -a %(snp_file)s  -b %(haplotype_file)s  -wb -loj |
    awk '{if($6 == -1) {print($1, $2, $3, $4)}
    else{printf("%%s\\t%%s\\t%%s\\t%%s\\n", $5, $6, $7, $4)}}'
    | sed 's/ /\\t/g' |
    gzip > %(outfile)s
    '''

    P.run()

# ---------------------------------------------------#
# ---------------------------------------------------#
# ---------------------------------------------------#
# Perform enrichment analysis of SNPset and annotation
# overlap with GAT

@follows(convertSet2Block,
         mergeBedAnnotations,
         mkdir("gat.dir"))
@product(convertSet2Block,
         formatter("(.bed.gz)$"),
         mergeBedAnnotations,
         formatter("(.bed.gz$)"),
         "gat.dir/"
         "{basename[0][0]}_vs_"
         "{basename[1][0]}.gat")
def genomicAsssociationTest(infiles, outfile):
    '''
    Use GAT to test for enrichment of overlap
    between SNP sets and annotations of interest.

    GAT must be in the $PATH variable

    '''

    job_memory = "10G"
    snpset = infiles[0]
    annotations = infiles[1]

    statement = '''
    gat-run.py
    --segments=%(snpset)s
    --annotations=%(annotations)s
    --workspace=%(gat_ungapped)s
    --workspace=%(gat_mappability)s
    --ignore-segment-tracks
    --num-samples=%(gat_samples)s
    --log=%(outfile)s.log
    > %(outfile)s
    '''

    P.run()


@follows(genomicAsssociationTest)
@collate(genomicAsssociationTest,
         regex("gat.dir/(.+)\.(.+).bed_vs_(.+).bed.gat"),
         r"gat.dir/\1-\2.gat")
def mergeGatResults(infiles, outfile):
    '''
    Merge output from GAT into a single table
    '''

    job_memory = "1G"

    infiles = " ".join(infiles)

    statement = '''
    python %(scriptsdir)s/combine_tables.py
    --columns=2
    --take=8,9,10,11
    --add-file-prefix
    --regex-filename="_vs_(.+).bed.gat"
    --log=%(outfile)s.log
    %(infiles)s
    > %(outfile)s
    '''

    P.run()


# need to intersect enriched annotations with SNPs of interest
@follows(mergeGatResults,
         mkdir("overlaps.dir"))
@transform(mergeGatResults,
           regex("gat.dir/(.+)-(.+).gat"),
           add_inputs(r"snpsets.dir/\1.snpset.bed.gz"),
           r"overlaps.dir/\1-SNP_overlap.bed.gz")
def overlapSnpsWithAnnotations(infiles, outfile):
    '''
    Get SNPs that overlap enriched annotations
    across all cell types
    '''

    annotation_dir = "bed.dir"
    gat_results = infiles[0]
    snp_bed = infiles[1]

    job_memory = "1G"

    # set q-value and log2 fold enrichment in the config
    statement = '''
    python /ifs/devel/projects/proj045/enrichment_pipeline/snp2annotation.py
    --log=%(outfile)s.log
    --annotation-dir=%(annotation_dir)s
    --q-value-threshold=%(gat_qval_thresh)f
    --l2fold-threshold=%(gat_l2fold_thresh)f
    --snp-bed=%(snp_bed)s
    %(gat_results)s
    | gzip > %(outfile)s
    '''

    P.run()


# -------------------------------------------------------- #
# GoShifter SNP-annotation enrichment
# -------------------------------------------------------- #

@follows(convertSNPstoSNPset,
         mkdir("goshifter.dir"))
@product(convertSNPstoSNPset,
         formatter("(.snpmap$)"),
         "cell_types.dir/*.bed.gz",
         formatter("(.bed.gz)"),
         "goshifter.dir/"
         "{basename[0][0]}_vs_"
         "{basename[1][0]}.goshift")
def goShifter(infiles, outfile):
    '''
    Test enrichment of overlap between SNP sets
    and individual annotations using
    GoShifter.
    '''

    job_memory = "12G"

    snpset = infiles[0]
    annotations = infiles[1]

    statement = '''
    goshifter.py
    --snpmap %(snpset)s
    --annotation %(annotations)s
    --permute %(goshifter_perms)s
    --ld %(goshifter_ld)s
    --out %(outfile)s
    > %(outfile)s
    '''

    P.run()


@follows(goShifter)
@transform(goShifter,
           regex("goshifter.dir/(.+).goshift"),
           r"goshifter.dir/\1.pval")
def extractPvalueGoshifter(infile, outfile):
    '''
    Extract the GoShifter enrichment p-value from
    the log output.
    '''

    job_memory = "1G"

    statement = '''
    tail -n 10 %(infile)s | grep "p-value" |
    cut -f 2 -d "=" | sed 's/ //g' > %(outfile)s
    '''

    P.run()


@follows(extractPvalueGoshifter)
@transform(goShifter,
           regex("goshifter.dir/(.+).goshift"),
           r"goshifter.dir/\1.boundary")
def extractAnnotationBoundaryGoshifter(infile, outfile):
    '''
    Extract the extension size of the LD boundary
    '''

    job_memory = "1G"

    statement = '''
    tail -n 10 %(infile)s | grep "extended by" |
    cut -f 8 -d " " > %(outfile)s
    '''

    P.run()


@follows(extractAnnotationBoundaryGoshifter,
         goShifter)
@transform("goshifter.dir/*.enrich",
           regex("goshifter.dir/(.+).goshift.(.+).enrich"),
           r"goshifter.dir/\1.summary")
def extractEnrichmentSummary(infile, outfile):
    '''
    Calculate the median enrichment score across SNPs
    for each annotation tested
    '''

    job_memory = "1G"

    statement = '''
    python /ifs/devel/projects/proj045/enrichment_pipeline/enrich2stats.py
    --method=stat
    --statistic=summary
    --data-column=3
    --log=%(outfile)s.log
    %(infile)s
    > %(outfile)s
    '''

    P.run()


@follows(goShifter,
         extractPvalueGoshifter,
         extractAnnotationBoundaryGoshifter,
         extractEnrichmentSummary)
@collate([extractPvalueGoshifter,
          extractEnrichmentSummary,
          extractAnnotationBoundaryGoshifter],
         regex("goshifter.dir/(.+).bed.(.+)"),
         r"goshifter.dir/\1.results")
def mergeGoshifterResults(infiles, outfile):
    '''
    Merge output from Goshifter into a single table
    for each tested annotation
    '''

    job_memory = "1G"

    infiles = ",".join(infiles)

    statement = '''
    python /ifs/devel/projects/proj045/enrichment_pipeline/enrich2stats.py
    --method=merge
    --log=%(outfile)s.log
    %(infiles)s
    > %(outfile)s
    '''

    P.run()


@follows(mergeGoshifterResults)
@collate(mergeGoshifterResults,
         regex("goshifter.dir/(.+)_vs_(.+)-(.+).results"),
         r"goshifter.dir/\1.goshifter")
def mergeAllGoshifterResults(infiles, outfile):
    '''
    Merge together all of the GoShifter summary
    results over all tested annotations and cell types
    '''

    job_memory = "4G"

    infiles = ",".join(infiles)

    statement = '''
    python /ifs/devel/projects/proj045/enrichment_pipeline/enrich2stats.py
    --method=append
    --log=%(outfile)s.log   
    %(infiles)s
    > %(outfile)s
    '''

    P.run()


@follows(mergeAllGoshifterResults)
@collate("goshifter.dir/*.locusscore",
         regex("goshifter.dir/(.+)_vs_(.+).bed.(.+).locusscore"),
         r"goshifter.dir/\1.snpscores")
def mergeSnpScores(infiles, outfile):
    '''
    Merge together SNP scores for each
    cell type - annotation combination
    '''

    job_memory = "6G"

    infiles = " ".join(infiles)

    statement = '''
    python %(scriptsdir)s/combine_tables.py
    --columns=1
    --take=3
    --add-file-prefix
    --regex-filename="_vs_(.+).bed.goshift.nperm(\d+).locusscore"
    --log=%(outfile)s.log
    %(infiles)s
    > %(outfile)s
    '''

    P.run()


# --------------------------------------------------- #
# Test enriched annotation intervals for motif enrichment
# using MEME-suite tools (Centrimo)

# need the user to define which cell type(s) and
# annotation are of interest
@follows(overlapSnpsWithAnnotations,
         mkdir("fasta.dir"))
@transform(overlapSnpsWithAnnotations,
           regex("overlaps.dir/(.+)-SNP_overlap.bed.gz"),
           r"overlaps.dir/\1-intervals.bed.gz")
def getAnnotationsOverlapWithSnps(infile, outfile):
    '''
    Get the enriched annotation intervals that overlap
    SNPs.
    '''

    job_memory = "1G"

    statement = '''
    zcat %(infile)s | grep -P %(annotations_regex)s
    | grep -P %(annotations_cell_regex)s
    | cut -f 5-9
    | awk '{printf("%%s\\t%%s\\t%%s\\t%%s_%%s\\n", $1,$2,$3,$5,$4)}'
    | gzip > %(outfile)s
    '''

    P.run()


@follows(getAnnotationsOverlapWithSnps,
         mkdir("enriched_annotation.dir"))
@subdivide(getAnnotationsOverlapWithSnps,
           regex("overlaps.dir/(.+)-intervals.bed.gz"),
           r"enriched_annotation.dir/split-enriched.txt")
def splitEnrichedAnnotations(infile, outfile):
    '''
    Split enriched annotations overlapping SNPs
    into separate BED files
    '''

    job_memory = "1G"

    # what an awk statement!
    # it redirects each line to a file that matches
    # the cell type_annotation field in column 4
    

    statement = '''
    zcat %(infile)s |
    awk '{if(NR == 1) {s = $4; print $0 > "enriched_annotation.dir/"s".bed";} 
    else {s_1 = $4;} {if(s == s_1) {print $0 > "enriched_annotation.dir/"s".bed";} 
    else {s = s_1; print $0 > "enriched_annotation.dir/"s_1".bed"}}}'
    '''

    P.run()
    P.touch(outfile)


@follows(splitEnrichedAnnotations)
@transform("enriched_annotation.dir/*.bed",
           regex("enriched_annotation.dir/(.+).bed"),
           r"enriched_annotation.dir/\1-slop.bed.gz")
def slopIntervals(infile, outfile):
    '''
    Set all annotation intervals to a specific size
    for MEME suite tools <- use bedtools slop
    '''

    job_memory = "2G"

    interval = PARAMS['bed_interval']/2.0

    genome_contigs = "/".join([PARAMS['annotations_dir'],
                               PARAMS['annotations_contigs']])

    # need to set the interval to the midpoint of the input interval
    # before slopping with bedtools
    # add a numeric id to interval name
    statement = '''
    cat %(infile)s |
    awk '{printf("%%s\\t%%i\\t%%i\\t%%s_%%i\\n", $1, ($2 + ($3 - $2)/2),
    ($2 + ($3 - $2)/2), $4, NR)}'
    | bedtools slop -l %(interval)s  -r %(interval)s -i -
    -g %(genome_contigs)s
    | gzip > %(outfile)s
    '''
    
    P.run()


@follows(splitEnrichedAnnotations,
         slopIntervals,
         mkdir("fasta.dir"))
@transform("enriched_annotation.dir/*.bed.gz",
           regex("enriched_annotation.dir/(.+)-slop.bed.gz"),
           r"fasta.dir/\1.fa")
def getIntervalFasta(infile, outfile):
    '''
    Retrieve the genome sequence as FASTA
    for slopped intervals <- required
    input for MEME-chip
    '''

    job_memory = "4G"

    statement = '''
    zcat %(infile)s |
    python %(scriptsdir)s/bed2fasta.py
    --genome=%(genome_fasta)s
    --log=%(outfile)s.log
    > %(outfile)s
    '''

    P.run()


@follows(getIntervalFasta,
         mkdir("meme.dir"))
@transform(getIntervalFasta,
           regex("fasta.dir/(.+).fa"),
           r"meme.dir/\1.dir/motif_alignment.txt")
def runMemeChip(infile, outfile):
    '''
    Run the MEME-suite tools on the
    input interval sequences to test for
    enriched motifs.  If there are very
    few intervals there is unlikely to be much
    power to detect enrichments
    '''

    job_memory = "4G"

    out_dir = "/".join(outfile.split("/")[:-1])
    # meme-chip needs to be in the $PATH
    # variable
    statement = '''
    meme-chip %(infile)s
    -oc %(out_dir)s
    -db %(motifs_db)s
    '''

    P.run()


# find SNPs overlapping enriched annotations
# at test for motif disruption

@follows(runMemeChip,
         overlapSnpsWithAnnotations,
         mkdir("plots.dir", "motifs.dir"))
@transform(overlapSnpsWithAnnotations,
           regex("overlaps.dir/(.+)-SNP_overlap.bed.gz"),
           r"motifs.dir/\1-enriched-disrupted.tsv")
def testMotifDisruptingSnpsEnrichedAnnotations(infile, outfile):
    '''
    test SNPs for motif disrupting effects using
    motifbreakR
    '''

    job_memory = "6G"

    tmp = P.getTempFilename(shared=True)

    statement = '''
    zcat %(infile)s
    | grep -P %(annotations_regex)s
    | grep -P %(annotations_cell_regex)s
    > %(tmp)s; checkpoint;
    python /ifs/devel/projects/proj045/enrichment_pipeline/snps2motif.py
    --log=%(outfile)s.log
    --snp-column=3
    --R-scripts-directory=%(r_scripts)s
    --R-script=%(motifs_script)s
    --additional-motif=%(motifs_pwms)s
    --image-directory=plots.dir
    %(tmp)s
    > %(outfile)s; checkpoint;
    rm -f %(tmp)s'''

    P.run()


@follows(runMemeChip,
         overlapSnpsWithAnnotations,
         testMotifDisruptingSnpsEnrichedAnnotations,
         mkdir("plots.dir", "motifs.dir"))
@transform("snpsets.dir/*-gat.tsv",
           regex("snpsets.dir/(.+)-gat.tsv"),
           add_inputs(overlapSnpsWithAnnotations),
           r"motifs.dir/\1-notenriched-disrupted.tsv")
def testMotifDisruptingSnpsNotEnriched(infiles, outfile):
    '''
    test SNPs for motif disrupting effects using
    motifbreakR
    '''

    infile = infiles[0]
    annot_file = infiles[1]
    job_memory = "6G"

    tmp = P.getTempFilename(shared=True)

    statement = '''
    comm -23 %(infile)s <(zcat %(annot_file)s | cut -f 4 | sort)
    > %(tmp)s; checkpoint;
    python /ifs/devel/projects/proj045/enrichment_pipeline/snps2motif.py
    --log=%(outfile)s.log
    --snp-column=0
    --R-scripts-directory=%(r_scripts)s
    --R-script=%(motifs_script)s
    --additional-motif=%(motifs_pwms)s
    --image-directory=plots.dir
    %(tmp)s
    > %(outfile)s; checkpoint;
    rm -f %(tmp)s'''

    P.run()


# ---------------------------------------------------
# Generic pipeline tasks
@follows(convertSet2Block,
         mergeBedAnnotations)
def process_samples():
    pass

@follows(overlapSnpsWithAnnotations,
         mergeSnpScores)
def test_enrichment():
    pass

@follows(runMemeChip,
         testMotifDisruptingSnpsEnrichedAnnotations,
         testMotifDisruptingSnpsNotEnriched)
def find_motifs():
    pass

@follows(process_samples,
         test_enrichment,
         find_motifs)
def full():
    pass


@follows(mkdir("report"))
def build_report():
    '''build report from scratch.

    Any existing report will be overwritten.
    '''

    E.info("starting report build process from scratch")
    P.run_report(clean=True)


@follows(mkdir("report"))
def update_report():
    '''update report.

    This will update a report with any changes inside the report
    document or code. Note that updates to the data will not cause
    relevant sections to be updated. Use the cgatreport-clean utility
    first.
    '''

    E.info("updating report")
    P.run_report(clean=False)


@follows(update_report)
def publish_report():
    '''publish report in the CGAT downloads directory.'''

    E.info("publishing report")
    P.publish_report()

if __name__ == "__main__":
    sys.exit(P.main(sys.argv))