def generate_feed(req):
prefix = lib.absolute_prefix()
news = _enum_news()
feed = ""
lastmodified = ""
for id in news:
filename = lib.valid_dir(djoin(dirname(__file__), basename(id)))
filename = filename + ".md" if exists(filename +
".md") else filename + ".news"
if id == news[0]: lastmodified = lib.get_time_for_file(filename, True)
content, headline, author = parse_news(id)
content = sub('style=".*?"', "", escape(content))
uri = djoin(lib.get_config("CanonicalName"), prefix, "News",
"?id=" + id)
feed += lib.get_template("feedentry") % {
"uri": uri,
"title": headline,
"mtime": lib.get_time_for_file(filename, True),
"content": content,
"author": author
}
req.content_type = "application/atom+xml; charset=UTF-8"
return lib.get_template("feed") % {
"uri":
djoin(lib.get_config("CanonicalName"), prefix),
"self":
djoin(lib.get_config("CanonicalName"), prefix, "News", "?feed=true"),
"mtime":
lastmodified,
"content":
feed
}
def test_create_puppet_change(self):
hostci = CI(name='s11401.dc2', uid='mm-1')
hostci.type_id = CI_TYPES.DEVICE.id
hostci.save()
p = PuppetAgentsImporter()
yaml = open(
djoin(CURRENT_DIR, 'cmdb/tests/samples/canonical.yaml')
).read()
p.import_contents(yaml)
yaml = open(
djoin(CURRENT_DIR, 'cmdb/tests/samples/canonical_unchanged.yaml')
).read()
p.import_contents(yaml)
chg = CIChange.objects.get(type=CI_CHANGE_TYPES.CONF_AGENT.id)
logs = PuppetLog.objects.filter(
cichange__host='s11401.dc2').order_by('id')
self.assertEqual(chg.content_object.host, u's11401.dc2')
self.assertEqual(chg.content_object.kind, u'apply')
self.assertEqual(chg.ci, hostci)
self.assertEqual(chg.type, 2)
# check parsed logs
self.assertEqual(len(logs), 16)
time_iso = logs[0].time.isoformat().split('.')[0]
self.assertEqual(time_iso, datetime.datetime(
2010, 12, 31, 0, 56, 37).isoformat())
# should not import puppet report which has 'unchanged' status
self.assertEqual(
CIChangePuppet.objects.filter(status='unchanged').count(), 0)
def output(self):
return [
luigi.LocalTarget(djoin(gp.read_dir,
gp.final_prefix + '.R1.fastq')),
luigi.LocalTarget(djoin(gp.read_dir,
gp.final_prefix + '.R2.fastq'))
]
def run(self):
fastq_to_table(
djoin(gp.read_dir,
'.'.join([gp.final_prefix, self.direction, 'fastq'])),
djoin(gp.analyses_dir,
'.'.join([gp.gstd_prefix, self.direction, 'csv'])),
gp.analyses_dir)
def run(self):
if self.enhance_mode:
yield Single_Enhanced_FastQ(enh_csv=djoin(
gp.enhd_cld_dir, '.'.join(['bsort', self.direction, 'csv'])),
direction=self.direction)
barcode_sorter(
djoin(self.in_dir, '.'.join([self.prefix, self.direction,
'fastq'])), gp.read_dir, False)
def run(self):
for i, j in enumerate(['R1', 'R2']):
enhd_file_path = '.'.join([self.enhd_file_prefix, j, 'csv'])
with open(enhd_file_path, 'w') as outf:
for chunk_file in listdir(djoin(gp.enhd_cld_dir, j)):
with open(djoin(gp.enhd_cld_dir, j, chunk_file)) as f:
for l in f:
outf.write(l)
def _enum_news(req=None):
if req: return
newsfiles = [
basename(file).split(".")[0]
for file in glob(djoin(dirname(__file__), "*.md")) +
glob(djoin(dirname(__file__), "*.news"))
]
return sorted(newsfiles,
lambda a, b: cmp(int(a.split(".")[0]), int(b.split(".")[0])),
reverse=True)
def run(self):
# complete_reads(djoin(gp.read_dir, '.'.join([gp.prefix + '_bsort', self.direction, 'fastq'])),
# djoin(gp.cs_init_dir, 'K55', '.'.join([gp.search_dist, self.direction, 'fastq'])),
# djoin(gp.ariadne_dir, '.'.join([gp.search_dist + '_full', self.direction, 'fastq'])))
# yield Single_Sort_FastQ(in_dir = gp.ariadne_dir, prefix = gp.search_dist + '_full', direction = self.direction,
# out_dir = gp.ariadne_dir)
with self.output().open('w') as out_fq:
with open(djoin(gp.cs_init_dir, 'K55', '.'.join([gp.search_dist, self.direction, 'fastq']))) as ari_fq, \
open(djoin(gp.read_dir, '.'.join([gp.prefix + '_nobc', self.direction, 'fastq']))) as nobc_fq:
out_fq.write(ari_fq.read())
out_fq.write(nobc_fq.read())
def run(self):
fastq_prefix = djoin(gp.cs_init_dir, 'K55', gp.search_dist)
retcode = subprocess.run(['/home/lam4003/bin/spades/assembler/spades.py', '--meta', '--only-assembler', \
'--gemcode1-1', djoin(gp.ariadne_dir, gp.ariadne_prefix + '.R1.fastq'), \
'--gemcode1-2', djoin(gp.ariadne_dir, gp.ariadne_prefix + '.R2.fastq'), \
'--search-distance', '0', '--size-cutoff', '6', '-t', gp.num_threads, '-m', gp.memory, '-o', gp.cs_final_dir])
# '-k', '55', '--assembly-graph', djoin(gp.cs_init_dir, 'assembly_graph.fastg')])
spades_mem = int(gp.memory) + 100
while retcode.returncode:
retcode = subprocess.run(['/home/lam4003/bin/spades/assembler/spades.py', '--restart-from', 'k55', '-m', str(spades_mem), '-o', gp.cs_final_dir])
spades_mem += 100
shutil.copy(djoin(gp.cs_final_dir, 'scaffolds.fasta'), self.output().path)
def run(self):
for i, j in enumerate(['R1', 'R2']):
enhd_file_path = '.'.join([self.enhd_file_prefix, j, 'csv'])
with open(enhd_file_path, 'w') as outf:
for chunk_file in listdir(djoin(gp.enhd_cld_dir, j)):
with open(djoin(gp.enhd_cld_dir, j, chunk_file)) as f:
for l in f:
outf.write(l)
if gp.deconv_type == gp.gstd_prefix:
shutil.copy(
enhd_file_path,
djoin(gp.analyses_dir, '.'.join([gp.gstd_prefix, j,
'csv'])))
def run(self):
fastq_enhance(
djoin(
gp.read_dir,
'.'.join([gp.base_prefix + '_bsort', self.direction,
'fastq'])), self.enh_csv, gp.enhd_cld_dir,
gp.edit_prefix)
def run(self):
srt_bam = djoin(gp.work_dir, 'bsort.bam')
srt_step = subprocess.run(
['samtools', 'sort',
self.input().path, '-o', srt_bam])
idx_step = subprocess.run(['samtools', 'index', srt_bam])
bam_to_annotate(srt_bam, self.ids_to_names, gp.work_dir, False)
def parse_news(id, req=None):
if not type(id) is str: return
newstpl = lib.get_template("news")
filename = lib.valid_dir(djoin(dirname(__file__), basename(id)))
if exists(filename + ".news"):
filename += ".news"
mode = "plain"
elif exists(filename + ".md"):
filename += ".md"
mode = "markdown"
else:
raise Exception()
time = lib.get_time_for_file(filename)
with open(filename) as newsfile:
author = lib.escape(newsfile.readline().replace("\n", ""))
headline = lib.escape(newsfile.readline().replace("\n", ""))
newscontent = lib.escape("\n".join(newsfile.readlines()))
if mode is "markdown":
newscontent = markdown(newscontent)
return (newstpl % {
"headline": headline,
"author": author,
"date": time,
"id": id,
"content": newscontent,
"uri": lib.ljoin("News", "?id=" + id)
}, headline, author)
def run(self):
subprocess.run(['/home/lam4003/bin/ema/ema', 'align', '-d', '-r', self.reference + '.fna', '-s', \
djoin(gp.bin_dir, self.ema_bin_file), '-o', self.out_prefix + '.sam'])
subprocess.run([
'samtools', 'view', '-hb', '-f', '0', '-F', '256',
self.out_prefix + '.sam', '-o',
self.output().path
])
def run(self):
if exists(djoin(gp.cs_init_dir, 'K33', 'assembly_graph.fastg')): # The previous iteration exists, run from the last one.
retcode = subprocess.run(['/home/lam4003/bin/spades/assembler/spades.py', '--restart-from', 'k55', '-o', gp.cs_init_dir])
else: # The initial cloudSPAdes iterations do not exist.
retcode = subprocess.run(['/home/lam4003/bin/spades/assembler/spades.py', '--meta', '--only-assembler', '--gemcode1-1',
self.input()[0].path, '--gemcode1-2', self.input()[1].path, '--search-distance', gp.search_dist,
'--size-cutoff', '6', '-t', gp.num_threads, '-m', gp.memory, '-o', gp.cs_init_dir])
spades_mem = int(gp.memory) + 100
while retcode.returncode:
retcode = subprocess.run(['/home/lam4003/bin/spades/assembler/spades.py', '--restart-from', 'k55', '-m', str(spades_mem), '-o', gp.cs_init_dir])
spades_mem += 100
def run(self):
subprocess.run([
'/home/lam4003/bin/spades/assembler/spades.py', '--meta',
'--only-assembler', '--gemcode1-1',
self.input()[0].path, '--gemcode1-2',
self.input()[1].path, '--search-distance', '0', '--size-cutoff',
'6', '-t', gp.num_threads, '-m', self.memory, '-o',
gp.cldspades_dir
])
shutil.copy(djoin(gp.cldspades_dir, 'scaffolds.fasta'),
self.output().path)
def run(self):
raw_sam = djoin(gp.work_dir, 'bsort.sam')
raw_bam = djoin(gp.work_dir, 'bsort_raw.bam')
srt_bam = djoin(gp.work_dir, 'bsort.bam')
map_step = subprocess.run([
'bowtie2', '--sensitive-local', '-p', gp.num_threads, '-x',
self.reference, '-1',
self.input()[0].path, '-2',
self.input()[1].path, '-S', raw_sam
])
bam_step = subprocess.run([
'samtools', 'view', '-hb', '-f', '0', '-F', '256', raw_sam, '-o',
raw_bam
]) # Primary alignments only
srt_step = subprocess.run(['samtools', 'sort', raw_bam, '-o', srt_bam])
idx_step = subprocess.run(['samtools', 'index', srt_bam])
bam_to_annotate(srt_bam, self.ids_to_names, gp.work_dir,
self.fragments)
add_barcodes(self.input()[0].path, djoin(gp.work_dir, 'bsort.csv'),
gp.work_dir)
def index (req,page=None):
entries = lib.get_config ("Entries","Static")
pagemeta = None
for entry in entries:
if entry["uri"] == page and "src" in entry:
pagemeta = entry
break
else:
req.headers_out["location"] = lib.get_config ("SitePrefix").encode()
req.status = apache.HTTP_MOVED_TEMPORARILY
return
with open ( lib.valid_dir (djoin (dirname(__file__),pagemeta["src"])) ) as pagefile:
text = lib.escape ("\n".join (pagefile.readlines()))
if ".md" in pagemeta["src"]:
text = markdown (text)
text = lib.get_template ("static") % {"content": text}
return lib.respond (req, text, pagemeta["title"],pagemeta["caption"],pagemeta["description"],module_info(active=page))
def test_get_new_issues(self):
dp1_1 = DeploymentPoll(
key='RALPH-341',
date=datetime.datetime.strptime('1-1-2012 1:10', '%d-%m-%Y %H:%M')
)
dp1_1.save()
dp1_2 = DeploymentPoll(
key='RALPH-341',
date=datetime.datetime.strptime('1-1-2012 1:20', '%d-%m-%Y %H:%M'))
dp1_2.save()
dp2_1 = DeploymentPoll(
key='RALPH-342',
date=datetime.datetime.strptime('2-2-2012 2:10', '%d-%m-%Y %H:%M'),
checked=False)
dp2_1.save()
dp2_2 = DeploymentPoll(
key='RALPH-342',
date=datetime.datetime.strptime('2-2-2012 2:20', '%d-%m-%Y %H:%M'),
checked=False)
dp2_2.save()
dp3_1 = DeploymentPoll(
key='RALPH-343',
date=datetime.datetime.strptime('3-3-2012 3:10', '%d-%m-%Y %H:%M')
)
dp3_1.save()
dp3_2 = DeploymentPoll(
key='RALPH-343',
date=datetime.datetime.strptime('3-3-2012 3:20', '%d-%m-%Y %H:%M'))
dp3_2.save()
dp4_1 = DeploymentPoll(
key='RALPH-344',
date=datetime.datetime.strptime('4-4-2012 5:10', '%d-%m-%Y %H:%M'))
dp4_1.save()
x = JiraRSS(tracker_name='JIRA')
rss = open(
djoin(CURRENT_DIR, 'cmdb/tests/samples/jira_rss.xml')
).read()
x.rss_url = rss
self.assertEquals(
sorted(x.get_new_issues()), [
'RALPH-341', 'RALPH-342', 'RALPH-343', 'RALPH-344']
)
def index(req, page=None):
entries = lib.get_config("Entries", "Static")
pagemeta = None
for entry in entries:
if entry["uri"] == page and "src" in entry:
pagemeta = entry
break
else:
req.headers_out["location"] = lib.get_config("SitePrefix").encode()
req.status = apache.HTTP_MOVED_TEMPORARILY
return
with open(lib.valid_dir(djoin(dirname(__file__),
pagemeta["src"]))) as pagefile:
text = lib.escape("\n".join(pagefile.readlines()))
if ".md" in pagemeta["src"]:
text = markdown(text)
text = lib.get_template("static") % {"content": text}
return lib.respond(req, text, pagemeta["title"], pagemeta["caption"],
pagemeta["description"], module_info(active=page))
def test_get_new_issues(self):
dp1_1 = DeploymentPoll(key="RALPH-341", date=datetime.datetime.strptime("1-1-2012 1:10", "%d-%m-%Y %H:%M"))
dp1_1.save()
dp1_2 = DeploymentPoll(key="RALPH-341", date=datetime.datetime.strptime("1-1-2012 1:20", "%d-%m-%Y %H:%M"))
dp1_2.save()
dp2_1 = DeploymentPoll(
key="RALPH-342", date=datetime.datetime.strptime("2-2-2012 2:10", "%d-%m-%Y %H:%M"), checked=False
)
dp2_1.save()
dp2_2 = DeploymentPoll(
key="RALPH-342", date=datetime.datetime.strptime("2-2-2012 2:20", "%d-%m-%Y %H:%M"), checked=False
)
dp2_2.save()
dp3_1 = DeploymentPoll(key="RALPH-343", date=datetime.datetime.strptime("3-3-2012 3:10", "%d-%m-%Y %H:%M"))
dp3_1.save()
dp3_2 = DeploymentPoll(key="RALPH-343", date=datetime.datetime.strptime("3-3-2012 3:20", "%d-%m-%Y %H:%M"))
dp3_2.save()
dp4_1 = DeploymentPoll(key="RALPH-344", date=datetime.datetime.strptime("4-4-2012 5:10", "%d-%m-%Y %H:%M"))
dp4_1.save()
x = JiraRSS(tracker_name="JIRA")
rss = open(djoin(CURRENT_DIR, "cmdb/tests/samples/jira_rss.xml")).read()
x.rss_url = rss
self.assertEquals(sorted(x.get_new_issues()), ["RALPH-341", "RALPH-342", "RALPH-343", "RALPH-344"])
def output(self):
return luigi.LocalTarget(
djoin(gp.enhd_cld_dir,
'.'.join([gp.edit_prefix, self.direction, 'fastq'])))
def get_details(self, *args, **kwargs):
xml = open(
djoin(CURRENT_DIR + 'cmdb/tests/samples/fisheye_details.xml')
).read()
return objectify.fromstring(xml)
def _enum_files(req=None):
if req: return
files = glob(djoin(dirname(__file__), "_mainpage", "*.md"))
return sorted(files,
lambda a, b: cmp(int(a.split(".")[0]), int(b.split(".")[0])))
def _enum_files (req=None):
if req: return
files = glob (djoin (dirname(__file__),"_mainpage","*.md"))
return sorted(files, lambda a,b: cmp(int(a.split(".")[0]), int(b.split(".")[0])))
def _latest_ver (req=None,repo=None):
if req or not repo: return
git_dir = djoin (lib.get_config ("RepoPath","Downloads"),repo["repo"])
return Popen (["git", "--git-dir="+git_dir, "show-ref"],stdout=PIPE).communicate()[0].split("\n")[-2].split("/")[-1]
def get_changes(self, *args, **kwargs):
xml = open(djoin(CURRENT_DIR, "cmdb/tests/samples/fisheye_changesets.xml")).read()
return objectify.fromstring(xml)
def run(self):
concat_annotations(djoin(gp.orig_map_dir, self.chunk_file_name),
gp.enhd_cld_dir, self.fragments)
def output(self):
return luigi.LocalTarget(djoin(gp.work_dir, 'bsort.final.csv'))
def output(self):
return luigi.LocalTarget(
djoin(gp.enhd_cld_dir, 'R1',
self.chunk_file_name)), luigi.LocalTarget(
djoin(gp.enhd_cld_dir, 'R2', self.chunk_file_name))
def output(self):
return luigi.LocalTarget(
djoin(gp.read_dir,
'.'.join([self.prefix + '_bsort', self.direction, 'fastq'])))
def output(self):
return luigi.LocalTarget(
djoin(gp.analyses_dir, gp.final_prefix + '.scaffolds.fasta'))
import sys, os
from markdown import markdown
from mod_python import apache
from glob import glob
from math import ceil
from re import sub
from xml.sax.saxutils import escape
from os.path import dirname, basename, exists
from os.path import join as djoin
sys.path.append(djoin(dirname(__file__), ".."))
import lib
def index(req, id=-1, page="0", feed=False):
if id >= 0:
try:
content, title, author = parse_news(str(int(id)), req)
return lib.respond(req, content, title, "News", title,
module_info())
except:
return lib.e404(req, "Could not find matching news!",
module_info())
if feed:
return generate_feed(req)
return lib.respond(req, generate_page(int(page), 1), "News", "News",
"News", module_info())
def check_make(curr_dir, sub):
outdir = djoin(curr_dir, sub)
if not exists(outdir):
makedirs(outdir)
return outdir
import sys
from mod_python import apache
from os.path import dirname,basename,exists
from os.path import join as djoin
from markdown import markdown
sys.path.append (djoin (dirname(__file__),".."))
import lib
def index (req,page=None):
entries = lib.get_config ("Entries","Static")
pagemeta = None
for entry in entries:
if entry["uri"] == page and "src" in entry:
pagemeta = entry
break
else:
req.headers_out["location"] = lib.get_config ("SitePrefix").encode()
req.status = apache.HTTP_MOVED_TEMPORARILY
return
with open ( lib.valid_dir (djoin (dirname(__file__),pagemeta["src"])) ) as pagefile:
text = lib.escape ("\n".join (pagefile.readlines()))
if ".md" in pagemeta["src"]:
text = markdown (text)
text = lib.get_template ("static") % {"content": text}
return lib.respond (req, text, pagemeta["title"],pagemeta["caption"],pagemeta["description"],module_info(active=page))
def generate_entries (req=None,current=None):
if req: return
def __init__(self, *args, **kwargs):
super(Generate_Enhanced_Chunks, self).__init__(*args, **kwargs)
self.enhd_file_prefix = djoin(gp.enhd_cld_dir, 'bsort')
def output(self):
return luigi.LocalTarget(
djoin(gp.analyses_dir,
'.'.join([gp.final_prefix, self.direction, 'csv'])))
from os.path import join as djoin
from json import dumps as jdumps
from flask import Flask
from flask import render_template
from flask import request
from flask import url_for
from urllib.parse import unquote_plus
app = Flask(__name__)
app.jinja_env.add_extension('pypugjs.ext.jinja.PyPugJSExtension')
#Edit this as you require
filepath = 'title.txt'
folderpath = os.path.dirname(
os.path.realpath(__file__)) or '/foo/bar/path/to/file/'
fullpath = djoin(folderpath, filepath)
_PLAYER = 'spotify'
@app.context_processor
def override_url_for():
return dict(url_for=dated_url_for)
def dated_url_for(endpoint, **values):
if endpoint == 'static':
filename = values.get('filename', None)
if filename:
file_path = os.path.join(app.root_path, endpoint, filename)
values['q'] = int(os.stat(file_path).st_mtime)
return url_for(endpoint, **values)
def output(self):
return luigi.LocalTarget(
djoin(gp.analyses_dir, gp.final_prefix + '.statistics.csv'))