def render_and_write(filename, **kw):
with open(join(TEMPLATES_DIR, filename)) as f:
template = Template(f.read())
html = template.render(kw)
html = strip_whitespace_from_links(html)
with open(join(BUILD_DIR, filename), 'w') as f:
f.write(html)
def _main():
# start with clear build dir
if exists(BUILD_DIR):
rmtree(BUILD_DIR)
mkdir(BUILD_DIR)
mkdir(TAGS_DIR)
# prepare to count tag occurances
tags_count = defaultdict(int)
# render posts creating tag pages as you go by
dates = []
for post in get_posts():
dates.append(post.date)
with open(post.path, 'w') as post_f:
post_f.write(render(post))
for link in post.links:
post_f.write(render(link))
for tag in link.tags[:-1]:
tags_count[tag.name] += 1
is_new = not exists(tag.path)
with open(tag.path, 'a') as tag_f:
if is_new:
tag_f.write(render(tag))
tag_f.write(render(link, tag))
# prepare to create a tag cloud
min_count = min(tags_count.itervalues())
max_count = min([200, max(tags_count.itervalues())])
min_scale = 1
max_scale = 3
get_scale = lambda count: '%.2f' % \
(min_scale + (max_scale - min_scale) * \
(min([count, max_count]) - min_count) / (max_count - min_count))
# render tag cloud
render_and_write('tags.html', tags=[
{'name': tag, 'scale': get_scale(count)} \
for tag, count in sorted(tags_count.iteritems())
])
# render feed
render_and_write('feed.xml', dates=dates)
def __init__(self, date, prev, next, src):
self.date = date
self.prev = '/%s' % prev if prev else None
self.next = '/%s' % next if next else None
self.url = '/%s' % date
self.path = join(BUILD_DIR, '%s.html' % date)
self.links = (Link(dct, self) for dct in yaml.load(open(src)))
def _load_map(self):
stream = common.open('level.txt')
lines = stream.readlines()
stream.close()
self.moves = lines[1].rstrip()
self.timeline = lines[2].rstrip()
self.moves_count = int(self.moves.replace(" ", ""))
if len(self.moves) != len(self.timeline):
#TODO: lanzar una excepción, tal vez...
print "eh!, la lista de movimientos y linea de tiempo difieren."
import yaml
from common import open
# set paths
CURR_DIR = dirname(realpath(__file__))
BUILD_DIR = join(CURR_DIR, 'build')
POSTS_DIR = join(CURR_DIR, '..', 'posts')
TAGS_DIR = join(BUILD_DIR, 'tags')
TEMPLATES_DIR = join(CURR_DIR, 'templates')
# keep templates in memory
templates = {}
for name in ['link', 'post', 'tag', 'tags']:
with open(join(TEMPLATES_DIR, '%s.html' % name)) as f:
templates[name] = Template(f.read())
class Link(object):
def __init__(self, dct, post):
# render markdown
desc = dct.get('desc', None)
if desc:
dct['desc'] = improve_typography(render_markdown(desc))
# improve typography in title
dct['title'] = improve_typography(dct['title'])
self._dct = dct
def takeaway(path):
with open(join(BUILD_DIR, path)) as f:
return f.read()
TEMPLATES_DIR = join(CURR_DIR, 'templates')
# list available website paths
PATHS = ['feed']
ext = '.html'
ext_len = len(ext)
for prefix in ['', 'tags/']:
for name in listdir(join(BUILD_DIR, prefix)):
if name.endswith(ext):
PATHS.append(prefix + name[:-ext_len])
# get latest post's path
LATEST_POST_PATH = sorted(filter(lambda s: s[0] == '2', PATHS))[-1]
# get wrapping template
with open(join(TEMPLATES_DIR, 'wrapper.html')) as f:
WRAPPER = Template(f.read())
# read 404 page
with open(join(TEMPLATES_DIR, '404.html')) as f:
html_for_404 = f.read()
# create wsgi app
app = Flask(__name__)
@app.route('/')
def flipflop_ninja():
return redirect(LATEST_POST_PATH)
def run_qsub_mr(options):
# Record current working directory. Both mapper and reducer commands will run under this directory.
cwd = os.getcwd()
# Find input files.
input_files = glob.glob(options.input)
input_files.sort()
common.check(input_files, 'input pattern does not list any file')
common.check(len(set([basename(i) for i in input_files])) == len(input_files), 'input pattern matches input files with duplicate basename')
if options.verbose:
common.info('input pattern %s matched %d files', repr(options.input), len(input_files))
# Check then create output directory.
common.check(options.retry or not os.path.exists(options.output), 'output directory already exists')
workdir = os.path.join(options.output, '.mr')
if not os.path.exists(workdir):
os.makedirs(workdir)
if options.verbose:
common.info('workdir: %s', workdir)
# Write out script to rerun this command.
with common.open(os.path.join(workdir, 'rerun.bash'), 'w') as f:
f.write('''# Run this script with --rerun is equivalent to running the mapreduction (assuming the input hasn't changed)
cd %(cwd)s
%(args)s "$@"
''' % {'cwd': pipes.quote(cwd),
'args': ' '.join(map(pipes.quote, sys.argv))})
# Write out mapper script to run on nodes.
with common.open(os.path.join(workdir, 'map.node.bash'), 'w') as f:
f.write(bash_start + options.mapper)
# Write out mapper script to submit to qsub.
with common.open(os.path.join(workdir, 'map.qsub.bash'), 'w') as f:
f.write(bash_start + '''
cd %(cwd)s
if [ -e %(workdir)s/map.${PBS_ARRAYID}.err ]; then
mv %(workdir)s/map.${PBS_ARRAYID}.err %(workdir)s/map.${PBS_ARRAYID}.err.old
fi
bash %(workdir)s/map.${PBS_ARRAYID}.bash 2> %(workdir)s/map.${PBS_ARRAYID}.err
''' % {'cwd': pipes.quote(cwd), 'workdir': pipes.quote(workdir)})
# Write out individual mapper command scripts.
for i, path in enumerate(input_files):
with common.open(os.path.join(workdir, 'map.%d.bash' % i), 'w') as f:
f.write(bash_start + '''
cd %(cwd)s
%(cat)s %(path)s | bash %(workdir)s/map.node.bash | gzip - > %(workdir)s/map.%(task_id)d.out.gz
touch %(workdir)s/map.%(task_id)d.success
''' % {'cwd': pipes.quote(cwd),
'cat': choose_cat(path),
'path': pipes.quote(path),
'workdir': pipes.quote(workdir),
'task_id': i})
# Write out reducer script.
if options.reducer != 'NONE':
with common.open(os.path.join(workdir, 'reduce.node.bash'), 'w') as f:
f.write(bash_start + options.reducer)
with common.open(os.path.join(workdir, 'reduce.qsub.bash'), 'w') as f:
f.write(bash_start + '''
cd %(cwd)s
if [ -e %(workdir)s/reduce.err ]; then
mv %(workdir)s/reduce.err %(workdir)s/reduce.err.old
fi
find %(workdir)s -name 'map.*.out.gz' -print0 2>> %(workdir)s/reduce.err | \\
xargs -0 zcat 2>> %(workdir)s/reduce.err | \\
LC_ALL=C sort %(sort_options)s 2>> %(workdir)s/reduce.err | \\
bash %(workdir)s/reduce.node.bash 2>> %(workdir)s/reduce.err | \\
gzip - > %(workdir)s/reduce.out.gz 2>> %(workdir)s/reduce.err
touch %(workdir)s/reduce.success 2>> %(workdir)s/reduce.err
''' % {'cwd': pipes.quote(cwd),
'workdir': pipes.quote(workdir),
'sort_options': '-n' if options.numerical_sort else ''})
# Run mapper jobs.
for i in range(options.max_tries):
# Find tasks to run.
task_ids = []
for task_id in range(len(input_files)):
if not os.path.exists(os.path.join(workdir, 'map.%d.success' % task_id)):
task_ids.append(task_id)
if not task_ids:
break
qsub_args = ['-N', '%s-map' % options.name, '-q', options.mapper_queue,
'-l', 'pmem=%s,walltime=%s' % (options.mapper_pmem, options.mapper_walltime),
'-t', format_task_ids(task_ids), '-o', os.devnull, '-e', os.devnull] + \
options.qsub_args + [os.path.join(workdir, 'map.qsub.bash')]
if options.verbose:
common.info('map try %d of %d: need to run %d tasks', i + 1, options.max_tries, len(task_ids))
common.info('map try %d of %d: qsub_args is %s', i + 1, options.max_tries, repr(qsub_args))
wait_qsub(qsub_args)
if options.verbose:
common.info('map try %d of %d: finished', i + 1, options.max_tries)
map_success = 0
for task_id in range(len(input_files)):
if os.path.exists(os.path.join(workdir, 'map.%d.success' % task_id)):
map_success += 1
if i > 0 and options.verbose:
common.info('map success: %d / %d', map_success, len(input_files))
common.check(map_success == len(input_files), 'map failed after %d tries', options.max_tries)
# Run reducer jobs.
if options.reducer != 'NONE':
for i in range(options.max_tries):
if os.path.exists(os.path.join(workdir, 'reduce.success')):
break
qsub_args = ['-N', '%s-reduce' % options.name, '-q', options.reducer_queue,
'-l', 'pmem=%s,walltime=%s' % (options.reducer_pmem, options.reducer_walltime),
'-o', os.devnull, '-e', os.devnull] + options.qsub_args + \
[os.path.join(workdir, 'reduce.qsub.bash')]
if options.verbose:
common.info('reduce try %d of %d: started', i + 1, options.max_tries)
common.info('reduce try %d of %d: qsub_args is %s', i + 1, options.max_tries, repr(qsub_args))
wait_qsub(qsub_args)
if options.verbose:
common.info('reduce try %d of %d: finished', i + 1, options.max_tries)
common.check(os.path.exists(os.path.join(workdir, 'reduce.success')), 'reduce failed after %d tries', options.max_tries)
# Move output.
if options.reducer != 'NONE':
src = os.path.join(workdir, 'reduce.out.gz')
dst = os.path.join(options.output, 'reduce.out.gz')
subprocess.check_call(['rm', '-f', dst])
if options.keep_workdir:
os.symlink(os.path.abspath(src), dst)
else:
os.rename(src, dst)
else:
for i, path in enumerate(input_files):
src = os.path.join(workdir, 'map.%d.out.gz' % i)
dst = os.path.join(options.output, 'map.%s.out.gz' % basename(path))
subprocess.check_call(['rm', '-f', dst])
if options.keep_workdir:
os.symlink(os.path.abspath(src), dst)
else:
os.rename(src, dst)
# Remove workdir
if not options.keep_workdir:
subprocess.call(['rm', '-rf', workdir])
