def get_local_stories(path, render=True, cache={}):
if render:
stories = []; append = stories.append
else:
stories = {}
for file in listdir(path):
if file.startswith('.') or not file.endswith('.txt'):
continue
filepath = join(path, file)
text = read(filepath)
m_time = stat(filepath).st_mtime
yaml = match_yaml_frontmatter(text)
if not yaml:
continue
story = decode_yaml(yaml.group(1))
id = story["id"]
text = replace_yaml_frontmatter('', text)
if render:
if id not in cache or m_time != cache[id][0]:
if text:
text = render_text(text)
cache[id] = (m_time, text)
else:
text = cache[id][1]
story["text"] = render_text(text)
append(story)
else:
story["text"] = text
stories[id] = file[:-4], story
return stories
def __init__(self, path, profile='default', force=None):
data_dir = join(
gettempdir(), 'assetgen-%s' % sha1(path).hexdigest()[:12]
)
if not isdir(data_dir):
makedirs(data_dir)
lock_path = join(data_dir, 'lock')
lock(lock_path, path)
self.config_path = path
self.data_path = data_path = join(data_dir, 'data')
self.force = force
if isfile(data_path):
data_file = open(data_path, 'rb')
try:
self.data = load(data_file)
except Exception:
self.data = {}
data_file.close()
else:
self.data = {}
config_file = open(path, 'rb')
config_data = config_file.read() % os.environ
self.config = config = decode_yaml(config_data)
config_file.close()
if not config:
exit("No config found at %s" % path)
if not isinstance(config, dict):
exit("Config at %s is not a dict mapping." % path)
for key in config.keys():
if key.startswith('profile.'):
if key == 'profile.%s' % profile:
profile_conf = config.pop(key)
config.update(profile_conf)
else:
config.pop(key)
for key in DEFAULTS:
if key not in config:
config[key] = DEFAULTS[key]
if 'env' in config:
env = config['env']
for key, val in env.iteritems():
if '.' in key:
key, action = key.split('.', 1)
existing = environ.get(key)
if action == 'prefix':
if existing:
environ[key] = "%s:%s" % (val, existing)
continue
elif action == 'append':
if existing:
environ[key] = "%s:%s" % (existing, val)
continue
environ[key] = val
self.base_dir = base_dir = dirname(path)
output_dir = config.pop('output.directory', None)
if not output_dir:
exit("No value found for output.directory in %s." % path)
self.output_dir = output_dir = join(base_dir, output_dir)
self.output_template = config.pop('output.template')
self.hashed = config.pop('output.hashed', False)
manifest_path = config.pop('output.manifest', None)
if manifest_path:
self.manifest_path = join(base_dir, manifest_path)
def expand_src(source):
source = join(base_dir, source)
if '*' not in source:
return [source]
root = split(source.partition('*')[0])[0]
sources = []; new_source = sources.append
for directory, _, files in walk(root):
for file in files:
path = join(directory, file)
if fnmatch(path, source):
new_source(path)
return sources
for key in ('prereqs', 'generate'):
listing = config.pop(key, None)
if not listing:
if key == 'prereqs':
self.prereqs = []
continue
exit("No value found for %s in %s." % (key, path))
assets = []
add_asset = assets.append
setattr(self, key, assets)
for info in listing:
output, spec = info.items()[0]
if 'type' in spec:
type = spec.pop('type')
else:
if '.' not in output:
exit("Couldn't determine asset type for %r" % output)
type = output.rsplit('.', 1)[1]
if type not in HANDLERS:
exit("No handler found for asset type %r." % type)
prefix = '%s.' % type
for conf_key in config:
if conf_key.startswith(prefix):
spec_key = conf_key.split('.', 1)[1]
if spec_key not in spec:
spec[spec_key] = config[conf_key]
for key in spec.keys():
if key.startswith('profile.'):
if key == 'profile.%s' % profile:
profile_conf = spec.pop(key)
spec.update(profile_conf)
else:
spec.pop(key)
_sources = spec.pop('source', None)
if not _sources:
exit("No 'source' defined for %s" % output)
if not isinstance(_sources, list):
_sources = [_sources]
_depends = spec.pop('depends', [])
if isinstance(_depends, basestring):
_depends = [_depends]
depends = []
for source in _depends:
depends.extend(expand_src(source))
if output.endswith('/*'):
io = []; add_io = io.append
oprefix = output[:-1]
for source in _sources:
if isinstance(source, Raw):
exit("Source for %r cannot be raw text." % output)
if not source.endswith('/*'):
exit("Glob source %r must end in /* too." % source)
source = join(base_dir, source[:-1])
src_len = len(source)
for directory, _, files in walk(source):
for file in files:
path = join(directory, file)
_src = [path]
_dep = depends + _src
add_io((_src, _dep, oprefix + path[src_len:]))
else:
sources = []
for source in _sources:
if isinstance(source, basestring):
sources.extend(expand_src(source))
else:
sources.append(Raw(source['raw']))
depends = depends + [
source for source in sources
if not isinstance(source, Raw)
]
io = [(sources, depends, output)]
for sources, depends, output in io:
if DEBUG:
print depends, '->', output
add_asset(
HANDLERS[type](self, output, sources, depends, spec)
)
role_file = join(path, role) + '.yaml'
if isfile(role_file):
break
else:
exit("ERROR: Couldn't find a data file for the %r role." % role)
try:
role_file = open(role_file, 'rb')
except IOError, error:
exit("ERROR: %s: %s" % (error[1], error.filename))
role_data = role_file.read()
role_file.close()
try:
role_data = decode_yaml(role_data)
except Exception:
exit("ERROR: Couldn't decode the JSON input: %s" % role_file.name)
packages = set(role_data['packages'])
for package in packages:
install_package(package)
if 'requires' in role_data:
packages.update(load_role(role_data['requires']))
if role == 'base':
for package in packages:
BASE_PACKAGES.update([package])
BASE_PACKAGES.update(get_dependencies(package))
def __init__(self, path, profile='default', force=None):
data_dir = join(gettempdir(),
'assetgen-%s' % sha1(path).hexdigest()[:12])
if not isdir(data_dir):
makedirs(data_dir)
lock_path = join(data_dir, 'lock')
lock(lock_path, path)
self.config_path = path
self.data_path = data_path = join(data_dir, 'data')
self.force = force
if isfile(data_path):
data_file = open(data_path, 'rb')
try:
self.data = load(data_file)
except Exception:
self.data = {}
data_file.close()
else:
self.data = {}
config_file = open(path, 'rb')
config_data = config_file.read() % os.environ
self.config = config = decode_yaml(config_data)
config_file.close()
if not config:
exit("No config found at %s" % path)
if not isinstance(config, dict):
exit("Config at %s is not a dict mapping." % path)
for key in config.keys():
if key.startswith('profile.'):
if key == 'profile.%s' % profile:
profile_conf = config.pop(key)
config.update(profile_conf)
else:
config.pop(key)
for key in DEFAULTS:
if key not in config:
config[key] = DEFAULTS[key]
if 'env' in config:
env = config['env']
for key, val in env.iteritems():
if '.' in key:
key, action = key.split('.', 1)
existing = environ.get(key)
if action == 'prefix':
if existing:
environ[key] = "%s:%s" % (val, existing)
continue
elif action == 'append':
if existing:
environ[key] = "%s:%s" % (existing, val)
continue
environ[key] = val
self.base_dir = base_dir = dirname(path)
output_dir = config.pop('output.directory', None)
if not output_dir:
exit("No value found for output.directory in %s." % path)
self.output_dir = output_dir = join(base_dir, output_dir)
self.output_template = config.pop('output.template')
self.hashed = config.pop('output.hashed', False)
manifest_path = config.pop('output.manifest', None)
if manifest_path:
self.manifest_path = join(base_dir, manifest_path)
self.manifest_force = config.pop('output.manifest_force', False)
def expand_src(source):
if source.startswith('http://'):
return get_downloaded_source(source)
if source.startswith('https://'):
return get_downloaded_source(source, 1)
source = join(base_dir, source)
if '*' not in source:
return [source]
root = split(source.partition('*')[0])[0]
sources = []
new_source = sources.append
for directory, _, files in walk(root):
for file in files:
path = join(directory, file)
if fnmatch(path, source):
new_source(path)
return sources
for key in ('prereqs', 'generate'):
listing = config.pop(key, None)
if not listing:
if key == 'prereqs':
self.prereqs = []
continue
exit("No value found for %s in %s." % (key, path))
assets = []
add_asset = assets.append
setattr(self, key, assets)
for info in listing:
output, spec = info.items()[0]
if 'type' in spec:
type = spec.pop('type')
else:
if '.' not in output:
exit("Couldn't determine asset type for %r" % output)
type = output.rsplit('.', 1)[1]
if type not in HANDLERS:
exit("No handler found for asset type %r." % type)
prefix = '%s.' % type
for conf_key in config:
if conf_key.startswith(prefix):
spec_key = conf_key.split('.', 1)[1]
if spec_key not in spec:
spec[spec_key] = config[conf_key]
for key in spec.keys():
if key.startswith('profile.'):
if key == 'profile.%s' % profile:
profile_conf = spec.pop(key)
spec.update(profile_conf)
else:
spec.pop(key)
_sources = spec.pop('source', None)
if not _sources:
exit("No 'source' defined for %s" % output)
if not isinstance(_sources, list):
_sources = [_sources]
_depends = spec.pop('depends', [])
if isinstance(_depends, basestring):
_depends = [_depends]
depends = []
for source in _depends:
depends.extend(expand_src(source))
if output.endswith('/*'):
io = []
add_io = io.append
oprefix = output[:-1]
for source in _sources:
if isinstance(source, Raw):
exit("Source for %r cannot be raw text." % output)
if not source.endswith('/*'):
exit("Glob source %r must end in /* too." % source)
source = join(base_dir, source[:-1])
src_len = len(source)
for directory, _, files in walk(source):
for file in files:
path = join(directory, file)
_src = [path]
_dep = depends + _src
add_io((_src, _dep, oprefix + path[src_len:]))
else:
sources = []
for source in _sources:
if isinstance(source, basestring):
sources.extend(expand_src(source))
else:
sources.append(Raw(source['raw']))
depends = depends + [
source
for source in sources if not isinstance(source, Raw)
]
io = [(sources, depends, output)]
for sources, depends, output in io:
if DEBUG:
if len(depends) > 1:
log.info("Combined:\n%s -> %s" %
(pformat(depends), output))
elif len(depends) == 1:
log.info("%s -> %s" %
(pformat(depends[0]), output))
else:
log.info("%s -> %s" % (depends, output))
add_asset(HANDLERS[type](self, output, sources, depends,
spec))
# -----------------------------------------------------------------------------
# Environ Loading
# -----------------------------------------------------------------------------
ENVIRON = environ.get("REDPILL_ENVIRON")
if not ENVIRON:
exit("ERROR: The $REDPILL_ENVIRON directory variable hasn't been specified.")
conf_path = join(ENVIRON, 'redpill.yaml')
try:
conf_file = open(conf_path, 'rb')
except Exception, err:
exit("ERROR: Couldn't open the redpill.yaml file: %s" % err)
conf = decode_yaml(conf_file.read())
conf_file.close()
if not conf:
exit("ERROR: Empty config found in %s" % conf_path)
sentinel = object()
def get_conf(key, default=sentinel, conf=conf, path=conf_path):
value = conf.get(key, default)
if value is sentinel:
exit("ERROR: Config value for %s not found in %s" % (key, path))
return value
del conf_file
del conf_path
