def connect(self):
self._append_counter = 0
if self.wrapper.state in ('walked', 'checked', 'running'):
if file_exists(self.this_data_file()):
self.connected_to = 'existing'
self._storage = sqlite3.connect(self.this_data_file())
self._cursor = self._storage.cursor()
elif file_exists(self.last_data_file()):
msg = "Should not only have last data file %s"
msgargs = (self.last_data_file())
raise dexy.exceptions.InternalDexyProblem(msg % msgargs)
else:
assert not os.path.exists(self.working_file())
assert os.path.exists(os.path.dirname(self.working_file()))
self.connected_to = 'working'
self._storage = sqlite3.connect(self.working_file())
self._cursor = self._storage.cursor()
self._cursor.execute(
"CREATE TABLE kvstore (key TEXT, value TEXT)")
elif self.wrapper.state == 'walked':
raise dexy.exceptions.InternalDexyProblem(
"connect should not be called in 'walked' state")
else:
if file_exists(self.last_data_file()):
self._storage = sqlite3.connect(self.last_data_file())
self._cursor = self._storage.cursor()
elif file_exists(self.this_data_file()):
self._storage = sqlite3.connect(self.this_data_file())
self._cursor = self._storage.cursor()
else:
raise dexy.exceptions.InternalDexyProblem("no data for %s" %
self.storage_key)
def connect(self):
self._append_counter = 0
if self.wrapper.state in ('walked', 'checked', 'running'):
if file_exists(self.this_data_file()):
self.connected_to = 'existing'
self._storage = sqlite3.connect(self.this_data_file())
self._cursor = self._storage.cursor()
elif file_exists(self.last_data_file()):
msg ="Should not only have last data file %s"
msgargs=(self.last_data_file())
raise dexy.exceptions.InternalDexyProblem(msg % msgargs)
else:
assert not os.path.exists(self.working_file())
assert os.path.exists(os.path.dirname(self.working_file()))
self.connected_to = 'working'
self._storage = sqlite3.connect(self.working_file())
self._cursor = self._storage.cursor()
self._cursor.execute("CREATE TABLE kvstore (key TEXT, value TEXT)")
elif self.wrapper.state == 'walked':
raise dexy.exceptions.InternalDexyProblem("connect should not be called in 'walked' state")
else:
if file_exists(self.last_data_file()):
self._storage = sqlite3.connect(self.last_data_file())
self._cursor = self._storage.cursor()
elif file_exists(self.this_data_file()):
self._storage = sqlite3.connect(self.this_data_file())
self._cursor = self._storage.cursor()
else:
raise dexy.exceptions.InternalDexyProblem("no data for %s" % self.storage_key)
def process(self):
self.populate_workspace()
wd = self.parent_work_dir()
env = self.setup_env()
latex_command = self.command_string()
if any(doc.output_data().ext == '.bib'
for doc in self.doc.walk_input_docs()):
bibtex_command = "bibtex %s" % os.path.splitext(
self.output_data.basename())[0]
else:
bibtex_command = None
def run_cmd(command):
self.log_debug("running %s in %s" % (command, os.path.abspath(wd)))
proc = subprocess.Popen(command,
shell=True,
cwd=wd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
stdout, stderr = proc.communicate()
self.log_debug(stdout)
if bibtex_command and self.setting('run-bibtex'):
run_cmd(latex_command) #generate aux
run_cmd(bibtex_command) #generate bbl
n = self.setting('times-to-run-latex')
for i in range(n):
self.log_debug("running latex time %s (out of %s)" % (i + 1, n))
run_cmd(latex_command)
if not file_exists(os.path.join(wd, self.output_data.basename())):
log_file_path = os.path.join(
wd,
self.output_data.basename().replace(".pdf", ".log"))
if file_exists(log_file_path):
msg = "Latex file not generated. Look for information in latex log %s"
msgargs = log_file_path
else:
msg = "Latex file not generated. Look for information in latex log in %s directory."
msgargs = os.path.abspath(wd)
raise dexy.exceptions.UserFeedback(msg % msgargs)
if self.setting('add-new-files'):
self.log_debug("adding new files found in %s for %s" %
(wd, self.key))
self.add_new_files()
self.copy_canonical_file()
def process(self):
self.populate_workspace()
wd = self.parent_work_dir()
env = self.setup_env()
latex_command = self.command_string()
if any(doc.output_data().ext == '.bib' for doc in self.doc.walk_input_docs()):
bibtex_command = "bibtex %s" % os.path.splitext(self.output_data.basename())[0]
else:
bibtex_command = None
def run_cmd(command):
self.log_debug("running %s in %s" % (command, os.path.abspath(wd)))
proc = subprocess.Popen(command, shell=True,
cwd=wd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
stdout, stderr = proc.communicate()
self.log_debug(stdout)
if bibtex_command and self.setting('run-bibtex'):
run_cmd(latex_command) #generate aux
run_cmd(bibtex_command) #generate bbl
n = self.setting('times-to-run-latex')
for i in range(n):
self.log_debug("running latex time %s (out of %s)" % (i+1, n))
run_cmd(latex_command)
if not file_exists(os.path.join(wd, self.output_data.basename())):
log_file_path = os.path.join(wd, self.output_data.basename().replace(".pdf", ".log"))
if file_exists(log_file_path):
msg = "Latex file not generated. Look for information in latex log %s"
msgargs = log_file_path
else:
msg = "Latex file not generated. Look for information in latex log in %s directory."
msgargs = os.path.abspath(wd)
raise dexy.exceptions.UserFeedback(msg % msgargs)
if self.setting('add-new-files'):
self.log_debug("adding new files found in %s for %s" % (wd, self.key))
self.add_new_files()
self.copy_canonical_file()
def document_config_file(self, exists_already=True):
postfix_config_filename = "%s%s" % (os.path.splitext(self.output_data.name)[0], self.setting('document-api-config-postfix'))
if self.setting('document-api-config-file'):
dir_based_config_filename = os.path.join(self.output_data.parent_dir(), self.setting('document-api-config-file'))
else:
dir_based_config_filename = None
if file_exists(postfix_config_filename):
return postfix_config_filename
elif dir_based_config_filename and file_exists(dir_based_config_filename):
return dir_based_config_filename
elif not exists_already:
return postfix_config_filename
def template_file_and_path(self, doc):
ws_template = doc.setting('ws-template')
if ws_template and not isinstance(ws_template, bool):
template_file = ws_template
else:
template_file = self.setting('default-template')
template_path = None
for subpath in reverse_iter_paths(doc.name):
template_path = os.path.join(subpath, template_file)
if file_exists(template_path):
break
if not template_path:
msg = "no template path for %s" % doc.key
raise dexy.exceptions.UserFeedback(msg)
else:
msg = " using template %s for %s"
msgargs = (template_path, doc.key)
self.log_debug(msg % msgargs)
return (
template_file,
template_path,
)
def gen_command(
plugins='', # extra python packages to load so plugins will register with dexy
d=None, # The directory to place generated files in, must not exist.
t=False, # Shorter alternative to --template.
template=DEFAULT_TEMPLATE, # The alias of the template to use.
**kwargs # Additional kwargs passed to template's run() method.
):
"""
Generate a new dexy project in the specified directory, using the template.
"""
wrapper = init_wrapper(locals())
if t and (template == DEFAULT_TEMPLATE):
template = t
elif t and template != DEFAULT_TEMPLATE:
raise dexy.exceptions.UserFeedback("Only specify one of --t or --template, not both.")
if not template in dexy.template.Template.plugins:
print "Can't find a template named '%s'. Run 'dexy templates' for a list of templates." % template
sys.exit(1)
template_instance = dexy.template.Template.create_instance(template)
template_instance.generate(d, **kwargs)
# We run dexy setup. This will respect any dexy.conf file in the template
# but passing command line options for 'setup' to 'gen' currently not supported.
os.chdir(d)
wrapper.create_dexy_dirs()
print "Success! Your new dexy project has been created in directory '%s'" % d
if file_exists("README"):
print "\n--------------------------------------------------"
with open("README", "r") as f:
print f.read()
print "\n--------------------------------------------------"
print "\nThis information is in the 'README' file for future reference."
def process(self):
command = self.command_string()
proc, stdout = self.run_command(command, self.setup_env())
self.handle_subprocess_proc_return(command, proc.returncode, stdout)
if not self.output_data.is_cached():
# Find the first page
for pagenum in (
'1',
'01',
'001',
'0001',
):
basename = os.path.join(self.workspace(),
self.output_data.name)
first_page_file = "%s-%s.png" % (basename, pagenum)
if file_exists(first_page_file):
print "Copy from '%s'" % first_page_file
self.output_data.copy_from_file(first_page_file)
break
assert self.output_data.is_cached()
if self.setting('add-new-files'):
self.log_debug("adding new files found in %s for %s" %
(self.workspace(), self.key))
self.add_new_files()
def conf_command(
conf=defaults['config_file'], # name of config file to write to
p=False # whether to print to stdout rather than write to file
):
"""
Write a config file containing dexy's defaults.
"""
if file_exists(conf) and not p:
print "Config file %s already exists, will print conf to stdout instead..." % conf
p = True
config = default_config()
# No point specifying config file name in config file.
del config['conf']
YAML_HELP = """# YAML config file for dexy.
# You can delete any lines you don't wish to customize.
# Options are same as command line options, for more info run 'dexy help -on dexy'.\n"""
if p:
print yaml.dump(config, default_flow_style=False)
else:
with open(conf, "wb") as f:
if conf.endswith(".yaml") or conf.endswith(".conf"):
f.write(YAML_HELP)
f.write(yaml.dump(config, default_flow_style=False))
elif conf.endswith(".json"):
json.dump(config, f, sort_keys=True, indent=4)
else:
raise dexy.exceptions.UserFeedback("Don't know how to write config file '%s'" % conf)
print "Config file has been written to '%s'" % conf
def iter_dexy_dirs(self):
"""
Iterate over the required dirs (e.g. artifacts, logs)
"""
for d in self.__class__._required_dirs:
dirpath = self.__dict__[d]
safety_filepath = os.path.join(dirpath, self.safety_filename)
try:
stat = os.stat(dirpath)
except OSError:
stat = None
if stat:
if not file_exists(safety_filepath):
msg = (
s(
"""You need to manually delete the '%s' directory
and then run 'dexy setup' to create new directories. This
should just be a once-off issue due to a change in dexy to
prevent accidentally deleting directories which dexy does
not create.
"""
)
% dirpath
)
raise UserFeedback(msg)
yield (dirpath, safety_filepath, stat)
def conf_command(
conf=defaults['config_file'], # name of config file to write to
p=False # whether to print to stdout rather than write to file
):
"""
Write a config file containing dexy's defaults.
"""
if file_exists(conf) and not p:
print "Config file %s already exists, will print conf to stdout instead..." % conf
p = True
config = default_config()
# No point specifying config file name in config file.
del config['conf']
YAML_HELP = """# YAML config file for dexy.
# You can delete any lines you don't wish to customize.
# Options are same as command line options, for more info run 'dexy help -on dexy'.\n"""
if p:
print yaml.dump(config, default_flow_style=False)
else:
with open(conf, "wb") as f:
if conf.endswith(".yaml") or conf.endswith(".conf"):
f.write(YAML_HELP)
f.write(yaml.dump(config, default_flow_style=False))
elif conf.endswith(".json"):
json.dump(config, f, sort_keys=True, indent=4)
else:
raise dexy.exceptions.UserFeedback(
"Don't know how to write config file '%s'" % conf)
print "Config file has been written to '%s'" % conf
def read_param(self, param_name):
param_value = None
for filename in self.api_key_locations():
if "~" in filename:
filename = os.path.expanduser(filename)
if file_exists(filename):
with open(filename, "r") as f:
params = json.load(f)
if params.has_key(self.setting('api-key-name')):
param_value = params[self.setting('api-key-name')].get(param_name)
if param_value:
break
if param_value and isinstance(param_value, basestring) and param_value.startswith("$"):
# need to get value of bash variable
param_value_from_env = os.getenv(param_value.lstrip("$"))
if not param_value_from_env:
raise Exception("Bash variable %s not defined in this environment!" % param_value)
param_value = param_value_from_env
if param_value:
return param_value
else:
msg = "Could not find %s for %s in: %s" % (param_name, self.setting('api-key-name'), ", ".join(self.api_key_locations()))
raise Exception(msg)
def create_keyfile(self, keyfilekey):
"""
Creates a key file.
"""
key_filename = os.path.expanduser(self.setting(keyfilekey))
if file_exists(key_filename):
raise dexy.exceptions.UserFeedback("File %s already exists!" %
key_filename)
keyfile_content = {}
for filter_instance in dexy.filter.Filter:
if isinstance(
filter_instance,
ApiFilter) and not filter_instance.__class__ == ApiFilter:
api_key_name = filter_instance.setting('api-key-name')
# We want to create a keyfile for this filter instance.
keyfile_content[api_key_name] = {}
# Get all the entries we want in the keyfile.
for k, v in filter_instance.setting_values().iteritems():
if k.startswith("api_"):
keyfile_content[api_key_name][k.replace("api_",
"")] = "TODO"
with open(key_filename, "wb") as f:
json.dump(keyfile_content, f, sort_keys=True, indent=4)
def gen_command(
plugins='', # extra python packages to load so plugins will register with dexy
d=None, # The directory to place generated files in, must not exist.
t=False, # Shorter alternative to --template.
template=DEFAULT_TEMPLATE, # The alias of the template to use.
**kwargs # Additional kwargs passed to template's run() method.
):
"""
Generate a new dexy project in the specified directory, using the template.
"""
wrapper = init_wrapper(locals())
if t and (template == DEFAULT_TEMPLATE):
template = t
elif t and template != DEFAULT_TEMPLATE:
raise dexy.exceptions.UserFeedback("Only specify one of --t or --template, not both.")
if not template in dexy.template.Template.plugins:
print("Can't find a template named '%s'. Run 'dexy templates' for a list of templates." % template)
sys.exit(1)
template_instance = dexy.template.Template.create_instance(template)
template_instance.generate(d, **kwargs)
# We run dexy setup. This will respect any dexy.conf file in the template
# but passing command line options for 'setup' to 'gen' currently not supported.
os.chdir(d)
wrapper.create_dexy_dirs()
print("Success! Your new dexy project has been created in directory '%s'" % d)
if file_exists("README"):
print("\n--------------------------------------------------")
with open("README", "r") as f:
print(f.read())
print("\n--------------------------------------------------")
print("\nThis information is in the 'README' file for future reference.")
def read_param(self, param_name, default=False):
param_value = None
for filename in self.api_key_locations():
if "~" in filename:
filename = os.path.expanduser(filename)
if file_exists(filename):
with open(filename, "r") as f:
params = json.load(f)
if self.setting('api-key-name') in params:
param_value = params[self.setting('api-key-name')].get(param_name)
if param_value:
break
if param_value and isinstance(param_value, str) and param_value.startswith("$"):
# need to get value of bash variable
param_value_from_env = os.getenv(param_value.lstrip("$"))
if not param_value_from_env:
raise KeyError("Bash variable %s not defined in this environment!" % param_value)
param_value = param_value_from_env
if param_value:
return param_value
elif default:
return default
else:
msg = "Could not find %s for %s in: %s" % (param_name, self.setting('api-key-name'), ", ".join(self.api_key_locations()))
raise KeyError(msg)
def read_document_config(self):
document_config = self.document_config_file()
if file_exists(document_config):
with open(document_config, "r") as f:
return json.load(f)
else:
msg = "Filter %s needs a file %s, couldn't find it."
raise dexy.exceptions.UserFeedback(msg % (self.alias, document_config))
def document_config_file(self):
postfix_config_filename = "%s%s" % (
os.path.splitext(self.output_data.name)[0],
self.setting('document-api-config-postfix'))
if file_exists(postfix_config_filename):
return postfix_config_filename
else:
return os.path.join(self.output_data.parent_dir(),
self.setting('document-api-config-file'))
def copy_canonical_file(self):
canonical_file = os.path.join(self.workspace(), self.output_data.name)
if not self.output_data.is_cached() and file_exists(canonical_file):
self.output_data.copy_from_file(canonical_file)
if self.setting('add-new-files') and self.setting('make-dummy-output'):
if not self.output_data.is_cached():
self.output_data.set_data("ok")
self.output_data.update_settings({"canonical-output": False})
def read_document_config(self):
document_config = self.document_config_file()
if file_exists(document_config):
with open(document_config, "r") as f:
return json.load(f)
else:
msg = "Filter %s needs a file %s, couldn't find it."
raise dexy.exceptions.UserFeedback(msg %
(self.alias, document_config))
def create_reports_dir(self):
if not self.report_dir():
return False
if not file_exists(self.report_dir()):
os.makedirs(self.report_dir())
self.write_safety_file()
if self.readme_filepath():
self.write_readme_file()
def create_reports_dir(self):
if not self.report_dir():
return False
if not file_exists(self.report_dir()):
os.makedirs(self.report_dir())
self.write_safety_file()
if self.readme_filepath():
self.write_readme_file()
def serve_command(
port=-1,
reporters=['ws',
'output'], # Reporters whose output to try to serve (in order).
directory=False # Custom directory to be served.
):
"""
Runs a simple web server on dexy-generated files. Will look first to see if
the Website Reporter has run, if so this content is served. If not the
standard output/ directory contents are served. You can also specify
another directory to be served. The port defaults to 8085, this can also be
customized.
"""
if not directory:
for alias in reporters:
reporter = dexy.reporter.Reporter.create_instance(alias)
if file_exists(reporter.setting('dir')):
directory = reporter.setting('dir')
break
if not directory:
print NO_OUTPUT_MSG
sys.exit(1)
os.chdir(directory)
if port < 0:
ports = range(8085, 8100)
else:
ports = [port]
p = None
for p in ports:
try:
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
httpd = SocketServer.TCPServer(("", p), Handler)
except socket.error:
print "port %s already in use" % p
p = None
else:
break
if p:
print "serving contents of %s on http://localhost:%s" % (directory, p)
print "type ctrl+c to stop"
try:
httpd.serve_forever()
except KeyboardInterrupt:
sys.exit(1)
else:
print "could not find a free port to serve on, tried", ports
sys.exit(1)
def serve_command(
port=-1,
reporters=['ws', 'output'], # Reporters whose output to try to serve (in order).
directory=False # Custom directory to be served.
):
"""
Runs a simple web server on dexy-generated files. Will look first to see if
the Website Reporter has run, if so this content is served. If not the
standard output/ directory contents are served. You can also specify
another directory to be served. The port defaults to 8085, this can also be
customized.
"""
if not directory:
for alias in reporters:
reporter = dexy.reporter.Reporter.create_instance(alias)
if file_exists(reporter.setting('dir')):
directory = reporter.setting('dir')
break
if not directory:
print NO_OUTPUT_MSG
sys.exit(1)
os.chdir(directory)
if port < 0:
ports = range(8085, 8100)
else:
ports = [port]
p = None
for p in ports:
try:
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
httpd = SocketServer.TCPServer(("", p), Handler)
except socket.error:
print "port %s already in use" % p
p = None
else:
break
if p:
print "serving contents of %s on http://localhost:%s" % (directory, p)
print "type ctrl+c to stop"
try:
httpd.serve_forever()
except KeyboardInterrupt:
sys.exit(1)
else:
print "could not find a free port to serve on, tried", ports
sys.exit(1)
def remove_reports_dir(self, wrapper, keep_empty_dir=False):
self.wrapper = wrapper
if not self.report_dir():
return False
if file_exists(self.report_dir()) and not file_exists(self.safety_filepath()):
msg = "Please remove directory %s, Dexy wants to write a report here but there's already a file or directory in this location."
msgargs = (os.path.abspath(self.report_dir()),)
raise dexy.exceptions.UserFeedback(msg % msgargs)
elif file_exists(self.report_dir()):
if keep_empty_dir:
# Does not remove the base directory, useful if you are running
# a process (like 'dexy serve') from inside that directory
for f in os.listdir(self.report_dir()):
if not f in self.setting('no-delete'):
path = os.path.join(self.report_dir(), f)
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.remove(path)
self.write_safety_file()
else:
shutil.rmtree(self.report_dir())
def remove_reports_dir(self, wrapper, keep_empty_dir=False):
self.wrapper = wrapper
if not self.report_dir():
return False
if file_exists(self.report_dir()) and not file_exists(self.safety_filepath()):
msg = "Please remove directory %s, Dexy wants to write a report here but there's already a file or directory in this location."
msgargs = (os.path.abspath(self.report_dir()),)
raise dexy.exceptions.UserFeedback(msg % msgargs)
elif file_exists(self.report_dir()):
if keep_empty_dir:
# Does not remove the base directory, useful if you are running
# a process (like 'dexy serve') from inside that directory
for f in os.listdir(self.report_dir()):
if not f in self.setting('no-delete'):
path = os.path.join(self.report_dir(), f)
if os.path.isdir(path):
shutil.rmtree(path)
else:
os.remove(path)
self.write_safety_file()
else:
shutil.rmtree(self.report_dir())
def write_params(self, params):
api_key_name = self.setting('api-key-name')
location = self.setting('project-api-key-file')
if file_exists(location):
with open(location, 'r') as f:
existing_config = json.load(f)
else:
existing_config = {api_key_name : {}}
existing_config[api_key_name].update(params)
with open(location, 'w') as f:
json.dump(existing_config, f, indent=4, sort_keys=True)
def config_args(modargs):
cliargs = modargs.get("__cli_options", {})
kwargs = modargs.copy()
config_file = modargs.get('conf', dexy.utils.defaults['config_file'])
# Update from config file
if file_exists(config_file):
with open(config_file, "r") as f:
if config_file.endswith(".conf"):
try:
conf_args = parse_yaml(f.read())
except dexy.exceptions.UserFeedback as yaml_exception:
try:
conf_args = parse_json(f.read())
except dexy.exceptions.UserFeedback as json_exception:
print("--------------------------------------------------")
print("Tried to parse YAML:")
print(yaml_exception)
print("--------------------------------------------------")
print("Tried to parse JSON:")
print(json_exception)
print("--------------------------------------------------")
raise dexy.exceptions.UserFeedback("Unable to parse config file '%s' as YAML or as JSON." % config_file)
elif config_file.endswith(".yaml"):
conf_args = parse_yaml(f.read())
elif config_file.endswith(".json"):
conf_args = parse_json(f.read())
else:
raise dexy.exceptions.UserFeedback("Don't know how to load config from '%s'" % config_file)
if conf_args:
# TODO raise error if
kwargs.update(conf_args)
if cliargs: # cliargs may be False
for k in list(cliargs.keys()):
try:
kwargs[k] = modargs[k]
except KeyError:
msg = "This command does not take a '--%s' argument." % k
raise dexy.exceptions.UserFeedback(msg)
# TODO allow updating from env variables, e.g. DEXY_ARTIFACTS_DIR
return kwargs
def config_args(modargs):
cliargs = modargs.get("__cli_options", {})
kwargs = modargs.copy()
config_file = modargs.get('conf', dexy.utils.defaults['config_file'])
# Update from config file
if file_exists(config_file):
with open(config_file, "rb") as f:
if config_file.endswith(".conf"):
try:
conf_args = parse_yaml(f.read())
except dexy.exceptions.UserFeedback as yaml_exception:
try:
conf_args = parse_json(f.read())
except dexy.exceptions.UserFeedback as json_exception:
print "--------------------------------------------------"
print "Tried to parse YAML:"
print yaml_exception
print "--------------------------------------------------"
print "Tried to parse JSON:"
print json_exception
print "--------------------------------------------------"
raise dexy.exceptions.UserFeedback("Unable to parse config file '%s' as YAML or as JSON." % config_file)
elif config_file.endswith(".yaml"):
conf_args = parse_yaml(f.read())
elif config_file.endswith(".json"):
conf_args = parse_json(f.read())
else:
raise dexy.exceptions.UserFeedback("Don't know how to load config from '%s'" % config_file)
if conf_args:
kwargs.update(conf_args)
if cliargs: # cliargs may be False
for k in cliargs.keys():
try:
kwargs[k] = modargs[k]
except KeyError:
msg = "This command does not take a '--%s' argument." % k
raise dexy.exceptions.UserFeedback(msg)
# TODO allow updating from env variables, e.g. DEXY_ARTIFACTS_DIR
return kwargs
def config_args(modargs):
cliargs = modargs.get("__cli_options", {})
kwargs = modargs.copy()
config_file = modargs.get('conf', dexy.utils.defaults['config_file'])
# Update from config file
if file_exists(config_file):
with open(config_file, "rb") as f:
if config_file.endswith(".conf"):
try:
conf_args = parse_yaml(f.read())
except dexy.exceptions.UserFeedback as yaml_exception:
try:
conf_args = parse_json(f.read())
except dexy.exceptions.UserFeedback as json_exception:
print "--------------------------------------------------"
print "Tried to parse YAML:"
print yaml_exception
print "--------------------------------------------------"
print "Tried to parse JSON:"
print json_exception
print "--------------------------------------------------"
raise dexy.exceptions.UserFeedback(
"Unable to parse config file '%s' as YAML or as JSON."
% config_file)
elif config_file.endswith(".yaml"):
conf_args = parse_yaml(f.read())
elif config_file.endswith(".json"):
conf_args = parse_json(f.read())
else:
raise dexy.exceptions.UserFeedback(
"Don't know how to load config from '%s'" % config_file)
if conf_args:
kwargs.update(conf_args)
if cliargs: # cliargs may be False
for k in cliargs.keys():
kwargs[k] = modargs[k]
# TODO allow updating from env variables, e.g. DEXY_ARTIFACTS_DIR
return kwargs
def process(self):
command = self.command_string()
proc, stdout = self.run_command(command, self.setup_env())
self.handle_subprocess_proc_return(command, proc.returncode, stdout)
if not self.output_data.is_cached():
# Find the first page
for pagenum in ('1', '01', '001', '0001',):
basename = os.path.join(self.workspace(), self.output_data.name)
first_page_file = "%s-%s.png" % (basename, pagenum)
if file_exists(first_page_file):
print "Copy from '%s'" % first_page_file
self.output_data.copy_from_file(first_page_file)
break
assert self.output_data.is_cached()
if self.setting('add-new-files'):
self.log_debug("adding new files found in %s for %s" % (self.workspace(), self.key))
self.add_new_files()
def create_keyfile(self, keyfilekey):
"""
Creates a key file.
"""
key_filename = os.path.expanduser(self.setting(keyfilekey))
if file_exists(key_filename):
raise dexy.exceptions.UserFeedback("File %s already exists!" % key_filename)
keyfile_content = {}
for filter_instance in dexy.filter.Filter:
if isinstance(filter_instance, ApiFilter) and not filter_instance.__class__ == ApiFilter:
api_key_name = filter_instance.setting('api-key-name')
# We want to create a keyfile for this filter instance.
keyfile_content[api_key_name] = {}
# Get all the entries we want in the keyfile.
for k, v in filter_instance.setting_values().iteritems():
if k.startswith("api_"):
keyfile_content[api_key_name][k.replace("api_", "")] = "TODO"
with open(key_filename, "wb") as f:
json.dump(keyfile_content, f, sort_keys = True, indent=4)
def template_file_and_path(self, doc):
ws_template = doc.setting('ws-template')
if ws_template and not isinstance(ws_template, bool):
template_file = ws_template
else:
template_file = self.setting('default-template')
template_path = None
for subpath in reverse_iter_paths(doc.name):
template_path = os.path.join(subpath, template_file)
if file_exists(template_path):
break
if not template_path:
msg = "no template path for %s" % doc.key
raise dexy.exceptions.UserFeedback(msg)
else:
msg = " using template %s for %s"
msgargs = (template_path, doc.key)
self.log_debug(msg % msgargs)
return (template_file, template_path,)
def iter_dexy_dirs(self):
"""
Iterate over the required dirs (e.g. artifacts, logs)
"""
for d in self.__class__._required_dirs:
dirpath = self.__dict__[d]
safety_filepath = os.path.join(dirpath, self.safety_filename)
try:
stat = os.stat(dirpath)
except OSError:
stat = None
if stat:
if not file_exists(safety_filepath):
msg = s("""You need to manually delete the '%s' directory
and then run 'dexy setup' to create new directories. This
should just be a once-off issue due to a change in dexy to
prevent accidentally deleting directories which dexy does
not create.
""") % dirpath
raise UserFeedback(msg)
yield (dirpath, safety_filepath, stat)
def serve_command(
port=-1,
reporters=['ws', 'output'], # Reporters whose output to try to serve (in order).
username='', # http auth username to use (if provided)
password='', # http auth password to use (if provided)
realm='Dexy', # http auth realm to use (if username and password are provided)
directory=False, # Custom directory to be served.
**kwargs
):
"""
Runs a simple web server on dexy-generated files.
Will look first to see if the Website Reporter has run, if so this content
is served. If not the standard output/ directory contents are served. You
can also specify another directory to be served. The port defaults to 8085,
this can also be customized. If a username and password are provided, uses
HTTP auth to access pages.
"""
if not directory:
wrapper = init_wrapper(locals(), True)
for alias in reporters:
report_dir = dexy.reporter.Reporter.create_instance(alias).setting("dir")
print "report dir", report_dir
if report_dir and file_exists(report_dir):
directory = report_dir
break
if not directory:
print NO_OUTPUT_MSG
sys.exit(1)
os.chdir(directory)
if port < 0:
ports = range(8085, 8100)
else:
ports = [port]
p = None
for p in ports:
try:
if username and password:
import base64
authcode = base64.b64encode("%s:%s" % (username, password))
Handler = SimpleHTTPAuthRequestHandler
Handler.authcode = authcode
Handler.realm = realm
else:
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
httpd = SocketServer.TCPServer(("", p), Handler)
except socket.error:
print "port %s already in use" % p
p = None
else:
break
if p:
print "serving contents of %s on http://localhost:%s" % (directory, p)
if username and password and Handler.authcode:
print "username '%s' and password '%s' are required to access contents" % (username, password)
print "type ctrl+c to stop"
try:
httpd.serve_forever()
except KeyboardInterrupt:
sys.exit(1)
else:
print "could not find a free port to serve on, tried", ports
sys.exit(1)
def apply_and_render_template(self, doc):
# Figure out which template to use.
ws_template = doc.setting('ws-template')
if ws_template and not isinstance(ws_template, bool):
template_file = ws_template
else:
template_file = self.setting('default-template')
# Look for a file named template_file in nearest parent dir to document.
template_path = None
for subpath in reverse_iter_paths(doc.name):
template_path = os.path.join(subpath, template_file)
if file_exists(template_path):
break
if not template_path:
raise dexy.exceptions.UserFeedback("no template path for %s" %
doc.key)
else:
self.log_debug(" using template %s for %s" %
(template_path, doc.key))
# Populate template environment
env = Environment(undefined=jinja2.StrictUndefined)
dirs = [".", os.path.dirname(__file__), os.path.dirname(template_path)]
env.loader = FileSystemLoader(dirs)
self.log_debug(" loading template at %s" % template_path)
template = env.get_template(template_path)
current_dir = posixpath.dirname(doc.output_data().output_name())
parent_dir = os.path.split(current_dir)[0]
env_data = {}
for alias in self.setting('plugins'):
plugin = dexy.filters.templating_plugins.TemplatePlugin.create_instance(
alias)
env_data.update(plugin.run())
navigation = {}
env_data.update({
'attrgetter': operator.attrgetter,
'itemgetter': operator.itemgetter,
'content': doc.output_data(),
'locals': locals,
'navigation': navigation,
'nav': self._navobj.nodes["/%s" % current_dir],
'root': self._navobj.nodes["/"],
'navobj': self._navobj,
'page_title': doc.output_data().title(),
'parent_dir': parent_dir,
'current_dir': current_dir,
's': doc.output_data(),
'source': doc.output_data().output_name(),
'template_source': template_path,
'wrapper': self.wrapper,
'year': datetime.now().year
})
if self.wrapper.globals:
env_data.update(dict_from_string(self.wrapper.globals))
fp = os.path.join(self.setting('dir'),
doc.output_data().output_name()).replace(
".json", ".html")
parent_dir = os.path.dirname(fp)
try:
os.makedirs(os.path.dirname(fp))
except os.error:
pass
self.log_debug(" writing to %s" % (fp))
template.stream(env_data).dump(fp, encoding="utf-8")
def create_cache_reports_dir(self):
if not file_exists(self.cache_reports_dir()):
os.makedirs(self.cache_reports_dir())
def document_config_file(self):
postfix_config_filename = "%s%s" % (os.path.splitext(self.output_data.name)[0], self.setting('document-api-config-postfix'))
if file_exists(postfix_config_filename):
return postfix_config_filename
else:
return os.path.join(self.output_data.parent_dir(), self.setting('document-api-config-file'))
def dexy_dirs_exist(self):
"""
Returns a boolean indicating whether dexy dirs exist.
"""
return all(file_exists(d[0]) for d in self.iter_dexy_dirs())
def copy_canonical_file(self):
canonical_file = os.path.join(self.workspace(), self.output_data.name)
if not self.output_data.is_cached() and file_exists(canonical_file):
self.output_data.copy_from_file(canonical_file)
def create_cache_reports_dir(self):
if not file_exists(self.cache_reports_dir()):
os.makedirs(self.cache_reports_dir())
def serve_command(
port=-1,
reporters=['ws', 'output'
], # Reporters whose output to try to serve (in order).
username='', # http auth username to use (if provided)
password='', # http auth password to use (if provided)
realm='Dexy', # http auth realm to use (if username and password are provided)
directory=False, # Custom directory to be served.
**kwargs):
"""
Runs a simple web server on dexy-generated files.
Will look first to see if the Website Reporter has run, if so this content
is served. If not the standard output/ directory contents are served. You
can also specify another directory to be served. The port defaults to 8085,
this can also be customized. If a username and password are provided, uses
HTTP auth to access pages.
"""
if not directory:
wrapper = init_wrapper(locals(), True)
for alias in reporters:
report_dir = dexy.reporter.Reporter.create_instance(alias).setting(
"dir")
print "report dir", report_dir
if report_dir and file_exists(report_dir):
directory = report_dir
break
if not directory:
print NO_OUTPUT_MSG
sys.exit(1)
os.chdir(directory)
if port < 0:
ports = range(8085, 8100)
else:
ports = [port]
p = None
for p in ports:
try:
if username and password:
import base64
authcode = base64.b64encode("%s:%s" % (username, password))
Handler = SimpleHTTPAuthRequestHandler
Handler.authcode = authcode
Handler.realm = realm
else:
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
httpd = SocketServer.TCPServer(("", p), Handler)
except socket.error:
print "port %s already in use" % p
p = None
else:
break
if p:
print "serving contents of %s on http://localhost:%s" % (directory, p)
if username and password and Handler.authcode:
print "username '%s' and password '%s' are required to access contents" % (
username, password)
print "type ctrl+c to stop"
try:
httpd.serve_forever()
except KeyboardInterrupt:
sys.exit(1)
else:
print "could not find a free port to serve on, tried", ports
sys.exit(1)
def apply_and_render_template(self, doc):
# Figure out which template to use.
ws_template = doc.setting('ws-template')
if ws_template and not isinstance(ws_template, bool):
template_file = ws_template
else:
template_file = self.setting('default-template')
# Look for a file named template_file in nearest parent dir to document.
template_path = None
for subpath in reverse_iter_paths(doc.name):
template_path = os.path.join(subpath, template_file)
if file_exists(template_path):
break
if not template_path:
raise dexy.exceptions.UserFeedback("no template path for %s" % doc.key)
else:
self.log_debug(" using template %s for %s" % (template_path, doc.key))
# Populate template environment
env = Environment(undefined=jinja2.StrictUndefined)
dirs = [".", os.path.dirname(__file__), os.path.dirname(template_path)]
env.loader = FileSystemLoader(dirs)
self.log_debug(" loading template at %s" % template_path)
template = env.get_template(template_path)
current_dir = posixpath.dirname(doc.output_data().output_name())
parent_dir = os.path.split(current_dir)[0]
env_data = {}
for alias in self.setting('plugins'):
plugin = dexy.filters.templating_plugins.TemplatePlugin.create_instance(alias)
env_data.update(plugin.run())
navigation = {
}
env_data.update({
'attrgetter' : operator.attrgetter,
'itemgetter' : operator.itemgetter,
'content' : doc.output_data(),
'locals' : locals,
'navigation' : navigation,
'nav' : self._navobj.nodes["/%s" % current_dir],
'root' : self._navobj.nodes["/"],
'navobj' : self._navobj,
'page_title' : doc.output_data().title(),
'parent_dir' : parent_dir,
'current_dir' : current_dir,
's' : doc.output_data(),
'source' : doc.output_data().output_name(),
'template_source' : template_path,
'wrapper' : self.wrapper,
'year' : datetime.now().year
})
if self.wrapper.globals:
env_data.update(dict_from_string(self.wrapper.globals))
fp = os.path.join(self.setting('dir'), doc.output_data().output_name()).replace(".json", ".html")
parent_dir = os.path.dirname(fp)
try:
os.makedirs(os.path.dirname(fp))
except os.error:
pass
self.log_debug(" writing to %s" % (fp))
template.stream(env_data).dump(fp, encoding="utf-8")
def copy_canonical_file(self):
canonical_file = os.path.join(self.workspace(), self.output_data.name)
if not self.output_data.is_cached() and file_exists(canonical_file):
self.output_data.copy_from_file(canonical_file)
def dexy_dirs_exist(self):
"""
Returns a boolean indicating whether dexy dirs exist.
"""
return all(file_exists(d[0]) for d in self.iter_dexy_dirs())
