def make_files(self):
"""
Makes the files for building this corpus:
directories, the original corpus and the makefile
"""
self.change_status(Status.Init)
# Make directories
map(mkdir, [self.original_dir, self.annotations_dir])
# Make makefile
with open(self.makefile, 'w') as f:
f.write(makefile(self.settings))
# Write settings file
with open(self.settings_file, 'w') as f:
import json
f.write(json.dumps(self.settings, indent=2))
if os.path.isfile(self.text_file):
# This file has probably been built by a previous incarnation of the pipeline
# (index.wsgi script has been restarted)
log.info("File exists and is not rewritten: %s" % self.build_hash)
else:
with open(self.text_file, 'w') as f:
f.write(self.text)
def get_makefile():
"""Handler for returning the makefile."""
try:
lang = request.values.get('language', 'sv')
mode = request.values.get('mode', 'plain')
settings, incremental = get_settings(lang, mode)
log.info("Returning makefile")
return Response(makefile(settings), mimetype='text/plain')
except:
trace = make_trace()
log.exception("Error in /makefile")
res = '<result>\n<trace>' + escape(trace) + '</trace>\n</result>\n'
return Response(res, mimetype='application/xml')
def handle(builds, environ, cmd=None):
error = None
try:
settings = json.loads(query(environ, 'settings', '{}'))
except:
log.exception("Error in json parsing the settings variable")
error = escape(make_trace())
settings = {}
for e in sorted(settings_validator.iter_errors(settings)):
if error is None:
error = ""
error += str(e) + "\n"
if error is not None:
log.error("Errors from schema: " + error)
yield '<result>\n<error>' + error + '</error>\n</result>\n'
else:
incremental = query(environ, 'incremental', '')
incremental = incremental.lower() == 'true'
try:
if cmd == "makefile":
log.info("Returning makefile")
yield makefile(settings)
elif cmd == "join":
log.info("Joining existing build")
yield "<result>\n"
hashnumber = query(environ, 'hash', '')
for k in join_from_hash(builds, hashnumber, incremental):
yield k
else:
log.info("Starting a new build")
yield "<result>\n"
for k in build(builds, text(environ), settings, incremental, "xml"):
yield k
except:
trace = make_trace()
log.exception("Error in handle")
yield '<trace>' + escape(trace) + '</trace>\n'
yield '</result>\n'
def __init__(self, text, settings, init_from_hash=None):
"""
Creates the necessary directories and the makefile for this
text and the JSON settings.
"""
self.status = None
self.queues = []
if init_from_hash:
self.build_hash = init_from_hash
else:
self.text = text
self.makefile_contents = makefile(settings)
self.build_hash = make_hash(self.text, self.makefile_contents)
self.settings = settings
# Directories
self.directory = os.path.join(Config.directory, self.build_hash)
self.original_dir = os.path.join(self.directory, 'original')
self.annotations_dir = os.path.join(self.directory, 'annotations')
self.export_dir = os.path.join(self.directory, 'export')
self.warnings_log_file = os.path.join(self.directory,'warnings.log');
# and files
self.makefile = os.path.join(self.directory, 'Makefile')
self.settings_file = os.path.join(self.directory, 'settings.json')
self.text_file = os.path.join(self.original_dir, 'text.xml')
# Deem this build as accessed now.
self.access()
# Output from make, line by line
self.make_out = []
# Set increments to dummy values
self.command = ""
self.steps = 0
self.step = 0
def make_files(self):
"""
Make the files for building this corpus:
directories, the original corpus and the makefile
"""
self.change_status(Status.Init)
self.access()
# Make directories
# map(mkdir, [self.directory, self.original_dir, self.annotations_dir, self.export_dir])
for i in [self.directory, self.original_dir, self.annotations_dir, self.export_dir]:
mkdir(i)
# Make makefile
with open(self.makefile, 'w') as f:
f.write(makefile(self.settings))
# Write settings file
with open(self.settings_file, 'w') as f:
import json
f.write(json.dumps(self.settings, indent=2))
# for file upload
if self.files:
for filename, text in self.files:
infile = os.path.join(self.original_dir, filename + ".xml")
with open(infile, 'w') as f:
f.write(text)
else:
if os.path.isfile(self.text_file):
# This file has probably been built by a previous incarnation of the pipeline
# (index.wsgi script has been restarted)
log.info("File exists and is not rewritten: %s" % self.build_hash)
else:
with open(self.text_file, 'w') as f:
f.write(self.text)
def __init__(self, text, settings, files=None, init_from_hash=None, resuming=False):
"""
Create the necessary directories and the makefile for this
text and the JSON settings.
"""
self.status = None
self.queues = []
self.files = files
if init_from_hash:
self.build_hash = init_from_hash
# File upload:
if init_from_hash.endswith(Config.fileupload_ext):
original_dir = os.path.join(os.path.join(Config.builds_dir, self.build_hash), 'original')
filelist = []
for root, dirs, files in os.walk(original_dir):
for infile in files:
with open(os.path.join(root, infile), "r") as f:
text = f.read()
fname = infile[:infile.rfind(".")]
filelist.append((fname, text))
self.files = filelist
else:
self.text = text
self.filename = 'text'
else:
self.makefile_contents = makefile(settings)
self.settings = settings
# File upload
if files:
self.text = "\n".join(text for _fn, text in files)
filenames = " ".join(fn for fn, _text in files)
self.build_hash = make_hash(self.text, self.makefile_contents, filenames) + Config.fileupload_ext
else:
self.text = text
self.filename = 'text'
self.build_hash = make_hash(self.text, self.makefile_contents)
# Directories
self.directory = os.path.join(Config.builds_dir, self.build_hash)
self.original_dir = os.path.join(self.directory, 'original')
self.annotations_dir = os.path.join(self.directory, 'annotations')
self.export_dir = os.path.join(self.directory, 'export.original')
# Files
self.makefile = os.path.join(self.directory, 'Makefile')
self.warnings_log_file = os.path.join(self.directory, 'warnings.log')
self.accessed_file = os.path.join(self.directory, 'accessed')
self.settings_file = os.path.join(self.directory, 'settings.json')
self.zipfpath = os.path.join(self.directory, "export.zip")
self.zipfile = "export.zip"
if not files:
self.text_file = os.path.join(self.original_dir, self.filename + '.xml')
self.result_file_path = os.path.join(self.export_dir, self.filename + '.xml')
self.result_file = self.filename + '.xml'
# Deem this build as accessed now, unless resuming an old build
self.access(resuming)
# Output from make, line by line
self.make_out = []
# Set increments to dummy values
self.command = ""
self.steps = 0
self.step = 0
