def make_files(self): """ Makes the files for building this corpus: directories, the original corpus and the makefile """ self.change_status(Status.Init) # Make directories map(mkdir, [self.original_dir, self.annotations_dir]) # Make makefile with open(self.makefile, 'w') as f: f.write(makefile(self.settings)) # Write settings file with open(self.settings_file, 'w') as f: import json f.write(json.dumps(self.settings, indent=2)) if os.path.isfile(self.text_file): # This file has probably been built by a previous incarnation of the pipeline # (index.wsgi script has been restarted) log.info("File exists and is not rewritten: %s" % self.build_hash) else: with open(self.text_file, 'w') as f: f.write(self.text)
def get_makefile(): """Handler for returning the makefile.""" try: lang = request.values.get('language', 'sv') mode = request.values.get('mode', 'plain') settings, incremental = get_settings(lang, mode) log.info("Returning makefile") return Response(makefile(settings), mimetype='text/plain') except: trace = make_trace() log.exception("Error in /makefile") res = '<result>\n<trace>' + escape(trace) + '</trace>\n</result>\n' return Response(res, mimetype='application/xml')
def handle(builds, environ, cmd=None): error = None try: settings = json.loads(query(environ, 'settings', '{}')) except: log.exception("Error in json parsing the settings variable") error = escape(make_trace()) settings = {} for e in sorted(settings_validator.iter_errors(settings)): if error is None: error = "" error += str(e) + "\n" if error is not None: log.error("Errors from schema: " + error) yield '<result>\n<error>' + error + '</error>\n</result>\n' else: incremental = query(environ, 'incremental', '') incremental = incremental.lower() == 'true' try: if cmd == "makefile": log.info("Returning makefile") yield makefile(settings) elif cmd == "join": log.info("Joining existing build") yield "<result>\n" hashnumber = query(environ, 'hash', '') for k in join_from_hash(builds, hashnumber, incremental): yield k else: log.info("Starting a new build") yield "<result>\n" for k in build(builds, text(environ), settings, incremental, "xml"): yield k except: trace = make_trace() log.exception("Error in handle") yield '<trace>' + escape(trace) + '</trace>\n' yield '</result>\n'
def __init__(self, text, settings, init_from_hash=None): """ Creates the necessary directories and the makefile for this text and the JSON settings. """ self.status = None self.queues = [] if init_from_hash: self.build_hash = init_from_hash else: self.text = text self.makefile_contents = makefile(settings) self.build_hash = make_hash(self.text, self.makefile_contents) self.settings = settings # Directories self.directory = os.path.join(Config.directory, self.build_hash) self.original_dir = os.path.join(self.directory, 'original') self.annotations_dir = os.path.join(self.directory, 'annotations') self.export_dir = os.path.join(self.directory, 'export') self.warnings_log_file = os.path.join(self.directory,'warnings.log'); # and files self.makefile = os.path.join(self.directory, 'Makefile') self.settings_file = os.path.join(self.directory, 'settings.json') self.text_file = os.path.join(self.original_dir, 'text.xml') # Deem this build as accessed now. self.access() # Output from make, line by line self.make_out = [] # Set increments to dummy values self.command = "" self.steps = 0 self.step = 0
def make_files(self): """ Make the files for building this corpus: directories, the original corpus and the makefile """ self.change_status(Status.Init) self.access() # Make directories # map(mkdir, [self.directory, self.original_dir, self.annotations_dir, self.export_dir]) for i in [self.directory, self.original_dir, self.annotations_dir, self.export_dir]: mkdir(i) # Make makefile with open(self.makefile, 'w') as f: f.write(makefile(self.settings)) # Write settings file with open(self.settings_file, 'w') as f: import json f.write(json.dumps(self.settings, indent=2)) # for file upload if self.files: for filename, text in self.files: infile = os.path.join(self.original_dir, filename + ".xml") with open(infile, 'w') as f: f.write(text) else: if os.path.isfile(self.text_file): # This file has probably been built by a previous incarnation of the pipeline # (index.wsgi script has been restarted) log.info("File exists and is not rewritten: %s" % self.build_hash) else: with open(self.text_file, 'w') as f: f.write(self.text)
def __init__(self, text, settings, files=None, init_from_hash=None, resuming=False): """ Create the necessary directories and the makefile for this text and the JSON settings. """ self.status = None self.queues = [] self.files = files if init_from_hash: self.build_hash = init_from_hash # File upload: if init_from_hash.endswith(Config.fileupload_ext): original_dir = os.path.join(os.path.join(Config.builds_dir, self.build_hash), 'original') filelist = [] for root, dirs, files in os.walk(original_dir): for infile in files: with open(os.path.join(root, infile), "r") as f: text = f.read() fname = infile[:infile.rfind(".")] filelist.append((fname, text)) self.files = filelist else: self.text = text self.filename = 'text' else: self.makefile_contents = makefile(settings) self.settings = settings # File upload if files: self.text = "\n".join(text for _fn, text in files) filenames = " ".join(fn for fn, _text in files) self.build_hash = make_hash(self.text, self.makefile_contents, filenames) + Config.fileupload_ext else: self.text = text self.filename = 'text' self.build_hash = make_hash(self.text, self.makefile_contents) # Directories self.directory = os.path.join(Config.builds_dir, self.build_hash) self.original_dir = os.path.join(self.directory, 'original') self.annotations_dir = os.path.join(self.directory, 'annotations') self.export_dir = os.path.join(self.directory, 'export.original') # Files self.makefile = os.path.join(self.directory, 'Makefile') self.warnings_log_file = os.path.join(self.directory, 'warnings.log') self.accessed_file = os.path.join(self.directory, 'accessed') self.settings_file = os.path.join(self.directory, 'settings.json') self.zipfpath = os.path.join(self.directory, "export.zip") self.zipfile = "export.zip" if not files: self.text_file = os.path.join(self.original_dir, self.filename + '.xml') self.result_file_path = os.path.join(self.export_dir, self.filename + '.xml') self.result_file = self.filename + '.xml' # Deem this build as accessed now, unless resuming an old build self.access(resuming) # Output from make, line by line self.make_out = [] # Set increments to dummy values self.command = "" self.steps = 0 self.step = 0