def skipparse(self, dataset, size): """Parse dataset as an N-Triples/N3 file.""" if not hasattr(dataset, 'read'): raise ParseError("Item to parse must be a file-like object.") self.file = codecs.getreader('utf-8')( dataset ) # since N-Triples 1.1 files can and should be utf-8 encoded self.buffer = '' line = '' nb_lines = 0 logger.info(f'Start parsing the RDF file... {nb_lines} line written') while size == -1 or nb_lines < size: line = self.readline() self.line = line if self.line is None: break try: self.parseline() nb_lines += 1 if nb_lines % 10000 == 0: logger.info( f'Parsing the RDF file... {nb_lines} lines written') except (ParseError, UnicodeEncodeError) as exception: logger.warning( f"parse error: dropping {line}. Reason {exception}") continue logger.info(f'Parsing complete... {nb_lines} lines written') return self.sink
def parseline(self) -> Generator: """ Parse each line and yield triples. Parameters ---------- Generator A generator """ print(self.line) self.eat(r_wspace) if self.line or not self.line.startswith('#'): subject = self.subject() self.eat(r_wspaces) predicate = self.predicate() self.eat(r_wspaces) object = self.object() self.eat(r_tail) if self.line: raise ParseError("Trailing garbage") return self.sink.triple(subject, predicate, object)
def eat(self, pattern): m = pattern.match(self.line) if not m: # @@ Why can't we get the original pattern? # print(dir(pattern)) # print repr(self.line), type(self.line) raise ParseError('Failed to eat %s at %s' % (pattern.pattern, self.line)) self.line = self.line[m.end():] return m
def parse_literal(strng): m = r_literal.match(strng) if not m: m = r_literal.match(strng.replace("\\", "")) if not m: raise ParseError("Failed to eat %s at %s" % (r_literal.pattern, strng)) lit, lang, dtype = m.groups() if lang: lang = lang else: lang = None if dtype: dtype = dtype else: dtype = None if lang and dtype: raise ParseError("Can't have both a language and a datatype") lit = unquote(lit) return Literal(lit, lang, dtype)
def parseline(self): subject = self.subject() self.eat(r_wspace) predicate = self.predicate() self.eat(r_wspace) obj = self.object() self.eat(r_tail) if self.line: raise ParseError("Trailing garbage") return subject, predicate, obj
def parse(self, inputsource, sink, **kwargs): """Parse f as an N-Triples file.""" assert sink.store.context_aware, ("NQuadsParser must be given" " a context aware store.") self.sink = sink source = inputsource.getByteStream() if not hasattr(source, 'read'): raise ParseError("Item to parse must be a file-like object.") self.file = source self.buffer = '' while True: self.line = __line = self.readline() if self.line is None: break try: self.parseline() except ParseError, msg: raise ParseError("Invalid line (%s):\n%r" % (msg, __line))
def iter(self, inputsource): """Iter f as an N-Quads file.""" inputsource = create_input_source(source=inputsource, format='nquads') source = inputsource.getByteStream() if not hasattr(source, 'read'): raise ParseError("Item to parse must be a file-like object.") source = getreader('utf-8')(source) self.file = source self.buffer = '' while True: self.line = __line = self.readline() if self.line is None: break self.eat(r_wspace) if (not self.line) or self.line.startswith(('#')): continue # The line is empty or a comment try: yield self.parseline() except ParseError as msg: raise ParseError("Invalid line (%s):\n%r" % (msg, __line))
def parse(self, inputsource, sink, bnode_context=None, **kwargs): """ Parse inputsource as an N-Quads file. :type inputsource: `rdflib.parser.InputSource` :param inputsource: the source of N-Quads-formatted data :type sink: `rdflib.graph.Graph` :param sink: where to send parsed triples :type bnode_context: `dict`, optional :param bnode_context: a dict mapping blank node identifiers to `~rdflib.term.BNode` instances. See `.NTriplesParser.parse` """ assert sink.store.context_aware, ("NQuadsParser must be given" " a context aware store.") self.sink = ConjunctiveGraph(store=sink.store, identifier=sink.identifier) source = inputsource.getCharacterStream() if not source: source = inputsource.getByteStream() source = getreader("utf-8")(source) if not hasattr(source, "read"): raise ParseError("Item to parse must be a file-like object.") self.file = source self.buffer = "" while True: self.line = __line = self.readline() if self.line is None: break try: self.parseline(bnode_context) except ParseError as msg: raise ParseError("Invalid line (%s):\n%r" % (msg, __line)) return self.sink
def parse(self, filename: str) -> Generator: """ Parses an N-Triples file and yields triples. Parameters ---------- filename: str The filename to parse Returns ------- Generator A generator for triples """ if not hasattr(filename, "read"): raise ParseError("Item to parse must be a file-like object.") # since N-Triples 1.1 files can and should be utf-8 encoded f = codecs.getreader("utf-8")(filename) self.file = f self.buffer = "" while True: self.line = self.readline() if self.line is None: break if self.line == "": raise ParseError(f"Empty line encountered in {filename}. " f"Ensure that no leading or trailing empty lines persist " f"in the N-Triples file.") break try: yield from self.parseline() except ParseError: raise ParseError("Invalid line: %r" % self.line)
def get_triples(self): self.mode = 'parse' parser = NTriplesParser_() parser.sink = self.Sink(self) self.triple = None while True: parser.line = self._stream.readline().strip().decode('utf-8') if not parser.line: break try: parser.parseline() except ParseError: raise ParseError("Invalid line: %r" % parser.line) if self.triple: yield self.triple self.triple = None
def literal(self): if self.peek('"'): lit, lang, dtype = self.eat(r_literal).groups() if lang: lang = lang else: lang = None if dtype: dtype = dtype else: dtype = None if lang and dtype: raise ParseError("Can't have both a language and a datatype") lit = unquote(lit) return rdfuri.Literal(lit, dtype, lang) return False
def parseline(self): self.eat(r_wspace) if (not self.line) or self.line.startswith(b('#')): return # The line is empty or a comment subject = self.subject() self.eat(r_wspaces) predicate = self.predicate() self.eat(r_wspaces) obj = self.object() self.eat(r_wspaces) context = self.uriref() self.eat(r_tail) if self.line: raise ParseError("Trailing garbage") # Must have a context aware store - add on a normal Graph # discards anything where the ctx != graph.identifier self.sink.store.add((subject, predicate, obj), context)
def skipparse(self, f): """Parse f as an N-Triples file.""" if not hasattr(f, 'read'): raise ParseError("Item to parse must be a file-like object.") # since N-Triples 1.1 files can and should be utf-8 encoded f = codecs.getreader('utf-8')(f) self.file = f self.buffer = '' while True: self.line = self.readline() if self.line is None: break try: self.parseline() except ParseError: logger.warning( f"parse error: dropping {self.line}. Reason {sys.exc_info()[0]}" ) continue # raise ParseError("Invalid line: %r" % self.line) return self.sink
def context(self): context = self.uriref() if not context: raise ParseError("Context must be a uriref") return context