def __init__(self, url=None, local_directory=None, container_dir=None, **kwargs): self.local_directory = pick(local_directory, CFG.LOCAL_DATA_PATH) self.ext_url = pick(url, self.default_url) self.local_url = pick(kwargs.get('local_url'), self.default_url) self.url = None self.active = kwargs.get('active', True) if not kwargs.get('delay_check'): self.check_status if self.url: kwargs['es_url'] = self.url else: kwargs['es_url'] = self.ext_url super(Elastic, self).__init__(**kwargs) self.container_dir = container_dir if self.ext_url is None: msg = ["A Elasticsearch url must be defined. Either pass 'url'", "or initialize the 'RdfConfigManager'"] raise AttributeError(" ".join(msg))
def __init__(self, url=None, namespace=None, namespace_params=None, local_directory=None, container_dir=None, graph=None, **kwargs): self.local_directory = pick(local_directory, CFG.dirs.data) self.ext_url = pick(url, self.default_url) self.local_url = pick(kwargs.get('local_url'), self.default_url) self.log_level = log.level log.setLevel(kwargs.get("log_level", log.level)) self.namespace = pick(namespace, self.default_ns) self.namespace_params = namespace_params self.container_dir = container_dir self.graph = pick(graph, self.default_graph) self.url = None self.active = kwargs.get('active', True) if self.ext_url is None: msg = [ "A Blazegraph url must be defined. Either pass 'url'", "or initialize the 'RdfConfigManager'" ] raise AttributeError(" ".join(msg)) if not kwargs.get('delay_check'): self.check_status self.__set_mgr__(**kwargs)
def __init__(self, url=None, local_directory=None, container_dir=None, **kwargs): self.local_directory = pick(local_directory, CFG.LOCAL_DATA_PATH) self.ext_url = pick(url, self.default_url) self.local_url = pick(kwargs.get('local_url'), self.default_url) self.url = None self.active = kwargs.get('active', True) if not kwargs.get('delay_check'): self.check_status if self.url: kwargs['es_url'] = self.url else: kwargs['es_url'] = self.ext_url super(Elastic, self).__init__(**kwargs) self.container_dir = container_dir if self.ext_url is None: msg = [ "A Elasticsearch url must be defined. Either pass 'url'", "or initialize the 'RdfConfigManager'" ] raise AttributeError(" ".join(msg))
def create_namespace(self, namespace=None, params=None): """ Creates a namespace in the triplestore args: namespace: the name of the namspace to create params: Dictionary of Blazegraph paramaters. defaults are: {'axioms': 'com.bigdata.rdf.axioms.NoAxioms', 'geoSpatial': False, 'isolatableIndices': False, 'justify': False, 'quads': False, 'rdr': False, 'textIndex': False, 'truthMaintenance': False} """ namespace = pick(namespace, self.namespace) params = pick(params, self.namespace_params) if not namespace: raise ReferenceError("No 'namespace' specified") _params = { 'axioms': 'com.bigdata.rdf.axioms.NoAxioms', 'geoSpatial': False, 'isolatableIndices': False, 'justify': False, 'namespace': namespace, 'quads': True, 'rdr': False, 'textIndex': False, 'truthMaintenance': False } if params: _params.update(params) content_type = "text/plain" url = self._make_url("prepareProperties").replace("/sparql", "") params = ["%s=%s" % (map_val, json.dumps(_params[map_key]).replace("\"", "")) \ for map_key, map_val in self.ns_property_map.items()] params = "\n".join(params) result = requests.post(url=url, headers={"Content-Type": content_type}, data=params) data = result.text content_type = "application/xml" url = self._make_url("x").replace("/x/sparql", "") result = requests.post(url=url, headers={"Content-Type": content_type}, data=data) if result.status_code == 201: log.warning(result.text) return result.text else: raise RuntimeError(result.text)
def reset_namespace(self, namespace=None, params=None): """ Will delete and recreate specified namespace args: namespace(str): Namespace to reset params(dict): params used to reset the namespace """ namespace = pick(namespace, self.namespace) params = pick(params, self.namespace_params) log.warning(" Reseting namespace '%s' at host: %s", namespace, self.url) try: self.delete_namespace(namespace) except KeyError: pass self.create_namespace(namespace, params)
def load_data(self, data, datatype="ttl", namespace=None, graph=None, is_file=False, **kwargs): """ loads data via file stream from python to triplestore Args: data: The data or filepath to load datatype(['ttl', 'xml', 'rdf']): the type of data to load namespace: the namespace to use graph: the graph to load the data to. is_file(False): If true python will read the data argument as a filepath, determine the datatype from the file extension, read the file and send it to blazegraph as a datastream """ if kwargs.get('debug'): log.setLevel(logging.DEBUG) time_start = datetime.datetime.now() datatype_map = { 'ttl': 'turtle', 'xml': 'xml', 'rdf': 'xml', 'nt': 'nt', 'n3': 'n3', 'nquads': 'nquads', 'hturtle': 'hturtle' } if is_file: datatype = data.split(os.path.extsep)[-1] file_name = data log.debug('starting data load of %s', file_name) data = open(data, 'rb').read() try: content_type = datatype_map[datatype] except KeyError: raise NotImplementedError("'%s' is not an implemented data fromat", datatype) conn = self.conn if namespace: conn = self.tstore.get_namespace(namespace) else: namespace = self.namespace graph = pick(graph, self.graph) start = datetime.datetime.now() try: result = conn.parse(data=data, publicID=graph, format=content_type) except: if is_file: print("Datafile ", file_name) raise if is_file: log.info(" loaded %s into rdflib namespace '%s'", file_name, namespace) else: log.info(" loaded data into rdflib namespace '%s' in time: %s", namespace, (datetime.datetime.now() - start)) return result
def load_local_file(self, file_path, namespace=None, graph=None, **kwargs): """ Uploads data to the Blazegraph Triplestore that is stored in files in directory that is available locally to blazegraph args: file_path: full path to the file namespace: the Blazegraph namespace to load the data graph: uri of the graph to load the data. Default is None kwargs: container_dir: the directory as seen by blazegraph - defaults to instance attribute if not passed """ time_start = datetime.datetime.now() url = self._make_url(namespace) params = {} if graph: params['context-uri'] = graph new_path = [] container_dir = pick(kwargs.get('container_dir'), self.container_dir) if container_dir: new_path.append(self.container_dir) new_path.append(file_path) params['uri'] = "file:///%s" % os.path.join(*new_path) log.debug(" loading %s into blazegraph", file_path) result = requests.post(url=url, params=params) if result.status_code > 300: raise SyntaxError(result.text) log.info("loaded '%s' in time: %s blazegraph response: %s", file_path, datetime.datetime.now() - time_start, self.format_response(result.text)) return result
def load_data(self, data, datatype="ttl", namespace=None, graph=None, is_file=False, **kwargs): """ Loads data via file stream from python to triplestore Args: ----- data: The data or filepath to load datatype(['ttl', 'xml', 'rdf']): the type of data to load namespace: the namespace to use graph: the graph to load the data to. is_file(False): If true python will read the data argument as a filepath, determine the datatype from the file extension, read the file and send it to blazegraph as a datastream """ log.setLevel(kwargs.get("log_level", self.log_level)) time_start = datetime.datetime.now() datatype_map = { 'ttl': 'text/turtle', 'xml': 'application/rdf+xml', 'rdf': 'application/rdf+xml', 'nt': 'text/plain' } if is_file: datatype = data.split(os.path.extsep)[-1] file_name = data log.debug('starting data load of %s', file_name) data = open(data, 'rb').read() else: try: data = data.encode('utf-8') except AttributeError: # data already encoded pass try: content_type = datatype_map[datatype] except KeyError: raise NotImplementedError("'%s' is not an implemented data format", datatype) context_uri = pick(graph, self.graph) result = requests.post(url=self._make_url(namespace), headers={"Content-Type": content_type}, params={"context-uri": context_uri}, data=data) if result.status_code == 200: if is_file: log.info(" loaded %s into blazegraph - %s", file_name, self.format_response(result.text)) else: log.info(" loaded data - %s", self.format_response(result.text)) log.setLevel(self.log_level) return result else: raise SyntaxError(result.text)
def reset_namespace(self, namespace=None, params=None): """ Will delete and recreate specified namespace args: namespace(str): Namespace to reset params(dict): params used to reset the namespace """ namespace = pick(namespace, self.namespace) params = pick(params, self.namespace_params) log.warning(" Reseting namespace '%s' at host: %s", namespace, self.url) try: self.delete_namespace(namespace) except KeyError: pass self.create_namespace(namespace, params)
def reset_namespace(self, namespace=None, params=None): """ Will delete and recreate specified namespace args: namespace(str): Namespace to reset params(dict): params used to reset the namespace """ log = logging.getLogger("%s.%s" % (self.log_name, inspect.stack()[0][3])) log.setLevel(self.log_level) namespace = pick(namespace, self.namespace) params = pick(params, self.namespace_params) log.warning(" Reseting namespace '%s' at host: %s", namespace, self.url) try: self.delete_namespace(namespace) except RuntimeError: pass self.create_namespace(namespace, params)
def __init__(self, url=None, namespace=None, namespace_params=None, local_directory=None, container_dir=None, graph=None, **kwargs): self.active = kwargs.get('active', True) self.local_directory = pick(local_directory, CFG.LOCAL_DATA_PATH, "") self.url = "No Url for Rdflib tstore" self.namespace = pick(namespace, self.default_ns) self.namespace_params = namespace_params self.container_dir = container_dir self.graph = pick(graph, self.default_graph) try: self.conn = self.tstore.get_namespace(self.namespace) except KeyError: self.tstore.create_namespace(self.namespace) self.conn = self.tstore.get_namespace(self.namespace) self.__set_mgr__(**kwargs)
def __init__(self, url=None, namespace=None, namespace_params=None, local_directory=None, container_dir=None, graph=None, **kwargs): self.active = kwargs.get('active', True) self.local_directory = pick(local_directory, CFG.LOCAL_DATA_PATH, "") self.url = "No Url for Rdflib tstore" self.namespace = pick(namespace, self.default_ns) self.namespace_params = namespace_params self.container_dir = container_dir self.graph = pick(graph, self.default_graph) try: self.conn = self.tstore.get_namespace(self.namespace) except KeyError: self.tstore.create_namespace(self.namespace) self.conn = self.tstore.get_namespace(self.namespace) self.__set_mgr__(**kwargs)
def _make_url(self, namespace=None, url=None, **kwargs): """ Creates the REST Url based on the supplied namespace args: namespace: string of the namespace kwargs: check_status_call: True/False, whether the function is called from check_status. Used to avoid recurrsion error """ if not kwargs.get("check_status_call"): if not self.url: self.check_status rtn_url = self.url if url: rtn_url = url if rtn_url is None: rtn_url = self.ext_url namespace = pick(namespace, self.namespace) if namespace: rtn_url = os.path.join(rtn_url.replace("sparql", ""), "namespace", namespace, "sparql").replace("\\", "/") elif not rtn_url.endswith("sparql"): rtn_url = os.path.join(rtn_url, "sparql").replace("\\", "/") return rtn_url
def __predicate_object_map__(self, map_iri): """Iterates through rr:predicateObjectMaps for this TripleMap creating a SimpleNamespace for each triple map and assigning the constant, template, parentTripleMap, reference as properties. Args: ----- map_iri: rdflib.URIRef, TripleMap IRI Returns: -------- list: List of predicate_object Namespace objects """ pred_obj_maps = [] for pred_obj_map_bnode in self.rml.objects( subject=map_iri, predicate=NS_MGR.rr.predicateObjectMap.rdflib): pred_obj_map = SimpleNamespace() pred_obj_map.predicate = self.rml.value( subject=pred_obj_map_bnode, predicate=NS_MGR.rr.predicate.rdflib) obj_map_bnode = self.rml.value( subject=pred_obj_map_bnode, predicate=NS_MGR.rr.objectMap.rdflib) if obj_map_bnode is None: continue pred_obj_map.constant = self.rml.value( subject=obj_map_bnode, predicate=NS_MGR.rr.constant.rdflib) pred_obj_map.template = self.rml.value( subject=obj_map_bnode, predicate=NS_MGR.rr.template.rdflib) pred_obj_map.parentTriplesMap = self.rml.value( subject=obj_map_bnode, predicate=NS_MGR.rr.parentTriplesMap.rdflib) if pred_obj_map.parentTriplesMap is not None: self.parents.add(str(pred_obj_map.parentTriplesMap)) pred_obj_map.reference = self.rml.value( subject=obj_map_bnode, predicate=NS_MGR.rr.reference.rdflib) pred_obj_map.datatype = self.rml.value( subject=obj_map_bnode, predicate=NS_MGR.rr.datatype.rdflib) pred_obj_map.query = self.rml.value( subject=obj_map_bnode, predicate=NS_MGR.rml.query.rdflib) pred_obj_map.json_query = self.rml.value( subject=obj_map_bnode, predicate=NS_MGR.rml.reference.rdflib) json_key = None if hasattr(self.triple_maps[str(map_iri)].logicalSource, 'json_key'): json_key = self.triple_maps[str( map_iri)].logicalSource.json_key pred_obj_map.json_key = pick( self.rml.value(subject=obj_map_bnode, predicate=NS_MGR.rml.key.rdflib), json_key) # BIBCAT Extensions pred_obj_map.delimiters = [] if pred_obj_map.json_query: self.use_json_qry = True for obj in self.rml.objects(subject=obj_map_bnode, predicate=NS_MGR.kds.delimiter.rdflib): pred_obj_map.delimiters.append(obj) pred_obj_maps.append(pred_obj_map) return pred_obj_maps
def query(self, sparql, mode="get", namespace=None, rtn_format="json", **kwargs): """ Runs a sparql query and returns the results Args: ----- sparql: the sparql query to run namespace: the namespace to run the sparql query against mode: ['get'(default), 'update'] the type of sparql query rtn_format: ['json'(default), 'xml'] format of query results Kwargs: ------- debug(bool): If True sets logging level to debug """ namespace = pick(namespace, self.namespace) if kwargs.get("log_level"): log.setLevel(kwargs['log_level']) if kwargs.get("debug"): log.setLevel(logging.DEBUG) if rtn_format not in self.qry_formats: raise KeyError("rtn_format was '%s'. Allowed values are %s" % \ (rtn_format, self.qry_results_formats)) url = self._make_url(namespace) if 'prefix' not in sparql.lower(): sparql = "%s\n%s" % (NSM.prefix(), sparql) if mode == "get": data = {"query": sparql} #, "format": rtn_format} elif mode == "update": data = {"update": sparql} else: raise NotImplementedError("'mode' != to ['get', 'update']") headers = {'Accept': self.qry_formats[rtn_format]} start = datetime.datetime.now() try: result = requests.post(url, data=data, headers=headers) except requests.exceptions.ConnectionError: result = requests.post(self._make_url(namespace, self.local_url), data=data, headers=headers) log.debug( format_multiline([ "", "url='{url}'", """mode='{mode}', namespace='{namespace}', rtn_format='{rtn_format}'""", "**** SPAQRL QUERY ****", "", "{sparql}", "Query Time: {q_time}" ], url=url, mode=mode, namespace=namespace, rtn_format=rtn_format, sparql=sparql, q_time=(datetime.datetime.now() - start), **kwargs)) if result.status_code == 200: try: if rtn_format == "json": bindings = result.json().get('results', {}).get('bindings', []) elif rtn_format == 'xml': xml_doc = etree.XML(result.text) bindings = xml_doc.findall("results/bindings") else: bindings = result.text try: log.debug("result count: %s", len(bindings)) except TypeError: pass return bindings except json.decoder.JSONDecodeError: if mode == 'update': return BeautifulSoup(result.text, 'lxml').get_text() return result.text else: raise SyntaxError("%s\n\n%s\n\n%s" % (sparql, add_sparql_line_nums(sparql), result.text[result.text.find("java."):]))
def load_data(self, data, datatype="ttl", namespace=None, graph=None, is_file=False, **kwargs): """ loads data via file stream from python to triplestore Args: data: The data or filepath to load datatype(['ttl', 'xml', 'rdf']): the type of data to load namespace: the namespace to use graph: the graph to load the data to. is_file(False): If true python will read the data argument as a filepath, determine the datatype from the file extension, read the file and send it to blazegraph as a datastream """ if kwargs.get('debug'): log.setLevel(logging.DEBUG) time_start = datetime.datetime.now() datatype_map = { 'ttl': 'turtle', 'xml': 'xml', 'rdf': 'xml', 'nt': 'nt', 'n3': 'n3', 'nquads': 'nquads', 'hturtle': 'hturtle' } if is_file: datatype = data.split(os.path.extsep)[-1] file_name = data log.debug('starting data load of %s', file_name) data = open(data, 'rb').read() try: content_type = datatype_map[datatype] except KeyError: raise NotImplementedError("'%s' is not an implemented data fromat", datatype) conn = self.conn if namespace: conn = self.tstore.get_namespace(namespace) else: namespace = self.namespace graph = pick(graph, self.graph) start = datetime.datetime.now() try: result = conn.parse(data=data, publicID=graph, format=content_type) except: if is_file: print("Datafile ", file_name) raise if is_file: log.info (" loaded %s into rdflib namespace '%s'", file_name, namespace) else: log.info(" loaded data into rdflib namespace '%s' in time: %s", namespace, (datetime.datetime.now() - start)) return result