def testInitExistingRepo(self): init_repository(self.local, False) conf = QuitConfiguration(configfile=self.localConfigFile) conf.initgraphconfig() self.assertEqual(sorted(conf.getfiles()), ['example1.nq', 'example2.nt']) conf = QuitConfiguration(repository='assests/configuration_test', configfile=self.localConfigFile, configmode='repoconfig') conf.initgraphconfig() self.assertEqual(sorted(conf.getfiles()), ['example1.nq', 'example2.nt']) conf = QuitConfiguration(configfile=self.localConfigFile, configmode='localconfig') conf.initgraphconfig() self.assertEqual(sorted(conf.getfiles()), ['example1.nq', 'example2.nt'])
def testConfigParams(self): init_repository(self.local, False) # no params given conf = QuitConfiguration(configfile=self.localConfigFile) self.assertTrue(conf.isversioningon) # all params set conf = QuitConfiguration(versioning=True, configfile=self.localConfigFile) self.assertTrue(conf.isversioningon) # all params unset conf = QuitConfiguration(versioning=False, configfile=self.localConfigFile) self.assertTrue(conf.isversioningon)
def testGraphConfigRemoteConfig(self): conf = QuitConfiguration(configmode='repoconfig', configfile=self.localConfigFile, namespace=self.ns) conf.initgraphconfig() graphs = conf.getgraphs() self.assertEqual(sorted([str(x) for x in graphs]), ['http://example.org/1/', 'http://example.org/2/']) files = conf.getfiles() self.assertEqual(sorted(files), ['example1.nq', 'example2.nt']) serialization = conf.getserializationoffile('example1.nq') self.assertEqual(serialization, 'nquads') gfMap = conf.getgraphurifilemap() self.assertEqual( gfMap, { rdflib.term.URIRef('http://example.org/1/'): 'example1.nq', rdflib.term.URIRef('http://example.org/2/'): 'example2.nt' }) self.assertEqual( [str(x) for x in conf.getgraphuriforfile('example1.nq')], ['http://example.org/1/']) self.assertEqual( [str(x) for x in conf.getgraphuriforfile('example2.nt')], ['http://example.org/2/']) self.assertEqual(conf.getfileforgraphuri('http://example.org/1/'), 'example1.nq') self.assertEqual(conf.getfileforgraphuri('http://example.org/2/'), 'example2.nt')
def testGraphConfigGraphFiles(self): conf = QuitConfiguration(configmode='graphfiles', configfile=self.localConfigFile) conf.initgraphconfig() graphs = conf.getgraphs() self.assertEqual( sorted(graphs), ['http://example.org/2/', 'http://example.org/discovered/']) files = conf.getfiles() self.assertEqual(sorted(files), ['example1.nq', 'example2.nt']) serialization = conf.getserializationoffile('example1.nq') self.assertEqual(serialization, 'nquads') gfMap = conf.getgraphurifilemap() self.assertEqual( gfMap, { 'http://example.org/discovered/': 'example1.nq', 'http://example.org/2/': 'example2.nt' }) self.assertEqual(conf.getgraphuriforfile('example1.nq'), ['http://example.org/discovered/']) self.assertEqual(conf.getgraphuriforfile('example2.nt'), ['http://example.org/2/']) self.assertEqual( conf.getfileforgraphuri('http://example.org/discovered/'), 'example1.nq') self.assertEqual(conf.getfileforgraphuri('http://example.org/2/'), 'example2.nt')
def testInitWithMissingGraphFiles(self): # Mode: fallback to graphfiles remove(join(self.local, 'example1.nq')) remove(join(self.local, 'example2.nt')) conf = QuitConfiguration(configfile=self.remoteConfigFile, namespace=self.ns) conf.initgraphconfig() files = conf.getfiles() # no files to use self.assertEqual(sorted(files), []) # Mode: graphfiles conf = QuitConfiguration(configfile=self.localConfigFile, configmode='graphfiles', namespace=self.ns) conf.initgraphconfig() files = conf.getfiles() # no files to use self.assertEqual(sorted(files), []) # Mode: local config file conf = QuitConfiguration(configfile=self.remoteConfigFile, configmode='localconfig', namespace=self.ns) conf.initgraphconfig() files = conf.getfiles() # deleted files should be created self.assertEqual(sorted(files), ['example1.nq', 'example2.nt']) # Mode: remote config file remove(join(self.local, 'example1.nq')) remove(join(self.local, 'example2.nt')) conf = QuitConfiguration(repository='assests/configuration_test', configfile=self.localConfigFile, configmode='repoconfig', namespace=self.ns) conf.initgraphconfig() files = conf.getfiles() # deleted files should be created self.assertEqual(sorted(files), ['example1.nq', 'example2.nt'])
def testStoreConfig(self): init_repository(self.local, False) conf = QuitConfiguration(configfile=self.localConfigFile) self.assertEqual(conf.getRepoPath(), self.local) self.assertEqual(conf.getOrigin(), 'git://github.com/aksw/QuitStore.git') allFiles = conf.getgraphsfromdir() self.assertEqual( sorted(allFiles), sorted(['config.ttl', 'example1.nq', 'example2.nt', 'example3.nq']))
def testNamespace(self): init_repository(self.local, False) # missing namespace self.assertRaises(InvalidConfigurationError, QuitConfiguration, 'configfile', self.localConfigFile) good = [ 'http://example.org/thing#', 'https://example.org/', 'http://example.org/things/' ] bad = ['file:///home/quit/', 'urn:graph/', 'urn:graph', '../test'] # good namespaces for uri in good: conf = QuitConfiguration(configfile=self.localConfigFile, namespace=uri) self.assertEqual(conf.namespace, uri) # bad namespaces for uri in bad: self.assertRaises(InvalidConfigurationError, QuitConfiguration, 'configfile', self.localConfigFile, 'namespace', uri)
def initialize(args): """Build all needed objects. Returns: A dictionary containing the store object and git repo object. """ if args.verbose: ch.setLevel(logging.INFO) logger.addHandler(ch) logger.debug('Loglevel: INFO') if args.verboseverbose: ch.setLevel(logging.DEBUG) logger.addHandler(ch) logger.debug('Loglevel: DEBUG') logger.debug("Parsed args: {}".format(args)) # add the handlers to the logger if args.logfile: try: fh = logging.FileHandler(args.logfile) fh.setLevel(logging.DEBUG) fh.setFormatter(formatter) logger.addHandler(fh) logger.debug("Logfile: {}".format(args.logfile)) except FileNotFoundError: logger.error("Logfile not found: {}".format(args.logfile)) sys.exit('Exiting quit') except PermissionError: logger.error("Can not create logfile: {}".format(args.logfile)) sys.exit('Exiting quit') if args.garbagecollection: logger.info( "Please use the option \"--feature garbagecollection\" instead of " + "\"-gc\" or \"--garbagecollection\".") args.features |= Feature.GarbageCollection try: config = QuitConfiguration( configfile=args.configfile, targetdir=args.targetdir, repository=args.repourl, configmode=args.configmode, features=args.features, namespace=args.namespace, ) except InvalidConfigurationError as e: logger.error(e) sys.exit('Exiting quit') # since repo is handled, we can add graphs to config config.initgraphconfig() logger.info('QuitStore successfully running.') logger.info('Known graphs: ' + str(config.getgraphs())) logger.info('Known files: ' + str(config.getfiles())) logger.debug('Path of Gitrepo: ' + config.getRepoPath()) logger.debug('Config mode: ' + str(config.getConfigMode())) logger.debug('All RDF files found in Gitepo:' + str(config.getgraphsfromdir())) return {'config': config}
def testInitExistingFolder(self): conf = QuitConfiguration(configfile=self.localConfigFile) self.assertEqual(conf.getRepoPath(), self.local)
def testInitMissingConfiguration(self): init_repository(self.local, False) with self.assertRaises(InvalidConfigurationError): QuitConfiguration(configfile='no.config')
def initialize(args): """Build all needed objects. Returns: A dictionary containing the store object and git repo object. """ if args.verbose: ch.setLevel(logging.INFO) logger.addHandler(ch) logger.debug('Loglevel: INFO') if args.verboseverbose: ch.setLevel(logging.DEBUG) logger.addHandler(ch) logger.debug('Loglevel: DEBUG') # add the handlers to the logger if args.logfile: try: fh = logging.FileHandler(args.logfile) fh.setLevel(logging.DEBUG) fh.setFormatter(formatter) logger.addHandler(fh) logger.debug("Logfile: {}".format(args.logfile)) except FileNotFoundError: logger.error("Logfile not found: {}".format(args.logfile)) sys.exit('Exiting quit') except PermissionError: logger.error("Can not create logfile: {}".format(args.logfile)) sys.exit('Exiting quit') if args.disableversioning: logger.info('Versioning: disabled') v = False else: logger.info('Versioning: enabled') v = True try: config = QuitConfiguration( versioning=v, configfile=args.configfile, targetdir=args.targetdir, repository=args.repourl, configmode=args.configmode, ) except InvalidConfigurationError as e: logger.error(e) sys.exit('Exiting quit') gitrepo = GitRepo( path=config.getRepoPath(), origin=config.getOrigin() ) try: gitrepo = GitRepo( path=config.getRepoPath(), origin=config.getOrigin() ) except Exception as e: raise InvalidConfigurationError(e) if args.garbagecollection: try: with subprocess.Popen( ["git", "config", "gc.auto"], stdout=subprocess.PIPE, cwd=config.getRepoPath() ) as gcAutoThresholdProcess: stdout, stderr = gcAutoThresholdProcess.communicate() gcAutoThreshold = stdout.decode("UTF-8").strip() if not gcAutoThreshold: gcAutoThreshold = 256 subprocess.Popen( ["git", "config", "gc.auto", str(gcAutoThreshold)], cwd=config.getRepoPath() ) logger.info("Set default gc.auto threshold {}".format(gcAutoThreshold)) gitrepo.gc = True logger.info( "Garbage Collection is enabled with gc.auto threshold {}".format( gcAutoThreshold ) ) except Exception as e: # Disable garbage collection for the rest of the run because it # is likely that git is not available logger.info('Git garbage collection could not be configured and was disabled') logger.debug(e) # since repo is handled, we can add graphs to config config.initgraphconfig() store = initializeMemoryStore(config) # Save file objects per file filereferences = {} for file in config.getfiles(): graphs = config.getgraphuriforfile(file) content = [] for graph in graphs: content += store.getgraphcontent(graph) fileobject = FileReference(join(config.getRepoPath(), file)) # TODO: Quick Fix, add sorting to FileReference fileobject.setcontent(sorted(content)) filereferences[file] = fileobject logger.info('QuitStore successfully running.') logger.info('Known graphs: ' + str(config.getgraphs())) logger.info('Known files: ' + str(config.getfiles())) logger.debug('Path of Gitrepo: ' + config.getRepoPath()) logger.debug('Config mode: ' + str(config.getConfigMode())) logger.debug('All RDF files found in Gitepo:' + str(config.getgraphsfromdir())) updateConfig(store, config, gitrepo, filereferences)