def testStoreConfig(self):
init_repository(self.local, False)
conf = QuitConfiguration(configfile=self.localConfigFile)
self.assertEqual(conf.getRepoPath(), self.local)
self.assertEqual(conf.getOrigin(),
'git://github.com/aksw/QuitStore.git')
allFiles = conf.getgraphsfromdir()
self.assertEqual(
sorted(allFiles),
sorted(['config.ttl', 'example1.nq', 'example2.nt',
'example3.nq']))
def testConfigParams(self):
init_repository(self.local, False)
# no params given
conf = QuitConfiguration(configfile=self.localConfigFile)
self.assertTrue(conf.isversioningon)
# all params set
conf = QuitConfiguration(versioning=True,
configfile=self.localConfigFile)
self.assertTrue(conf.isversioningon)
# all params unset
conf = QuitConfiguration(versioning=False,
configfile=self.localConfigFile)
self.assertTrue(conf.isversioningon)
def testNamespace(self):
init_repository(self.local, False)
# missing namespace
self.assertRaises(InvalidConfigurationError, QuitConfiguration,
'configfile', self.localConfigFile)
good = [
'http://example.org/thing#', 'https://example.org/',
'http://example.org/things/'
]
bad = ['file:///home/quit/', 'urn:graph/', 'urn:graph', '../test']
# good namespaces
for uri in good:
conf = QuitConfiguration(configfile=self.localConfigFile,
namespace=uri)
self.assertEqual(conf.namespace, uri)
# bad namespaces
for uri in bad:
self.assertRaises(InvalidConfigurationError, QuitConfiguration,
'configfile', self.localConfigFile, 'namespace',
uri)
def initialize(args):
"""Build all needed objects.
Returns:
A dictionary containing the store object and git repo object.
"""
if args.verbose:
ch.setLevel(logging.INFO)
logger.addHandler(ch)
logger.debug('Loglevel: INFO')
if args.verboseverbose:
ch.setLevel(logging.DEBUG)
logger.addHandler(ch)
logger.debug('Loglevel: DEBUG')
logger.debug("Parsed args: {}".format(args))
# add the handlers to the logger
if args.logfile:
try:
fh = logging.FileHandler(args.logfile)
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
logger.addHandler(fh)
logger.debug("Logfile: {}".format(args.logfile))
except FileNotFoundError:
logger.error("Logfile not found: {}".format(args.logfile))
sys.exit('Exiting quit')
except PermissionError:
logger.error("Can not create logfile: {}".format(args.logfile))
sys.exit('Exiting quit')
if args.garbagecollection:
logger.info(
"Please use the option \"--feature garbagecollection\" instead of "
+ "\"-gc\" or \"--garbagecollection\".")
args.features |= Feature.GarbageCollection
try:
config = QuitConfiguration(
configfile=args.configfile,
targetdir=args.targetdir,
repository=args.repourl,
configmode=args.configmode,
features=args.features,
namespace=args.namespace,
)
except InvalidConfigurationError as e:
logger.error(e)
sys.exit('Exiting quit')
# since repo is handled, we can add graphs to config
config.initgraphconfig()
logger.info('QuitStore successfully running.')
logger.info('Known graphs: ' + str(config.getgraphs()))
logger.info('Known files: ' + str(config.getfiles()))
logger.debug('Path of Gitrepo: ' + config.getRepoPath())
logger.debug('Config mode: ' + str(config.getConfigMode()))
logger.debug('All RDF files found in Gitepo:' +
str(config.getgraphsfromdir()))
return {'config': config}
def testInitExistingRepo(self):
init_repository(self.local, False)
conf = QuitConfiguration(configfile=self.localConfigFile)
conf.initgraphconfig()
self.assertEqual(sorted(conf.getfiles()),
['example1.nq', 'example2.nt'])
conf = QuitConfiguration(repository='assests/configuration_test',
configfile=self.localConfigFile,
configmode='repoconfig')
conf.initgraphconfig()
self.assertEqual(sorted(conf.getfiles()),
['example1.nq', 'example2.nt'])
conf = QuitConfiguration(configfile=self.localConfigFile,
configmode='localconfig')
conf.initgraphconfig()
self.assertEqual(sorted(conf.getfiles()),
['example1.nq', 'example2.nt'])
def testInitExistingFolder(self):
conf = QuitConfiguration(configfile=self.localConfigFile)
self.assertEqual(conf.getRepoPath(), self.local)
def testGraphConfigGraphFiles(self):
conf = QuitConfiguration(configmode='graphfiles',
configfile=self.localConfigFile)
conf.initgraphconfig()
graphs = conf.getgraphs()
self.assertEqual(
sorted(graphs),
['http://example.org/2/', 'http://example.org/discovered/'])
files = conf.getfiles()
self.assertEqual(sorted(files), ['example1.nq', 'example2.nt'])
serialization = conf.getserializationoffile('example1.nq')
self.assertEqual(serialization, 'nquads')
gfMap = conf.getgraphurifilemap()
self.assertEqual(
gfMap, {
'http://example.org/discovered/': 'example1.nq',
'http://example.org/2/': 'example2.nt'
})
self.assertEqual(conf.getgraphuriforfile('example1.nq'),
['http://example.org/discovered/'])
self.assertEqual(conf.getgraphuriforfile('example2.nt'),
['http://example.org/2/'])
self.assertEqual(
conf.getfileforgraphuri('http://example.org/discovered/'),
'example1.nq')
self.assertEqual(conf.getfileforgraphuri('http://example.org/2/'),
'example2.nt')
def testInitWithMissingGraphFiles(self):
# Mode: fallback to graphfiles
remove(join(self.local, 'example1.nq'))
remove(join(self.local, 'example2.nt'))
conf = QuitConfiguration(configfile=self.remoteConfigFile)
conf.initgraphconfig()
files = conf.getfiles()
# no files to use
self.assertEqual(sorted(files), [])
# Mode: graphfiles
conf = QuitConfiguration(configfile=self.localConfigFile,
configmode='graphfiles')
conf.initgraphconfig()
files = conf.getfiles()
# no files to use
self.assertEqual(sorted(files), [])
# Mode: local config file
conf = QuitConfiguration(configfile=self.remoteConfigFile,
configmode='localconfig')
conf.initgraphconfig()
files = conf.getfiles()
# deleted files should be created
self.assertEqual(sorted(files), ['example1.nq', 'example2.nt'])
# Mode: remote config file
remove(join(self.local, 'example1.nq'))
remove(join(self.local, 'example2.nt'))
conf = QuitConfiguration(repository='assests/configuration_test',
configfile=self.localConfigFile,
configmode='repoconfig')
conf.initgraphconfig()
files = conf.getfiles()
# deleted files should be created
self.assertEqual(sorted(files), ['example1.nq', 'example2.nt'])
def testInitMissingConfiguration(self):
init_repository(self.local, False)
with self.assertRaises(InvalidConfigurationError):
QuitConfiguration(configfile='no.config')
def testGraphConfigGraphFiles(self):
conf = QuitConfiguration(configmode='graphfiles',
configfile=self.localConfigFile,
namespace=self.ns)
conf.initgraphconfig()
graphs = conf.getgraphs()
self.assertEqual(
sorted([str(x) for x in graphs]),
['http://example.org/2/', 'http://example.org/discovered/'])
files = conf.getfiles()
self.assertEqual(sorted(files), ['example1.nq', 'example2.nt'])
serialization = conf.getserializationoffile('example1.nq')
self.assertEqual(serialization, 'nquads')
gfMap = conf.getgraphurifilemap()
self.assertEqual(
gfMap, {
rdflib.term.URIRef('http://example.org/discovered/'):
'example1.nq',
rdflib.term.URIRef('http://example.org/2/'): 'example2.nt'
})
self.assertEqual(
[str(x) for x in conf.getgraphuriforfile('example1.nq')],
['http://example.org/discovered/'])
self.assertEqual(
[str(x) for x in conf.getgraphuriforfile('example2.nt')],
['http://example.org/2/'])
self.assertEqual(
conf.getfileforgraphuri('http://example.org/discovered/'),
'example1.nq')
self.assertEqual(conf.getfileforgraphuri('http://example.org/2/'),
'example2.nt')
def initialize(args):
"""Build all needed objects.
Returns:
A dictionary containing the store object and git repo object.
"""
if args.verbose:
ch.setLevel(logging.INFO)
logger.addHandler(ch)
logger.debug('Loglevel: INFO')
if args.verboseverbose:
ch.setLevel(logging.DEBUG)
logger.addHandler(ch)
logger.debug('Loglevel: DEBUG')
# add the handlers to the logger
if args.logfile:
try:
fh = logging.FileHandler(args.logfile)
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
logger.addHandler(fh)
logger.debug("Logfile: {}".format(args.logfile))
except FileNotFoundError:
logger.error("Logfile not found: {}".format(args.logfile))
sys.exit('Exiting quit')
except PermissionError:
logger.error("Can not create logfile: {}".format(args.logfile))
sys.exit('Exiting quit')
if args.disableversioning:
logger.info('Versioning: disabled')
v = False
else:
logger.info('Versioning: enabled')
v = True
try:
config = QuitConfiguration(
versioning=v,
configfile=args.configfile,
targetdir=args.targetdir,
repository=args.repourl,
configmode=args.configmode,
)
except InvalidConfigurationError as e:
logger.error(e)
sys.exit('Exiting quit')
gitrepo = GitRepo(
path=config.getRepoPath(),
origin=config.getOrigin()
)
try:
gitrepo = GitRepo(
path=config.getRepoPath(),
origin=config.getOrigin()
)
except Exception as e:
raise InvalidConfigurationError(e)
if args.garbagecollection:
try:
with subprocess.Popen(
["git", "config", "gc.auto"],
stdout=subprocess.PIPE,
cwd=config.getRepoPath()
) as gcAutoThresholdProcess:
stdout, stderr = gcAutoThresholdProcess.communicate()
gcAutoThreshold = stdout.decode("UTF-8").strip()
if not gcAutoThreshold:
gcAutoThreshold = 256
subprocess.Popen(
["git", "config", "gc.auto", str(gcAutoThreshold)],
cwd=config.getRepoPath()
)
logger.info("Set default gc.auto threshold {}".format(gcAutoThreshold))
gitrepo.gc = True
logger.info(
"Garbage Collection is enabled with gc.auto threshold {}".format(
gcAutoThreshold
)
)
except Exception as e:
# Disable garbage collection for the rest of the run because it
# is likely that git is not available
logger.info('Git garbage collection could not be configured and was disabled')
logger.debug(e)
# since repo is handled, we can add graphs to config
config.initgraphconfig()
store = initializeMemoryStore(config)
# Save file objects per file
filereferences = {}
for file in config.getfiles():
graphs = config.getgraphuriforfile(file)
content = []
for graph in graphs:
content += store.getgraphcontent(graph)
fileobject = FileReference(join(config.getRepoPath(), file))
# TODO: Quick Fix, add sorting to FileReference
fileobject.setcontent(sorted(content))
filereferences[file] = fileobject
logger.info('QuitStore successfully running.')
logger.info('Known graphs: ' + str(config.getgraphs()))
logger.info('Known files: ' + str(config.getfiles()))
logger.debug('Path of Gitrepo: ' + config.getRepoPath())
logger.debug('Config mode: ' + str(config.getConfigMode()))
logger.debug('All RDF files found in Gitepo:' + str(config.getgraphsfromdir()))
updateConfig(store, config, gitrepo, filereferences)