def parse_args(self):
parser = ArgumentParser()
parser.add_argument('base_path')
parser.add_argument('-d',
'--debug',
action='store_true',
default=False)
parser.add_argument('-p',
'--progress',
action='store_true',
default=False)
parser.add_argument('-c', '--collection', action='append')
parser.add_argument('-f', '--format', action='append')
parser.add_argument('-s', '--source', action='append', required=True)
# Discover installed plugins
PluginManager.discover()
# Add arguments from updater sources
for _, source in PluginManager.list_ordered('database-updater'):
for argument in source.__parameters__:
if 'name' not in argument:
log.warn('Invalid source argument: %r', argument)
continue
parser.add_argument(
'--%s-%s' % (source.__key__, argument['name']),
**argument.get('kwargs', {}))
# Parse command line arguments
return parser.parse_args()
def __init__(self, sources, collections=None, formats=None):
self.collections = collections
# Discover installed plugins
PluginManager.discover()
# Retrieve formats
self.formats = dict(
self._load_plugins('format', formats or DEFAULT_FORMATS))
# Retrieve sources
self.sources = dict(
self._load_plugins('database-updater', sources, construct=False))
def _construct_storage(self, storage_or_key):
if isinstance(storage_or_key, ProviderStorage):
# Use provided source
storage = storage_or_key
elif PluginManager.has('storage', storage_or_key):
# Construct source by key
storage = PluginManager.get('storage', storage_or_key)()
else:
raise ValueError('Unknown storage interface: %r' % storage_or_key)
# Initialize source
storage.initialize(self._client)
return storage
def __init__(self, sources, collections=None, formats=None):
self.collections = collections
# Discover installed plugins
PluginManager.discover()
# Retrieve formats
self.formats = dict(self._load_plugins(
'format', formats or DEFAULT_FORMATS
))
# Retrieve sources
self.sources = dict(self._load_plugins(
'database-updater', sources,
construct=False
))
def initialize(self, client):
super(ReleaseProvider, self).initialize(client)
# Construct format
cls = PluginManager.get('format', self._fmt)
if cls is None:
raise ValueError('Unable to find format: %r' % self._fmt)
self.format = cls()
def parse_args(self):
parser = ArgumentParser()
parser.add_argument('base_path')
parser.add_argument('-d', '--debug', action='store_true', default=False)
parser.add_argument('-p', '--progress', action='store_true', default=False)
parser.add_argument('-c', '--collection', action='append')
parser.add_argument('-f', '--format', action='append')
parser.add_argument('-s', '--source', action='append', required=True)
# Discover installed plugins
PluginManager.discover()
# Add arguments from updater sources
for _, source in PluginManager.list_ordered('database-updater'):
for argument in source.__parameters__:
if 'name' not in argument:
log.warn('Invalid source argument: %r', argument)
continue
parser.add_argument('--%s-%s' % (source.__key__, argument['name']), **argument.get('kwargs', {}))
# Parse command line arguments
return parser.parse_args()
def _load_plugins(kind, keys, construct=True):
if not keys:
return
for name in keys:
cls = PluginManager.get(kind, name)
if cls is None:
log.warn('Unable to find plugin: %r', name)
continue
if not cls.available:
log.warn('Plugin %r is not available', name)
continue
if construct:
yield cls.__key__, cls()
else:
yield cls.__key__, cls
def _load_plugins(kind, keys, construct=True):
if not keys:
return
for name in keys:
cls = PluginManager.get(kind, name)
if cls is None:
log.warn('Unable to find plugin: %r', name)
continue
if not cls.available:
log.warn('Plugin %r is not available', name)
continue
if construct:
yield cls.__key__, cls()
else:
yield cls.__key__, cls
def _construct_provider(self, provider_or_key):
if isinstance(provider_or_key, Provider):
# Class
provider = provider_or_key
elif isinstance(provider_or_key, six.string_types):
# Identifier
provider = PluginManager.get('client-provider', provider_or_key)
if provider is None:
raise ValueError('Unable to find provider: %r' % provider_or_key)
else:
raise ValueError('Unknown provider: %r' % provider_or_key)
# Ensure provider has been constructed
if inspect.isclass(provider):
provider = provider()
# Initialize provider
provider.initialize(self)
return provider
def _construct_provider(self, provider_or_key):
if isinstance(provider_or_key, Provider):
# Class
provider = provider_or_key
elif isinstance(provider_or_key, six.string_types):
# Identifier
provider = PluginManager.get('client-provider', provider_or_key)
if provider is None:
raise ValueError('Unable to find provider: %r' %
provider_or_key)
else:
raise ValueError('Unknown provider: %r' % provider_or_key)
# Ensure provider has been constructed
if inspect.isclass(provider):
provider = provider()
# Initialize provider
provider.initialize(self)
return provider
def run(self, base_path, **kwargs):
if not os.path.exists(base_path):
log.error('Path %r doesn\'t exist', base_path)
return False
for _, cls in self.sources.items():
# Retrieve collections
collections = cls.__collections__
# Filter `collections` by provided list
if self.collections is not None:
collections = [
key for key in collections
if key in self.collections
]
# Run updater on each collection
for source, target in collections:
# Build collection path
database_path = os.path.join(
base_path,
self._build_package_name(source, target),
self._build_module_name(source, target)
)
# Ensure database exists
if not os.path.exists(database_path):
log.warn('Unknown collection: %r -> %r', source, target)
continue
# Run updater on database for each format
for fmt in self.formats.itervalues():
if fmt.__construct__ is False:
continue
# Build updater client
client = UpdaterClient(fmt)
# Build database storage interface
storage = PluginManager.get('storage', 'file/database')(
self, source, target,
path=database_path
)
storage.initialize(client)
# Load database
database = Database.load(storage, source, target)
# Load database collections
database.load_collections([
(source, target),
(target, source)
])
for collection in database.collections.itervalues():
# Run updater on collection
try:
s = cls(collection, kwargs)
if not s.run():
return False
except Exception as ex:
log.warn(
'Unable to run updater on %r (format: %r) - %s',
database_path, fmt, ex,
exc_info=True
)
return False
# Write collection to disk
try:
collection.write()
except Exception as ex:
log.warn('Unable to write collection to disk - %s', ex, exc_info=True)
return False
return True
def run(self, base_path, **kwargs):
if not os.path.exists(base_path):
log.error('Path %r doesn\'t exist', base_path)
return False
for _, cls in self.sources.items():
# Retrieve collections
collections = cls.__collections__
# Filter `collections` by provided list
if self.collections is not None:
collections = [
key for key in collections if key in self.collections
]
# Run updater on each collection
for source, target in collections:
# Build collection path
database_path = os.path.join(
base_path, self._build_package_name(source, target),
self._build_module_name(source, target))
# Ensure database exists
if not os.path.exists(database_path):
log.warn('Unknown collection: %r -> %r', source, target)
continue
# Run updater on database for each format
for fmt in self.formats.itervalues():
if fmt.__construct__ is False:
continue
# Build updater client
client = UpdaterClient(fmt)
# Build database storage interface
storage = PluginManager.get('storage', 'file/database')(
self, source, target, path=database_path)
storage.initialize(client)
# Load database
database = Database.load(storage, source, target)
# Load database collections
database.load_collections([(source, target),
(target, source)])
for collection in database.collections.itervalues():
# Run updater on collection
try:
s = cls(collection, kwargs)
if not s.run():
return False
except Exception as ex:
log.warn(
'Unable to run updater on %r (format: %r) - %s',
database_path,
fmt,
ex,
exc_info=True)
return False
# Write collection to disk
try:
collection.write()
except Exception as ex:
log.warn('Unable to write collection to disk - %s',
ex,
exc_info=True)
return False
return True