def initialize(self, client):
super(ReleaseProvider, self).initialize(client)
# Construct format
cls = PluginManager.get('format', self._fmt)
if cls is None:
raise ValueError('Unable to find format: %r' % self._fmt)
self.format = cls()
def _construct_storage(self, storage_or_key):
if isinstance(storage_or_key, ProviderStorage):
# Use provided source
storage = storage_or_key
elif PluginManager.has('storage', storage_or_key):
# Construct source by key
storage = PluginManager.get('storage', storage_or_key)()
else:
raise ValueError('Unknown storage interface: %r' % storage_or_key)
# Initialize source
storage.initialize(self._client)
return storage
def _load_plugins(kind, keys, construct=True):
if not keys:
return
for name in keys:
cls = PluginManager.get(kind, name)
if cls is None:
log.warn('Unable to find plugin: %r', name)
continue
if not cls.available:
log.warn('Plugin %r is not available', name)
continue
if construct:
yield cls.__key__, cls()
else:
yield cls.__key__, cls
def _load_plugins(kind, keys, construct=True):
if not keys:
return
for name in keys:
cls = PluginManager.get(kind, name)
if cls is None:
log.warn('Unable to find plugin: %r', name)
continue
if not cls.available:
log.warn('Plugin %r is not available', name)
continue
if construct:
yield cls.__key__, cls()
else:
yield cls.__key__, cls
def _construct_provider(self, provider_or_key):
if isinstance(provider_or_key, Provider):
# Class
provider = provider_or_key
elif isinstance(provider_or_key, six.string_types):
# Identifier
provider = PluginManager.get('client-provider', provider_or_key)
if provider is None:
raise ValueError('Unable to find provider: %r' % provider_or_key)
else:
raise ValueError('Unknown provider: %r' % provider_or_key)
# Ensure provider has been constructed
if inspect.isclass(provider):
provider = provider()
# Initialize provider
provider.initialize(self)
return provider
def _construct_provider(self, provider_or_key):
if isinstance(provider_or_key, Provider):
# Class
provider = provider_or_key
elif isinstance(provider_or_key, six.string_types):
# Identifier
provider = PluginManager.get('client-provider', provider_or_key)
if provider is None:
raise ValueError('Unable to find provider: %r' %
provider_or_key)
else:
raise ValueError('Unknown provider: %r' % provider_or_key)
# Ensure provider has been constructed
if inspect.isclass(provider):
provider = provider()
# Initialize provider
provider.initialize(self)
return provider
def run(self, base_path, **kwargs):
if not os.path.exists(base_path):
log.error('Path %r doesn\'t exist', base_path)
return False
for _, cls in self.sources.items():
# Retrieve collections
collections = cls.__collections__
# Filter `collections` by provided list
if self.collections is not None:
collections = [
key for key in collections
if key in self.collections
]
# Run updater on each collection
for source, target in collections:
# Build collection path
database_path = os.path.join(
base_path,
self._build_package_name(source, target),
self._build_module_name(source, target)
)
# Ensure database exists
if not os.path.exists(database_path):
log.warn('Unknown collection: %r -> %r', source, target)
continue
# Run updater on database for each format
for fmt in self.formats.itervalues():
if fmt.__construct__ is False:
continue
# Build updater client
client = UpdaterClient(fmt)
# Build database storage interface
storage = PluginManager.get('storage', 'file/database')(
self, source, target,
path=database_path
)
storage.initialize(client)
# Load database
database = Database.load(storage, source, target)
# Load database collections
database.load_collections([
(source, target),
(target, source)
])
for collection in database.collections.itervalues():
# Run updater on collection
try:
s = cls(collection, kwargs)
if not s.run():
return False
except Exception as ex:
log.warn(
'Unable to run updater on %r (format: %r) - %s',
database_path, fmt, ex,
exc_info=True
)
return False
# Write collection to disk
try:
collection.write()
except Exception as ex:
log.warn('Unable to write collection to disk - %s', ex, exc_info=True)
return False
return True
def run(self, base_path, **kwargs):
if not os.path.exists(base_path):
log.error('Path %r doesn\'t exist', base_path)
return False
for _, cls in self.sources.items():
# Retrieve collections
collections = cls.__collections__
# Filter `collections` by provided list
if self.collections is not None:
collections = [
key for key in collections if key in self.collections
]
# Run updater on each collection
for source, target in collections:
# Build collection path
database_path = os.path.join(
base_path, self._build_package_name(source, target),
self._build_module_name(source, target))
# Ensure database exists
if not os.path.exists(database_path):
log.warn('Unknown collection: %r -> %r', source, target)
continue
# Run updater on database for each format
for fmt in self.formats.itervalues():
if fmt.__construct__ is False:
continue
# Build updater client
client = UpdaterClient(fmt)
# Build database storage interface
storage = PluginManager.get('storage', 'file/database')(
self, source, target, path=database_path)
storage.initialize(client)
# Load database
database = Database.load(storage, source, target)
# Load database collections
database.load_collections([(source, target),
(target, source)])
for collection in database.collections.itervalues():
# Run updater on collection
try:
s = cls(collection, kwargs)
if not s.run():
return False
except Exception as ex:
log.warn(
'Unable to run updater on %r (format: %r) - %s',
database_path,
fmt,
ex,
exc_info=True)
return False
# Write collection to disk
try:
collection.write()
except Exception as ex:
log.warn('Unable to write collection to disk - %s',
ex,
exc_info=True)
return False
return True
