def synchronize_by_projects(projects=None, fetcher=None, progress=True,
download=None):
if fetcher is None:
fetcher = PyPIFetcher()
# Grab the current datetime
current = fetcher.current()
# Synchronize all the classifiers with PyPI
synchronize_classifiers(fetcher)
if not projects:
# Grab a list of projects from PyPI
projects = fetcher.projects()
# We are not synchronizing a subset of projects, so we can check for
# any deletions (if required) and yank them.
diff.projects(projects)
# Commit our yanked projects
db.session.commit()
if progress:
bar = ShadyBar("Processing Projects", max=len(projects))
else:
bar = DummyBar()
for project in bar.iter(projects):
synchronize_project(project, fetcher, download=download)
logger.info("Finished processing projects at %s", current)
return current
def synchronize_by_projects(projects=None,
fetcher=None,
progress=True,
download=None):
if fetcher is None:
fetcher = PyPIFetcher()
# Grab the current datetime
current = fetcher.current()
# Synchronize all the classifiers with PyPI
synchronize_classifiers(fetcher)
if not projects:
# Grab a list of projects from PyPI
projects = fetcher.projects()
# We are not synchronizing a subset of projects, so we can check for
# any deletions (if required) and yank them.
diff.projects(projects)
# Commit our yanked projects
db.session.commit()
if progress:
bar = ShadyBar("Processing Projects", max=len(projects))
else:
bar = DummyBar()
for project in bar.iter(projects):
synchronize_project(project, fetcher, download=download)
logger.info("Finished processing projects at %s", current)
return current
class ProgressBarArgument(FlagArgument):
"""Manage a progress bar"""
def __init__(self, help='', parsed_name='', default=True):
self.suffix = '%(percent)d%%'
super(ProgressBarArgument, self).__init__(help, parsed_name, default)
def clone(self):
"""Get a modifiable copy of this bar"""
newarg = ProgressBarArgument(self.help, self.parsed_name, self.default)
newarg._value = self._value
return newarg
def get_generator(self,
message,
message_len=25,
countdown=False,
timeout=100):
"""Get a generator to handle progress of the bar (gen.next())"""
if self.value:
return None
try:
self.bar = KamakiProgressBar(message.ljust(message_len),
max=timeout or 100)
except NameError:
self.value = None
return self.value
if countdown:
bar_phases = list(self.bar.phases)
self.bar.empty_fill, bar_phases[0] = bar_phases[-1], ''
bar_phases.reverse()
self.bar.phases = bar_phases
self.bar.bar_prefix = ' '
self.bar.bar_suffix = ' '
self.bar.suffix = '%(remaining)ds to timeout'
else:
self.bar.suffix = '%(percent)d%% - %(eta)ds'
self.bar.start()
def progress_gen(n):
for i in self.bar.iter(range(int(n))):
yield
yield
return progress_gen
def finish(self):
"""Stop progress bar, return terminal cursor to user"""
if self.value:
return
mybar = getattr(self, 'bar', None)
if mybar:
mybar.finish()
class ProgressBarArgument(FlagArgument):
"""Manage a progress bar"""
def __init__(self, help='', parsed_name='', default=True):
self.suffix = '%(percent)d%%'
super(ProgressBarArgument, self).__init__(help, parsed_name, default)
def clone(self):
"""Get a modifiable copy of this bar"""
newarg = ProgressBarArgument(
self.help, self.parsed_name, self.default)
newarg._value = self._value
return newarg
def get_generator(
self, message, message_len=25, countdown=False, timeout=100):
"""Get a generator to handle progress of the bar (gen.next())"""
if self.value:
return None
try:
self.bar = KamakiProgressBar(
message.ljust(message_len), max=timeout or 100)
except NameError:
self.value = None
return self.value
if countdown:
bar_phases = list(self.bar.phases)
self.bar.empty_fill, bar_phases[0] = bar_phases[-1], ''
bar_phases.reverse()
self.bar.phases = bar_phases
self.bar.bar_prefix = ' '
self.bar.bar_suffix = ' '
self.bar.suffix = '%(remaining)ds to timeout'
else:
self.bar.suffix = '%(percent)d%% - %(eta)ds'
self.bar.start()
def progress_gen(n):
for i in self.bar.iter(range(int(n))):
yield
yield
return progress_gen
def finish(self):
"""Stop progress bar, return terminal cursor to user"""
if self.value:
return
mybar = getattr(self, 'bar', None)
if mybar:
mybar.finish()
def synchronize_by_journals(since=None, fetcher=None, progress=True,
download=None):
if fetcher is None:
fetcher = PyPIFetcher()
# Grab the current datetime
current = fetcher.current()
# Synchronize all the classifiers with PyPI
synchronize_classifiers(fetcher)
# Grab the journals since `since`
journals = fetcher.journals(since=since)
# Storage for projects that have been updated or deleted
updated = set()
deleted = set()
# Check if we have anything to process before attempting to
if journals:
if progress:
bar = ShadyBar("Processing Journals", max=len(journals))
else:
bar = DummyBar()
# Handle Renames, these need to occur first because PyPI retroactively
# changes journal names to the new project, which if we experience
# any of these prior to handling a rename it'll trigger a sync which
# will act like it's a new project and not a renamed project.
if since is not None:
for journal in journals:
if journal.action.lower().startswith("rename from "):
_, _, previous = journal.action.split(" ", 2)
proj = Project.get(previous)
proj.rename(journal.name)
# Commit the renames
db.session.commit()
for journal in bar.iter(journals):
if redis.sismember(REDIS_JOURNALS_KEY, journal.id):
# We've already processed this entry, so skip to the next one
continue
created = datetime.datetime.utcfromtimestamp(journal.timestamp)
Journal.create(
name=journal.name,
version=journal.version,
created=created,
action=journal.action,
pypi_id=journal.id,
)
if (journal.action.lower() == "remove" and
journal.version is None):
# Delete the entire project
if journal.name not in deleted:
updated.discard(journal.name)
deleted.add(journal.name)
# Actually yank the project
Project.yank(journal.name, synchronize=False)
elif journal.action.lower().startswith("rename from "):
_, _, previous = journal.action.split(" ", 2)
# Do nothing for right now, eventually we'll use this spot for
# creating a history event
else:
# Process the update
if journal.name not in updated:
deleted.discard(journal.name)
updated.add(journal.name)
# Actually synchronize the project
synchronize_project(journal.name,
fetcher,
download=download,
)
try:
# Add this ID to our list of IDs we've processed in Redis
redis.sadd(REDIS_JOURNALS_KEY, journal.id)
# Commit any changes made from this journal entry
db.session.commit()
except:
# If any exception occured during committing remove the id
# from redis
redis.srem(REDIS_JOURNALS_KEY, journal.id)
raise
logger.info(
"Finished processing journals at %s; updated %s and deleted %s",
current, len(updated), len(deleted),
)
return current
def synchronize_by_journals(since=None,
fetcher=None,
progress=True,
download=None):
if fetcher is None:
fetcher = PyPIFetcher()
# Grab the current datetime
current = fetcher.current()
# Synchronize all the classifiers with PyPI
synchronize_classifiers(fetcher)
# Grab the journals since `since`
journals = fetcher.journals(since=since)
# Storage for projects that have been updated or deleted
updated = set()
deleted = set()
# Check if we have anything to process before attempting to
if journals:
if progress:
bar = ShadyBar("Processing Journals", max=len(journals))
else:
bar = DummyBar()
# Handle Renames, these need to occur first because PyPI retroactively
# changes journal names to the new project, which if we experience
# any of these prior to handling a rename it'll trigger a sync which
# will act like it's a new project and not a renamed project.
if since is not None:
for journal in journals:
if journal.action.lower().startswith("rename from "):
_, _, previous = journal.action.split(" ", 2)
proj = Project.get(previous)
proj.rename(journal.name)
# Commit the renames
db.session.commit()
for journal in bar.iter(journals):
if redis.sismember(REDIS_JOURNALS_KEY, journal.id):
# We've already processed this entry, so skip to the next one
continue
created = datetime.datetime.utcfromtimestamp(journal.timestamp)
Journal.create(
name=journal.name,
version=journal.version,
created=created,
action=journal.action,
pypi_id=journal.id,
)
if (journal.action.lower() == "remove"
and journal.version is None):
# Delete the entire project
if journal.name not in deleted:
updated.discard(journal.name)
deleted.add(journal.name)
# Actually yank the project
Project.yank(journal.name, synchronize=False)
elif journal.action.lower().startswith("rename from "):
_, _, previous = journal.action.split(" ", 2)
# Do nothing for right now, eventually we'll use this spot for
# creating a history event
else:
# Process the update
if journal.name not in updated:
deleted.discard(journal.name)
updated.add(journal.name)
# Actually synchronize the project
synchronize_project(
journal.name,
fetcher,
download=download,
)
try:
# Add this ID to our list of IDs we've processed in Redis
redis.sadd(REDIS_JOURNALS_KEY, journal.id)
# Commit any changes made from this journal entry
db.session.commit()
except:
# If any exception occured during committing remove the id
# from redis
redis.srem(REDIS_JOURNALS_KEY, journal.id)
raise
logger.info(
"Finished processing journals at %s; updated %s and deleted %s",
current,
len(updated),
len(deleted),
)
return current
