def synchronize_by_projects(projects=None, fetcher=None, progress=True, download=None): if fetcher is None: fetcher = PyPIFetcher() # Grab the current datetime current = fetcher.current() # Synchronize all the classifiers with PyPI synchronize_classifiers(fetcher) if not projects: # Grab a list of projects from PyPI projects = fetcher.projects() # We are not synchronizing a subset of projects, so we can check for # any deletions (if required) and yank them. diff.projects(projects) # Commit our yanked projects db.session.commit() if progress: bar = ShadyBar("Processing Projects", max=len(projects)) else: bar = DummyBar() for project in bar.iter(projects): synchronize_project(project, fetcher, download=download) logger.info("Finished processing projects at %s", current) return current
def synchronize_by_projects(projects=None, fetcher=None, progress=True, download=None): if fetcher is None: fetcher = PyPIFetcher() # Grab the current datetime current = fetcher.current() # Synchronize all the classifiers with PyPI synchronize_classifiers(fetcher) if not projects: # Grab a list of projects from PyPI projects = fetcher.projects() # We are not synchronizing a subset of projects, so we can check for # any deletions (if required) and yank them. diff.projects(projects) # Commit our yanked projects db.session.commit() if progress: bar = ShadyBar("Processing Projects", max=len(projects)) else: bar = DummyBar() for project in bar.iter(projects): synchronize_project(project, fetcher, download=download) logger.info("Finished processing projects at %s", current) return current
class ProgressBarArgument(FlagArgument): """Manage a progress bar""" def __init__(self, help='', parsed_name='', default=True): self.suffix = '%(percent)d%%' super(ProgressBarArgument, self).__init__(help, parsed_name, default) def clone(self): """Get a modifiable copy of this bar""" newarg = ProgressBarArgument(self.help, self.parsed_name, self.default) newarg._value = self._value return newarg def get_generator(self, message, message_len=25, countdown=False, timeout=100): """Get a generator to handle progress of the bar (gen.next())""" if self.value: return None try: self.bar = KamakiProgressBar(message.ljust(message_len), max=timeout or 100) except NameError: self.value = None return self.value if countdown: bar_phases = list(self.bar.phases) self.bar.empty_fill, bar_phases[0] = bar_phases[-1], '' bar_phases.reverse() self.bar.phases = bar_phases self.bar.bar_prefix = ' ' self.bar.bar_suffix = ' ' self.bar.suffix = '%(remaining)ds to timeout' else: self.bar.suffix = '%(percent)d%% - %(eta)ds' self.bar.start() def progress_gen(n): for i in self.bar.iter(range(int(n))): yield yield return progress_gen def finish(self): """Stop progress bar, return terminal cursor to user""" if self.value: return mybar = getattr(self, 'bar', None) if mybar: mybar.finish()
class ProgressBarArgument(FlagArgument): """Manage a progress bar""" def __init__(self, help='', parsed_name='', default=True): self.suffix = '%(percent)d%%' super(ProgressBarArgument, self).__init__(help, parsed_name, default) def clone(self): """Get a modifiable copy of this bar""" newarg = ProgressBarArgument( self.help, self.parsed_name, self.default) newarg._value = self._value return newarg def get_generator( self, message, message_len=25, countdown=False, timeout=100): """Get a generator to handle progress of the bar (gen.next())""" if self.value: return None try: self.bar = KamakiProgressBar( message.ljust(message_len), max=timeout or 100) except NameError: self.value = None return self.value if countdown: bar_phases = list(self.bar.phases) self.bar.empty_fill, bar_phases[0] = bar_phases[-1], '' bar_phases.reverse() self.bar.phases = bar_phases self.bar.bar_prefix = ' ' self.bar.bar_suffix = ' ' self.bar.suffix = '%(remaining)ds to timeout' else: self.bar.suffix = '%(percent)d%% - %(eta)ds' self.bar.start() def progress_gen(n): for i in self.bar.iter(range(int(n))): yield yield return progress_gen def finish(self): """Stop progress bar, return terminal cursor to user""" if self.value: return mybar = getattr(self, 'bar', None) if mybar: mybar.finish()
def synchronize_by_journals(since=None, fetcher=None, progress=True, download=None): if fetcher is None: fetcher = PyPIFetcher() # Grab the current datetime current = fetcher.current() # Synchronize all the classifiers with PyPI synchronize_classifiers(fetcher) # Grab the journals since `since` journals = fetcher.journals(since=since) # Storage for projects that have been updated or deleted updated = set() deleted = set() # Check if we have anything to process before attempting to if journals: if progress: bar = ShadyBar("Processing Journals", max=len(journals)) else: bar = DummyBar() # Handle Renames, these need to occur first because PyPI retroactively # changes journal names to the new project, which if we experience # any of these prior to handling a rename it'll trigger a sync which # will act like it's a new project and not a renamed project. if since is not None: for journal in journals: if journal.action.lower().startswith("rename from "): _, _, previous = journal.action.split(" ", 2) proj = Project.get(previous) proj.rename(journal.name) # Commit the renames db.session.commit() for journal in bar.iter(journals): if redis.sismember(REDIS_JOURNALS_KEY, journal.id): # We've already processed this entry, so skip to the next one continue created = datetime.datetime.utcfromtimestamp(journal.timestamp) Journal.create( name=journal.name, version=journal.version, created=created, action=journal.action, pypi_id=journal.id, ) if (journal.action.lower() == "remove" and journal.version is None): # Delete the entire project if journal.name not in deleted: updated.discard(journal.name) deleted.add(journal.name) # Actually yank the project Project.yank(journal.name, synchronize=False) elif journal.action.lower().startswith("rename from "): _, _, previous = journal.action.split(" ", 2) # Do nothing for right now, eventually we'll use this spot for # creating a history event else: # Process the update if journal.name not in updated: deleted.discard(journal.name) updated.add(journal.name) # Actually synchronize the project synchronize_project(journal.name, fetcher, download=download, ) try: # Add this ID to our list of IDs we've processed in Redis redis.sadd(REDIS_JOURNALS_KEY, journal.id) # Commit any changes made from this journal entry db.session.commit() except: # If any exception occured during committing remove the id # from redis redis.srem(REDIS_JOURNALS_KEY, journal.id) raise logger.info( "Finished processing journals at %s; updated %s and deleted %s", current, len(updated), len(deleted), ) return current
def synchronize_by_journals(since=None, fetcher=None, progress=True, download=None): if fetcher is None: fetcher = PyPIFetcher() # Grab the current datetime current = fetcher.current() # Synchronize all the classifiers with PyPI synchronize_classifiers(fetcher) # Grab the journals since `since` journals = fetcher.journals(since=since) # Storage for projects that have been updated or deleted updated = set() deleted = set() # Check if we have anything to process before attempting to if journals: if progress: bar = ShadyBar("Processing Journals", max=len(journals)) else: bar = DummyBar() # Handle Renames, these need to occur first because PyPI retroactively # changes journal names to the new project, which if we experience # any of these prior to handling a rename it'll trigger a sync which # will act like it's a new project and not a renamed project. if since is not None: for journal in journals: if journal.action.lower().startswith("rename from "): _, _, previous = journal.action.split(" ", 2) proj = Project.get(previous) proj.rename(journal.name) # Commit the renames db.session.commit() for journal in bar.iter(journals): if redis.sismember(REDIS_JOURNALS_KEY, journal.id): # We've already processed this entry, so skip to the next one continue created = datetime.datetime.utcfromtimestamp(journal.timestamp) Journal.create( name=journal.name, version=journal.version, created=created, action=journal.action, pypi_id=journal.id, ) if (journal.action.lower() == "remove" and journal.version is None): # Delete the entire project if journal.name not in deleted: updated.discard(journal.name) deleted.add(journal.name) # Actually yank the project Project.yank(journal.name, synchronize=False) elif journal.action.lower().startswith("rename from "): _, _, previous = journal.action.split(" ", 2) # Do nothing for right now, eventually we'll use this spot for # creating a history event else: # Process the update if journal.name not in updated: deleted.discard(journal.name) updated.add(journal.name) # Actually synchronize the project synchronize_project( journal.name, fetcher, download=download, ) try: # Add this ID to our list of IDs we've processed in Redis redis.sadd(REDIS_JOURNALS_KEY, journal.id) # Commit any changes made from this journal entry db.session.commit() except: # If any exception occured during committing remove the id # from redis redis.srem(REDIS_JOURNALS_KEY, journal.id) raise logger.info( "Finished processing journals at %s; updated %s and deleted %s", current, len(updated), len(deleted), ) return current