def handle(self, *args, **options): if options['quiet']: self.verbosity = 0 else: self.verbosity = int(options['verbosity']) self.client = ForgeClient(api_url=options['api_url'], throttle=options['throttle']) # Sync authors first, then modules, and finally create releases # after downloading the module tarballs. self.sync_authors() self.sync_modules() self.sync_releases()
class Command(BaseCommand): help = ( 'Syncs with another Puppet Forge.' ) option_list = BaseCommand.option_list + ( make_option( '--api-url', action='store', dest='api_url', default=constants.PUPPETLABS_FORGE_API_URL, help=("Puppet Forge URL to mirror, defaults to: %s." % constants.PUPPETLABS_FORGE_API_URL), ), make_option( '-q', '--quiet', action='store_true', dest='quiet', default=False, help=('Silence all output (sets verbosity to 0).'), ), make_option( '-t', '--throttle', action='store', dest='throttle', default=0.5, type='float', help=('Time (in seconds) to wait between API requests.'), ), make_option( '-s', '--supported_only', action='store_true', dest='supported_only', default=False, help=('Download only supported modules'), ), ) def handle(self, *args, **options): if options['quiet']: self.verbosity = 0 else: self.verbosity = int(options['verbosity']) self.client = ForgeClient(api_url=options['api_url'], throttle=options['throttle']) self.supported_only = bool(getattr(options,'supported_only', False)) # Sync authors first, then modules, and finally create releases # after downloading the module tarballs. self.sync_authors() self.sync_modules(supported_only=self.supported_only) self.sync_releases(supported_only=self.supported_only) def log(self, msg, error=False, verbosity_level=1): if error: logger.error(msg) else: logger.info(msg) if self.verbosity >= verbosity_level: sys.stdout.write('%s\n' % msg) def sync_authors(self): self.users_api = ForgeAPI('users', client=self.client) for user in self.users_api: if user['module_count'] > 0: author, created = Author.objects.get_or_create( name=user['username'] ) if created: self.log('Created Author: %s' % author) def sync_modules(self, supported_only=False): self.modules_api = ForgeAPI('modules', client=self.client) for mod in self.modules_api: if supported_only and getattr(mod, 'supported', False) is False: #Skip unsupported modules where the option is set continue module, created = Module.objects.get_or_create( author=Author.objects.get_by_natural_key(mod['owner']['username']), name=mod['name'], supported=mod['supported'] ) if created: msg = 'Created Module: %s' % module self.log(msg) desc = mod['current_release']['metadata'].get('description', '') tags = ' '.join(mod['current_release']['tags']) if tags != module.tags or desc != module.desc: if not created: if tags != module.tags: self.log( '\n'.join( [' Tags Differ:', ' Old: %s' % module.tags, ' New: %s' % tags]), verbosity_level=2 ) if desc != module.desc: self.log( '\n'.join( [' Descriptions Differ:', ' Old: %s' % module.desc, ' New: %s' % desc]), verbosity_level=2 ) module.tags = tags module.desc = desc module.save() if not created: self.log('Updated Module: %s' % module) def sync_releases(self, supported_only=False): # Only synchronize releases from authors that have released at least # one Puppet module. Querying the releases by author should make it # so that less total API calls are requested of the remote forge. module_authors = Author.objects.annotate( num_modules=Count('module') ).filter(num_modules__gt=0).distinct() for author in module_authors.iterator(): author_name = author.name.lower() alpha = author_name[0].lower() releases_api = ForgeAPI( 'releases', client=self.client, query={'owner': author_name, 'sort_by': 'release_date'} ) for rel in releases_api: if supported_only and rel['supported'] is False: #Skip unsupported modules where the option is set continue tarball = os.path.basename(rel['file_uri']) # TODO: Change to v3 compatible file structure, this is using the # the same structure that v1 does. upload_to = '/'.join([alpha, author_name, tarball]) destination = os.path.join(settings.MEDIA_ROOT, upload_to) destination_tmp = destination + '.tmp' dest_dir = os.path.dirname(destination) if not os.path.isdir(dest_dir): os.makedirs(dest_dir, mode=0755) if not os.path.isfile(destination): tarball_url = urlparse.urljoin( self.client.api_url, rel['file_uri'] ) file_md5 = hashlib.md5() with open(destination_tmp, 'wb') as tb_h: with closing(self.client.get(tarball_url, stream=True)) as req: for chunk in req: if chunk: tb_h.write(chunk) file_md5.update(chunk) if file_md5.hexdigest() == rel['file_md5']: os.rename(destination_tmp, destination) self.log('Downloaded Release: %s' % tarball) else: os.remove(destination_tmp) self.log( 'Downloaded corrupt data from: %s' % tarball_url, error=True ) continue # Creating Release now download is completed. try: # Get corresponding module. module = Module.objects.get( author=author, name=rel['module']['name'] ) release, created = Release.objects.get_or_create( module=module, version=rel['version'], tarball=upload_to, supported=rel['supported'] ) if created: self.log('Created Release: %s' % release) except Exception as e: err_msg = ( 'Could not create release for: %s version %s\n' % (module, rel['version']) ) self.log(err_msg, error=True)