def make_api_version(version_data): from builds.models import Version for key in ['resource_uri', 'absolute_url']: if key in version_data: del version_data[key] project_data = version_data['project'] project = make_api_project(project_data) version_data['project'] = project ver = Version(**version_data) ver.save = _new_save return ver
def make_api_version(version_data): from builds.models import Version for key in ['resource_uri', 'absolute_url', 'downloads']: if key in version_data: del version_data[key] project_data = version_data['project'] project = make_api_project(project_data) version_data['project'] = project ver = Version(**version_data) ver.save = _new_save return ver
def make_api_version(version_data): from builds.models import Version del version_data['resource_uri'] project_data = version_data['project'] project = make_api_project(project_data) version_data['project'] = project ver = Version(**version_data) return ver
def api_versions(self): from builds.models import Version ret = [] for version_data in api.version.get(project=self.pk, active=True)['objects']: del version_data['resource_uri'] project_data = version_data['project'] del project_data['users'] del project_data['resource_uri'] del project_data['absolute_url'] project = Project(**project_data) version_data['project'] = project ret.append(Version(**version_data)) return sort_version_aware(ret)
def update_intersphinx(version_pk): version_data = api.version(version_pk).get() del version_data['resource_uri'] project_data = version_data['project'] del project_data['users'] del project_data['resource_uri'] del project_data['absolute_url'] project = Project(**project_data) version_data['project'] = project version = Version(**version_data) try: object_file = version.project.find('objects.inv', version.slug)[0] except IndexError, e: print "Failed to find objects file" return None
def update_intersphinx(version_pk): version_data = api.version(version_pk).get() del version_data['resource_uri'] project_data = version_data['project'] del project_data['users'] del project_data['resource_uri'] del project_data['absolute_url'] project = Project(**project_data) version_data['project'] = project version = Version(**version_data) object_file = version.project.find('objects.inv', version.slug)[0] path = version.project.rtd_build_path(version.slug) if not path: log.warning("%s has no path" % version) return None app = DictObj() app.srcdir = path try: inv = fetch_inventory(app, path, object_file) except TypeError, e: log.error("Failed to fetch inventory for %s" % version, exc_info=True) return None
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False, **kwargs): """ The main entry point for updating documentation. It handles all of the logic around whether a project is imported or we created it. Then it will build the html docs and other requested parts. It also handles clearing the varnish cache. `pk` Primary key of the project to update `record` Whether or not to keep a record of the update in the database. Useful for preventing changes visible to the end-user when running commands from the shell, for example. """ ### # Handle passed in arguments ### update_output = kwargs.get('update_output', {}) project_data = api.project(pk).get() del project_data['users'] del project_data['resource_uri'] del project_data['absolute_url'] project = Project(**project_data) # Prevent saving the temporary Project instance def new_save(*args, **kwargs): log.warning("Called save on a non-real object.") return 0 project.save = new_save log.info("Building %s" % project) if version_pk: version_data = api.version(version_pk).get() del version_data['resource_uri'] else: #Create or use the 'latest' branch, which is the default for a project. branch = project.default_branch or project.vcs_repo().fallback_branch try: version_data = api.version( project.slug).get(slug='latest')['objects'][0] del version_data['resource_uri'] except (slumber.exceptions.HttpClientError, IndexError) as exc: #if exc.response.status_code in [404,500]: version_data = dict( project='/api/v1/project/%s/' % project.pk, slug='latest', active=True, verbose_name='latest', identifier=branch, ) try: version_data = api.version.post(version_data) del version_data['resource_uri'] except Exception as e: log.info("Exception in creating version: %s" % e) #raise e version_data['project'] = project version = Version(**version_data) version.save = new_save if not version_pk: #Lots of course correction. to_save = False if not version.verbose_name: version_data['verbose_name'] = 'latest' to_save = True if not version.active: version_data['active'] = True to_save = True if version.identifier != branch: version_data['identifier'] = branch to_save = True if to_save: version_data[ 'project'] = "/api/v1/version/%s/" % version_data['project'].pk api.version(version.pk).put(version_data) if record: #Create Build Object. build = api.build.post( dict( project='/api/v1/project/%s/' % project.pk, version='/api/v1/version/%s/' % version.pk, type='html', state='triggered', )) else: build = {} #Make Dirs path = project.doc_path if not os.path.exists(path): os.makedirs(path) with project.repo_lock(30): if project.is_imported: try: update_output = update_imported_docs(project, version) except ProjectImportError, err: log.error("Failed to import project; skipping build.", exc_info=True) build['state'] = 'finished' build[ 'setup_error'] = 'Failed to import project; skipping build.\nPlease make sure your repo is correct and you have a conf.py' api.build(build['id']).put(build) return False else:
def update_imported_docs(project, version): """ Check out or update the given project's repository. """ update_docs_output = {} if not project.vcs_repo(): raise ProjectImportError("Repo type '{repo_type}' unknown".format( repo_type=project.repo_type)) if version: log.info('Checking out version {slug}: {identifier}'.format( slug=version.slug, identifier=version.identifier)) version_slug = version.slug version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.checkout( version.identifier) else: log.info('Updating to latest revision') version_slug = 'latest' version_repo = project.vcs_repo(version_slug) update_docs_output['checkout'] = version_repo.update() # Ensure we have a conf file (an exception is raised if not) conf_file = project.conf_file(version.slug) #Do Virtualenv bits: if project.use_virtualenv: update_docs_output['venv'] = run( '{cmd} --distribute --no-site-packages {path}'.format( cmd='virtualenv', path=project.venv_path(version=version_slug))) update_docs_output['sphinx'] = run( '{cmd} install -U hg+http://bitbucket.org/birkenfeld/sphinx/@d4c6ac1fcc9c#egg=Sphinx virtualenv==1.8.2 distribute==0.6.28 docutils==0.8.1' .format(cmd=project.venv_bin(version=version_slug, bin='pip'))) if project.requirements_file: os.chdir(project.checkout_path(version_slug)) update_docs_output['requirements'] = run( '{cmd} install -r {requirements}'.format( cmd=project.venv_bin(version=version_slug, bin='pip'), requirements=project.requirements_file)) os.chdir(project.checkout_path(version_slug)) update_docs_output['install'] = run( '{cmd} setup.py install --force'.format( cmd=project.venv_bin(version=version_slug, bin='python'))) # check tags/version #XXX:dc: what in this block raises the values error? try: old_versions = [ obj['identifier'] for obj in api.version.get( project__slug=project.slug, limit=500)['objects'] ] if version_repo.supports_tags: transaction.enter_transaction_management(True) tags = version_repo.tags for tag in tags: if tag.identifier in old_versions: continue slug = slugify_uniquely(Version, tag.verbose_name, 'slug', 255, project=project) try: version_data = api.version.post( dict(project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=tag.identifier, verbose_name=tag.verbose_name)) del version_data['resource_uri'] project_data = version_data['project'] del project_data['users'] del project_data['resource_uri'] del project_data['absolute_url'] version_data['project'] = project ver = Version(**version_data) log.info("New tag found: {0}".format(tag.identifier)) ver, highest = project.highest_version[1] ver_obj = mkversion(ver) #TODO: Handle updating higher versions automatically. #This never worked very well, anyways. if highest and ver_obj and ver_obj > highest: log.info("Highest verison known, building docs") update_docs.delay(ver.project.pk, version_pk=ver.pk) except Exception, e: log.error("Failed to create version (tag)", exc_info=True) transaction.rollback() #break here to stop updating tags when they will all fail. break transaction.leave_transaction_management() if version_repo.supports_branches: transaction.enter_transaction_management(True) branches = version_repo.branches for branch in branches: if branch.identifier in old_versions: continue slug = slugify_uniquely(Version, branch.verbose_name, 'slug', 255, project=project) try: api.version.post( dict(project="/api/v1/project/%s/" % project.pk, slug=slug, identifier=branch.identifier, verbose_name=branch.verbose_name)) log.info("New branch found: {0}".format(branch.identifier)) except Exception, e: log.error("Failed to create version (branch)", exc_info=True) transaction.rollback() #break here to stop updating branches when they will all fail. break transaction.leave_transaction_management()
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False, **kwargs): """ The main entry point for updating documentation. It handles all of the logic around whether a project is imported or we created it. Then it will build the html docs and other requested parts. It also handles clearing the varnish cache. `pk` Primary key of the project to update `record` Whether or not to keep a record of the update in the database. Useful for preventing changes visible to the end-user when running commands from the shell, for example. """ ### # Handle passed in arguments ### update_output = kwargs.get('update_output', {}) project_data = api.project(pk).get() del project_data['users'] del project_data['resource_uri'] del project_data['absolute_url'] project = Project(**project_data) # Prevent saving the temporary Project instance def new_save(*args, **kwargs): log.warning("Called save on a non-real object.") return 0 project.save = new_save log.info("Building %s" % project) if version_pk: version_data = api.version(version_pk).get() del version_data['resource_uri'] else: #Create or use the 'latest' branch, which is the default for a project. branch = project.default_branch or project.vcs_repo().fallback_branch try: version_data = api.version(project.slug).get(slug='latest')['objects'][0] del version_data['resource_uri'] except (slumber.exceptions.HttpClientError, IndexError) as exc: #if exc.response.status_code in [404,500]: version_data = dict( project='/api/v1/project/%s/' % project.pk, slug='latest', active=True, verbose_name='latest', identifier=branch, ) try: version_data = api.version.post(version_data) del version_data['resource_uri'] except Exception as e: raise e version_data['project'] = project version = Version(**version_data) version.save = new_save if not version_pk: #Lots of course correction. to_save = False if not version.verbose_name: version_data['verbose_name'] = 'latest' to_save = True if not version.active: version_data['active'] = True to_save = True if version.identifier != branch: version_data['identifier'] = branch to_save = True if to_save: version_data['project'] = "/api/v1/version/%s/" % version_data['project'].pk api.version(version.pk).put(version_data) if record: #Create Build Object. build = api.build.post(dict( project= '/api/v1/project/%s/' % project.pk, version= '/api/v1/version/%s/' % version.pk, type='html', state='triggered', )) else: build = {} #Make Dirs path = project.doc_path if not os.path.exists(path): os.makedirs(path) with project.repo_lock(30): if project.is_imported: try: update_output = update_imported_docs(project, version) except ProjectImportError, err: log.error("Failed to import project; skipping build.", exc_info=True) return False else:
def setUp(self): self.project = Project.objects.get(slug='pip') self.version = Version(slug='foo', verbose_name='foobar') self.project.versions.add(self.version)
def update_docs(pk, record=True, pdf=True, man=True, epub=True, version_pk=None, force=False, **kwargs): """ The main entry point for updating documentation. It handles all of the logic around whether a project is imported or we created it. Then it will build the html docs and other requested parts. It also handles clearing the varnish cache. """ ### # Handle passed in arguments ### update_output = kwargs.get("update_output", {}) project_data = api.project(pk).get() del project_data["users"] del project_data["resource_uri"] del project_data["absolute_url"] project = Project(**project_data) def new_save(*args, **kwargs): # fields = [(field, field.value_to_string(self)) for field in self._meta.fields] print "*** Called save on a non-real object." # print fields # raise TypeError('Not a real model') return 0 project.save = new_save print "Building %s" % project if version_pk: version_data = api.version(version_pk).get() del version_data["resource_uri"] else: # Create or use the 'latest' branch, which is the default for a project. branch = project.default_branch or project.vcs_repo().fallback_branch try: version_data = api.version(project.slug).get(slug="latest")["objects"][0] del version_data["resource_uri"] except (slumber.exceptions.HttpClientError, IndexError) as exc: # if exc.response.status_code in [404,500]: version_data = dict( project="/api/v1/project/%s/" % project.pk, slug="latest", active=True, verbose_name="latest", identifier=branch, ) try: version_data = api.version.post(version_data) del version_data["resource_uri"] except Exception as e: raise e version_data["project"] = project version = Version(**version_data) version.save = new_save if not version_pk: # Lots of course correction. to_save = False if not version.verbose_name: version_data["verbose_name"] = "latest" to_save = True if not version.active: version_data["active"] = True to_save = True if version.identifier != branch: version_data["identifier"] = branch to_save = True if to_save: version_data["project"] = "/api/v1/version/%s/" % version_data["project"].pk api.version(version.pk).put(version_data) if record: # Create Build Object. build = api.build.post( dict( project="/api/v1/project/%s/" % project.pk, version="/api/v1/version/%s/" % version.pk, type="html", state="triggered", ) ) else: build = {} # Make Dirs path = project.doc_path if not os.path.exists(path): os.makedirs(path) with project.repo_lock(30): if project.is_imported: try: update_output = update_imported_docs(project, version) except ProjectImportError, err: print ("Error importing project: %s. Skipping build." % err) return False # scrape_conf_file(version) else: