def _load_configuration(self):
self.p4_user = getattr(settings, 'P4USER')
if self.p4_user == "NOTSPECIFIED":
raise ProjectImportError("Perforce username must be configured")
self.p4_pass = getattr(settings, 'P4PASSWD')
if self.p4_pass == "NOTSPECIFIED":
raise ProjectImportError(
"Perforce user credentials must be configured")
def _pull(self):
retcode = self._run_command('hg', 'pull')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (hg pull): %s" % (self.repo_url, retcode)
)
retcode = self._run_command('hg', 'update', '-C')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (hg update): %s" % (self.repo_url, retcode)
)
def _up(self):
retcode = self._run_command('svn', 'revert', '--recursive', '.')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (svn revert): %s" %
(self.repo_url, retcode))
retcode = self._run_command('svn', 'up', '--accept', 'theirs-full')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (svn up): %s" %
(self.repo_url, retcode))
def _up(self):
retcode = self._run_command('bzr', 'revert')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (bzr revert): %s" %
(self.repo_url, retcode))
retcode = self._run_command('bzr', 'up')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (bzr up): %s" %
(self.repo_url, retcode))
def up(self):
retcode = self.run('svn', 'revert', '--recursive', '.')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (svn revert): %s" %
(self.repo_url, retcode))
retcode = self.run('svn', 'up', '--accept', 'theirs-full',
'--trust-server-cert', '--non-interactive')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (svn up): %s" %
(self.repo_url, retcode))
def _sync(self):
retcode = self._run_on_client('sync')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to sync client '{workspace}'".format(
workspace=self._get_workspace_name()))
# Force sync conf files because they get modified by the doc builder
retcode = self._run_on_client('sync', '-f', '//...conf.py')[0]
if retcode != 0:
raise ProjectImportError("Failed to force sync .conf files")
def pull(self):
pull_output = self.run('hg', 'pull')
if pull_output[0] != 0:
raise ProjectImportError(
("Failed to get code from '%s' (hg pull): %s" %
(self.repo_url, pull_output[0])))
update_output = self.run('hg', 'update', '-C')[0]
if update_output[0] != 0:
raise ProjectImportError(
("Failed to get code from '%s' (hg update): %s" %
(self.repo_url, pull_output[0])))
return update_output
def up(self):
retcode = self.run('bzr', 'revert')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (bzr revert): %s" % (self.repo_url, retcode)
)
up_output = self.run('bzr', 'up')
if up_output[0] != 0:
raise ProjectImportError(
"Failed to get code from '%s' (bzr up): %s" % (self.repo_url, retcode)
)
return up_output
def clone(self):
self.make_clean_working_dir()
retcode = self.run('bzr', 'checkout', self.repo_url, '.')[0]
if retcode != 0:
raise ProjectImportError(
("Failed to get code from '%s' (bzr checkout): %s" %
(self.repo_url, retcode)))
def _co(self):
retcode = self._run_command('svn', 'checkout', '--quiet',
self.repo_url, '.')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (svn checkout): %s" %
(self.repo_url, retcode))
def fetch(self):
code, out, err = self.run('git', 'fetch', '--prune')
if code != 0:
raise ProjectImportError(
"Failed to get code from '%s' (git fetch): %s\n\nStderr:\n\n%s\n\n" % (
self.repo_url, code, err)
)
def _pull(self):
retcode = self._run_command('git', 'fetch')[0]
retcode = self._run_command('git', 'fetch', '-t')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (git fetch): %s" % (self.repo_url, retcode)
)
def clone(self):
output = self.run('hg', 'clone', self.repo_url, '.')
if output[0] != 0:
raise ProjectImportError(
"Failed to get code from '%s' (hg clone): %s" %
(self.repo_url, output[0]))
return output
def clone(self):
code, out, err = self.run('git', 'clone', '--recursive', '--quiet',
self.repo_url, '.')
if code != 0:
raise ProjectImportError(
"Failed to get code from '%s' (git clone): %s" %
(self.repo_url, code))
def conf_file(self, version=LATEST):
if self.conf_py_file:
conf_path = os.path.join(self.checkout_path(version),
self.conf_py_file)
if os.path.exists(conf_path):
log.info('Inserting conf.py file path from model')
return conf_path
else:
log.warning("Conf file specified on model doesn't exist")
files = self.find('conf.py', version)
print files
if not files:
files = self.full_find('conf.py', version)
print files
if len(files) == 1:
return files[0]
for file in files:
if file.find('doc', 70) != -1:
return file
# Having this be translatable causes this odd error:
# ProjectImportError(<django.utils.functional.__proxy__ object at
# 0x1090cded0>,)
raise ProjectImportError(
u"Conf File Missing. Please make sure you have a conf.py in your project."
)
def pull(self):
code, out, err = self.run('git', 'fetch')
code, out, err = self.run('git', 'fetch', '-t')
if code != 0:
raise ProjectImportError(
"Failed to get code from '%s' (git fetch): %s" %
(self.repo_url, code))
def _clone(self):
retcode = self._run_command(
'git', 'clone', '--quiet',
'%s.git' % self.repo_url.replace('.git', ''), '.')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (git clone): %s" %
(self.repo_url, retcode))
def clone(self):
self.make_clean_working_dir()
output = self.run('hg', 'clone', self.repo_url, '.')
if output[0] != 0:
raise ProjectImportError(
("Failed to get code from '%s' (hg clone): %s" %
(self.repo_url, output[0])))
return output
def _reset(self):
branch = self.fallback_branch
if self.project.default_branch:
branch = self.project.default_branch
retcode = self._run_command('git', 'reset', '--hard', 'origin/%s' % branch)[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (git reset): %s" % (self.repo_url, retcode)
)
def co(self, identifier=None):
if identifier:
url = self.base_url + identifier
else:
url = self.repo_url
retcode = self.run('svn', 'checkout', '--quiet', url, '.')[0]
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (svn checkout): %s" %
(url, retcode))
def _update_client(self):
try:
# Change client root for current project
client = self.p4.fetch_client()
client['Root'] = self.working_dir
self.p4.save_client(client)
except P4Exception as err:
raise ProjectImportError(
'Failed to update client "{}".\n{}\n'.format(
self.p4.client, err))
def __init__(self, project, version):
super(Backend, self).__init__(project, version)
p4 = P4() # P4 Python API
try:
if not p4.connected():
p4.connect()
except P4Exception as err:
raise ProjectImportError(
'Cannot connect to Perforce server.\n{}'.format(err))
try:
p4.run_login()
except P4Exception as err:
# Oops... any problems о_О
raise ProjectImportError(
'Cannot login to Perforce server.\n{}'.format(err))
self.p4 = p4
def conf_file(self, version='latest'):
files = self.find('conf.py', version)
if not files:
files = self.full_find('conf.py', version)
if len(files) == 1:
return files[0]
elif len(files) > 1:
for file in files:
if file.find('doc', 70) != -1:
return file
else:
raise ProjectImportError("Conf File Missing.")
def co(self, identifier=None):
self.make_clean_working_dir()
if identifier:
url = self.base_url + identifier
else:
url = self.repo_url
retcode, out, err = self.run('svn', 'checkout', '--quiet', url, '.')
if retcode != 0:
raise ProjectImportError(
"Failed to get code from '%s' (svn checkout): %s" %
(url, retcode))
return retcode, out, err
def conf_file(self, version='latest'):
if self.conf_py_file:
log.debug('Inserting conf.py file path from model')
return os.path.join(self.checkout_path(version), self.conf_py_file)
files = self.find('conf.py', version)
if not files:
files = self.full_find('conf.py', version)
if len(files) == 1:
return files[0]
elif len(files) > 1:
for file in files:
if file.find('doc', 70) != -1:
return file
else:
raise ProjectImportError(_("Conf File Missing."))
def conf_file(self, version='latest'):
if self.conf_py_file:
log.debug('Inserting conf.py file path from model')
return os.path.join(self.checkout_path(version), self.conf_py_file)
files = self.find('conf.py', version)
if not files:
files = self.full_find('conf.py', version)
if len(files) == 1:
return files[0]
for file in files:
if file.find('doc', 70) != -1:
return file
# Having this be translatable causes this odd error:
# ProjectImportError(<django.utils.functional.__proxy__ object at
# 0x1090cded0>,)
raise ProjectImportError(
u"Conf File Missing. Please make sure you have a conf.py in your project.")
def _create_workspace(self):
workspace_name = self._get_workspace_name()
filled_template = WORKSPACE_TEMPLATE.format(client_name=workspace_name,
owner=self.p4_user,
depot_path=self.repo_url,
root=self.working_dir,
hostname=gethostname(),
project_name=self.name)
ps = subprocess.Popen(
['p4', '-u', self.p4_user, '-P', self.p4_ticket, 'client', '-i'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
log.debug(
['p4', '-u', self.p4_user, '-P', self.p4_ticket, 'client', '-i'])
out, err = ps.communicate(filled_template)
if err:
raise ProjectImportError(err)
def _sync(self, identifier):
depot_url = self._get_depot_url()
if identifier:
# Use a specified changelist or label
depot_url += identifier
try:
info = self.p4.run_sync('-f', depot_url)
except P4Exception as err:
raise ProjectImportError(
"Failed to sync code from '{}'. Error:\n{}\n".format(
depot_url, err))
for rootdir, dirnames, filenames in os.walk(self.working_dir):
for fname in filenames:
pname = os.path.join(rootdir, fname)
os.chmod(pname, 0o777)
return info
def update_imported_docs(version_pk, api=None):
"""
Check out or update the given project's repository.
"""
if api is None:
api = tastyapi.api
version_data = api.version(version_pk).get()
version = make_api_version(version_data)
project = version.project
ret_dict = {}
# Make Dirs
if not os.path.exists(project.doc_path):
os.makedirs(project.doc_path)
if not project.vcs_repo():
raise ProjectImportError(
("Repo type '{0}' unknown".format(project.repo_type)))
with project.repo_nonblockinglock(version=version,
max_lock_age=getattr(
settings, 'REPO_LOCK_SECONDS', 30)):
before_vcs.send(sender=version)
# Get the actual code on disk
if version:
log.info(
LOG_TEMPLATE.format(
project=project.slug,
version=version.slug,
msg='Checking out version {slug}: {identifier}'.format(
slug=version.slug, identifier=version.identifier)))
version_slug = version.slug
version_repo = project.vcs_repo(version_slug)
ret_dict['checkout'] = version_repo.checkout(version.identifier, )
else:
# Does this ever get called?
log.info(
LOG_TEMPLATE.format(project=project.slug,
version=version.slug,
msg='Updating to latest revision'))
version_slug = 'latest'
version_repo = project.vcs_repo(version_slug)
ret_dict['checkout'] = version_repo.update()
after_vcs.send(sender=version)
# Update tags/version
version_post_data = {'repo': version_repo.repo_url}
if version_repo.supports_tags:
version_post_data['tags'] = [{
'identifier': v.identifier,
'verbose_name': v.verbose_name,
} for v in version_repo.tags]
if version_repo.supports_branches:
version_post_data['branches'] = [{
'identifier': v.identifier,
'verbose_name': v.verbose_name,
} for v in version_repo.branches]
try:
apiv2.project(project.pk).sync_versions.post(version_post_data)
except Exception, e:
print "Sync Versions Exception: %s" % e.message
def append_conf(self, **kwargs):
"""Modify the given ``conf.py`` file from a whitelisted user's project.
"""
# Pull config data
try:
conf_py_path = version_utils.get_conf_py_path(self.version)
except ProjectImportError:
self._write_config()
self.create_index(extension='rst')
project = self.version.project
# Open file for appending.
try:
outfile = codecs.open(project.conf_file(self.version.slug),
encoding='utf-8',
mode='a')
except IOError:
trace = sys.exc_info()[2]
raise ProjectImportError('Conf file not found'), None, trace
outfile.write("\n")
conf_py_path = version_utils.get_conf_py_path(self.version)
remote_version = version_utils.get_vcs_version_slug(self.version)
github_info = version_utils.get_github_username_repo(self.version)
bitbucket_info = version_utils.get_bitbucket_username_repo(
self.version)
if github_info[0] is None:
display_github = False
else:
display_github = True
if bitbucket_info[0] is None:
display_bitbucket = False
else:
display_bitbucket = True
rtd_ctx = Context({
'current_version':
self.version.slug,
'project':
project,
'settings':
settings,
'static_path':
STATIC_DIR,
'template_path':
TEMPLATE_DIR,
'conf_py_path':
conf_py_path,
'api_host':
getattr(settings, 'SLUMBER_API_HOST', 'https://readthedocs.org'),
# GitHub
'github_user':
github_info[0],
'github_repo':
github_info[1],
'github_version':
remote_version,
'display_github':
display_github,
# BitBucket
'bitbucket_user':
bitbucket_info[0],
'bitbucket_repo':
bitbucket_info[1],
'bitbucket_version':
remote_version,
'display_bitbucket':
display_bitbucket,
'commit':
self.version.project.vcs_repo(self.version.slug).commit,
})
# Avoid hitting database and API if using Docker build environment
if getattr(settings, 'DONT_HIT_API', False):
rtd_ctx['versions'] = project.active_versions()
rtd_ctx['downloads'] = self.version.get_downloads(pretty=True)
else:
rtd_ctx['versions'] = project.api_versions()
rtd_ctx['downloads'] = (apiv2.version(
self.version.pk).get()['downloads'])
rtd_string = template_loader.get_template(
'doc_builder/conf.py.tmpl').render(rtd_ctx)
outfile.write(rtd_string)
