def get_credentials(username=None,
password=None,
insecure=False,
apikey=None,
target_apikey=None):
""" A helper function to get credentials based on cmdline options.
Returns a tuple with 2 strings: (username, password).
When working with registries where only username matters:
missing password leads to auth request to registry authentication service
without 'account' query parameter which breaks login.
"""
if insecure:
if username or password:
raise BadParameterException(
'Insecure credentials not compatible with username/password')
return None, None
elif apikey:
return apikey, ' '
elif username:
if password is None:
raise BadParameterException(
'Password is required when passing username.')
return username, password
elif password:
raise BadParameterException(
'Username is required when passing password.')
return target_apikey, ' '
def validate_job_key(project_id, short_key):
job_key = "%s/%s" % (project_id, short_key)
if len(short_key.split("/")) != 2:
raise BadParameterException(
"keys must be defined as <spider_id>/<job_id>"
)
try:
return parse_job_key(job_key)
except ValueError as err:
raise BadParameterException(str(err))
except Exception as err:
raise SubcommandException(str(err))
def get_job_specs(job):
"""
Parse job identifier into valid job id and corresponding API key.
With projects default=10 and external=20 defined in config:
* 1/1 -> 10/1/1
* 2/2/2 -> 2/2/2
* external/2/2 -> 20/2/2
It also accepts job URLs from Scrapinghub.
"""
match = re.match(r'^((\w+)/)?(\d+/\d+)$', job)
if not match:
job_url_re = r'^https?://[^/]+/p/((\d+)/)job/(\d+/\d+).*'
match = re.match(job_url_re, job)
if not match:
raise BadParameterException(
"Job ID {} is invalid. Format should be spiderid/jobid (inside a "
"project) or target/spiderid/jobid, where target can be either a "
"project ID or an identifier defined in scrapinghub.yml."
"".format(job),
param_hint='job_id',
)
# XXX: Lazy import due to circular dependency
from shub.config import get_target_conf
targetconf = get_target_conf(match.group(2) or 'default')
return ("{}/{}".format(targetconf.project_id,
match.group(3)), targetconf.apikey)
def get_project(self, project):
"""
Given a project alias or a canonical project ID, return the
corresponding normalized configuration dictionary from
``self.projects``.
"""
if project in self.projects:
return self.normalized_projects[project]
try:
endpoint, proj_id = project.split('/')
except (ValueError, AttributeError):
endpoint, proj_id = 'default', project
try:
proj_id = int(proj_id)
except ValueError:
if project == 'default':
msg = ("Please specify target or configure a default target "
"in scrapinghub.yml.")
else:
msg = ("Could not find target \"%s\". Please define it in "
"your scrapinghub.yml or supply a numerical project ID."
"" % project)
raise BadParameterException(msg, param_hint='target')
for proj in self.normalized_projects.values():
if proj['id'] == proj_id and proj['endpoint'] == endpoint:
return proj
else:
return {'id': proj_id, 'endpoint': endpoint, 'apikey': endpoint}
def _checkout(repo, git_branch=None):
tmpdir = tempfile.mkdtemp(prefix='shub-deploy-egg-from-url')
click.echo("Cloning the repository to a tmp folder...")
os.chdir(tmpdir)
if (_run('git clone %s egg-tmp-clone' % repo) != 0
and _run('hg clone %s egg-tmp-clone' % repo) != 0
and _run('bzr branch %s egg-tmp-clone' % repo) != 0):
error = "\nERROR: The provided repository URL is not valid: %s\n"
raise BadParameterException(error % repo)
os.chdir('egg-tmp-clone')
if git_branch:
if _run('git checkout %s' % git_branch) != 0:
raise BadParameterException("Branch %s is not valid" % git_branch)
click.echo("%s branch was checked out" % git_branch)
def _checkout(repo, git_branch=None, target_dir='egg-tmp-clone'):
tmpdir = tempfile.mkdtemp(prefix='shub-deploy-egg-from-url')
click.echo("Cloning the repository to a tmp folder...")
os.chdir(tmpdir)
vcs_commands = [
['git', 'clone', repo, target_dir],
['hg', 'clone', repo, target_dir],
['bzr', 'branch', repo, target_dir],
]
missing_exes = []
for cmd in vcs_commands:
exe = find_executable(cmd[0])
if not exe:
missing_exes.append(cmd[0])
continue
try:
run_cmd([exe] + cmd[1:])
except SubcommandException:
pass
else:
break
else:
if missing_exes:
click.secho(
"shub was unable to find the following VCS executables and "
"could not try to check out your repository with these: %s"
"" % ', '.join(missing_exes),
fg='yellow')
raise BadParameterException(
"\nERROR: The provided repository URL is not valid: %s\n")
os.chdir(target_dir)
if git_branch:
try:
run_cmd([find_executable('git'), 'checkout', git_branch])
except SubcommandException:
raise BadParameterException("Branch %s is not valid" % git_branch)
click.echo("%s branch was checked out" % git_branch)
def cli(target, version, debug, egg, build_egg, verbose, keep_log,
ignore_size):
conf, image = load_shub_config(), None
if not build_egg:
create_scrapinghub_yml_wizard(conf, target=target)
image = conf.get_target_conf(target).image
if not image:
deploy_cmd(target, version, debug, egg, build_egg, verbose, keep_log,
conf=conf)
elif image.startswith(SH_IMAGES_REGISTRY):
upload_cmd(target, version)
else:
raise BadParameterException(
"Please use `shub image` commands to work with Docker registries "
"other than Scrapinghub default registry.")
def cli(project, target_dir):
target_dir = os.path.normpath(
os.path.join(os.getcwd(), target_dir or project))
if os.path.exists(target_dir):
raise BadParameterException(
"Target directory %s already exists, please delete it or supply a "
"non-existing target." % target_dir)
projects = get_available_projects()
if project not in projects:
raise NotFoundException(
"There is no example project named '%s'. Run 'shub bootstrap -l' "
"to get a list of all available projects." % project)
click.echo("Downloading custom image examples")
repo_zip = get_repo_zip(EXAMPLE_REPO)
click.echo("Cloning project '%s' into %s" % (project, target_dir))
unzip_project(repo_zip, project=projects[project], target_dir=target_dir)