def _upload_egg(target, eggpath, project, version):
data = {'project': project, 'version': version}
files = {'egg': ('project.egg', open(eggpath, 'rb'))}
url = _url(target, 'addversion.json')
auth = _get_auth(target)
log('Deploying to Scrapy Cloud project "%s"' % project)
make_deploy_request(url, data, files, auth)
def _upload_egg(target, eggpath, project, version):
data = {'project': project, 'version': version}
files = {'egg': ('project.egg', open(eggpath, 'rb'))}
url = _url(target, 'addversion.json')
auth = _get_auth(target)
log('Deploying to Scrapy Cloud project "%s"' % project)
make_deploy_request(url, data, files, auth)
def main(project_id, requirements_file):
target = scrapycfg.get_target('default')
project_id = scrapycfg.get_project(target, project_id)
apikey = find_api_key()
log('Deploying requirements to project "%s"' % project_id)
requirements_full_path = os.path.abspath(requirements_file)
eggs_tmp_dir = _mk_and_cd_eggs_tmpdir()
_download_egg_files(eggs_tmp_dir, requirements_full_path)
decompress_egg_files()
utils.build_and_deploy_eggs(project_id, apikey)
def build_and_deploy_egg(project_id, apikey):
"""Builds and deploys the current dir's egg"""
log("Building egg in: %s" % os.getcwd())
try:
run("python setup.py bdist_egg")
except CalledProcessError:
# maybe a C extension or distutils package, forcing bdist_egg
log("Couldn't build an egg with vanilla setup.py, trying with setuptools...")
run("python -c \"import setuptools; __file__='setup.py'; execfile('setup.py')\" bdist_egg")
_deploy_dependency_egg(apikey, project_id)
def _download_egg_files(eggs_dir, requirements_file):
editable_src_dir = tempfile.mkdtemp(prefix='pipsrc')
log('Downloading eggs...')
try:
pip_cmd = ("pip install -d {eggs_dir} -r {requirements_file}"
" --src {editable_src_dir} --no-deps --no-use-wheel")
log(run(pip_cmd.format(eggs_dir=eggs_dir,
editable_src_dir=editable_src_dir,
requirements_file=requirements_file)))
finally:
shutil.rmtree(editable_src_dir, ignore_errors=True)
def decompress_egg_files():
decompressor_by_ext = _build_decompressor_by_ext_map()
eggs = [f for ext in decompressor_by_ext for f in glob("*.%s" % ext)]
if not eggs:
files = glob("*")
err = "No egg files with a supported file extension were found. " "Files: %s" % ", ".join(files)
raise ClickException(err)
for egg in eggs:
log("Uncompressing: %s" % egg)
run("%s %s" % (decompressor_by_ext[_ext(egg)], egg))
def _deploy_dependency_egg(apikey, project_id):
name = _get_dependency_name()
version = _get_dependency_version(name)
egg_name, egg_path = _get_egg_info(name)
url = 'https://dash.scrapinghub.com/api/eggs/add.json'
data = {'project': project_id, 'name': name, 'version': version}
files = {'egg': (egg_name, open(egg_path, 'rb'))}
auth = (apikey, '')
log('Deploying dependency to Scrapy Cloud project "%s"' % project_id)
make_deploy_request(url, data, files, auth)
success = "Deployed eggs list at: https://dash.scrapinghub.com/p/%s/eggs"
log(success % project_id)
def _deploy_dependency_egg(shub_apikey, project_id):
name = _get_dependency_name()
version = _get_dependency_version(name)
egg_name, egg_path = _get_egg_info(name)
url = 'https://dash.scrapinghub.com/api/eggs/add.json'
data = {'project': project_id, 'name': name, 'version': version}
files = {'egg': (egg_name, open(egg_path, 'rb'))}
auth = (shub_apikey, '')
log('Deploying dependency to Scrapy Cloud project "%s"' % project_id)
make_deploy_request(url, data, files, auth)
success = "Deployed eggs list at: https://dash.scrapinghub.com/p/%s/eggs"
log(success % project_id)
def _deploy_dependency_egg(apikey, project_id):
name = _get_dependency_name()
version = _get_dependency_version(name)
egg_name, egg_path = _get_egg_info(name)
url = "https://dash.scrapinghub.com/api/eggs/add.json"
data = {"project": project_id, "name": name, "version": version}
files = {"egg": (egg_name, open(egg_path, "rb"))}
auth = (apikey, "")
log('Deploying dependency to Scrapy Cloud project "%s"' % project_id)
make_deploy_request(url, data, files, auth)
success = "Deployed eggs list at: https://dash.scrapinghub.com/p/%s/eggs"
log(success % project_id)
def fetch_eggs(project_id, api_key, destfile):
auth = (api_key, '')
url = "https://dash.scrapinghub.com/api/eggs/bundle.zip?project=%s" % project_id
rsp = requests.get(url=url, auth=auth, stream=True, timeout=300)
_assert_response_is_valid(rsp)
log("Downloading eggs to %s" % destfile)
with open(destfile, 'wb') as f:
for chunk in rsp.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
f.flush()
def cli(context):
if auth.get_key_netrc():
log("You're already logged in. To change credentials, use 'shub logout' first.")
return 0
cfg_key = _find_cfg_key()
key = _prompt_for_key(suggestion=cfg_key)
if not key and is_valid_key(cfg_key):
auth.write_key_netrc(cfg_key)
elif key and is_valid_key(key):
auth.write_key_netrc(key)
else:
context.fail('Invalid key. Tip: your key must have 32 characters.')
log('Success.')
def make_deploy_request(url, data, files, auth):
try:
rsp = requests.post(url=url, auth=auth, data=data, files=files, stream=True, timeout=300)
rsp.raise_for_status()
for line in rsp.iter_lines():
log(line)
return True
except requests.HTTPError as exc:
rsp = exc.response
if rsp.status_code == 403:
raise AuthException()
msg = "Deploy failed ({}):\n{}".format(rsp.status_code, rsp.text)
raise ClickException(msg)
except requests.RequestException as exc:
raise ClickException("Deploy failed: {}".format(exc))
def _checkout(repo, git_branch=None):
tmpdir = tempfile.mkdtemp(prefix='shub-deploy-egg-from-url')
log("Cloning the repository to a tmp folder...")
os.chdir(tmpdir)
if (_run('git clone %s egg-tmp-clone' % repo) != 0 and
_run('hg clone %s egg-tmp-clone' % repo) != 0 and
_run('bzr branch %s egg-tmp-clone' % repo) != 0):
error = "\nERROR: The provided repository URL is not valid: %s\n"
fail(error % repo)
os.chdir('egg-tmp-clone')
if git_branch:
if _run('git checkout %s' % git_branch) != 0:
fail("Branch %s is not valid" % git_branch)
log("%s branch was checked out" % git_branch)
def cli(target, project, version, list_targets, debug, egg, build_egg):
exitcode = 0
if not inside_project():
log("Error: no Scrapy project found in this location")
sys.exit(1)
if list_targets:
for name, target in _get_targets().items():
click.echo(name)
return
tmpdir = None
if build_egg:
egg, tmpdir = _build_egg()
log("Writing egg to %s" % build_egg)
shutil.copyfile(egg, build_egg)
else:
target = _get_target(target)
project = _get_project(target, project)
version = _get_version(target, version)
if egg:
log("Using egg: %s" % egg)
egg = egg
else:
log("Packing version %s" % version)
egg, tmpdir = _build_egg()
if _upload_egg(target, egg, project, version):
click.echo(
"Run your spiders at: https://dash.scrapinghub.com/p/%s/" %
project)
else:
exitcode = 1
if tmpdir:
if debug:
log("Output dir not removed: %s" % tmpdir)
else:
shutil.rmtree(tmpdir)
sys.exit(exitcode)
def cli(target, project, version, list_targets, debug, egg, build_egg):
if not inside_project():
log("Error: no Scrapy project found in this location")
sys.exit(1)
if list_targets:
for name, target in scrapycfg.get_targets().items():
click.echo(name)
return
tmpdir = None
try:
if build_egg:
egg, tmpdir = _build_egg()
log("Writing egg to %s" % build_egg)
shutil.copyfile(egg, build_egg)
else:
target = scrapycfg.get_target(target)
project = scrapycfg.get_project(target, project)
version = scrapycfg.get_version(target, version)
apikey = target.get('username') or find_api_key()
auth = (apikey, '')
if egg:
log("Using egg: %s" % egg)
egg = egg
else:
log("Packing version %s" % version)
egg, tmpdir = _build_egg()
_upload_egg(target, egg, project, version, auth)
click.echo("Run your spiders at: https://dash.scrapinghub.com/p/%s/" % project)
finally:
if tmpdir:
if debug:
log("Output dir not removed: %s" % tmpdir)
else:
shutil.rmtree(tmpdir, ignore_errors=True)
def cli(target, project, version, list_targets, debug, egg, build_egg):
exitcode = 0
if not inside_project():
log("Error: no Scrapy project found in this location")
sys.exit(1)
if list_targets:
for name, target in _get_targets().items():
click.echo(name)
return
tmpdir = None
if build_egg:
egg, tmpdir = _build_egg()
log("Writing egg to %s" % build_egg)
shutil.copyfile(egg, build_egg)
else:
target = _get_target(target)
project = _get_project(target, project)
version = _get_version(target, version)
if egg:
log("Using egg: %s" % egg)
egg = egg
else:
log("Packing version %s" % version)
egg, tmpdir = _build_egg()
if _upload_egg(target, egg, project, version):
click.echo("Run your spiders at: https://dash.scrapinghub.com/p/%s/" % project)
else:
exitcode = 1
if tmpdir:
if debug:
log("Output dir not removed: %s" % tmpdir)
else:
shutil.rmtree(tmpdir)
sys.exit(exitcode)
def _fetch_from_pypi(pkg):
tmpdir = tempfile.mkdtemp(prefix='shub-deploy-egg-from-pypi')
log('Fetching %s from pypi' % pkg)
pip_cmd = "pip install -d %s %s --no-deps --no-use-wheel" % (tmpdir, pkg)
log(run(pip_cmd))
log('Package fetched successfully')
os.chdir(tmpdir)
def make_deploy_request(url, data, files, auth):
try:
rsp = requests.post(url=url, auth=auth, data=data, files=files,
stream=True, timeout=300)
rsp.raise_for_status()
for line in rsp.iter_lines():
log(line)
return True
except requests.HTTPError as exc:
rsp = exc.response
log("Deploy failed ({}):".format(rsp.status_code))
log(rsp.text)
return False
except requests.RequestException as exc:
log("Deploy failed: {}".format(exc))
return False
