def _get_conf(scrapycfg_default_target):
with open(tmpfilepath, 'w') as f:
f.write(textwrap.dedent(scrapycfg_default_target))
f.write(textwrap.dedent(
"""
[deploy:prod]
project = 222
[deploy:otheruser]
project = 333
username = otherkey
[deploy:otherurl]
project = 444
url = http://dash.scrapinghub.com/api/scrapyd/
[deploy:external]
project = 555
url = external_endpoint
username = externalkey
"""
))
conf = ShubConfig()
conf.load_scrapycfg([tmpfilepath])
return conf
def _get_conf(scrapycfg_default_target):
with open(tmpfilepath, 'w') as f:
f.write(textwrap.dedent(scrapycfg_default_target))
f.write(
textwrap.dedent("""
[deploy:prod]
project = 222
[deploy:otheruser]
project = 333
username = otherkey
[deploy:otherurl]
project = 444
url = http://app.scrapinghub.com/api/scrapyd/
[deploy:external]
project = 555
url = external_endpoint
username = externalkey
[deploy:invalid_external]
project = non-numeric
url = external_endpoint
username = externalkey
"""))
conf = ShubConfig()
conf.load_scrapycfg([tmpfilepath])
return conf
def _test_wizard(self, mock_project_access, conf=None, target='default',
image=None, sh_yml=None, scrapy_cfg=True,
dockerfile=False, **kwargs):
if not conf:
conf = ShubConfig()
conf.apikeys = {'default': 'abcdef'}
mock_project_access.return_value = self.has_project_access
@click.command()
def call_wizard():
utils.create_scrapinghub_yml_wizard(
conf, target=target, image=image)
with self.runner.isolated_filesystem():
if scrapy_cfg:
open('scrapy.cfg', 'w').close()
if dockerfile:
open('Dockerfile', 'w').close()
if sh_yml:
with open('scrapinghub.yml', 'w') as f:
f.write(sh_yml)
conf.load_file('scrapinghub.yml')
result = self.runner.invoke(call_wizard, **kwargs)
if os.path.exists('scrapinghub.yml'):
with open('scrapinghub.yml', 'r') as f:
sh_yml = yaml.safe_load(f.read())
else:
sh_yml = None
return result, conf, sh_yml
def _test_wizard(self, mock_project_access, conf=None, target='default',
image=None, sh_yml=None, scrapy_cfg=True,
dockerfile=False, **kwargs):
if not conf:
conf = ShubConfig()
conf.apikeys = {'default': 'abcdef'}
mock_project_access.return_value = self.has_project_access
@click.command()
def call_wizard():
utils.create_scrapinghub_yml_wizard(
conf, target=target, image=image)
with self.runner.isolated_filesystem():
if scrapy_cfg:
open('scrapy.cfg', 'w').close()
if dockerfile:
open('Dockerfile', 'w').close()
if sh_yml:
with open('scrapinghub.yml', 'w') as f:
f.write(sh_yml)
conf.load_file('scrapinghub.yml')
result = self.runner.invoke(call_wizard, **kwargs)
if os.path.exists('scrapinghub.yml'):
with open('scrapinghub.yml', 'r') as f:
sh_yml = yaml.safe_load(f.read())
else:
sh_yml = None
return result, conf, sh_yml
def test_dont_leak_global_config(self):
conf = ShubConfig()
conf.projects = {'prod': 33333}
conf.apikeys = {'default': 'abc'}
result, conf, sh_yml = self._test_wizard(conf=conf, input='12345\n')
assert result.exit_code == 0
assert conf.projects == {'default': 12345, 'prod': 33333}
assert conf.apikeys == {'default': 'abc'}
assert sh_yml == {'project': 12345}
def test_save(self):
with CliRunner().isolated_filesystem():
self.conf.save('conf.yml')
loaded_conf = ShubConfig()
loaded_conf.load_file('conf.yml')
for option in ['projects', 'endpoints', 'apikeys', 'version',
'stacks', 'requirements_file', 'eggs', 'images']:
self.assertEqual(
getattr(self.conf, option), getattr(loaded_conf, option))
def test_dont_leak_global_config(self):
conf = ShubConfig()
conf.projects = {'prod': 33333}
conf.apikeys = {'default': 'abc'}
result, conf, sh_yml = self._test_wizard(conf=conf, input='12345\n')
assert result.exit_code == 0
assert conf.projects == {'default': 12345, 'prod': 33333}
assert conf.apikeys == {'default': 'abc'}
assert sh_yml == {'project': 12345}
class Migrator(object):
def __init__(self, mfile):
self.mfile = mfile
self.sh_yml = './scrapinghub.yml'
self.conf = ShubConfig()
self.conf.load_file(self.sh_yml)
self.req_content = to_unicode(self.mfile.read('requirements.txt'))
self.eggs = []
for filename in self.mfile.namelist():
if filename.endswith('.egg'):
self.eggs.append(filename)
def start(self):
if self.eggs:
self.migrate_eggs()
self.migrate_requirements_txt()
self.conf.save(self.sh_yml)
def migrate_eggs(self):
eggsdir = './eggs'
msg = "Eggs will be stored in {}, are you sure ? ".format(eggsdir)
click.confirm(msg)
try:
os.mkdir(eggsdir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
for filename in self.eggs:
filepath = os.path.join(eggsdir, filename)
if filepath in self.conf.eggs:
continue
self.conf.eggs.append(filepath)
self.mfile.extract(filename, eggsdir)
def migrate_requirements_txt(self):
req_file = self.conf.requirements_file or './requirements.txt'
if os.path.isfile(req_file):
y = click.confirm('requirements.txt already exists, '
'are you sure to override it ?')
if not y:
click.echo('Aborting')
return
self.conf.requirements_file = req_file
with open(self.conf.requirements_file, 'w') as reqfile:
reqfile.write(self.req_content)
def test_save(self):
with CliRunner().isolated_filesystem():
self.conf.save('conf.yml')
loaded_conf = ShubConfig()
loaded_conf.load_file('conf.yml')
for option in [
'projects', 'endpoints', 'apikeys', 'version', 'stacks',
'requirements_file', 'eggs', 'images'
]:
self.assertEqual(getattr(self.conf, option),
getattr(loaded_conf, option))
def __init__(self, mfile):
self.mfile = mfile
self.sh_yml = './scrapinghub.yml'
self.conf = ShubConfig()
self.conf.load_file(self.sh_yml)
self.req_content = to_unicode(self.mfile.read('requirements.txt'))
self.eggs = []
for filename in self.mfile.namelist():
if filename.endswith('.egg'):
self.eggs.append(filename)
def test_dont_leak_global_config_on_image(self):
conf = ShubConfig()
conf.projects = {'prod': 33333}
conf.apikeys = {'default': 'abc'}
result, conf, sh_yml = self._test_wizard(
conf=conf, scrapy_cfg=False, dockerfile=True,
input='12345\nrepo\n')
assert result.exit_code == 0
assert conf.projects == {'default': 12345, 'prod': 33333}
assert conf.apikeys == {'default': 'abc'}
assert conf.images == {'default': 'repo'}
assert sh_yml == {'project': 12345, 'image': 'repo'}
def test_dont_leak_global_config_on_image(self):
conf = ShubConfig()
conf.projects = {'prod': 33333}
conf.apikeys = {'default': 'abc'}
result, conf, sh_yml = self._test_wizard(
conf=conf, scrapy_cfg=False, dockerfile=True,
input='12345\nrepo\n')
assert result.exit_code == 0
assert conf.projects == {'default': 12345, 'prod': 33333}
assert conf.apikeys == {'default': 'abc'}
assert conf.images == {'default': 'repo'}
assert sh_yml == {'project': 12345, 'image': 'repo'}
def test_load_file(self):
tmpdir = tempfile.mkdtemp()
tmpfilepath = os.path.join(tmpdir, 'scrapinghub.yml')
with open(tmpfilepath, 'w') as f:
f.write(
textwrap.dedent("""
apikeys:
external: ext_endpoint
"""))
conf = ShubConfig()
conf.load_file(tmpfilepath)
shutil.rmtree(tmpdir)
self.assertEqual({'external': 'ext_endpoint'}, conf.apikeys)
def test_load_file(self):
tmpdir = tempfile.mkdtemp()
tmpfilepath = os.path.join(tmpdir, 'scrapinghub.yml')
with open(tmpfilepath, 'w') as f:
f.write(textwrap.dedent(
"""
apikeys:
external: ext_endpoint
"""
))
conf = ShubConfig()
conf.load_file(tmpfilepath)
shutil.rmtree(tmpdir)
self.assertEqual({'external': 'ext_endpoint'}, conf.apikeys)
def test_save_shortcut(self):
conf = ShubConfig()
conf.endpoints['ext'] = 'external'
conf.stacks['default'] = 'my_stack'
expected_yml_dict = {
# No shortcut
'endpoints': {
'default': ShubConfig.DEFAULT_ENDPOINT,
'ext': 'external',
},
# Shortcut
'stack': 'my_stack',
}
with CliRunner().isolated_filesystem():
conf.save('conf.yml')
with open('conf.yml', 'r') as f:
self.assertEqual(yaml.load(f), expected_yml_dict)
def _update_conf_file(filename, target, project, repository):
"""Load the given config file, update ``target`` with the given ``project``
and ``repository``, then save it. If the file does not exist, it will be
created."""
try:
# XXX: Runtime import to avoid circular dependency
from shub.config import ShubConfig
conf = ShubConfig()
if os.path.exists(filename):
conf.load_file(filename)
_update_conf(conf, target, project, repository)
conf.save(filename)
except Exception as e:
click.echo(
"There was an error while trying to write to %s: %s"
"" % (filename, e),
)
else:
click.echo("Saved to %s." % filename)
def _get_config(self):
conf = ShubConfig()
conf.load(StringIO(json.dumps(self._default_config())))
if 'SHUB_APIKEY' in os.environ:
conf.apikeys['default'] = os.environ['SHUB_APIKEY']
try:
conf.load(self.storage.open('scrapinghub.yml'))
except OSError:
raise ('Need a `scrapinghub.yml` file to identify which project '
'to deploy to. Find more information at: {}'.format(
self.SHUB_DOCS_URL
))
return conf
def test_save_partial_options(self):
OLD_YML = """\
projects:
default: 12345
prod: 33333
stack: custom-stack
"""
with CliRunner().isolated_filesystem():
with open('conf.yml', 'w') as f:
f.write(textwrap.dedent(OLD_YML))
conf = ShubConfig()
conf.load_file('conf.yml')
del conf.projects['prod']
del conf.stacks['default']
conf.save('conf.yml', options=['projects'])
with open('conf.yml', 'r') as f:
self.assertEqual(yaml.load(f), {
'project': 12345,
'stack': 'custom-stack'
})
conf.save('conf.yml')
with open('conf.yml', 'r') as f:
self.assertEqual(yaml.load(f), {'project': 12345})
def test_save_shortcut(self):
conf = ShubConfig()
conf.endpoints['ext'] = 'external'
conf.stacks['default'] = 'my_stack'
expected_yml_dict = {
# No shortcut
'endpoints': {
'default': ShubConfig.DEFAULT_ENDPOINT,
'ext': 'external',
},
# Shortcut
'stack': 'my_stack',
}
with CliRunner().isolated_filesystem():
conf.save('conf.yml')
with open('conf.yml', 'r') as f:
self.assertEqual(yaml.load(f), expected_yml_dict)
def _update_conf_file(filename, target, project, repository):
"""Load the given config file, update ``target`` with the given ``project``
and ``repository``, then save it. If the file does not exist, it will be
created."""
try:
# XXX: Runtime import to avoid circular dependency
from shub.config import ShubConfig
conf = ShubConfig()
if os.path.exists(filename):
conf.load_file(filename)
_update_conf(conf, target, project, repository)
conf.save(filename)
except Exception as e:
click.echo(
"There was an error while trying to write to %s: %s"
"" % (filename, e), )
else:
click.echo("Saved to %s." % filename)
def test_save_partial_options(self):
OLD_YML = """\
projects:
default: 12345
prod: 33333
stack: custom-stack
"""
with CliRunner().isolated_filesystem():
with open('conf.yml', 'w') as f:
f.write(textwrap.dedent(OLD_YML))
conf = ShubConfig()
conf.load_file('conf.yml')
del conf.projects['prod']
del conf.stacks['default']
conf.save('conf.yml', options=['projects'])
with open('conf.yml', 'r') as f:
self.assertEqual(
yaml.load(f),
{'project': 12345, 'stack': 'custom-stack'})
conf.save('conf.yml')
with open('conf.yml', 'r') as f:
self.assertEqual(yaml.load(f), {'project': 12345})
def test_save_skip_defaults(self):
conf = ShubConfig()
with CliRunner().isolated_filesystem():
conf.save('conf.yml')
with open('conf.yml', 'r') as f:
self.assertEqual(yaml.load(f), None)
def _get_conf_with_yml(self, yml):
conf = ShubConfig()
conf.load(StringIO.StringIO(textwrap.dedent(yml)))
return conf
def test_not_logged_in(self):
conf = ShubConfig()
result, _, sh_yml = self._test_wizard(conf=conf)
assert result.exit_code == MissingAuthException.exit_code
assert sh_yml is None
def test_save_shortcut_updated(self):
OLD_YML = """\
projects:
default: 12345
prod: 33333
"""
runner = CliRunner()
with runner.isolated_filesystem():
with open('scrapinghub.yml', 'w') as f:
f.write(textwrap.dedent(OLD_YML))
conf = ShubConfig()
conf.load_file('scrapinghub.yml')
del conf.projects['prod']
print(conf.projects)
conf.save('scrapinghub.yml')
with open('scrapinghub.yml', 'r') as f:
new_yml = yaml.safe_load(f)
# Should not contain 'projects'
self.assertEqual(new_yml, {'project': 12345})
conf = ShubConfig()
conf.load_file('scrapinghub.yml')
# Should also work in reverse
conf.projects['prod'] = 33333
conf.save('scrapinghub.yml')
with open('scrapinghub.yml', 'r') as f:
new_yml = yaml.safe_load(f)
# Should not contain 'project' singleton
self.assertEqual(
new_yml,
{'projects': {'default': 12345, 'prod': 33333}},
)
# Make sure it is readable again
ShubConfig().load_file('scrapinghub.yml')
def test_save_skip_defaults(self):
conf = ShubConfig()
with CliRunner().isolated_filesystem():
conf.save('conf.yml')
with open('conf.yml', 'r') as f:
self.assertEqual(yaml.load(f), None)
def test_init_sets_default(self):
conf = ShubConfig()
self.assertIn('default', conf.endpoints)
self.assertEqual(conf.version, 'AUTO')
def _get_conf_with_yml(self, yml):
conf = ShubConfig()
conf.load(six.StringIO(textwrap.dedent(yml)))
return conf
def test_save_shortcut_updated(self):
OLD_YML = """\
projects:
default: 12345
prod: 33333
"""
runner = CliRunner()
with runner.isolated_filesystem():
with open('scrapinghub.yml', 'w') as f:
f.write(textwrap.dedent(OLD_YML))
conf = ShubConfig()
conf.load_file('scrapinghub.yml')
del conf.projects['prod']
print(conf.projects)
conf.save('scrapinghub.yml')
with open('scrapinghub.yml', 'r') as f:
new_yml = yaml.safe_load(f)
# Should not contain 'projects'
self.assertEqual(new_yml, {'project': 12345})
conf = ShubConfig()
conf.load_file('scrapinghub.yml')
# Should also work in reverse
conf.projects['prod'] = 33333
conf.save('scrapinghub.yml')
with open('scrapinghub.yml', 'r') as f:
new_yml = yaml.safe_load(f)
# Should not contain 'project' singleton
self.assertEqual(
new_yml,
{'projects': {
'default': 12345,
'prod': 33333
}},
)
# Make sure it is readable again
ShubConfig().load_file('scrapinghub.yml')
