def test_save_shortcut_updated(self): OLD_YML = """\ projects: default: 12345 prod: 33333 """ runner = CliRunner() with runner.isolated_filesystem(): with open('scrapinghub.yml', 'w') as f: f.write(textwrap.dedent(OLD_YML)) conf = ShubConfig() conf.load_file('scrapinghub.yml') del conf.projects['prod'] print(conf.projects) conf.save('scrapinghub.yml') with open('scrapinghub.yml', 'r') as f: new_yml = yaml.safe_load(f) # Should not contain 'projects' self.assertEqual(new_yml, {'project': 12345}) conf = ShubConfig() conf.load_file('scrapinghub.yml') # Should also work in reverse conf.projects['prod'] = 33333 conf.save('scrapinghub.yml') with open('scrapinghub.yml', 'r') as f: new_yml = yaml.safe_load(f) # Should not contain 'project' singleton self.assertEqual( new_yml, {'projects': {'default': 12345, 'prod': 33333}}, ) # Make sure it is readable again ShubConfig().load_file('scrapinghub.yml')
def _test_wizard(self, mock_project_access, conf=None, target='default', image=None, sh_yml=None, scrapy_cfg=True, dockerfile=False, **kwargs): if not conf: conf = ShubConfig() conf.apikeys = {'default': 'abcdef'} mock_project_access.return_value = self.has_project_access @click.command() def call_wizard(): utils.create_scrapinghub_yml_wizard( conf, target=target, image=image) with self.runner.isolated_filesystem(): if scrapy_cfg: open('scrapy.cfg', 'w').close() if dockerfile: open('Dockerfile', 'w').close() if sh_yml: with open('scrapinghub.yml', 'w') as f: f.write(sh_yml) conf.load_file('scrapinghub.yml') result = self.runner.invoke(call_wizard, **kwargs) if os.path.exists('scrapinghub.yml'): with open('scrapinghub.yml', 'r') as f: sh_yml = yaml.safe_load(f.read()) else: sh_yml = None return result, conf, sh_yml
def test_save(self): with CliRunner().isolated_filesystem(): self.conf.save('conf.yml') loaded_conf = ShubConfig() loaded_conf.load_file('conf.yml') for option in ['projects', 'endpoints', 'apikeys', 'version', 'stacks', 'requirements_file', 'eggs', 'images']: self.assertEqual( getattr(self.conf, option), getattr(loaded_conf, option))
def test_save(self): with CliRunner().isolated_filesystem(): self.conf.save('conf.yml') loaded_conf = ShubConfig() loaded_conf.load_file('conf.yml') for option in [ 'projects', 'endpoints', 'apikeys', 'version', 'stacks', 'requirements_file', 'eggs', 'images' ]: self.assertEqual(getattr(self.conf, option), getattr(loaded_conf, option))
class Migrator(object): def __init__(self, mfile): self.mfile = mfile self.sh_yml = './scrapinghub.yml' self.conf = ShubConfig() self.conf.load_file(self.sh_yml) self.req_content = to_unicode(self.mfile.read('requirements.txt')) self.eggs = [] for filename in self.mfile.namelist(): if filename.endswith('.egg'): self.eggs.append(filename) def start(self): if self.eggs: self.migrate_eggs() self.migrate_requirements_txt() self.conf.save(self.sh_yml) def migrate_eggs(self): eggsdir = './eggs' msg = "Eggs will be stored in {}, are you sure ? ".format(eggsdir) click.confirm(msg) try: os.mkdir(eggsdir) except OSError as e: if e.errno != errno.EEXIST: raise for filename in self.eggs: filepath = os.path.join(eggsdir, filename) if filepath in self.conf.eggs: continue self.conf.eggs.append(filepath) self.mfile.extract(filename, eggsdir) def migrate_requirements_txt(self): req_file = self.conf.requirements_file or './requirements.txt' if os.path.isfile(req_file): y = click.confirm('requirements.txt already exists, ' 'are you sure to override it ?') if not y: click.echo('Aborting') return self.conf.requirements_file = req_file with open(self.conf.requirements_file, 'w') as reqfile: reqfile.write(self.req_content)
def test_load_file(self): tmpdir = tempfile.mkdtemp() tmpfilepath = os.path.join(tmpdir, 'scrapinghub.yml') with open(tmpfilepath, 'w') as f: f.write( textwrap.dedent(""" apikeys: external: ext_endpoint """)) conf = ShubConfig() conf.load_file(tmpfilepath) shutil.rmtree(tmpdir) self.assertEqual({'external': 'ext_endpoint'}, conf.apikeys)
def test_load_file(self): tmpdir = tempfile.mkdtemp() tmpfilepath = os.path.join(tmpdir, 'scrapinghub.yml') with open(tmpfilepath, 'w') as f: f.write(textwrap.dedent( """ apikeys: external: ext_endpoint """ )) conf = ShubConfig() conf.load_file(tmpfilepath) shutil.rmtree(tmpdir) self.assertEqual({'external': 'ext_endpoint'}, conf.apikeys)
def _update_conf_file(filename, target, project, repository): """Load the given config file, update ``target`` with the given ``project`` and ``repository``, then save it. If the file does not exist, it will be created.""" try: # XXX: Runtime import to avoid circular dependency from shub.config import ShubConfig conf = ShubConfig() if os.path.exists(filename): conf.load_file(filename) _update_conf(conf, target, project, repository) conf.save(filename) except Exception as e: click.echo( "There was an error while trying to write to %s: %s" "" % (filename, e), ) else: click.echo("Saved to %s." % filename)
def test_save_partial_options(self): OLD_YML = """\ projects: default: 12345 prod: 33333 stack: custom-stack """ with CliRunner().isolated_filesystem(): with open('conf.yml', 'w') as f: f.write(textwrap.dedent(OLD_YML)) conf = ShubConfig() conf.load_file('conf.yml') del conf.projects['prod'] del conf.stacks['default'] conf.save('conf.yml', options=['projects']) with open('conf.yml', 'r') as f: self.assertEqual( yaml.load(f), {'project': 12345, 'stack': 'custom-stack'}) conf.save('conf.yml') with open('conf.yml', 'r') as f: self.assertEqual(yaml.load(f), {'project': 12345})
def test_save_shortcut_updated(self): OLD_YML = """\ projects: default: 12345 prod: 33333 """ runner = CliRunner() with runner.isolated_filesystem(): with open('scrapinghub.yml', 'w') as f: f.write(textwrap.dedent(OLD_YML)) conf = ShubConfig() conf.load_file('scrapinghub.yml') del conf.projects['prod'] print(conf.projects) conf.save('scrapinghub.yml') with open('scrapinghub.yml', 'r') as f: new_yml = yaml.safe_load(f) # Should not contain 'projects' self.assertEqual(new_yml, {'project': 12345}) conf = ShubConfig() conf.load_file('scrapinghub.yml') # Should also work in reverse conf.projects['prod'] = 33333 conf.save('scrapinghub.yml') with open('scrapinghub.yml', 'r') as f: new_yml = yaml.safe_load(f) # Should not contain 'project' singleton self.assertEqual( new_yml, {'projects': { 'default': 12345, 'prod': 33333 }}, ) # Make sure it is readable again ShubConfig().load_file('scrapinghub.yml')
def test_save_partial_options(self): OLD_YML = """\ projects: default: 12345 prod: 33333 stack: custom-stack """ with CliRunner().isolated_filesystem(): with open('conf.yml', 'w') as f: f.write(textwrap.dedent(OLD_YML)) conf = ShubConfig() conf.load_file('conf.yml') del conf.projects['prod'] del conf.stacks['default'] conf.save('conf.yml', options=['projects']) with open('conf.yml', 'r') as f: self.assertEqual(yaml.load(f), { 'project': 12345, 'stack': 'custom-stack' }) conf.save('conf.yml') with open('conf.yml', 'r') as f: self.assertEqual(yaml.load(f), {'project': 12345})