def validate_expect(self, resp, expect): # Print as a tuple, so literal "\n"s don't produde line-breaks in the log output self.log.info("Response: %s", (resp, )) assert isinstance(expect, list), "Expect must be a list!" for expect_val in expect: if isinstance(expect_val, str): if isinstance(resp, bool): if self.debug: import pdb pdb.set_trace() raise marshaller_exceptions.InvalidDeployResponse( "Expected '%s' response: '%s'" % (expect_val, resp)) elif isinstance(resp, str): if not expect_val in resp: if self.debug: import pdb pdb.set_trace() raise marshaller_exceptions.InvalidDeployResponse( "Expected '%s' in response '%s'" % (expect_val, resp)) else: self.log.info("Found %s in response!", expect) elif isinstance(resp, dict): if not expect_val in str(resp): if self.debug: import pdb pdb.set_trace() raise marshaller_exceptions.InvalidDeployResponse( "Expected '%s' in response '%s'" % (expect_val, resp)) else: self.log.info("Found %s in response!", expect) else: self.log.error("Unknown response type: %s!", type(resp)) else: self.log.error("Unknown expect type: %s!", type(resp)) # if self.debug: # import pdb # pdb.set_trace() raise marshaller_exceptions.InvalidExpectParameter( "Invalid expect parameter: '%s' (type: %s)." % (expect_val, type(expect_val))) self.log.info("Command response passed validation.")
def configure_client(self, clientname, client_idx, provider=None, provider_kwargs=None): assert "_" not in clientname, "VM names cannot contain _ on digital ocean, I think?" self.log.info("Configuring client") # [Command, [shell command], {execution context}, ['strings', 'expected', 'in', 'response']], commands = [ ['cmd.run', [ "bash -c \'whoami\'", ], {}, ['root']], ['cmd.run', [ 'mkdir -p .ssh/', ], {}, None], ['cmd.run', [ dirmake_oneliner, ], {}, None], ['cmd.run', [ dirmake_ssh_oneliner, ], {}, None], ['cmd.run', [ "bash -c \"ls /\"", ], {}, [ 'scraper', ]], ['cmd.run', [ 'bash -c \"pwd\"', ], {}, ['/root']], ['cmd.run', [ 'bash -c \"ls -la\"', ], {}, None], ['pkg.refresh_db', [], {}, None], ['pkg.install', [ 'software-properties-common', ], {}, None], # splat in public keys. [ 'cmd.run', [ 'echo ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCoNUeZ/L6QYntVXtBCdFLk3L7X1Smio+pKi/63W4i9VQdocxY7zl3fCyu5LsPzVQUBU5n' + 'LKb/iJkABH+hxq8ZL7kXiKuGgeHsI60I2wECMxg17Qs918ND626AkXqlMIUW1SchcAi3rYRMVY0OaGSOutIcjR+mJ6liogTv1DLRD0eRbuollz7XsYz4ILb' + 'i9kEsqwaly92vK6vlIVlAWtDoNf95c6jk/lh0M5p1LV0lwrEtfCreuv1rrOldUdwgU4wCFgRI+p6FXs69+OsNWxZSOSr28eE9sbsHxIxthcRHMtsnDxzeJ1' + 'PVhvC4JclFEHEZSlYaemI6zOezVuBuipwSv Neko@ODO | tee -a .ssh/authorized_keys', ], {}, None ], [ 'cmd.run', [ 'echo ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCi7/9jOHVJj0ODnPqinqFbOaErT2pNaeq0pYKapcG2DHGrvVlX3ZUO8z7uY1QZX0OiC3y' + '7rv4c7NEl7/OtmRDfNPd5YgpAuXelbwu5Pj1BjQq1pn3CeP4zhw4gcEPx2UAc5Rw1jzH8vE7NMf2iReBiHr2SfSLh8T/jt5bEAVDCnhMS/8YvoPLLftESiL' + 'oi+TU6Y9/zw4zac3AyJJ02tHpHLSpWWPPLi31ASEu/p+lWynUd+dSTMbwmc3hwBQkZTrK6P1I3431eQqYVNOyWJe+GeCXLaw5CvO8qlE7Nj3Z+dics3Bq0F' + '7ugDC+27qWk7m5soPfbZ8qlQz4CWFv01GHWdWwdHh9SR2bplNZ6MDuED91mu7gxyx2Wyo2AIiKsLcpGOIdLnIvrSA9VGpdgKbflbnqtfyIm6gloPpITnAJX' + 'imWSvIxF76PVFjdZa86jAx7JZfBfirvtRg6/qXbDUDAErF3OllqxBvuGOzHptDDgha/29tabzxUIxhpBrG0TiRTMDmmqgM+b9kANgzEe4Yef2w/IaTC96D/' + 'oLxRHmRBbof8GIMlNZjFlVw8XIyzYxnvALwCE7gRubba13f6qU0lT56be9HKYrSvHVy9/855lKlLwTCePaHK0EPBGuMWZOBexGKyxTFXmA+oqkBg5zFnZLy' + 'xcsaVZQtZRnDU4Cu4jyQ== [email protected] | tee -a .ssh/authorized_keys', ], {}, None ], [ 'cmd.run', [ 'echo ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDMv9sMkEOjqsFu8wPiG2jUP62qKEpxmvQ7IiYaLKogW/LQlLhKP7KCIE2MVUmctwdvyEF' + 'rXGOXDCVgFMFLEZiCi2+B7itMcFBlxJsZYQ9Y5FzXg/xK/Xld9rZu2ST+4z9xrX03n0rrsvO3HgpbNoIWF1LbXrc8L80CUCf13GWkZErhzc4mcd44McLVxX' + 'q+hcikMguVdOcejpLJTQkq2LRLEx2zhrz+CfNe9AQ0I5AOsh8Os3rrILFs0t290hejMX82nwJUCIcODTBJqR0o7qs/Tt8zLy3YKnAN3eGqfO7tw/d75AD/n' + 'ENup5kJscpVb/6v3xfWnjgAjalj/hw2bwoc3SE+Y3u2wmyuhrJcSy6rw/IltFc+BaZamZMBW/si8tW+xo9rb903GXANJbjVOABECJSp2i03xtPfYfk9KqZb' + '/vUkpYTmwRQGvDK9u1viIF8nIomE4omN6buFktvVjH1IG6bOPeMi4Y0zBNds7Q1W28Um1ygaBU+NCalep8UDEWInNkfYe1E/hj5A5EaMPaRjnPhXJqUzglO' + 'l1O2Tco2FYhfvCiyZvAHv25LLrGzePidR59SzTP7/fLxK7FgmH0m79AOKvjuZaNjb7njmgDhyQggOLU6bJwiiJ7MqldPlic2qCKyQVavLv2nXGIGVXEovtM' + '9YfgSYuglkiYmbs6LU0w== durr@mainnas | tee -a .ssh/authorized_keys', ], {}, None ], ['cmd.run', [ "chmod 0600 .ssh/authorized_keys", ], {}, None], [ 'cmd.run', [ "cat .ssh/authorized_keys", ], {}, [ ' Neko@ODO', ' [email protected]', ' durr@mainnas', ] ], # So something is missing some of the keys, somehow ['cmd.run', [ "eval ssh-agent $SHELL", ], {}, None], ['cmd.run', [ "ssh-add .ssh/authorized_keys", ], {}, None], ['cmd.run', [ "ssh-add -l", ], {}, None], [ 'cmd.run', [ "eval ssh-agent $SHELL; ssh-add .ssh/authorized_keys; ssh-add -l", ], {}, None ], ['cmd.run', [ "apt-get update", ], {}, None], # So trying to have salt update itself makes it poop itself, # and never come back. # Siiiiiigh. # Anyways, I moved this command to my custom bootstrap script. # ['cmd.run', ["apt-get dist-upgrade -y", ], {'env' : {'DEBIAN_FRONTEND' : 'noninteractive'}}, None], # Apparently at least one VPS host has separated git from build-essential? # ['pkg.install', ['build-essential', 'locales', 'git', 'libfontconfig', 'wget', 'htop', 'libxml2', 'libxslt1-dev', # 'python3-dev', 'python3-dbg', 'python3-distutils', 'libz-dev', 'curl', 'screen'], {}, None], # ['pkg.install', ['libasound2', 'libatk1.0-0', 'libc6', 'libcairo2', 'libcups2', 'libdbus-1-3', 'libexpat1', 'libfontconfig1', 'libgcc1', # 'libgconf-2-4', 'libgdk-pixbuf2.0-0', 'libglib2.0-0', 'libgtk-3-0', 'libnspr4', 'libpango-1.0-0', 'libpangocairo-1.0-0', 'libstdc++6', # 'libx11-6', 'libx11-xcb1', 'libxcb1', 'libxcursor1', 'libxdamage1', 'libxext6', 'libxfixes3', 'libxi6', 'libxrandr2', 'libxrender1', # 'libxss1', 'libxtst6', 'libnss3'], {}, None], ['pkg.install', [ 'build-essential', ], {}, None], ['pkg.install', [ 'locales', ], {}, None], ['pkg.install', [ 'git', ], {}, None], ['pkg.install', [ 'libfontconfig', ], {}, None], ['pkg.install', [ 'wget', ], {}, None], ['pkg.install', [ 'htop', ], {}, None], ['pkg.install', [ 'libxml2', ], {}, None], ['pkg.install', [ 'libxslt1-dev', ], {}, None], ['pkg.install', [ 'python3-dev', ], {}, None], ['pkg.install', [ 'python3-dbg', ], {}, None], ['pkg.install', [ 'python3-distutils', ], {}, None], ['pkg.install', [ 'libz-dev', ], {}, None], ['pkg.install', [ 'curl', ], {}, None], ['pkg.install', ['screen'], {}, None], ['pkg.install', [ 'libasound2', ], {}, None], ['pkg.install', [ 'libatk1.0-0', ], {}, None], ['pkg.install', [ 'libc6', ], {}, None], ['pkg.install', [ 'libcairo2', ], {}, None], ['pkg.install', [ 'libcups2', ], {}, None], ['pkg.install', [ 'libdbus-1-3', ], {}, None], ['pkg.install', [ 'libexpat1', ], {}, None], ['pkg.install', [ 'libfontconfig1', ], {}, None], ['pkg.install', [ 'libgcc1', ], {}, None], ['pkg.install', [ 'libgconf-2-4', ], {}, None], ['pkg.install', [ 'libgdk-pixbuf2.0-0', ], {}, None], ['pkg.install', [ 'libglib2.0-0', ], {}, None], ['pkg.install', [ 'libgtk-3-0', ], {}, None], ['pkg.install', [ 'libnspr4', ], {}, None], ['pkg.install', [ 'libpango-1.0-0', ], {}, None], ['pkg.install', [ 'libpangocairo-1.0-0', ], {}, None], ['pkg.install', [ 'libstdc++6', ], {}, None], ['pkg.install', [ 'libx11-6', ], {}, None], ['pkg.install', [ 'libx11-xcb1', ], {}, None], ['pkg.install', [ 'libxcb1', ], {}, None], ['pkg.install', [ 'libxcursor1', ], {}, None], ['pkg.install', [ 'libxdamage1', ], {}, None], ['pkg.install', [ 'libxext6', ], {}, None], ['pkg.install', [ 'libxfixes3', ], {}, None], ['pkg.install', [ 'libxi6', ], {}, None], ['pkg.install', [ 'libxrandr2', ], {}, None], ['pkg.install', [ 'libxrender1', ], {}, None], ['pkg.install', [ 'libxss1', ], {}, None], ['pkg.install', [ 'libxtst6', ], {}, None], ['pkg.install', ['libnss3'], {}, None], ['pkg.install', [ "xvfb", ], {}, None], # Adblocking. Lower the chrome cpu costs decently # So long hosts files cause things to explode, so we turn it off. # ['cmd.run', ["curl https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/social/hosts | tee -a /etc/hosts", ], {}, None], # Make swap so [ 'cmd.run', [ "dd if=/dev/zero of=/swapfile bs=1M count=4096", ], {}, None ], ['cmd.run', [ "mkswap /swapfile", ], {}, None], ['cmd.run', [ "chmod 0600 /swapfile", ], {}, None], ['cmd.run', [ "swapon /swapfile", ], {}, None], # Needed to make GCE play nice. I think they just flat-out don't preinstall a locale # ['cmd.run', ["sudo apt-get install language-pack-en -y", ], {}, ['The following NEW packages will be installed:', 'language-pack-en-base']], # Shit to make the tty work in UTF-8. Otherwise, the logging can asplode # and break all the things. [ 'cmd.run', [ 'echo LANG=\"en_US.UTF-8\" >> /etc/default/locale', ], {}, None ], [ 'cmd.run', [ 'echo LC_ALL=\"en_US.UTF-8\" >> /etc/default/locale', ], {}, None ], [ 'cmd.run', [ 'echo "LC_ALL=en_US.UTF-8" >> /etc/environment', ], {}, None ], [ 'cmd.run', [ 'echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen', ], {}, None ], [ 'cmd.run', [ 'echo "LANG=en_US.UTF-8" > /etc/locale.conf', ], {}, None ], [ 'cmd.run', [ "dpkg-reconfigure -f noninteractive locales", ], {}, ['en_US.UTF-8'] ], ['cmd.run', [ "locale", ], {}, None], ['cmd.run', [ "bash -c \"locale\"", ], {}, None], # Clone and Install settings ['cmd.run', [ "bash -c \"ls /\"", ], {}, [ 'scraper', ]], [ 'cmd.run', [ "git clone https://github.com/fake-name/AutoTriever.git /scraper" ], {}, [] ], [ 'cmd.run', [ "cat << EOF > /scraper/settings.json \n{content}\nEOF". format( content=self.__make_conf_file(clientname, client_idx)), ], {}, None ], # Make sure it all checked out at least somewhat [ 'cmd.run', [ "bash -c \"ls /scraper\"", ], {}, ['configure.sh', 'run.sh', 'settings.json'] ], # Finally, run the thing [ 'cmd.run', [ "adduser scrapeworker --disabled-password --gecos \"\"", ], {}, ["Adding user `scrapeworker'"] ], ['cmd.run', [ "usermod -a -G sudo scrapeworker", ], {}, None], [ 'cmd.run', [ "echo 'scrapeworker ALL=(ALL) NOPASSWD: ALL' | tee -a /etc/sudoers", ], {}, None ], [ 'cmd.run', [ "wget https://raw.githubusercontent.com/solarkennedy/instant-py-bt/master/py-bt -O /usr/local/bin/py-bt", ], {}, None ], [ 'cmd.run', [ "wget https://raw.githubusercontent.com/aurora/rmate/master/rmate -O /usr/local/bin/rmate", ], {}, None ], ['cmd.run', [ "chmod +x /usr/local/bin/py-bt", ], {}, None], ['cmd.run', [ "chmod +x /usr/local/bin/rmate", ], {}, None], [ 'cmd.run', [ "chown -R scrapeworker:scrapeworker /scraper", ], {}, None ], [ 'cmd.run', [ "./configure.sh", ], { 'env': "DEBIAN_FRONTEND=noninteractive", "cwd": '/scraper', 'runas': 'scrapeworker' }, ['Setup OK! System is configured for launch'] ], ] failures = 0 err = None for command, args, kwargs, expect in commands: while True: self.log.info( "Executing command '%s', args: '%s', kwargs: '%s'", command, args, kwargs) resp = self.local.cmd(clientname, fun=command, arg=args, kwarg=kwargs) self.log.info("Command executed. Clientname in response: %s", clientname in resp) try: if clientname in resp: if expect: self.validate_expect(resp[clientname], expect) elif resp is False: raise marshaller_exceptions.InvalidDeployResponse( "Command returned a failure result: '%s' (%s, %s)" % (command, args, kwargs)) failures = 0 break except (marshaller_exceptions.InvalidDeployResponse, marshaller_exceptions.InvalidExpectParameter) as e: failures += 1 err = e except Exception as e: failures += 1 err = e tries = 3 self.log.error("Command failed (attempt %s of %s)!", failures, tries) self.log.error("Response:") self.log.error("%s", resp) if failures > tries: if err is not None: raise err else: raise marshaller_exceptions.VmCreateFailed( "Failed to create VM!") if resp[clientname]: if isinstance(resp[clientname], dict): text = pprint.pformat(resp[clientname]) for line in text.split("\n"): self.log.info(" %s", line) elif isinstance(resp[clientname], str): for line in resp[clientname].split("\n"): self.log.info(" %s", line) else: self.log.warning("Unknown type: %s", type(resp[clientname])) text = pprint.pformat(resp[clientname]) for line in text.split("\n"): self.log.warning(" %s", line) else: self.log.info("Received empty response") if resp[clientname] and not resp[clientname].strip().endswith( 'Setup OK! System is configured for launch'): raise VmInitError("Setup command did not return success!") self.log.info("Node configured! Starting scraper client!") jobid = self.local.cmd_async(tgt=clientname, fun='cmd.run', arg=[ "screen -d -m ./run.sh", ], kwarg={ "cwd": '/scraper', 'runas': 'scrapeworker' }) self.log.info("Job id: '%s'", jobid)