def make_client(self, clientname, provider, kwargs):
self.log.info("Creating instance...")
self.log.info(" Client name: '%s'", clientname)
self.log.info(" using provider: '%s'", provider)
self.log.info(" kwargs: '%s'", kwargs)
try:
ret = self.cc.create(names=[clientname],
provider=provider,
**kwargs)
# self.log.info("Response: %s", ret)
self.log.info("Instance created!")
if provider == 'linode' and clientname in ret and ret[
clientname] is False:
raise marshaller_exceptions.VmCreateFailed(
"Failed when creating Linode VM?")
except Exception as e:
traceback.format_exc()
# DO Doesn't list the lcoation where they temporarily disabled VM creation, so if that specific thing
# happens, raise a more specific error.
if " The specified location, " in str(
e) and ", could not be found." in str(
e) and provider == 'digitalocean':
raise marshaller_exceptions.LocationNotAvailableResponse(
"Location error when creating VM. Exception: %s" % e)
raise marshaller_exceptions.VmCreateFailed(
"Failed when creating VM? Exception: %s" % e)
return provider, kwargs
def generate_vultr_conf(self):
provider = "vultr"
try:
planid, place = self.get_5_dollar_vultr_meta()
except TypeError as e:
raise marshaller_exceptions.VmCreateFailed(
"Failed when generating VM configuration? Exception: %s" % e)
if not planid:
raise marshaller_exceptions.VmCreateFailed(
"No vultr plan available?")
scriptname = "bootstrap-salt-delay.sh"
scriptdir = os.path.dirname(os.path.realpath(__file__))
fqscript = os.path.join(scriptdir, scriptname)
kwargs = {
'image': 'Ubuntu 18.04 x64',
'private_networking': False,
'size': planid,
'location': place,
'script': fqscript,
# 'script_args' : "-D",
}
return provider, kwargs
def generate_do_conf(self):
try:
planid, place = self.get_5_dollar_do_meta()
except TypeError as e:
raise marshaller_exceptions.VmCreateFailed(
"Failed when generating VM configuration? Exception: %s" % e)
scriptname = "bootstrap-salt-delay.sh"
scriptdir = os.path.dirname(os.path.realpath(__file__))
fqscript = os.path.join(scriptdir, scriptname)
provider = "digitalocean"
kwargs = {
'image': 'ubuntu-18-04-x64',
'size': planid,
# 'vm_size': planid,
'private_networking': False,
'location': place,
'script': fqscript,
}
return provider, kwargs
def configure_client(self,
clientname,
client_idx,
provider=None,
provider_kwargs=None):
assert "_" not in clientname, "VM names cannot contain _ on digital ocean, I think?"
self.log.info("Configuring client")
# [Command, [shell command], {execution context}, ['strings', 'expected', 'in', 'response']],
commands = [
['cmd.run', [
"bash -c \'whoami\'",
], {}, ['root']],
['cmd.run', [
'mkdir -p .ssh/',
], {}, None],
['cmd.run', [
dirmake_oneliner,
], {}, None],
['cmd.run', [
dirmake_ssh_oneliner,
], {}, None],
['cmd.run', [
"bash -c \"ls /\"",
], {}, [
'scraper',
]],
['cmd.run', [
'bash -c \"pwd\"',
], {}, ['/root']],
['cmd.run', [
'bash -c \"ls -la\"',
], {}, None],
['pkg.refresh_db', [], {}, None],
['pkg.install', [
'software-properties-common',
], {}, None],
# splat in public keys.
[
'cmd.run',
[
'echo ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCoNUeZ/L6QYntVXtBCdFLk3L7X1Smio+pKi/63W4i9VQdocxY7zl3fCyu5LsPzVQUBU5n'
+
'LKb/iJkABH+hxq8ZL7kXiKuGgeHsI60I2wECMxg17Qs918ND626AkXqlMIUW1SchcAi3rYRMVY0OaGSOutIcjR+mJ6liogTv1DLRD0eRbuollz7XsYz4ILb'
+
'i9kEsqwaly92vK6vlIVlAWtDoNf95c6jk/lh0M5p1LV0lwrEtfCreuv1rrOldUdwgU4wCFgRI+p6FXs69+OsNWxZSOSr28eE9sbsHxIxthcRHMtsnDxzeJ1'
+
'PVhvC4JclFEHEZSlYaemI6zOezVuBuipwSv Neko@ODO | tee -a .ssh/authorized_keys',
], {}, None
],
[
'cmd.run',
[
'echo ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCi7/9jOHVJj0ODnPqinqFbOaErT2pNaeq0pYKapcG2DHGrvVlX3ZUO8z7uY1QZX0OiC3y'
+
'7rv4c7NEl7/OtmRDfNPd5YgpAuXelbwu5Pj1BjQq1pn3CeP4zhw4gcEPx2UAc5Rw1jzH8vE7NMf2iReBiHr2SfSLh8T/jt5bEAVDCnhMS/8YvoPLLftESiL'
+
'oi+TU6Y9/zw4zac3AyJJ02tHpHLSpWWPPLi31ASEu/p+lWynUd+dSTMbwmc3hwBQkZTrK6P1I3431eQqYVNOyWJe+GeCXLaw5CvO8qlE7Nj3Z+dics3Bq0F'
+
'7ugDC+27qWk7m5soPfbZ8qlQz4CWFv01GHWdWwdHh9SR2bplNZ6MDuED91mu7gxyx2Wyo2AIiKsLcpGOIdLnIvrSA9VGpdgKbflbnqtfyIm6gloPpITnAJX'
+
'imWSvIxF76PVFjdZa86jAx7JZfBfirvtRg6/qXbDUDAErF3OllqxBvuGOzHptDDgha/29tabzxUIxhpBrG0TiRTMDmmqgM+b9kANgzEe4Yef2w/IaTC96D/'
+
'oLxRHmRBbof8GIMlNZjFlVw8XIyzYxnvALwCE7gRubba13f6qU0lT56be9HKYrSvHVy9/855lKlLwTCePaHK0EPBGuMWZOBexGKyxTFXmA+oqkBg5zFnZLy'
+
'xcsaVZQtZRnDU4Cu4jyQ== [email protected] | tee -a .ssh/authorized_keys',
], {}, None
],
[
'cmd.run',
[
'echo ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDMv9sMkEOjqsFu8wPiG2jUP62qKEpxmvQ7IiYaLKogW/LQlLhKP7KCIE2MVUmctwdvyEF'
+
'rXGOXDCVgFMFLEZiCi2+B7itMcFBlxJsZYQ9Y5FzXg/xK/Xld9rZu2ST+4z9xrX03n0rrsvO3HgpbNoIWF1LbXrc8L80CUCf13GWkZErhzc4mcd44McLVxX'
+
'q+hcikMguVdOcejpLJTQkq2LRLEx2zhrz+CfNe9AQ0I5AOsh8Os3rrILFs0t290hejMX82nwJUCIcODTBJqR0o7qs/Tt8zLy3YKnAN3eGqfO7tw/d75AD/n'
+
'ENup5kJscpVb/6v3xfWnjgAjalj/hw2bwoc3SE+Y3u2wmyuhrJcSy6rw/IltFc+BaZamZMBW/si8tW+xo9rb903GXANJbjVOABECJSp2i03xtPfYfk9KqZb'
+
'/vUkpYTmwRQGvDK9u1viIF8nIomE4omN6buFktvVjH1IG6bOPeMi4Y0zBNds7Q1W28Um1ygaBU+NCalep8UDEWInNkfYe1E/hj5A5EaMPaRjnPhXJqUzglO'
+
'l1O2Tco2FYhfvCiyZvAHv25LLrGzePidR59SzTP7/fLxK7FgmH0m79AOKvjuZaNjb7njmgDhyQggOLU6bJwiiJ7MqldPlic2qCKyQVavLv2nXGIGVXEovtM'
+
'9YfgSYuglkiYmbs6LU0w== durr@mainnas | tee -a .ssh/authorized_keys',
], {}, None
],
['cmd.run', [
"chmod 0600 .ssh/authorized_keys",
], {}, None],
[
'cmd.run', [
"cat .ssh/authorized_keys",
], {},
[
' Neko@ODO',
' [email protected]',
' durr@mainnas',
]
],
# So something is missing some of the keys, somehow
['cmd.run', [
"eval ssh-agent $SHELL",
], {}, None],
['cmd.run', [
"ssh-add .ssh/authorized_keys",
], {}, None],
['cmd.run', [
"ssh-add -l",
], {}, None],
[
'cmd.run',
[
"eval ssh-agent $SHELL; ssh-add .ssh/authorized_keys; ssh-add -l",
], {}, None
],
['cmd.run', [
"apt-get update",
], {}, None],
# So trying to have salt update itself makes it poop itself,
# and never come back.
# Siiiiiigh.
# Anyways, I moved this command to my custom bootstrap script.
# ['cmd.run', ["apt-get dist-upgrade -y", ], {'env' : {'DEBIAN_FRONTEND' : 'noninteractive'}}, None],
# Apparently at least one VPS host has separated git from build-essential?
# ['pkg.install', ['build-essential', 'locales', 'git', 'libfontconfig', 'wget', 'htop', 'libxml2', 'libxslt1-dev',
# 'python3-dev', 'python3-dbg', 'python3-distutils', 'libz-dev', 'curl', 'screen'], {}, None],
# ['pkg.install', ['libasound2', 'libatk1.0-0', 'libc6', 'libcairo2', 'libcups2', 'libdbus-1-3', 'libexpat1', 'libfontconfig1', 'libgcc1',
# 'libgconf-2-4', 'libgdk-pixbuf2.0-0', 'libglib2.0-0', 'libgtk-3-0', 'libnspr4', 'libpango-1.0-0', 'libpangocairo-1.0-0', 'libstdc++6',
# 'libx11-6', 'libx11-xcb1', 'libxcb1', 'libxcursor1', 'libxdamage1', 'libxext6', 'libxfixes3', 'libxi6', 'libxrandr2', 'libxrender1',
# 'libxss1', 'libxtst6', 'libnss3'], {}, None],
['pkg.install', [
'build-essential',
], {}, None],
['pkg.install', [
'locales',
], {}, None],
['pkg.install', [
'git',
], {}, None],
['pkg.install', [
'libfontconfig',
], {}, None],
['pkg.install', [
'wget',
], {}, None],
['pkg.install', [
'htop',
], {}, None],
['pkg.install', [
'libxml2',
], {}, None],
['pkg.install', [
'libxslt1-dev',
], {}, None],
['pkg.install', [
'python3-dev',
], {}, None],
['pkg.install', [
'python3-dbg',
], {}, None],
['pkg.install', [
'python3-distutils',
], {}, None],
['pkg.install', [
'libz-dev',
], {}, None],
['pkg.install', [
'curl',
], {}, None],
['pkg.install', ['screen'], {}, None],
['pkg.install', [
'libasound2',
], {}, None],
['pkg.install', [
'libatk1.0-0',
], {}, None],
['pkg.install', [
'libc6',
], {}, None],
['pkg.install', [
'libcairo2',
], {}, None],
['pkg.install', [
'libcups2',
], {}, None],
['pkg.install', [
'libdbus-1-3',
], {}, None],
['pkg.install', [
'libexpat1',
], {}, None],
['pkg.install', [
'libfontconfig1',
], {}, None],
['pkg.install', [
'libgcc1',
], {}, None],
['pkg.install', [
'libgconf-2-4',
], {}, None],
['pkg.install', [
'libgdk-pixbuf2.0-0',
], {}, None],
['pkg.install', [
'libglib2.0-0',
], {}, None],
['pkg.install', [
'libgtk-3-0',
], {}, None],
['pkg.install', [
'libnspr4',
], {}, None],
['pkg.install', [
'libpango-1.0-0',
], {}, None],
['pkg.install', [
'libpangocairo-1.0-0',
], {}, None],
['pkg.install', [
'libstdc++6',
], {}, None],
['pkg.install', [
'libx11-6',
], {}, None],
['pkg.install', [
'libx11-xcb1',
], {}, None],
['pkg.install', [
'libxcb1',
], {}, None],
['pkg.install', [
'libxcursor1',
], {}, None],
['pkg.install', [
'libxdamage1',
], {}, None],
['pkg.install', [
'libxext6',
], {}, None],
['pkg.install', [
'libxfixes3',
], {}, None],
['pkg.install', [
'libxi6',
], {}, None],
['pkg.install', [
'libxrandr2',
], {}, None],
['pkg.install', [
'libxrender1',
], {}, None],
['pkg.install', [
'libxss1',
], {}, None],
['pkg.install', [
'libxtst6',
], {}, None],
['pkg.install', ['libnss3'], {}, None],
['pkg.install', [
"xvfb",
], {}, None],
# Adblocking. Lower the chrome cpu costs decently
# So long hosts files cause things to explode, so we turn it off.
# ['cmd.run', ["curl https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/social/hosts | tee -a /etc/hosts", ], {}, None],
# Make swap so
[
'cmd.run', [
"dd if=/dev/zero of=/swapfile bs=1M count=4096",
], {}, None
],
['cmd.run', [
"mkswap /swapfile",
], {}, None],
['cmd.run', [
"chmod 0600 /swapfile",
], {}, None],
['cmd.run', [
"swapon /swapfile",
], {}, None],
# Needed to make GCE play nice. I think they just flat-out don't preinstall a locale
# ['cmd.run', ["sudo apt-get install language-pack-en -y", ], {}, ['The following NEW packages will be installed:', 'language-pack-en-base']],
# Shit to make the tty work in UTF-8. Otherwise, the logging can asplode
# and break all the things.
[
'cmd.run',
[
'echo LANG=\"en_US.UTF-8\" >> /etc/default/locale',
], {}, None
],
[
'cmd.run',
[
'echo LC_ALL=\"en_US.UTF-8\" >> /etc/default/locale',
], {}, None
],
[
'cmd.run', [
'echo "LC_ALL=en_US.UTF-8" >> /etc/environment',
], {}, None
],
[
'cmd.run', [
'echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen',
], {}, None
],
[
'cmd.run', [
'echo "LANG=en_US.UTF-8" > /etc/locale.conf',
], {}, None
],
[
'cmd.run', [
"dpkg-reconfigure -f noninteractive locales",
], {}, ['en_US.UTF-8']
],
['cmd.run', [
"locale",
], {}, None],
['cmd.run', [
"bash -c \"locale\"",
], {}, None],
# Clone and Install settings
['cmd.run', [
"bash -c \"ls /\"",
], {}, [
'scraper',
]],
[
'cmd.run',
[
"git clone https://github.com/fake-name/AutoTriever.git /scraper"
], {}, []
],
[
'cmd.run',
[
"cat << EOF > /scraper/settings.json \n{content}\nEOF".
format(
content=self.__make_conf_file(clientname, client_idx)),
], {}, None
],
# Make sure it all checked out at least somewhat
[
'cmd.run', [
"bash -c \"ls /scraper\"",
], {}, ['configure.sh', 'run.sh', 'settings.json']
],
# Finally, run the thing
[
'cmd.run',
[
"adduser scrapeworker --disabled-password --gecos \"\"",
], {}, ["Adding user `scrapeworker'"]
],
['cmd.run', [
"usermod -a -G sudo scrapeworker",
], {}, None],
[
'cmd.run',
[
"echo 'scrapeworker ALL=(ALL) NOPASSWD: ALL' | tee -a /etc/sudoers",
], {}, None
],
[
'cmd.run',
[
"wget https://raw.githubusercontent.com/solarkennedy/instant-py-bt/master/py-bt -O /usr/local/bin/py-bt",
], {}, None
],
[
'cmd.run',
[
"wget https://raw.githubusercontent.com/aurora/rmate/master/rmate -O /usr/local/bin/rmate",
], {}, None
],
['cmd.run', [
"chmod +x /usr/local/bin/py-bt",
], {}, None],
['cmd.run', [
"chmod +x /usr/local/bin/rmate",
], {}, None],
[
'cmd.run', [
"chown -R scrapeworker:scrapeworker /scraper",
], {}, None
],
[
'cmd.run', [
"./configure.sh",
], {
'env': "DEBIAN_FRONTEND=noninteractive",
"cwd": '/scraper',
'runas': 'scrapeworker'
}, ['Setup OK! System is configured for launch']
],
]
failures = 0
err = None
for command, args, kwargs, expect in commands:
while True:
self.log.info(
"Executing command '%s', args: '%s', kwargs: '%s'",
command, args, kwargs)
resp = self.local.cmd(clientname,
fun=command,
arg=args,
kwarg=kwargs)
self.log.info("Command executed. Clientname in response: %s",
clientname in resp)
try:
if clientname in resp:
if expect:
self.validate_expect(resp[clientname], expect)
elif resp is False:
raise marshaller_exceptions.InvalidDeployResponse(
"Command returned a failure result: '%s' (%s, %s)"
% (command, args, kwargs))
failures = 0
break
except (marshaller_exceptions.InvalidDeployResponse,
marshaller_exceptions.InvalidExpectParameter) as e:
failures += 1
err = e
except Exception as e:
failures += 1
err = e
tries = 3
self.log.error("Command failed (attempt %s of %s)!", failures,
tries)
self.log.error("Response:")
self.log.error("%s", resp)
if failures > tries:
if err is not None:
raise err
else:
raise marshaller_exceptions.VmCreateFailed(
"Failed to create VM!")
if resp[clientname]:
if isinstance(resp[clientname], dict):
text = pprint.pformat(resp[clientname])
for line in text.split("\n"):
self.log.info(" %s", line)
elif isinstance(resp[clientname], str):
for line in resp[clientname].split("\n"):
self.log.info(" %s", line)
else:
self.log.warning("Unknown type: %s",
type(resp[clientname]))
text = pprint.pformat(resp[clientname])
for line in text.split("\n"):
self.log.warning(" %s", line)
else:
self.log.info("Received empty response")
if resp[clientname] and not resp[clientname].strip().endswith(
'Setup OK! System is configured for launch'):
raise VmInitError("Setup command did not return success!")
self.log.info("Node configured! Starting scraper client!")
jobid = self.local.cmd_async(tgt=clientname,
fun='cmd.run',
arg=[
"screen -d -m ./run.sh",
],
kwarg={
"cwd": '/scraper',
'runas': 'scrapeworker'
})
self.log.info("Job id: '%s'", jobid)
def configure_client(self, clientname, client_idx):
assert "_" not in clientname, "VM names cannot contain _ on digital ocean, I think?"
self.log.info("Configuring client")
commands = [
['cmd.run', [
dirmake_oneliner,
], {}],
[
'cmd.run', [
"apt-get install -y build-essential git screen",
], {}
],
[
'cmd.run',
[
"git clone https://github.com/fake-name/AutoTriever.git /scraper"
], {}
],
['cmd.run', [
"ls /scraper",
], {}],
['cmd.run', [
"whoami",
], {}],
# Make swap so
[
'cmd.run', [
"dd if=/dev/zero of=/swapfile bs=1M count=1024",
], {}
],
['cmd.run', [
"mkswap /swapfile",
], {}],
['cmd.run', [
"swapon /swapfile",
], {}],
['cmd.run', [
"dpkg-reconfigure locales",
], {}],
[
'cmd.run',
[
'echo LANG=\"en_US.UTF-8\" >> /etc/default/locale',
], {}
],
[
'cmd.run',
[
'echo LC_ALL=\"en_US.UTF-8\" >> /etc/default/locale',
], {}
],
['cmd.run', [
"dpkg-reconfigure locales",
], {}],
['cmd.run', [
"locale",
], {}],
['cmd.run', [
"bash -c locale",
], {}],
# Install settings
[
'cmd.run',
[
"cat << EOF > /scraper/settings.json \n{content}\nEOF".
format(
content=self.__make_conf_file(clientname, client_idx)),
], {}
],
# Finally, run the thing
['cmd.run', [
"./configure.sh",
], {
"cwd": '/scraper'
}],
]
failures = 0
for command, args, kwargs in commands:
while True:
self.log.info(
"Executing command '%s', args: '%s', kwargs: '%s'",
command, args, kwargs)
resp = self.local.cmd(clientname,
fun=command,
arg=args,
kwarg=kwargs)
self.log.info("Command executed. Clientname in response: %s",
clientname in resp)
if clientname in resp:
failures = 0
break
else:
failures += 1
self.log.error("Command failed!")
self.log.error("Response:")
self.log.error("%s", resp)
if failures > 25:
raise marshaller_exceptions.VmCreateFailed(
"Failed to create VM!")
if resp[clientname]:
for line in resp[clientname].split("\n"):
self.log.info(" %s", line)
else:
self.log.info("Received empty response")
if not resp[clientname].strip().endswith(
'Setup OK! System is configured for launch'):
raise VmInitError("Setup command did not return success!")
self.log.info("Node configured! Starting scraper client!")
jobid = self.local.cmd_async(tgt=clientname,
fun='cmd.run',
arg=[
"screen -d -m ./run.sh",
],
kwarg={"cwd": '/scraper'})
self.log.info("Job id: '%s'", jobid)
