def bootstrapFrontera():
if env.host not in common.HOSTS["frontera_spiders"] and env.host not in common.HOSTS["frontera_workers"]:
return
installDependencies(["build-essential", "libpython-dev", "python-dev", "python-pip", "python-twisted", "git",
"python-six", "libsnappy-dev"])
cloneFrontera()
prepareBundles()
deploySpiders()
sudo("pip install -q -r %s/requirements.txt" % FRONTERA_DEST_DIR)
if env.host in common.HOSTS["frontera_spiders"]:
installDependencies(["dnsmasq", "python-lxml", "python-openssl", "python-w3lib",
"python-cssselect"], pre_commands=False)
setupDnsmasq()
# manual nltk.download() is still needed there
sudo("pip install -q nltk scrapy==0.24.6 kazoo")
generateSpiderConfigs()
generateSpiderStartupScripts()
if env.host in common.HOSTS["frontera_workers"]:
installDependencies(["python-lxml", "python-w3lib", "python-cssselect"], pre_commands=False)
sudo("pip install -q nltk scrapy==0.24.6 kazoo")
generateWorkersConfigs()
generateWorkersStartupScripts()
def bootstrapFrontera():
if env.host not in common.HOSTS[
"frontera_spiders"] and env.host not in common.HOSTS[
"frontera_workers"]:
return
installDependencies([
"build-essential", "libpython-dev", "python-dev", "python-pip",
"python-twisted", "git", "python-six", "libsnappy-dev"
])
cloneFrontera()
prepareBundles()
deploySpiders()
sudo("pip install -q -r %s/requirements.txt" % FRONTERA_DEST_DIR)
if env.host in common.HOSTS["frontera_spiders"]:
installDependencies([
"dnsmasq", "python-lxml", "python-openssl", "python-w3lib",
"python-cssselect"
],
pre_commands=False)
setupDnsmasq()
# manual nltk.download() is still needed there
sudo("pip install -q nltk scrapy==0.24.6 kazoo")
generateSpiderConfigs()
generateSpiderStartupScripts()
if env.host in common.HOSTS["frontera_workers"]:
installDependencies(
["python-lxml", "python-w3lib", "python-cssselect"],
pre_commands=False)
sudo("pip install -q nltk scrapy==0.24.6 kazoo")
generateWorkersConfigs()
generateWorkersStartupScripts()
def bootstrap():
_prepareStorageDevices()
ensureImportantDirectoriesExist()
installDependencies()
installHadoop()
installZookeeper()
installHBase()
installKafka()
if common.isService('hadoop') or common.isService('hbase'):
_updateHadoopSiteValues()
setupEnvironment()
configHadoop()
configZookeeper()
configHBase()
configKafka()
setupHosts()
formatHdfs()
cleanup()
def bootstrap():
_prepareStorageDevices()
ensureImportantDirectoriesExist()
installDependencies()
installHadoop()
installZookeeper()
installHBase()
installKafka()
if common.isService("hadoop") or common.isService("hbase"):
_updateHadoopSiteValues()
setupEnvironment()
configHadoop()
configZookeeper()
configHBase()
configKafka()
setupHosts()
formatHdfs()
cleanup()
