def bootstrapFrontera(): if env.host not in common.HOSTS["frontera_spiders"] and env.host not in common.HOSTS["frontera_workers"]: return installDependencies(["build-essential", "libpython-dev", "python-dev", "python-pip", "python-twisted", "git", "python-six", "libsnappy-dev"]) cloneFrontera() prepareBundles() deploySpiders() sudo("pip install -q -r %s/requirements.txt" % FRONTERA_DEST_DIR) if env.host in common.HOSTS["frontera_spiders"]: installDependencies(["dnsmasq", "python-lxml", "python-openssl", "python-w3lib", "python-cssselect"], pre_commands=False) setupDnsmasq() # manual nltk.download() is still needed there sudo("pip install -q nltk scrapy==0.24.6 kazoo") generateSpiderConfigs() generateSpiderStartupScripts() if env.host in common.HOSTS["frontera_workers"]: installDependencies(["python-lxml", "python-w3lib", "python-cssselect"], pre_commands=False) sudo("pip install -q nltk scrapy==0.24.6 kazoo") generateWorkersConfigs() generateWorkersStartupScripts()
def bootstrapFrontera(): if env.host not in common.HOSTS[ "frontera_spiders"] and env.host not in common.HOSTS[ "frontera_workers"]: return installDependencies([ "build-essential", "libpython-dev", "python-dev", "python-pip", "python-twisted", "git", "python-six", "libsnappy-dev" ]) cloneFrontera() prepareBundles() deploySpiders() sudo("pip install -q -r %s/requirements.txt" % FRONTERA_DEST_DIR) if env.host in common.HOSTS["frontera_spiders"]: installDependencies([ "dnsmasq", "python-lxml", "python-openssl", "python-w3lib", "python-cssselect" ], pre_commands=False) setupDnsmasq() # manual nltk.download() is still needed there sudo("pip install -q nltk scrapy==0.24.6 kazoo") generateSpiderConfigs() generateSpiderStartupScripts() if env.host in common.HOSTS["frontera_workers"]: installDependencies( ["python-lxml", "python-w3lib", "python-cssselect"], pre_commands=False) sudo("pip install -q nltk scrapy==0.24.6 kazoo") generateWorkersConfigs() generateWorkersStartupScripts()
def bootstrap(): _prepareStorageDevices() ensureImportantDirectoriesExist() installDependencies() installHadoop() installZookeeper() installHBase() installKafka() if common.isService('hadoop') or common.isService('hbase'): _updateHadoopSiteValues() setupEnvironment() configHadoop() configZookeeper() configHBase() configKafka() setupHosts() formatHdfs() cleanup()
def bootstrap(): _prepareStorageDevices() ensureImportantDirectoriesExist() installDependencies() installHadoop() installZookeeper() installHBase() installKafka() if common.isService("hadoop") or common.isService("hbase"): _updateHadoopSiteValues() setupEnvironment() configHadoop() configZookeeper() configHBase() configKafka() setupHosts() formatHdfs() cleanup()