def install_hdfs_xml(self):
target = OsUtil.join(self.target, 'shared', 'resources', 'spark')
OsUtil.mkdir('-p {}'.format(target))
OsUtil.install(source=OsUtil.join(self.source, 'spark'),
target=target,
files=['hdfs-site.xml'],
file_suffix='.example')
def install_dockerfile(self):
target = OsUtil.join(self.target, 'shared', 'resources', 'jupyter')
OsUtil.mkdir('-p {}'.format(target))
OsUtil.install(source=OsUtil.join(self.source, 'jupyter'),
target=target,
files=['Dockerfile'],
file_suffix='.example')
def __init__(self, args):
InitScript.__init__(self,
args=args,
source=OsUtil.join(Paths.app_root(), 'shared', 'resources', 'spark'),
target=Paths.spark_conf_path(),
start_message='i| running DockerScript')
def install_hdfs_xml(self):
OsUtil.install(source=self.source, target=self.target, files=['hdfs-site.xml'], with_sudo=True)
def root():
return OsUtil.join(os.path.dirname(__file__))
def resources_path():
return OsUtil.join(Paths.root(), 'shared', 'resources')
def spark_conf_path():
return OsUtil.join(Paths.spark_path(), 'conf')
def install_docker_compose(self):
OsUtil.install(source=OsUtil.join(self.source, 'jupyter'),
target=self.target,
files=['docker-compose.yml'],
file_suffix='.example')
def install_requirements_file(self):
print('i| installing requirements')
with open(OsUtil.join(self.target, 'requirements.txt'), 'a') as _file:
_file.write("\neasy_etl==0.1.0\n")
def install(self):
OsUtil.mkdir('-p {}/{}'.format(self.target, 'etls'))
self.install_requirements_file()
# self.install_dockerfile()
self.install_docker_compose()
self.install_hdfs_xml()