def _generate_module_stop(silent=False):
'''Generates Spark-stop module from available sources.'''
generation_loc = fs.join(fs.dirname(fs.abspath(__file__)), 'internal', 'remoto', 'modules', 'generated', 'start_spark.py')
files = [
fs.join(fs.dirname(fs.abspath(__file__)), 'internal', 'util', 'printer.py'),
fs.join(fs.dirname(fs.abspath(__file__)), 'internal', 'remoto', 'modules', 'printer.py'),
fs.join(fs.dirname(fs.abspath(__file__)), 'internal', 'remoto', 'modules', 'spark_stop.py'),
fs.join(fs.dirname(fs.abspath(__file__)), 'internal', 'remoto', 'modules', 'remoto_base.py'),
]
ModuleGenerator().with_module(fs).with_files(*files).generate(generation_loc, silent)
return importer.import_full_path(generation_loc)
def generate(self, outputpath, allowed_imports=None, silent=False):
'''Generates the final, non-stl dependency-free module to be used with Remoto remote module execution.
Captures all import commands and ensures they are present only once for the entire module.
Warning: Removes all non-stl import statements.
Args:
outputpath (str): Location to store module, including output filename. Creates every directory that does not exist.
allowed_imports (optional iterable(str)): If set to an iterable, does not remove given import statements.
silent (optional bool): If set, skips printing warnings when non-standard imports are encountered.'''
dest_dir = fs.dirname(outputpath)
if not fs.isdir(dest_dir):
fs.mkdir(dest_dir, exist_ok=True)
stl_imports, stl_imports_from = self._read_imports(
allowed_imports=allowed_imports, silent=silent)
with open(outputpath, 'w') as f:
header = '''
################################################################################
# Generated by the meta modulegenerator
# Processed {} files/modules:
{}
################################################################################
'''.format(len(self._files),
'\n'.join('# {}'.format(x) for x in self._files))
f.write(header)
importstring = '\n' + '\n'.join('import {}'.format(name)
for name in stl_imports)
importstring += '\n'
importstring += '\n'.join(
'from {} import {} as {}'.format(*names) if len(names) == 3
and names[2] != None else 'from {} import {}'.format(*names)
for names in stl_imports_from)
f.write(importstring)
for x in self._files:
content = self._read_non_imports(x)
f.write('''
################################################################################
# Created from file {}
'''.format(x))
f.write(content)
f.write('''
################################################################################
''')
def _generate_module_java(silent=False):
generation_loc = fs.join(fs.dirname(fs.abspath(__file__)), 'internal',
'remoto', 'modules', 'generated',
'install_java.py')
files = [
fs.join(fs.dirname(fs.abspath(__file__)), 'internal', 'util',
'printer.py'),
fs.join(fs.dirname(fs.abspath(__file__)), 'internal', 'remoto',
'modules', 'printer.py'),
fs.join(fs.dirname(fs.abspath(__file__)), 'internal', 'remoto',
'env.py'),
fs.join(fs.dirname(fs.abspath(__file__)), 'internal', 'remoto',
'modules', 'java_install.py'),
fs.join(fs.dirname(fs.abspath(__file__)), 'internal', 'remoto',
'modules', 'remoto_base.py'),
]
ModuleGenerator().with_module(fs).with_files(*files).generate(
generation_loc, silent)
return importer.import_full_path(generation_loc)