def load_source(module_name, file_name):
"""Loads the given module from a Python source file.
Arguments:
module_name -- the name of the variable read the module into
file_name -- the file that contains the source for the module
"""
from com.twitter.pycascading import Util
cascading_jar = Util.getJarFolder()
tmp_dir = _remove_last_dir(_remove_last_dir(cascading_jar))
sys.path.extend((cascading_jar, tmp_dir + '/python',
tmp_dir + '/python/Lib'))
# Haha... it's necessary to put this here, otherwise simplejson won't work.
# Maybe it's automatically imported in the beginning of a Jython program,
# but since at that point the sys.path is not set yet to Lib, it will fail?
import encodings
return imp.load_source(module_name, file_name)
def load_source(module_name, file_name):
"""Loads the given module from a Python source file.
Arguments:
module_name -- the name of the variable read the module into
file_name -- the file that contains the source for the module
"""
from com.twitter.pycascading import Util
cascading_jar = Util.getJarFolder()
tmp_dir = _remove_last_dir(_remove_last_dir(cascading_jar))
sys.path.extend(
(cascading_jar, tmp_dir + '/python', tmp_dir + '/python/Lib'))
# Haha... it's necessary to put this here, otherwise simplejson won't work.
# Maybe it's automatically imported in the beginning of a Jython program,
# but since at that point the sys.path is not set yet to Lib, it will fail?
import encodings
return imp.load_source(module_name, file_name)
__author__ = 'Gabor Szabo'
import sys, imp
if __name__ == "__main__":
# The first command line parameter must be 'hadoop' or 'local'
# to indicate the running mode
running_mode = sys.argv[1]
from com.twitter.pycascading import Util
cascading_jar = Util.getCascadingJar()
# This is the folder where Hadoop extracted the jar file for execution
tmp_dir = Util.getJarFolder()
# The initial value of sys.path is JYTHONPATH plus whatever Jython appends
# to it (normally the Python standard libraries the come with Jython)
sys.path.extend((cascading_jar, '.', tmp_dir, tmp_dir + 'python',
tmp_dir + 'python/Lib'))
# Haha... it's necessary to put this here, otherwise simplejson won't work.
# Maybe it's automatically imported in the beginning of a Jython program,
# but since at that point the sys.path is not set yet to Lib, it will fail?
# Instead, we can use Java's JSON decoder...
# import encodings
m = imp.load_source('main', sys.argv[2])
# We need to explicitly inject running_mode into the tap module,
# otherwise we cannot import bootstrap from tap and use the
tmp_dir + '/python/Lib'))
# Haha... it's necessary to put this here, otherwise simplejson won't work.
# Maybe it's automatically imported in the beginning of a Jython program,
# but since at that point the sys.path is not set yet to Lib, it will fail?
import encodings
return imp.load_source(module_name, file_name)
if __name__ == "__main__":
running_mode = sys.argv[1]
from com.twitter.pycascading import Util
cascading_jar = Util.getJarFolder()
# This is the folder where Hadoop extracted the jar file for execution
tmp_dir = _remove_last_dir(_remove_last_dir(cascading_jar))
sys.path.extend((cascading_jar, '.', tmp_dir, tmp_dir + '/python',
tmp_dir + '/python/Lib'))
# Haha... it's necessary to put this here, otherwise simplejson won't work.
# Maybe it's automatically imported in the beginning of a Jython program,
# but since at that point the sys.path is not set yet to Lib, it will fail?
# Instead, we can use Java's JSON decoder...
import encodings
m = imp.load_source('main', sys.argv[2])
# We need to explicitly inject running_mode into the tap modules,
# otherwise we cannot import bootstrap from tap and use the
# bootstrap.running_mode like that
(cascading_jar, tmp_dir + '/python', tmp_dir + '/python/Lib'))
# Haha... it's necessary to put this here, otherwise simplejson won't work.
# Maybe it's automatically imported in the beginning of a Jython program,
# but since at that point the sys.path is not set yet to Lib, it will fail?
import encodings
return imp.load_source(module_name, file_name)
if __name__ == "__main__":
running_mode = sys.argv[1]
from com.twitter.pycascading import Util
cascading_jar = Util.getJarFolder()
# This is the folder where Hadoop extracted the jar file for execution
tmp_dir = _remove_last_dir(_remove_last_dir(cascading_jar))
sys.path.extend((cascading_jar, '.', tmp_dir, tmp_dir + '/python',
tmp_dir + '/python/Lib'))
# Haha... it's necessary to put this here, otherwise simplejson won't work.
# Maybe it's automatically imported in the beginning of a Jython program,
# but since at that point the sys.path is not set yet to Lib, it will fail?
# Instead, we can use Java's JSON decoder...
import encodings
m = imp.load_source('main', sys.argv[2])
# We need to explicitly inject running_mode into the tap modules,
# otherwise we cannot import bootstrap from tap and use the
# bootstrap.running_mode like that
running_mode = sys.argv[1]
# The second is the location of the PyCascading Python sources in local
# mode, and the PyCascading tarball in Hadoop mode
python_dir = sys.argv[2]
# Remove the first two arguments so that sys.argv will look like as
# if it was coming from a simple command line execution
# The further parameters are the command line parameters to the script
sys.argv = sys.argv[3:]
from com.twitter.pycascading import Util
cascading_jar = Util.getCascadingJar()
# This is the folder where Hadoop extracted the jar file for execution
tmp_dir = Util.getJarFolder()
Util.setPycascadingRoot(python_dir)
# The initial value of sys.path is JYTHONPATH plus whatever Jython appends
# to it (normally the Python standard libraries the come with Jython)
sys.path.extend((cascading_jar, '.', tmp_dir, python_dir + '/python',
python_dir + '/python/Lib'))
# Allow the importing of user-installed Jython packages
import site
site.addsitedir(python_dir + 'python/Lib/site-packages')
import os
import encodings
import pycascading.pipe, getopt
