def __init__(self, launch_monitor=False):
"""
Setup the direct access server
"""
# Currently we are pulling the driver config information from the driver tests. There is a better way to do
# this, but we are time crunched. TODO: Fix this!
generator = DriverGenerator(Metadata())
__import__(generator.test_modulename())
self.test_config = InstrumentDriverTestConfig()
# Test to ensure we have initialized our test config
if not self.test_config.initialized:
raise TestNotInitialized(msg="Tests non initialized. Missing InstrumentDriverTestCase.initalize(...)?")
self.launch_monitor = launch_monitor
def _driver_test_module(self):
generator = DriverGenerator(self.metadata)
return generator.test_modulename()
class EggGenerator:
"""
Generate driver egg
"""
def __init__(self, metadata):
"""
@brief Constructor
@param metadata IDK Metadata object
"""
self.metadata = metadata
self._bdir = None
if not self._tmp_dir():
raise InvalidParameters("missing tmp_dir configuration")
if not self._tmp_dir():
raise InvalidParameters("missing working_repo configuration")
self.generator = DriverGenerator(self.metadata)
test_import = __import__(self._test_module())
def _test_module(self):
return self.generator.test_modulename()
def _driver_module(self):
test_config = InstrumentDriverTestConfig()
return test_config.driver_module
def _driver_class(self):
test_config = InstrumentDriverTestConfig()
return test_config.driver_class
def _repo_dir(self):
return Config().get('working_repo')
def _tmp_dir(self):
return Config().get('tmp_dir')
def _setup_path(self):
return os.path.join(self._build_dir(), 'setup.py' )
def _setup_template_path(self):
return os.path.join(Config().template_dir(), 'setup.tmpl' )
def _main_path(self):
return os.path.join(self._build_dir(), 'mi/main.py' )
def _main_template_path(self):
return os.path.join(Config().template_dir(), 'main.tmpl' )
def _build_name(self):
return "%s_%s_%s_%s" % (
self.metadata.driver_make,
self.metadata.driver_model,
self.metadata.driver_name,
self.metadata.version,
)
def _build_dir(self):
if self._bdir:
return self._bdir
self._bdir = self._generate_build_dir()
log.info( "egg build dir: %s" % self._bdir)
return self._bdir
def _generate_build_dir(self):
original_dir = os.path.join(self._tmp_dir(), self._build_name())
build_dir = original_dir
build_count = 1
# Find a directory that doesn't exist
while os.path.exists(build_dir):
build_dir = "%s.%03d" % (original_dir, build_count)
log.debug("build dir test: %s" % build_dir)
build_count += 1
return build_dir
def _stage_files(self, files):
if not os.path.exists(self._build_dir()):
os.makedirs(self._build_dir())
for file in files:
dest = os.path.join(self._build_dir(), file)
destdir = dirname(dest)
source = os.path.join(self._repo_dir(), file)
log.debug(" Copy %s => %s" % (source, dest))
if not os.path.exists(destdir):
os.makedirs(destdir)
shutil.copy(source, dest)
def _get_template(self, template_file):
"""
@brief return a string.Template object constructed with the contents of template_file
@param template_file path to a file that containes a template
@retval string.Template object
"""
try:
infile = open(template_file)
tmpl_str = infile.read()
return Template(tmpl_str)
except IOError:
raise MissingTemplate(msg="Missing: %s" % template_file)
def _generate_setup_file(self):
if not os.path.exists(self._build_dir()):
os.makedirs(self._build_dir())
setup_file = self._setup_path()
setup_template = self._get_template(self._setup_template_path())
log.debug("Create setup.py file: %s" % setup_file )
log.debug("setup.py template file: %s" % self._setup_template_path())
log.debug("setup.py template date: %s" % self._setup_template_data())
ofile = open(setup_file, 'w')
code = setup_template.substitute(self._setup_template_data())
ofile.write(code)
ofile.close()
def _setup_template_data(self):
name = "%s_%s_%s" % (self.metadata.driver_make,
self.metadata.driver_model,
self.metadata.driver_name)
return {
'name': name,
'version': self.metadata.version,
'description': 'ooi core driver',
'author': self.metadata.author,
'email': self.metadata.email,
'url': 'http://www.oceanobservatories.org',
'driver_module': self._driver_module(),
'driver_class': self._driver_class(),
}
def _generate_main_file(self):
if not os.path.exists(self._build_dir()):
os.makedirs(self._build_dir())
main_file= self._main_path()
main_template = self._get_template(self._main_template_path())
log.debug("Create mi/main.py file: %s" % main_file )
log.debug("main.py template file: %s" % self._main_template_path())
log.debug("main.py template date: %s" % self._setup_template_data())
ofile = open(main_file, 'w')
code = main_template.substitute(self._setup_template_data())
ofile.write(code)
ofile.close()
def _verify_ready(self):
"""
verify that we have all the information and are in the correct state to build the egg
"""
self._verify_python()
self._verify_metadata()
self._verify_version()
self._verify_git()
def _verify_metadata(self):
"""
Ensure we have all the metadata we need to build the egg
"""
pass
def _verify_version(self, version = None):
"""
Ensure we have a good version number and that it has not already been packaged and published
"""
if version == None:
version = self.metadata.version
if not version:
raise ValidationFailure("Driver version required in metadata")
p = re.compile("^\d+\.\d+\.\d+$")
if not p.findall("%s" % version):
raise ValidationFailure("Version format incorrect '%s', should be x.x.x" % version)
#TODO Add check to see if there is already the same version package built
def _verify_git(self):
"""
Ensure the repository is up to date. All files should be committed and pushed. The local repo
should be in sync with the mainline
"""
#TODO Update this check
pass
def _verify_python(self):
"""
Ensure we build with the correct python version
"""
if sys.version_info < (2, 7) or sys.version_info >= (2, 8):
raise ValidationFailure("Egg generation required version 2.7 of python")
def _build_egg(self, files):
try:
self._verify_ready()
self._stage_files(files)
self._generate_setup_file()
self._generate_main_file()
cmd = "cd %s; python setup.py bdist_egg" % self._build_dir()
log.info("CMD: %s" % cmd)
os.system(cmd)
egg_file = "%s/dist/%s_%s_%s-%s-py2.7.egg" % (self._build_dir(),
self.metadata.driver_make,
self.metadata.driver_model,
self.metadata.driver_name,
self.metadata.version)
except ValidationFailure, e:
log.error("Failed egg verification: %s" % e )
return None
log.debug("Egg file created:: %s" % egg_file)
return egg_file
class EggGenerator:
"""
Generate driver egg
"""
def __init__(self, metadata):
"""
@brief Constructor
@param metadata IDK Metadata object
"""
self.metadata = metadata
self._bdir = None
if not self._tmp_dir():
raise InvalidParameters("missing tmp_dir configuration")
if not self._tmp_dir():
raise InvalidParameters("missing working_repo configuration")
self.generator = DriverGenerator(self.metadata)
test_import = __import__(self._test_module())
def _test_module(self):
return self.generator.test_modulename()
def _driver_module(self):
test_config = InstrumentDriverTestConfig()
return test_config.driver_module
def _driver_class(self):
test_config = InstrumentDriverTestConfig()
return test_config.driver_class
def _repo_dir(self):
return '/tmp/repoclone/marine-integrations'
def _res_dir(self):
return os.path.join(self._versioned_dir(), 'res')
def _res_config_dir(self):
return os.path.join(self._res_dir(), 'config')
def _tmp_dir(self):
return Config().get('tmp_dir')
def _setup_path(self):
return os.path.join(self._build_dir(), 'setup.py')
def _setup_template_path(self):
return os.path.join(Config().template_dir(), 'setup.tmpl')
def _main_path(self):
return os.path.join(self._versioned_dir(), 'mi/main.py')
def _main_template_path(self):
return os.path.join(Config().template_dir(), 'main.tmpl')
def _build_name(self):
return "%s_%s_%s_%s" % (
self.metadata.driver_make,
self.metadata.driver_model,
self.metadata.driver_name,
self.metadata.version.replace('.', '_'),
)
def _build_dir(self):
if self._bdir:
return self._bdir
self._bdir = self._generate_build_dir()
log.info("egg build dir: %s" % self._bdir)
return self._bdir
def _generate_build_dir(self):
build_dir = os.path.join(self._tmp_dir(), self._build_name())
# clean out an old build if it exists
if os.path.exists(build_dir):
shutil.rmtree(build_dir)
return build_dir
def _versioned_dir(self):
return self._build_dir()
# leave in plumbing for building namespaced eggs
#return os.path.join(self._build_dir(),
# self._build_name())
def _stage_files(self, files):
"""
Copy files from the original directory into two levels of versioned
directories within a staging directory, and replace the mi namespace
with the versioned driver name.mi to account for the new directory
(only the lower versioned dir is included in the egg)
@param files - a list of files to copy into the staging directory
"""
# make two levels of versioned file directories, i.e.
# driverA_0_1 (= build_dir)
# driverA_0_1 (= versioned_dir)
# then copy driverA files into the bottom versioned dir
if not os.path.exists(self._build_dir()):
os.makedirs(self._build_dir())
if not os.path.exists(self._versioned_dir()):
os.makedirs(self._versioned_dir())
for file in files:
dest = os.path.join(self._versioned_dir(), file)
destdir = dirname(dest)
source = os.path.join(self._repo_dir(), file)
# this one goes elsewhere so the InstrumentDict can find it
if basename(file) == 'strings.yml':
dest = os.path.join(self._res_dir(), basename(file))
destdir = dirname(dest)
log.debug(" Copy %s => %s" % (source, dest))
# make sure the destination directory exists, if it doesn't make it
if not os.path.exists(destdir):
os.makedirs(destdir)
shutil.copy(source, dest)
# replace mi in the copied files with the versioned driver module.mi
# this is necessary because the top namespace in the versioned files starts
# with the versioned driver name directory, not mi
#driver_file = open(dest, "r")
#contents = driver_file.read()
#driver_file.close()
#new_contents = re.sub(r'(^import |^from |\'|= )mi\.|res/config/mi-logging|\'mi\'',
# self._mi_replace,
# contents,
# count=0,
# flags=re.MULTILINE)
#driver_file = open(dest, "w")
#driver_file.write(new_contents)
#driver_file.close()
# need to add mi-logging.yml special because it is not in cloned repo, only in local repository
milog = "mi-logging.yml"
dest = os.path.join(self._res_config_dir(), milog)
destdir = dirname(dest)
source = os.path.join(Config().base_dir(), "res/config/" + milog)
log.debug(" Copy %s => %s" % (source, dest))
# make sure the destination directory exists, if it doesn't make it
if not os.path.exists(destdir):
os.makedirs(destdir)
# Now that it exists, make the package scanner find it
self._create_file(os.path.join(self._res_dir(), "__init__.py"))
self._create_file(os.path.join(self._res_config_dir(), "__init__.py"))
shutil.copy(source, dest)
# we need to make sure an init file is in the versioned dir and
# resource directories so that find_packages() will look in here
init_file_list = [
os.path.join(self._versioned_dir(), "__init__.py"),
os.path.join(self._versioned_dir(), "res", "__init__.py"),
os.path.join(self._versioned_dir(), "res", "config", "__init__.py")
]
for file in init_file_list:
self._create_file(file)
@staticmethod
def _create_file(file):
"""
Create a file if it isnt there already
"""
if not os.path.exists(file):
init_file = open(file, "w")
init_file.close()
def _mi_replace(self, matchobj):
"""
This function is used in regex sub to replace mi with the versioned
driver name followed by mi
@param matchobj - the match object from re.sub
"""
if matchobj.group(0) == 'res/config/mi-logging':
return self._build_name() + '/' + matchobj.group(0)
elif matchobj.group(0) == '\'mi\'':
return '\'' + self._build_name() + '.mi\''
else:
return matchobj.group(1) + self._build_name() + '.mi.'
def _get_template(self, template_file):
"""
@brief return a string.Template object constructed with the contents of template_file
@param template_file path to a file that containes a template
@retval string.Template object
"""
try:
infile = open(template_file)
tmpl_str = infile.read()
return Template(tmpl_str)
except IOError:
raise MissingTemplate(msg="Missing: %s" % template_file)
def _generate_setup_file(self):
if not os.path.exists(self._build_dir()):
os.makedirs(self._build_dir())
if not os.path.exists(self._build_dir()):
raise IDKException("failed to create build dir: %s" %
self._build_dir())
setup_file = self._setup_path()
setup_template = self._get_template(self._setup_template_path())
log.debug("Create setup.py file: %s", setup_file)
log.debug("setup.py template file: %s", self._setup_template_path())
log.debug("setup.py template date: %s", self._setup_template_data())
log.debug("setup.py template: %s", setup_template)
ofile = open(setup_file, 'w')
code = setup_template.substitute(self._setup_template_data())
log.debug("CODE: %s", code)
ofile.write(code)
ofile.close()
def _setup_template_data(self):
return {
'name': self._build_name(),
'version': self.metadata.version,
'description': 'ooi core driver',
'author': self.metadata.author,
'email': self.metadata.email,
'url': 'http://www.oceanobservatories.org',
'driver_module': self._driver_module(),
'driver_class': self._driver_class(),
'driver_path': self.metadata.relative_driver_path(),
'short_name':
self.metadata.relative_driver_path().replace('/', '_')
}
def _generate_main_file(self):
if not os.path.exists(self._versioned_dir()):
os.makedirs(self._versioned_dir())
main_file = self._main_path()
main_template = self._get_template(self._main_template_path())
log.debug("Create mi/main.py file: %s" % main_file)
log.debug("main.py template file: %s" % self._main_template_path())
log.debug("main.py template date: %s" % self._setup_template_data())
ofile = open(main_file, 'w')
code = main_template.substitute(self._setup_template_data())
ofile.write(code)
ofile.close()
def _verify_ready(self):
"""
verify that we have all the information and are in the correct state to build the egg
"""
self._verify_python()
self._verify_metadata()
self._verify_version()
def _verify_metadata(self):
"""
Ensure we have all the metadata we need to build the egg
"""
pass
def _verify_version(self, version=None):
"""
Ensure we have a good version number and that it has not already been packaged and published
"""
if version == None:
version = self.metadata.version
if not version:
raise ValidationFailure("Driver version required in metadata")
p = re.compile("^\d+\.\d+\.\d+$")
if not p.findall("%s" % version):
raise ValidationFailure(
"Version format incorrect '%s', should be x.x.x" % version)
def _verify_python(self):
"""
Ensure we build with the correct python version
"""
if sys.version_info < (2, 7) or sys.version_info >= (2, 8):
raise ValidationFailure(
"Egg generation required version 2.7 of python")
def _build_egg(self, files):
try:
self._verify_ready()
self._stage_files(files)
self._generate_setup_file()
self._generate_main_file()
cmd = "cd %s; python setup.py bdist_egg" % self._build_dir()
log.info("CMD: %s" % cmd)
os.system(cmd)
egg_file = "%s/dist/%s-%s-py2.7.egg" % (self._build_dir(
), self.metadata.relative_driver_path().replace(
'/', '_'), self.metadata.version)
# Remove all pyc files from the egg. There was a jira case that suggested
# including the compiled py files caused the drivers to run slower.
# https://jira.oceanobservatories.org/tasks/browse/OOIION-1167
cmd = "zip %s -d \*.pyc" % egg_file
log.info("CMD: %s" % cmd)
os.system(cmd)
except ValidationFailure, e:
log.error("Failed egg verification: %s" % e)
return None
log.debug("Egg file created: %s" % egg_file)
return egg_file
class EggGenerator:
"""
Generate driver egg
"""
def __init__(self, metadata, repo_dir=REPODIR):
"""
@brief Constructor
@param metadata IDK Metadata object
"""
self.metadata = metadata
self._bdir = None
self._repodir = repo_dir
if not self._tmp_dir():
raise InvalidParameters("missing tmp_dir configuration")
if not self._tmp_dir():
raise InvalidParameters("missing working_repo configuration")
self.generator = DriverGenerator(self.metadata)
test_import = __import__(self._test_module())
def _test_module(self):
return self.generator.test_modulename()
def _driver_module(self):
test_config = InstrumentDriverTestConfig()
return test_config.driver_module
def _driver_class(self):
test_config = InstrumentDriverTestConfig()
return test_config.driver_class
def _repo_dir(self):
return self._repodir
def _res_dir(self):
return os.path.join(self._versioned_dir(), 'res')
def _res_config_dir(self):
return os.path.join(self._res_dir(), 'config' )
def _tmp_dir(self):
return Config().get('tmp_dir')
def _setup_path(self):
return os.path.join(self._build_dir(), 'setup.py' )
def _setup_template_path(self):
return os.path.join(Config().template_dir(), 'setup.tmpl' )
def _main_path(self):
return os.path.join(self._versioned_dir(), 'mi/main.py' )
def _main_template_path(self):
return os.path.join(Config().template_dir(), 'main.tmpl' )
def _build_name(self):
return "%s_%s_%s_%s" % (
self.metadata.driver_make,
self.metadata.driver_model,
self.metadata.driver_name,
self.metadata.version.replace('.', '_'),
)
def _build_dir(self):
if self._bdir:
return self._bdir
self._bdir = self._generate_build_dir()
log.info( "egg build dir: %s" % self._bdir)
return self._bdir
def _generate_build_dir(self):
build_dir = os.path.join(self._tmp_dir(), self._build_name())
# clean out an old build if it exists
if os.path.exists(build_dir):
shutil.rmtree(build_dir)
return build_dir
def _versioned_dir(self):
return self._build_dir()
#return os.path.join(self._build_dir(),
# self._build_name())
def _stage_files(self, files):
"""
Copy files from the original directory into two levels of versioned
directories within a staging directory, and replace the mi namespace
with the versioned driver name.mi to account for the new directory
(only the lower versioned dir is included in the egg)
@param files - a list of files to copy into the staging directory
"""
log.error(repr(files))
# make two levels of versioned file directories, i.e.
# driverA_0_1 (= build_dir)
# driverA_0_1 (= versioned_dir)
# then copy driverA files into the bottom versioned dir
if not os.path.exists(self._build_dir()):
os.makedirs(self._build_dir())
if not os.path.exists(self._versioned_dir()):
os.makedirs(self._versioned_dir())
for file in files:
if file not in ['res/config/__init__.py', 'res/__init__.py']:
dest = os.path.join(self._versioned_dir(), file)
destdir = dirname(dest)
source = os.path.join(self._repo_dir(), file)
# this one goes elsewhere so the InstrumentDict can find it
if basename(file) == 'strings.yml':
dest = os.path.join(self._res_dir(), basename(file))
destdir = dirname(dest)
log.debug(" Copy %s => %s" % (source, dest))
# make sure the destination directory exists, if it doesn't make it
if not os.path.exists(destdir):
os.makedirs(destdir)
shutil.copy(source, dest)
# replace mi in the copied files with the versioned driver module.mi
# this is necessary because the top namespace in the versioned files starts
# with the versioned driver name directory, not mi
#driver_file = open(dest, "r")
#contents = driver_file.read()
#driver_file.close()
#new_contents = re.sub(r'(^import |^from |\'|= )mi\.|res/config/mi-logging|\'mi\'',
# self._mi_replace,
# contents,
# count=0,
# flags=re.MULTILINE)
#driver_file = open(dest, "w")
#driver_file.write(new_contents)
#driver_file.close()
# need to add mi-logging.yml special because it is not in cloned repo, only in local repository
milog = "mi-logging.yml"
dest = os.path.join(self._res_config_dir(), milog)
destdir = dirname(dest)
source = os.path.join(Config().base_dir(), "res/config/" + milog)
log.debug(" Copy %s => %s" % (source, dest))
# make sure the destination directory exists, if it doesn't make it
if not os.path.exists(destdir):
os.makedirs(destdir)
shutil.copy(source, dest)
# we need to make sure an init file is in the versioned dir and
# resource directories so that find_packages() will look in here
init_file_list = [os.path.join(self._versioned_dir(), "__init__.py"),
os.path.join(self._versioned_dir(), "res", "__init__.py"),
os.path.join(self._versioned_dir(), "res", "config", "__init__.py")]
for file in init_file_list:
self._create_file(file)
@staticmethod
def _create_file(file):
"""
Create a file if it isnt there already
"""
if not os.path.exists(file):
init_file = open(file, "w")
init_file.close()
def _mi_replace(self, matchobj):
"""
This function is used in regex sub to replace mi with the versioned
driver name followed by mi
@param matchobj - the match object from re.sub
"""
if matchobj.group(0) == 'res/config/mi-logging':
return self._build_name() + '/' + matchobj.group(0)
elif matchobj.group(0) == '\'mi\'':
return '\'' + self._build_name() + '.mi\''
else:
return matchobj.group(1) + self._build_name() + '.mi.'
def _get_template(self, template_file):
"""
@brief return a string.Template object constructed with the contents of template_file
@param template_file path to a file that containes a template
@retval string.Template object
"""
try:
infile = open(template_file)
tmpl_str = infile.read()
return Template(tmpl_str)
except IOError:
raise MissingTemplate(msg="Missing: %s" % template_file)
def _generate_setup_file(self):
if not os.path.exists(self._build_dir()):
os.makedirs(self._build_dir())
if not os.path.exists(self._build_dir()):
raise IDKException("failed to create build dir: %s" % self._build_dir())
setup_file = self._setup_path()
setup_template = self._get_template(self._setup_template_path())
log.debug("Create setup.py file: %s", setup_file )
log.debug("setup.py template file: %s", self._setup_template_path())
log.debug("setup.py template date: %s", self._setup_template_data())
log.debug("setup.py template: %s", setup_template)
ofile = open(setup_file, 'w')
code = setup_template.substitute(self._setup_template_data())
log.debug("CODE: %s", code)
ofile.write(code)
ofile.close()
def _setup_template_data(self):
return {
'name': self._build_name(),
'version': self.metadata.version,
'description': 'ooi core driver',
'author': self.metadata.author,
'email': self.metadata.email,
'url': 'http://www.oceanobservatories.org',
'driver_module': self._driver_module(),
'driver_class': self._driver_class(),
'driver_path': self.metadata.relative_driver_path(),
'short_name': self.metadata.relative_driver_path().replace('/', '_')
}
def _generate_main_file(self):
if not os.path.exists(self._versioned_dir()):
os.makedirs(self._versioned_dir())
main_file= self._main_path()
main_template = self._get_template(self._main_template_path())
log.debug("Create mi/main.py file: %s" % main_file )
log.debug("main.py template file: %s" % self._main_template_path())
log.debug("main.py template date: %s" % self._setup_template_data())
ofile = open(main_file, 'w')
code = main_template.substitute(self._setup_template_data())
ofile.write(code)
ofile.close()
def _verify_ready(self):
"""
verify that we have all the information and are in the correct state to build the egg
"""
self._verify_python()
self._verify_metadata()
self._verify_version()
def _verify_metadata(self):
"""
Ensure we have all the metadata we need to build the egg
"""
pass
def _verify_version(self, version = None):
"""
Ensure we have a good version number and that it has not already been packaged and published
"""
if version == None:
version = self.metadata.version
if not version:
raise ValidationFailure("Driver version required in metadata")
p = re.compile("^\d+\.\d+\.\d+$")
if not p.findall("%s" % version):
raise ValidationFailure("Version format incorrect '%s', should be x.x.x" % version)
def _verify_python(self):
"""
Ensure we build with the correct python version
"""
if sys.version_info < (2, 7) or sys.version_info >= (2, 8):
raise ValidationFailure("Egg generation required version 2.7 of python")
def _build_egg(self, files):
try:
self._verify_ready()
self._stage_files(files)
self._generate_setup_file()
self._generate_main_file()
cmd = "cd %s; python setup.py bdist_egg" % self._build_dir()
log.info("CMD: %s" % cmd)
os.system(cmd)
egg_file = "%s/dist/%s-%s-py2.7.egg" % (self._build_dir(),
self.metadata.relative_driver_path().replace('/', '_'),
self.metadata.version)
# Remove all pyc files from the egg. There was a jira case that suggested
# including the compiled py files caused the drivers to run slower.
# https://jira.oceanobservatories.org/tasks/browse/OOIION-1167
cmd = "zip %s -d \*.pyc" % egg_file
log.info("CMD: %s" % cmd)
os.system(cmd)
except ValidationFailure, e:
log.error("Failed egg verification: %s" % e )
return None
log.debug("Egg file created: %s" % egg_file)
return egg_file
def _driver_test_module(self):
generator = DriverGenerator(self.metadata)
return generator.test_modulename()
