def CreateConfigSymlink(launcher_config_file):
"""Creates symbolic link to default startup YAML config file."""
if not launcher_config_file.startswith(env.GetResourcesDir()):
raise ShopFloorLauncherException('Config file should be in %r' %
env.GetResourcesDir())
dest_file = os.path.join(env.runtime_dir, 'shopfloor.yaml')
if os.path.exist(dest_file):
os.unlink(dest_file)
os.symlink(launcher_config_file, dest_file)
def Handle(self, args, request):
"""Verifies the resource files and structure of YAML config file."""
cwd = request['cwd']
all_passed = True
local_config = set([
os.path.basename(res) for res in glob.glob(
os.path.join(cwd, 'resources', 'shopfloor.yaml#*'))
])
system_config = set([
os.path.basename(res) for res in glob.glob(
os.path.join(env.GetResourcesDir(), 'shopfloor.yaml#*'))
])
all_config = set().union(local_config, system_config)
config_name = os.path.basename(args[0])
if config_name not in all_config:
raise ShopFloorLauncherException('Config file not found %r' %
config_name)
response = ['YAML config file: %s' % config_name]
if config_name in system_config:
response.append(' installed')
resources = utils.ListResources(args[0])
for res in resources:
try:
utils.VerifyResource(res)
response.append(' OK: %s' % res)
except Exception:
response.append(' FAIL: %s' % res)
all_passed = False
if all_passed:
response.append('OK: all resources in YAML config are good.')
else:
response.append('ERROR: YAML config verify failed.')
return '\n'.join(response)
def _PrepareUpdateBundle(self, bundle):
bundle_file = os.path.join(env.GetResourcesDir(), bundle)
bundle_dir = os.path.join(env.GetUpdatesDir(), 'factory')
latest_md5file = os.path.join(bundle_dir, LATEST_MD5FILE)
latest_md5sum = None
bundle_md5sum = None
# Check latest deployed update bundle
if os.path.isfile(latest_md5file):
with open(latest_md5file, 'r') as f:
latest_md5sum = f.readline().strip()
if latest_md5sum[0:8] == bundle[-8:]:
return
# Check other deployed bundle
bundle_md5sum = Md5sum(bundle_file)
dest_dir = os.path.join(bundle_dir, bundle_md5sum)
if not os.path.isfile(os.path.join(dest_dir, 'factory', 'MD5SUM')):
if os.path.isdir(dest_dir):
shutil.rmtree(dest_dir)
TryMakeDirs(dest_dir)
logging.info('Stagging into %s', dest_dir)
try:
subprocess.check_call(['tar', '-xjf', bundle_file, '-C', dest_dir])
except subprocess.CalledProcessError as e:
logging.exception('Failed to extract update bundle %s to %s',
bundle, dest_dir)
raise e
with open(os.path.join(dest_dir, 'factory', 'MD5SUM'), 'w') as f:
f.write(bundle_md5sum)
with open(latest_md5file, 'w') as f:
f.write(bundle_md5sum)
def PrepareResources(resources):
"""Copies resource files to system resouce folder.
Resources are static files with MD5 checksum appended in filename. This
function verifies file integrity and copies them to system folder.
Args:
resources: list of full path name to resource files. If the sources
file does not exist but already in system resource folder,
this function verifies the target file integrity only.
Raises:
ShopFloorLauncherException: when the resource file does not exist,
nor in system resource folder.
"""
dest_dir = env.GetResourcesDir()
for pathname in resources:
if os.path.isfile(pathname):
VerifyResource(pathname)
# Copy the file and keep its state
shutil.copy2(pathname, dest_dir)
else:
fname = os.path.basename(pathname)
dest_resource_name = os.path.join(dest_dir, fname)
if os.path.isfile(dest_resource_name):
VerifyResource(dest_resource_name)
else:
raise ShopFloorLauncherException('File not found: %r' % fname)
def Handle(self, args, request):
"""Lists available configurations."""
response = ['OK: available YAML configurations:']
config_files = glob.glob(
os.path.join(env.GetResourcesDir(), "shopfloor.yaml#*"))
for config in config_files:
filename = os.path.basename(config)
yaml_config = LauncherYAMLConfig(config)
response.append(' %s - %s' %
(filename, yaml_config['info']['version']))
return '\n'.join(response)
def Handle(self, args, request):
"""Deploys new configuration file."""
new_config_file = os.path.join(env.GetResourcesDir(), args[0])
try:
# Update new configuration and restart services
utils.UpdateConfig(new_config_file)
except Exception as e:
return 'ERROR: failed to deploy new configuration %s' % e
if os.getuid() == 0:
utils.CreateConfigSymlink(new_config_file)
#TODO(rong): utils.CreateBinSymlink()
return "OK: %r deployed successfully." % new_config_file
def main():
parser = optparse.OptionParser()
parser.add_option('-c',
'--config',
dest='yaml_config',
default='shopfloor.yaml')
parser.add_option('-t',
'--test',
dest='test_run',
action='store_true',
default=False)
parser.add_option('-l',
'--local',
dest='local_dir',
action='store_true',
default=False)
(options, args) = parser.parse_args()
if args:
parser.error('Invalid args: %s' % ' '.join(args))
log_format = '%(asctime)s %(levelname)s '
log_verbosity = logging.INFO
if options.test_run:
log_format += '(%(filename)s:%(lineno)d) '
log_verbosity = logging.DEBUG
log_format += '%(message)s'
logging.basicConfig(level=log_verbosity, format=log_format)
server_path = os.path.realpath(__file__)
search_dirs = []
# Set runtime_dir when running locally.
if options.test_run and not server_path.startswith(
constants.SHOPFLOOR_INSTALL_DIR):
if _RESOURCE_FACTORY_PAR in server_path:
env.runtime_dir = server_path[0:server_path.
index(_RESOURCE_FACTORY_PAR)]
else:
env.runtime_dir = os.path.join(os.path.dirname(server_path),
'testdata')
search_dirs.append(os.path.dirname(server_path))
search_dirs += [env.runtime_dir, env.GetResourcesDir()]
config_file = utils.SearchFile(options.yaml_config, search_dirs)
if config_file and os.path.isfile(config_file):
Run(config_file)
else:
raise ShopFloorLauncherException(
'Launcher YAML config file not found: %s' % options.yaml_config)
def Deploy(args):
"""Deploys new shopfloor YAML configuration."""
res_dir = env.GetResourcesDir()
new_config_file = os.path.join(res_dir, args.config)
if not os.path.isfile(new_config_file):
logging.error('Config file not found: %s', new_config_file)
return
# Verify listed resources.
try:
resources = [
os.path.join(res_dir, res)
for res in utils.ListResources(new_config_file)
]
map(utils.VerifyResource, resources)
except (IOError, ShopFloorLauncherException) as err:
logging.exception('Verify resources failed: %s', err)
return
# Get new factory.par resource name from YAML config.
launcher_config = yamlconf.LauncherYAMLConfig(new_config_file)
new_factory_par = os.path.join(
res_dir, launcher_config['shopfloor']['factory_software'])
# Restart shopfloor daemon.
config_file = os.path.join(env.runtime_dir, CONFIG_FILE)
factory_par = os.path.join(env.runtime_dir, FACTORY_SOFTWARE)
prev_config_file = os.path.join(env.runtime_dir, PREV_CONFIG_FILE)
prev_factory_par = os.path.join(env.runtime_dir, PREV_FACTORY_SOFTWARE)
shopfloor_util = os.path.join(env.runtime_dir, SHOPFLOOR_UTIL)
shopfloor_daemon = os.path.join(env.runtime_dir, SHOPFLOOR_DAEMON)
StopShopfloord()
try:
file_utils.TryUnlink(prev_config_file)
file_utils.TryUnlink(prev_factory_par)
if os.path.isfile(config_file):
shutil.move(config_file, prev_config_file)
if os.path.isfile(factory_par):
shutil.move(factory_par, prev_factory_par)
os.symlink(new_factory_par, factory_par)
os.symlink(new_config_file, config_file)
if not os.path.isfile(shopfloor_util):
os.symlink(factory_par, shopfloor_util)
if not os.path.isfile(shopfloor_daemon):
os.symlink(factory_par, shopfloor_daemon)
except (OSError, IOError) as err:
logging.exception('Can not deploy new config: %s (%s)',
new_config_file, err)
logging.exception('Shopfloor didn\'t restart.')
return
StartShopfloord()
def Import(self):
"""Imports resources.
This function converts downloadble images, netboot kernel image, factory
update bundle to resources. Download configuration default.conf#[md5sum]
and shopfloor configuration shopfloor.yaml#[md5sum] will be generated.
Raises:
IOError: when disk full or copy failed.
ImporterError: when resources name conflict.
"""
# File tuple list generated from bundle (src_file, md5sum).
bundle_files = []
bundle_files.extend(self.download_files)
bundle_files.extend([
self.netboot_file, self.update_bundle, self.hwid_bundle,
self.factory_software
])
bundle_files = filter(None, bundle_files)
# File-resource tuple list (src_file, res_file).
copy_list = []
# Hash conflict tuble list (src_file, src_md5, dest_file, dest_md5).
conflict_list = []
# Unexpected error list of tuple (src_file, res_base_name, str(e)).
error_list = []
# Validate source files and destination resources
for file_tuple in bundle_files:
f, md5sum = file_tuple
res_base_name = self.GetResourceName(file_tuple)
dest_file = os.path.join(env.GetResourcesDir(), res_base_name)
if os.path.isfile(dest_file):
try:
utils.VerifyResource(dest_file)
logging.info(' - resource exists, skip: %s', dest_file)
except ShopFloorLauncherException:
copy_list.append(f, dest_file)
except Exception, e:
error_list.append((f, res_base_name, str(e)))
else:
# Do not trust os.state(), perform a non-shallow file compare.
if not filecmp.cmp(f, dest_file, shallow=False):
# 32bit hash conflict
conflict_list.append(
(f, md5sum, dest_file, utils.Md5sum(dest_file)))
else:
copy_list.append((f, dest_file))
def WriteDownloadConfig(self):
"""Generate download config and writes to resource folder.
Raises:
IOError: when write failed.
"""
if not self.download_files:
return
config = []
# Also prepare a dictionary based config for shopfloor.yaml .
# sf_config holds a partial shopfloor.yaml dict that indicates
# resources under:
# <root>
# |--network_install
# | |--board
# | | |--default # <== sf_config here
# | | | |--default
# | | | | |--config: default.conf#
# | | | | |--oem: oem.gz#
# . . . . .
sf_config = {}
config.append('# date: %s' % self.datetime)
config.append('# bundle: %s' % self.bundle)
for file_tuple in self.download_files:
f = file_tuple[0]
base_name = os.path.basename(f)
# Skip non-gzipped file and remove '.gz' from file name.
if not base_name.endswith('.gz'):
continue
base_name = base_name[:-3]
res_name = self.GetResourceName(file_tuple)
url_name = urllib.quote(res_name)
sha1sum = utils.B64Sha1(f)
channel = self.GetChannel(base_name)
config.append(':'.join([channel, url_name, sha1sum]))
sf_config[base_name] = res_name
default_conf = '\n'.join(config)
conf_md5 = hashlib.md5(default_conf).hexdigest() # pylint: disable=E1101
sf_config['config'] = self.GetResourceName(('default.conf', conf_md5))
self.download_config = os.path.join(env.GetResourcesDir(),
sf_config['config'])
open(self.download_config, 'w').write(default_conf)
self.download_config_dict = sf_config
def WriteShopfloorConfig(self):
"""Generates shopfloor.yaml from the factory bundle."""
config = self.ReadShopfloorTemplate()
# TODO(rongchang): import info from bundle MANIFEST.yaml
config['info']['version'] = os.path.basename(self.bundle)
config['info']['note'] = str(self.datetime)
# Patch download configuration
if self.download_config_dict:
config['network_install']['board'][
'default'] = self.download_config_dict
# Patch netboot kernel resource
if self.netboot_file:
config['network_install']['netboot_kernel'] = (
self.GetResourceName(self.netboot_file))
# Patch update bundle and HWID bundle
if 'updater' not in config:
config['updater'] = {}
if self.update_bundle:
config['updater']['update_bundle'] = self.GetResourceName(
self.update_bundle)
if self.hwid_bundle:
config['updater']['hwid_bundle'] = self.GetResourceName(
self.hwid_bundle)
# Patch factory software resource
if self.factory_software:
config['shopfloor']['factory_software'] = (self.GetResourceName(
self.factory_software))
yaml_text = yaml.dump(config, default_flow_style=False)
yaml_md5 = hashlib.md5(yaml_text).hexdigest() # pylint: disable=E1101
self.shopfloor_config = os.path.join(
env.GetResourcesDir(),
self.GetResourceName(('shopfloor.yaml', yaml_md5)))
open(self.shopfloor_config, 'w').write(yaml_text)
logging.info(
'Shopfloor import from bundle complete.\n\n\tNew config: %s\n',
os.path.basename(self.shopfloor_config))
def List(dummy_args):
"""Lists available configurations."""
file_list = glob.glob(
os.path.join(env.GetResourcesDir(), 'shopfloor.yaml#*'))
config = None
version = None
note = None
count = 0
for fn in file_list:
try:
config = yaml.load(open(fn, 'r'))
version = config['info']['version']
note = config['info']['note']
except: # pylint: disable=W0702
continue
logging.info(os.path.basename(fn))
logging.info(' - version: %s', version)
logging.info(' - note: %s', note)
count += 1
if count > 0:
logging.info('OK: found %d configuration(s).', count)
else:
logging.info('ERROR: no configuration found.')
def _GenerateConfigFile(self, yaml_conf, conf_file):
"""Generates lighty config from YamlConfig."""
httpd_port = yaml_conf['shopfloor']['port']
lighty_modules = [
'mod_access', 'mod_accesslog', 'mod_alias', 'mod_fastcgi',
'mod_rewrite', 'mod_redirect'
]
cpu_count = multiprocessing.cpu_count()
dashboard_dir = os.path.join(env.runtime_dir, 'dashboard')
pid_file = os.path.join(env.runtime_dir, 'run', 'httpd.pid')
access_log = os.path.join(env.runtime_dir, 'log', 'httpd_access.log')
error_log = os.path.join(env.runtime_dir, 'log', 'httpd_error.log')
# Verify port
self.CheckPortPrivilage(httpd_port)
self.CheckPortPrivilage(env.fcgi_port)
# A minimal lighty config
lighty_conf = {
# Server tag and modules
'server.tag': 'usf-httpd',
'server.modules': lighty_modules,
# Document root, files and dirs
'index-file.names': ['index.html'],
'dir-listing.activate': 'enable',
'server.follow-symlink': 'enable',
'server.range-requests': 'enable',
'server.document-root': dashboard_dir,
'server.pid-file': pid_file,
# Access log
'accesslog.filename': access_log,
'server.errorlog': error_log,
# Performance options
'server.max-worker': cpu_count * 2,
'server.max-fds': constants.HTTPD_MAX_FDS,
'server.max-connections': constants.HTTPD_MAX_CONN,
'connection.kbytes-per-second': 0,
'server.kbytes-per-second': 0,
# Network options
'server.bind': env.bind_address,
'server.port': httpd_port,
# Blocks section, keep the order
'alias.url': {
'/res': env.GetResourcesDir()
},
LightyConditionalType('$HTTP["url"] =~ "^/$"'): {
'fastcgi.server': {
'/': [{
'host': '127.0.0.1',
'port': env.fcgi_port,
'check-local': 'disable'
}]
}
}
}
self._WriteLightyConf(lighty_conf, conf_file)
if 'network_install' in yaml_conf:
download_port = yaml_conf['network_install']['port']
if httpd_port != download_port:
download_conf = {
LightyConditionalType('$SERVER["socket"] == ":%d"' % download_port):
{}
}
self._WriteLightyConf(download_conf, conf_file, append=True)
# Generate conditional HTTP accelerator blocks.
if 'reverse_proxies' in yaml_conf['network_install']:
map((lambda proxy: self._WriteLightyConf(
self._ProxyBlock(proxy), conf_file, append=True)),
yaml_conf['network_install']['reverse_proxies'])
# Generate default download conf symlink.
for board, resmap in yaml_conf['network_install'][
'board'].iteritems():
link_name = os.path.join(env.GetResourcesDir(),
board + '.conf')
conf_name = os.path.join(env.GetResourcesDir(),
resmap['config'])
try:
os.unlink(link_name)
except os.error:
pass
os.symlink(conf_name, link_name)
def _PrepareHwidBundle(self, bundle):
os.symlink(os.path.join(env.GetResourcesDir(), bundle),
os.path.join(env.GetUpdatesDir(), bundle[0:-9]))
