def update_manifest_repo(self, dir_name, repo_commit_message):
"""
Update manifest repository based on its contents and user arguments.
:param dir_name: The directory of the repository
:return: if repo is updated, return updated manifest file path and the manifest object
otherwise, return None, None
"""
url_short_name = self.__repo.split('/')[-1][:-4]
commit_message = "update to {0} */{1} commit #{2}\n{3}"\
.format(url_short_name, self.__branch, self.__commit, repo_commit_message)
if self.__manifest_file is not None:
path_name = os.path.join(dir_name, self.__manifest_file)
if os.path.isfile(path_name):
try:
manifest = Manifest(path_name, self.__git_credentials)
manifest.update_manifest(self.__repo, self.__branch,
self.__commit)
if manifest.changed:
manifest.write_manifest_file(dir_name, commit_message,
path_name, self.__dryrun)
return path_name, manifest
else:
print "No changes to {0}".format(manifest.name)
except KeyError as error:
self.cleanup_and_exit("Failed to create an Manifest instance for the manifest file {0}\nError:{1}"\
.format(self.__manifest_file, error.message),1)
except RuntimeError as error:
self.cleanup_and_exit(
"Failed to update manifest repo\nError:{0}".format(
error.message), 1)
else:
for item in os.listdir(dir_name):
path_name = os.path.join(dir_name, item)
if os.path.isfile(path_name):
try:
manifest = Manifest(path_name, self.__git_credentials)
manifest.update_manifest(self.__repo, self.__branch,
self.__commit)
if manifest.changed:
manifest.write_manifest_file(
dir_name, commit_message, path_name,
self.__dryrun)
return path_name, manifest
else:
print "No changes to {0}".format(manifest.name)
except KeyError as error:
self.cleanup_and_exit("Failed to create an Manifest instance for the manifest file {0}\nError:{1}"\
.format(path_name, error.message),1)
except RuntimeError as error:
self.cleanup_and_exit(
"Failed to update manifest repo\nError:{0}".format(
error.message), 1)
return None, None
def build_from(package_path,
src_folder,
manifest_path=None,
description_path=None,
files=[],
excludes=[],
storage_factory=ZipFileStorage):
if manifest_path:
with file(manifest_path) as f:
manifest = Manifest()
manifest.load(f)
else:
node = resolve_path(src_folder, MANIFEST_PATH)
if node:
with node.open() as f:
manifest = Manifest()
manifest.load(f)
else:
logger.error('%s: not found' % MANIFEST_PATH)
raise IOError('%s: not found' % MANIFEST_PATH)
if description_path:
with file(description_path) as f:
description = Description.parse(f)
else:
node = resolve_path(src_folder, DESCRIPTION_PATH)
if node:
with node.open() as f:
description = Description.parse(f)
else:
raise IOError('%s: not found' % DESCRIPTION_PATH)
package_path = make_output_path(package_path, description)
package_files = dict()
from itertools import chain
required_files = chain(manifest, description.required_files())
for path in required_files:
node = resolve_path(src_folder, path)
if node is None:
raise IOError('%s: not found' % path)
package_files[path] = node
files = ((path, resolve_path(src_folder, path)) for path in files)
files = expand_folders(files)
files = exclude_files(excludes, files)
package_files.update(files)
return build(package_path,
manifest,
description,
package_files,
storage_factory=storage_factory)
def run(self, **kwargs):
"""
QThread function that executes the class in a separate thread
:param kwargs:
:return: None
"""
install_type = kwargs.get("install_type", "install")
try:
self.set_tasks.emit(3)
# Checks which type of manifest to load
if install_type == "install":
# Installs will load the bundled install.man
self._zip.extract_all(self._install_temp)
man_path = os.path.join(self._install_temp, "install.man")
self._man = Manifest(man_path)
else:
# Uninstalls will attempt to load the uninstall.man in the installation directory
man_path = os.path.join(
self._dirs.get_render_farming_install(), "uninstall.man")
self._man = Manifest(man_path)
self._add()
# Translates the manifest to InstallerItems
self._man_translated = ManifestTranslator(self._dirs, self._man)
self._add()
# Retrieves these InstallerItems
self._items = self._man_translated.get_items()
self._add()
# Checks which functions to run
if install_type == "install":
self.run_installation()
elif install_type == "upgrade":
self.run_uninstallation()
else:
self.run_uninstallation()
self.run_cleaner()
# Catches errors and prints them to the UI rather than crashing
except (IOError, OSError, RuntimeError, WindowsError,
ManifestError) as e:
self.print_error.emit(str(e))
print(self._man_translated)
else:
# If no Errors, set UI to complete
self.complete.emit()
finally:
# Always terminate the QThread
self.terminate()
def __init__(self, manifest_path, cert_path):
rhnSQL.initDB()
self.manifest = Manifest(manifest_path)
# Satellite 5 certificate
c = open(cert_path, 'r')
try:
self.sat5_cert = SatelliteCert()
content = c.read()
self.sat5_cert.load(content)
finally:
if c is not None:
c.close()
# Channel families metadata
f = open(constants.CHANNEL_FAMILY_MAPPING_PATH, 'r')
try:
self.families = json.load(f)
finally:
if f is not None:
f.close()
# product to family mapping
p = open(constants.PRODUCT_FAMILY_MAPPING_PATH, 'r')
try:
self.products = json.load(p)
finally:
if p is not None:
p.close()
self.families_to_import = []
def __init__(self, database: DB):
if not isinstance(database, DB):
raise TypeError('Controller was not supplied a DB object')
self.manifest = Manifest()
self.request_queue = multiprocessing.Queue()
self.listener = Listener(self.request_queue)
self.database = database
def __init__(self, inst_dir, mapping_file_type, pdf_fn, sitename):
# initialize internal values
self.correct = 0
self.wrong = 0
self.total = 0
self.json = None
self.logger = None
self.result = ""
self.pdf_fn = None
self.inst_dir = None
self.gsw = None
self.testapi_result = dict()
# initialize the logger
self.start_logger()
# check if its a url or directory
self.check_and_download(inst_dir, pdf_fn)
# initialize few more values
self.site_sw_dir = os.path.join(self.inst_dir, 'site', sitename,
'software')
self.type_sw_dir = self.get_type_sw(self.inst_dir, sitename)
if self.json is None:
self.read_json(self.pdf_fn)
self.manifest = Manifest(inst_dir=os.path.join(self.inst_dir, 'site', sitename), \
mapping_file_dir=os.path.join("mapping", mapping_file_type), logger=self.logger)
def manifest():
return Manifest({
"JobID": 12345,
"geometry": {
"height": 1000,
"width": 1000,
"fov": [1.0, 1.0, 1.0],
"upperleft": {
"position": [250, 250],
"distance": 21.21320344
},
"upperright": {
"position": [750, 250],
"distance": 21.21320344
},
"lowerleft": {
"position": [250, 750],
"distance": 21.21320344
},
"lowerright": {
"position": [750, 750],
"distance": 21.21320344
}
}
})
def __init__(self, s3, manifestPath, localWorkingDir):
self.manifestPath = manifestPath
self.manifest = Manifest(manifestPath)
self.s3interface = S3Interface(s3, self.manifest.GetBucketName(), localWorkingDir)
metafac = InstanceMetadataFactory(self.manifest)
self.instanceManager = InstanceManager(self.s3interface, self.manifest, metafac)
self.manifestKey = "/".join([self.manifest.GetS3KeyPrefix(), "manifest.json"])
def show_main():
doc = '''Usage: oxt-pkg-show [options] <package-path>
--help Print this screen.
'''
from docopt import docopt
args = docopt(doc)
logging.basicConfig(level=logging.INFO)
package_path = args['<package-path>']
with open_storage(package_path) as pkg:
with resolve_path(pkg, MANIFEST_PATH).open() as f:
manifest = Manifest()
manifest.load(f)
with resolve_path(pkg, DESCRIPTION_PATH).open() as f:
description = Description.parse(f)
from description import print_human_readable
print_human_readable(description, pkg)
for path in manifest:
item = manifest[path]
print path, item['media-type'],
node = resolve_path(pkg, path)
if node:
print '-- OK'
else:
print '-- MISSING'
def paper_manifest():
return Manifest({
"JobID": 11111,
"geometry": {
"height": 875,
"width": 1167,
"fov": [4.0, 3.0, 3.0],
"upperleft": {
"position": [357, 674],
"distance": 83.25
},
"upperright": {
"position": [832, 674],
"distance": 83.25
},
"lowerleft": {
"position": [255, 766],
"distance": 61.25
},
"lowerright": {
"position": [939, 768],
"distance": 61.25
}
}
})
def create_manifest():
manifest: Manifest = Manifest(context=context,
manifest_files=manifest_files,
resource_factory=MockResource)
assert manifest.manifest_files == manifest_files
assert manifest.context == context
return manifest
def _read_manifest(self):
try:
with self._fetcher.retrieve_raw_file(_common._MANIFEST_NAME) as manifest_file:
self.manifest = Manifest(manifest_file)
self.fqrn = self.manifest.repository_name
except FileNotFoundInRepository, e:
raise RepositoryNotFound(self._storage_location)
def run(self):
try:
manifest = Manifest(Manifest.locate_file(self.config.cwd))
self.install(manifest)
except MissingNpmShrinkwrap as e:
Log.error(e.message)
sys.exit(1)
def build_manifest(
docker_obj,
skip_fs_dump,
kml,
output,
envs,
cmd,
) -> Manifest:
manifest = Manifest()
manifest.runtime.entry = (docker_obj['Config']['Entrypoint'] or
[]) + (docker_obj['Config']['Cmd'] or []) + cmd
manifest.runtime.envs = docker_obj['Config']['Env'] + envs
manifest.runtime.working_directory = docker_obj['Config']['WorkingDir']
manifest.linux_configuration.options, manifest.runtime.enabled_init_options = get_linux_options(
)
if manifest.runtime.enabled_init_options[0] == '/init':
print("Entry command cannot use /init", file=sys.stderr)
exit(1)
manifest.linux_configuration.kml = kml
manifest.filesystem = dump_fs(docker_obj['RepoTags'][0], skip_fs_dump,
output)
return manifest
def initiate_manifest(self):
"""
initial manifest and validate it
:return: None
"""
self._manifest = Manifest(self._source_manifest_file)
self._manifest.validate_manifest()
def main():
"""Main function for invoking the bootstrap process
:raises Exception: When the invoking user is not root and --dry-run isn't specified
"""
# Get the commandline arguments
opts = get_opts()
# Require root privileges, except when doing a dry-run where they aren't needed
import os
if os.geteuid() != 0 and not opts['--dry-run']:
raise Exception('This program requires root privileges.')
# Set up logging
setup_loggers(opts)
# Load the manifest
from manifest import Manifest
manifest = Manifest(path=opts['MANIFEST'])
# Everything has been set up, begin the bootstrapping process
run(manifest,
debug=opts['--debug'],
pause_on_error=opts['--pause-on-error'],
dry_run=opts['--dry-run'])
def write_downstream_parameter_file(build_directory, manifest_file, is_official_release, parameter_file):
try:
params = {}
# Add rackhd version to downstream parameters
rackhd_repo_dir = os.path.join(build_directory, "RackHD")
version_generator = VersionGenerator(rackhd_repo_dir)
rackhd_version = version_generator.generate_package_version(is_official_release)
if rackhd_version != None:
params['RACKHD_VERSION'] = rackhd_version
else:
raise RuntimeError("Version of {0} is None. Maybe the repository doesn't contain debian directory ".format(rackhd_repo_dir))
# Add the commit of repository RackHD/RackHD to downstream parameters
manifest = Manifest(manifest_file)
# commit of repository RackHD/RackHD
rackhd_commit = ''
for repo in manifest.repositories:
repository = repo['repository'].lower()
if repository.endswith('rackhd') or repository.endswith('rackhd.git'):
rackhd_commit = repo['commit-id']
if rackhd_commit != '':
params['RACKHD_COMMIT'] = rackhd_commit
else:
raise RuntimeError("commit-id of RackHD is None. Please check the manifest {0}".format(manifest_file))
# Write downstream parameters to downstream parameter file.
common.write_parameters(parameter_file, params)
except Exception, e:
raise RuntimeError("Failed to write downstream parameter file \ndue to {0}".format(e))
def get_modules(self, modules_dir, auto_install=True): print '[DEBUG] Detecting modules in %s' % modules_dir modules = [] if auto_install: parent_dir = os.path.dirname(os.path.abspath(modules_dir)) self.auto_install_modules(parent_dir) if not os.path.exists(modules_dir): return modules for platform in os.listdir(modules_dir): platform_dir = os.path.join(modules_dir, platform) if not os.path.isdir(platform_dir): continue if platform in ['osx', 'win32', 'linux']: continue # skip desktop modules # iterate through the platform directory so we can get versioned modules too for root, dirs, files in os.walk(platform_dir): dirs.sort(reverse=True) for module_dir in dirs: module_dir = os.path.join(root, module_dir) manifest_file = os.path.join(module_dir, 'manifest') if not os.path.exists(manifest_file): continue manifest = Manifest(manifest_file) print '[DEBUG] Detected module for %s: %s %s @ %s' % (manifest.platform, manifest.moduleid, manifest.version, module_dir) modules.append(Module(module_dir, manifest)) return modules
def test_errorThrownOnMissingRequiredS3DataInJob(self):
self.assertRaises(
ValueError, lambda: Manifest(
self.writeTestJsonFile({
"ProjectName":
"testProject",
"BucketName":
"bucket",
"Documents": [
{
"Name": "document",
"Direction": "AWSToLocal",
"LocalPath": "mylocalPath",
"AWSInstancePath": "awsinstancepath"
},
],
"InstanceJobs": [{
"Id":
1,
"RequiredS3Data": ["a_missing_document"],
"Commands": [{
"Command": "run.exe",
"Args": []
}]
}]
})))
def main(args):
global android_sdk
# command platform project_dir
command = args[1]
platform = args[2]
project_dir = os.path.expanduser(args[3])
manifest = Manifest(os.path.join(project_dir, 'manifest'))
error = False
if is_android(platform):
build_properties = read_properties(
open(os.path.join(project_dir, 'build.properties')))
android_sdk_path = os.path.dirname(
os.path.dirname(build_properties['android.platform']))
android_sdk = AndroidSDK(android_sdk_path)
if command == 'run':
def run_callback(gen_project_dir):
script = os.path.abspath(
os.path.join(template_dir, '..', platform, 'builder.py'))
script_args = [script, 'run', gen_project_dir]
if is_android(platform):
script_args.append(android_sdk.get_android_sdk())
rc = run_python(script_args)
# run the project
if rc == 1:
if is_ios(platform):
error = os.path.join(gen_project_dir, 'build', 'iphone',
'build', 'build.log')
print "[ERROR] Build Failed. See: %s" % os.path.abspath(
error)
else:
print "[ERROR] Build Failed."
stage(platform, project_dir, manifest, run_callback)
elif command == 'run-emulator':
if is_android(platform):
def run_emulator_callback(gen_project_dir):
script = os.path.abspath(
os.path.join(template_dir, '..', platform, 'builder.py'))
run_python([
script, 'run-emulator', gen_project_dir,
android_sdk.get_android_sdk()
])
stage(platform, project_dir, manifest, run_emulator_callback)
elif command == 'docgen':
if is_android(platform):
dest_dir = args[4]
docgen(project_dir, dest_dir)
if error:
sys.exit(1)
else:
sys.exit(0)
def _read_manifest(self):
try:
manifest_file = self.retrieve_file(_common._MANIFEST_NAME)
self.manifest = Manifest(manifest_file)
manifest_file.close()
self.fqrn = self.manifest.repository_name
except FileNotFoundInRepository, e:
raise RepositoryNotFound(self.endpoint)
def run(args):
"""Runs the bootstrapping process
Args:
args (dict): Dictionary of arguments from the commandline
"""
# Load the manifest
from manifest import Manifest
manifest = Manifest(args.manifest)
# Get the tasklist
from tasklist import TaskList
tasklist = TaskList()
# 'resolve_tasks' is the name of the function to call on the provider and plugins
tasklist.load('resolve_tasks', manifest)
# Create the bootstrap information object that'll be used throughout the bootstrapping process
from bootstrapinfo import BootstrapInformation
bootstrap_info = BootstrapInformation(manifest=manifest, debug=args.debug)
try:
# Run all the tasks the tasklist has gathered
tasklist.run(info=bootstrap_info, dry_run=args.dry_run)
# We're done! :-)
log.info('Successfully completed bootstrapping')
except (Exception, KeyboardInterrupt) as e:
# When an error occurs, log it and begin rollback
log.exception(e)
if args.pause_on_error:
# The --pause-on-error is useful when the user wants to inspect the volume before rollback
raw_input('Press Enter to commence rollback')
log.error('Rolling back')
# Create a new tasklist to gather the necessary tasks for rollback
rollback_tasklist = TaskList()
# Create a useful little function for the provider and plugins to use,
# when figuring out what tasks should be added to the rollback list.
def counter_task(task, counter):
"""counter_task() adds the second argument to the rollback tasklist
if the first argument is present in the list of completed tasks
Args:
task (Task): The task to look for in the completed tasks list
counter (Task): The task to add to the rollback tasklist
"""
if task in tasklist.tasks_completed and counter not in tasklist.tasks_completed:
rollback_tasklist.tasks.add(counter)
# Ask the provider and plugins for tasks they'd like to add to the rollback tasklist
# Any additional arguments beyond the first two are passed directly to the provider and plugins
rollback_tasklist.load('resolve_rollback_tasks', manifest,
counter_task)
# Run the rollback tasklist
rollback_tasklist.run(info=bootstrap_info, dry_run=args.dry_run)
log.info('Successfully completed rollback')
def test_GetBucketName(self):
m = Manifest(
self.writeTestJsonFile({
"ProjectName": "projectname",
"BucketName": "myBucketName",
"Documents": [],
"InstanceJobs": []
}))
self.assertEqual(m.GetBucketName(), "myBucketName")
def run(opts):
"""Runs the bootstrapping process
:params dict opts: Dictionary of options from the commandline
"""
# Load the manifest
from manifest import Manifest
manifest = Manifest(opts['MANIFEST'])
# Get the tasklist
from tasklist import load_tasks
from tasklist import TaskList
tasks = load_tasks('resolve_tasks', manifest)
tasklist = TaskList(tasks)
# 'resolve_tasks' is the name of the function to call on the provider and plugins
# Create the bootstrap information object that'll be used throughout the bootstrapping process
from bootstrapinfo import BootstrapInformation
bootstrap_info = BootstrapInformation(manifest=manifest,
debug=opts['--debug'])
try:
# Run all the tasks the tasklist has gathered
tasklist.run(info=bootstrap_info, dry_run=opts['--dry-run'])
# We're done! :-)
log.info('Successfully completed bootstrapping')
except (Exception, KeyboardInterrupt) as e:
# When an error occurs, log it and begin rollback
log.exception(e)
if opts['--pause-on-error']:
# The --pause-on-error is useful when the user wants to inspect the volume before rollback
raw_input('Press Enter to commence rollback')
log.error('Rolling back')
# Create a useful little function for the provider and plugins to use,
# when figuring out what tasks should be added to the rollback list.
def counter_task(taskset, task, counter):
"""counter_task() adds the second argument to the rollback tasklist
if the first argument is present in the list of completed tasks
:param set taskset: The taskset to add the rollback task to
:param Task task: The task to look for in the completed tasks list
:param Task counter: The task to add to the rollback tasklist
"""
if task in tasklist.tasks_completed and counter not in tasklist.tasks_completed:
taskset.add(counter)
# Ask the provider and plugins for tasks they'd like to add to the rollback tasklist
# Any additional arguments beyond the first two are passed directly to the provider and plugins
rollback_tasks = load_tasks('resolve_rollback_tasks', manifest,
tasklist.tasks_completed, counter_task)
rollback_tasklist = TaskList(rollback_tasks)
# Run the rollback tasklist
rollback_tasklist.run(info=bootstrap_info, dry_run=opts['--dry-run'])
log.info('Successfully completed rollback')
raise e
def getRPath(self):
if not self.manifest:
return False
manifest = Manifest(self.manifest)
path_ = self.build_path + "/gen/" + "/".join(
manifest.getPackageName().split(".")) + "/R.java"
if path.exists(path_):
return path_
return False
def test_errorThrownOnExtraJsonKeys(self):
self.assertRaises(
ValueError, lambda: Manifest(
self.writeTestJsonFile({
"ProjectName": "projectname",
"BucketName": "myBucketName",
"Documents": [],
"InstanceJobs": [],
"extra": 1
})))
def __init__(self, request_queue):
self.manifest = Manifest()
self.request_queue = request_queue
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,
1)
self.buffer_size = self.manifest.listener_buffer_size
self.port_number = self.manifest.port_number
self.server_ip = ''
self.req_count = 0
def test_remove_version():
m = Manifest()
m.create('manifest.json', 'test-guid', 'test-app-0', 'test-description',
'overview....', 'jesseward', 'utility')
m.add_version('test-app-0', '0.0.0', 1, 2, 3, 4, 5)
m.add_version('test-app-0', '1.0.0', 1, 2, 3, 4, 5)
assert len(m.versions('test-app-0')) == 2
# remove version at index 1
m.remove_version('test-app-0', '1.0.0')
assert len(m.versions('test-app-0')) == 1
m.close()
def test_remove_application():
# create a pair of applications in the manifest
m = Manifest()
m.create('manifest.json', 'test-guid', 'test-app-0', 'test-description',
'overview....', 'jesseward', 'utility')
m.add_application('test-guid', 'test-app-1', 'test-description',
'overview....', 'jesseward', 'utility')
assert len(m.applications()) == 2
# remove index 0 / application test-app-0
m.remove_application('test-app-0')
assert len(m.applications()) == 1
def get_repositories_under_test(manifest_file):
"""
get repositories whose commit-id is something like: origin/pr/111/merge
"""
manifest = Manifest(manifest_file)
repos_under_test = []
for repo in manifest.repositories:
if "under-test" in repo:
if repo["under-test"] is True:
repo_name = common.strip_suffix(
os.path.basename(repo["repository"]), ".git")
repos_under_test.append(repo_name)
return repos_under_test
