def _ValidateArchive(archive, actual_sha1, actual_size):
if actual_size != archive.size:
raise Error('Size mismatch on "%s". Expected %s but got %s bytes' %
(archive.name, archive.size, actual_size))
if actual_sha1 != archive.GetChecksum():
raise Error('SHA1 checksum mismatch on "%s". Expected %s but got %s' %
(archive.name, archive.GetChecksum(), actual_sha1))
def UpdateBundle():
'''Helper to install a bundle'''
archive = bundle.GetHostOSArchive()
(_, _, path, _, _, _) = urlparse.urlparse(archive['url'])
dest_filename = os.path.join(options.user_data_dir, path.split('/')[-1])
sha1, size = DownloadArchiveToFile(archive, dest_filename)
if sha1 != archive.GetChecksum():
raise Error("SHA1 checksum mismatch on '%s'. Expected %s but got %s" %
(bundle.name, archive.GetChecksum(), sha1))
if size != archive.size:
raise Error("Size mismatch on Archive. Expected %s but got %s bytes" %
(archive.size, size))
InfoPrint('Updating bundle %s to version %s, revision %s' % (
(bundle.name, bundle.version, bundle.revision)))
ExtractInstaller(dest_filename, bundle_update_path)
if bundle.name != SDK_TOOLS:
repath = bundle.get('repath', None)
if repath:
bundle_move_path = os.path.join(bundle_update_path, repath)
else:
bundle_move_path = bundle_update_path
RenameDir(bundle_move_path, bundle_path)
if os.path.exists(bundle_update_path):
RemoveDir(bundle_update_path)
os.remove(dest_filename)
local_manifest.MergeBundle(bundle)
WriteToFile(local_manifest_path, local_manifest)
def LoadConfig(raise_on_error=False):
path = os.path.join(USER_DATA_DIR, CONFIG_FILENAME)
cfg = config.Config()
if not os.path.exists(path):
return cfg
try:
try:
with open(path) as f:
file_data = f.read()
except IOError as e:
raise Error('Unable to read config from "%s".\n %s' % (path, e))
try:
cfg.LoadJson(file_data)
except Error as e:
raise Error('Parsing config file from "%s" failed.\n %s' % (path, e))
return cfg
except Error as e:
if raise_on_error:
raise
else:
logging.warn(str(e))
return cfg
def ExtractArchive(self, archive, extract_dir, rename_from_dir,
rename_to_dir):
tar_file = None
archive_path = os.path.join(self.user_data_dir, archive)
extract_path = os.path.join(self.install_dir, extract_dir)
rename_from_path = os.path.join(self.install_dir, rename_from_dir)
rename_to_path = os.path.join(self.install_dir, rename_to_dir)
# Extract to extract_dir, usually "<bundle name>_update".
# This way if the extraction fails, we haven't blown away the old bundle
# (if it exists).
sdk_update_common.RemoveDir(extract_path)
sdk_update_common.MakeDirs(extract_path)
curpath = os.getcwd()
tar_file = None
try:
try:
tar_file = cygtar.CygTar(archive_path, 'r', verbose=True)
except Exception as e:
raise Error('Can\'t open archive "%s".\n %s' %
(archive_path, e))
try:
logging.info('Changing the directory to %s' % (extract_path, ))
os.chdir(extract_path)
except Exception as e:
raise Error('Unable to chdir into "%s".\n %s' %
(extract_path, e))
logging.info('Extracting to %s' % (extract_path, ))
tar_file.Extract()
logging.info('Changing the directory to %s' % (curpath, ))
os.chdir(curpath)
logging.info('Renaming %s->%s' %
(rename_from_path, rename_to_path))
sdk_update_common.RenameDir(rename_from_path, rename_to_path)
finally:
# Change the directory back so we can remove the update directory.
os.chdir(curpath)
# Clean up the ..._update directory.
try:
sdk_update_common.RemoveDir(extract_path)
except Exception as e:
logging.error('Failed to remove directory \"%s\". %s' %
(extract_path, e))
if tar_file:
tar_file.Close()
# Remove the archive.
os.remove(archive_path)
def WriteConfig(cfg):
path = os.path.join(USER_DATA_DIR, CONFIG_FILENAME)
try:
sdk_update_common.MakeDirs(USER_DATA_DIR)
except Exception as e:
raise Error('Unable to create directory "%s".\n %s' % (USER_DATA_DIR, e))
cfg_json = cfg.ToJson()
try:
with open(path, 'w') as f:
f.write(cfg_json)
except IOError as e:
raise Error('Unable to write config to "%s".\n %s' % (path, e))
def WriteLocalManifest(manifest):
path = os.path.join(USER_DATA_DIR, MANIFEST_FILENAME)
try:
sdk_update_common.MakeDirs(USER_DATA_DIR)
except Exception as e:
raise Error('Unable to create directory "%s".\n %s' % (USER_DATA_DIR, e))
try:
manifest_json = manifest.GetDataAsString()
except Exception as e:
raise Error('Error encoding manifest "%s" to JSON.\n %s' % (path, e))
try:
with open(path, 'w') as f:
f.write(manifest_json)
except IOError as e:
raise Error('Unable to write manifest to "%s".\n %s' % (path, e))
def LoadConfig(raise_on_error=False):
path = os.path.join(USER_DATA_DIR, CONFIG_FILENAME)
if not os.path.exists(path):
return config.Config()
try:
try:
with open(path) as f:
return config.Config(json.loads(f.read()))
except IOError as e:
raise Error('Unable to read config from "%s".\n %s' % (path, e))
except Exception as e:
raise Error('Parsing config file from "%s" failed.\n %s' % (path, e))
except Error as e:
if raise_on_error:
raise
else:
logging.warn(str(e))
return config.Config()
def LoadLocalManifest(raise_on_error=False):
path = os.path.join(USER_DATA_DIR, MANIFEST_FILENAME)
manifest = manifest_util.SDKManifest()
try:
try:
with open(path) as f:
manifest_string = f.read()
except IOError as e:
raise Error('Unable to read manifest from "%s".\n %s' % (path, e))
try:
manifest.LoadDataFromString(manifest_string)
except Exception as e:
raise Error('Parsing local manifest "%s" failed.\n %s' % (path, e))
except Error as e:
if raise_on_error:
raise
else:
logging.warn(str(e))
return manifest
def LoadRemoteManifest(url):
manifest = manifest_util.SDKManifest()
url_stream = None
try:
manifest_stream = cStringIO.StringIO()
url_stream = download.UrlOpen(url)
download.DownloadAndComputeHash(url_stream, manifest_stream)
except urllib2.URLError as e:
raise Error('Unable to read remote manifest from URL "%s".\n %s' % (
url, e))
finally:
if url_stream:
url_stream.close()
try:
manifest.LoadDataFromString(manifest_stream.getvalue())
return manifest
except manifest_util.Error as e:
raise Error('Parsing remote manifest from URL "%s" failed.\n %s' % (
url, e,))
def DownloadToFile(self, url, dest_filename):
dest_path = os.path.join(self.archive_cache, dest_filename)
sdk_update_common.MakeDirs(os.path.dirname(dest_path))
out_stream = None
url_stream = None
try:
out_stream = open(dest_path, 'wb')
url_stream = download.UrlOpen(url)
content_length = int(url_stream.info()[HTTP_CONTENT_LENGTH])
progress = download.MakeProgressFunction(content_length)
sha1, size = download.DownloadAndComputeHash(url_stream, out_stream,
progress)
return sha1, size
except urllib2.URLError as e:
raise Error('Unable to read from URL "%s".\n %s' % (url, e))
except IOError as e:
raise Error('Unable to write to file "%s".\n %s' % (dest_filename, e))
finally:
if url_stream:
url_stream.close()
if out_stream:
out_stream.close()
def DownloadArchiveToFile(archive, dest_path):
'''Download the archive's data to a file at dest_path.
As a side effect, computes the sha1 hash and data size, both returned as a
tuple. Raises an Error if the url can't be opened, or an IOError exception if
dest_path can't be opened.
Args:
dest_path: Path for the file that will receive the data.
Return:
A tuple (sha1, size) with the sha1 hash and data size respectively.'''
sha1 = None
size = 0
with open(dest_path, 'wb') as to_stream:
from_stream = None
try:
from_stream = UrlOpen(archive.url)
except urllib2.URLError:
raise Error('Cannot open "%s" for archive %s' %
(archive.url, archive.host_os))
try:
content_length = int(from_stream.info()[HTTP_CONTENT_LENGTH])
progress_function = ProgressFunction(content_length).GetProgressFunction()
InfoPrint('Downloading %s' % archive.url)
sha1, size = manifest_util.DownloadAndComputeHash(
from_stream,
to_stream=to_stream,
progress_func=progress_function)
if size != content_length:
raise Error('Download size mismatch for %s.\n'
'Expected %s bytes but got %s' %
(archive.url, content_length, size))
finally:
if from_stream:
from_stream.close()
return sha1, size
def WriteToFile(path, obj):
'''Write |manifest| to a JSON file at |path|.'''
methodlist = [m for m in dir(obj) if callable(getattr(obj, m))]
if 'GetDataAsString' not in methodlist:
raise Error('Unable to write object to file')
json_string = obj.GetDataAsString()
# Write the JSON data to a temp file.
temp_file_name = None
# TODO(dspringer): Use file locks here so that multiple sdk_updates can
# run at the same time.
with tempfile.NamedTemporaryFile(mode='w', delete=False) as f:
f.write(json_string)
temp_file_name = f.name
# Move the temp file to the actual file.
if os.path.exists(path):
os.remove(path)
shutil.move(temp_file_name, path)
def LoadManifestFromURLs(urls):
'''Returns a manifest loaded from |urls|, merged into one manifest.'''
manifest = manifest_util.SDKManifest()
for url in urls:
try:
url_stream = UrlOpen(url)
except urllib2.URLError as e:
raise Error('Unable to open %s. [%s]' % (url, e))
manifest_stream = cStringIO.StringIO()
manifest_util.DownloadAndComputeHash(url_stream, manifest_stream)
temp_manifest = manifest_util.SDKManifest()
temp_manifest.LoadDataFromString(manifest_stream.getvalue())
manifest.MergeManifest(temp_manifest)
def BundleFilter(bundle):
# Only add this bundle if it's supported on this platform.
return bundle.GetHostOSArchive()
manifest.FilterBundles(BundleFilter)
return manifest
def LoadJson(self, json_data):
try:
self.update(json.loads(json_data))
except Exception as e:
raise Error('Error reading json config:\n%s' % str(e))
def Update(options, argv, config):
'''Usage: %prog [global_options] update [options] [target]
Updates the Native Client SDK to a specified version. By default, this
command updates all the recommended components. The update process works
like this:
1. Fetch the manifest from the mirror.
2. Load manifest from USER_DATA_DIR - if there is no local manifest file,
make an empty manifest object.
3. Update each the bundle:
for bundle in bundles:
# Compare bundle versions & revisions.
# Test if local version.revision < mirror OR local doesn't exist.
if local_manifest < mirror_manifest:
update(bundle)
update local_manifest with mirror_manifest for bundle
write manifest to disk. Use locks.
else:
InfoPrint('bundle is up-to-date')
Targets:
recommended: (default) Install/Update all recommended components
all: Install/Update all available components
bundle_name: Install/Update only the given bundle
'''
DebugPrint("Running Update command with: %s, %s" % (options, argv))
ALL = 'all' # Update all bundles
RECOMMENDED = 'recommended' # Only update the bundles with recommended=yes
parser = optparse.OptionParser(usage=Update.__doc__)
parser.add_option(
'-F', '--force', dest='force',
default=False, action='store_true',
help='Force updating existing components that already exist')
(update_options, args) = parser.parse_args(argv)
if len(args) == 0:
args = [RECOMMENDED]
manifest = LoadManifestFromURLs([options.manifest_url] + config.GetSources())
bundles = manifest.GetBundles()
local_manifest_path = os.path.join(options.user_data_dir,
options.manifest_filename)
local_manifest = LoadFromFile(local_manifest_path,
manifest_util.SDKManifest())
# Validate the arg list against the available bundle names. Raises an
# error if any invalid bundle names or args are detected.
valid_args = set([ALL, RECOMMENDED] + [bundle.name for bundle in bundles])
bad_args = set(args) - valid_args
if len(bad_args) > 0:
raise Error("Unrecognized bundle name or argument: '%s'" %
', '.join(bad_args))
if SDK_TOOLS in args and not options.update_sdk_tools:
# We only want sdk_tools to be updated by sdk_update.py. If the user
# tries to update directly, we just ignore the request.
InfoPrint('Updating sdk_tools happens automatically.\n'
'Ignoring manual update request.')
args.remove(SDK_TOOLS)
for bundle in bundles:
bundle_path = os.path.join(options.sdk_root_dir, bundle.name)
bundle_update_path = '%s_update' % bundle_path
if not (bundle.name in args or
ALL in args or (RECOMMENDED in args and
bundle[RECOMMENDED] == 'yes')):
continue
if bundle.name == SDK_TOOLS and not options.update_sdk_tools:
continue
def UpdateBundle():
'''Helper to install a bundle'''
archive = bundle.GetHostOSArchive()
(_, _, path, _, _, _) = urlparse.urlparse(archive['url'])
dest_filename = os.path.join(options.user_data_dir, path.split('/')[-1])
sha1, size = DownloadArchiveToFile(archive, dest_filename)
if sha1 != archive.GetChecksum():
raise Error("SHA1 checksum mismatch on '%s'. Expected %s but got %s" %
(bundle.name, archive.GetChecksum(), sha1))
if size != archive.size:
raise Error("Size mismatch on Archive. Expected %s but got %s bytes" %
(archive.size, size))
InfoPrint('Updating bundle %s to version %s, revision %s' % (
(bundle.name, bundle.version, bundle.revision)))
ExtractInstaller(dest_filename, bundle_update_path)
if bundle.name != SDK_TOOLS:
repath = bundle.get('repath', None)
if repath:
bundle_move_path = os.path.join(bundle_update_path, repath)
else:
bundle_move_path = bundle_update_path
RenameDir(bundle_move_path, bundle_path)
if os.path.exists(bundle_update_path):
RemoveDir(bundle_update_path)
os.remove(dest_filename)
local_manifest.MergeBundle(bundle)
WriteToFile(local_manifest_path, local_manifest)
# Test revision numbers, update the bundle accordingly.
# TODO(dspringer): The local file should be refreshed from disk each
# iteration thought this loop so that multiple sdk_updates can run at the
# same time.
if local_manifest.BundleNeedsUpdate(bundle):
if (not update_options.force and os.path.exists(bundle_path) and
bundle.name != SDK_TOOLS):
WarningPrint('%s already exists, but has an update available.\n'
'Run update with the --force option to overwrite the '
'existing directory.\nWarning: This will overwrite any '
'modifications you have made within this directory.'
% bundle.name)
else:
UpdateBundle()
else:
InfoPrint('%s is already up-to-date.' % bundle.name)
def AddSource(config, url):
try:
download.UrlOpen(url)
except Exception as e:
raise Error('Not adding %s, unable to load URL.\n %s' % (url, e))
config.AddSource(url)
def ExtractArchives(self, archives, extract_dir, rename_from_dir,
rename_to_dir):
tar_file = None
extract_path = os.path.join(self.install_dir, extract_dir)
rename_from_path = os.path.join(self.install_dir, rename_from_dir)
rename_to_path = os.path.join(self.install_dir, rename_to_dir)
# Extract to extract_dir, usually "<bundle name>_update".
# This way if the extraction fails, we haven't blown away the old bundle
# (if it exists).
sdk_update_common.RemoveDir(extract_path)
sdk_update_common.MakeDirs(extract_path)
curpath = os.getcwd()
tar_file = None
try:
try:
logging.info('Changing the directory to %s' % (extract_path, ))
os.chdir(extract_path)
except Exception as e:
raise Error('Unable to chdir into "%s".\n %s' %
(extract_path, e))
for i, archive in enumerate(archives):
archive_path = os.path.join(self.archive_cache, archive)
if len(archives) > 1:
print '(file %d/%d - "%s")' % (
i + 1, len(archives), os.path.basename(archive_path))
logging.info('Extracting to %s' % (extract_path, ))
if sys.platform == 'win32':
try:
logging.info('Opening file %s (%d/%d).' %
(archive_path, i + 1, len(archives)))
try:
tar_file = cygtar.CygTar(archive_path,
'r',
verbose=True)
except Exception as e:
raise Error("Can't open archive '%s'.\n %s" %
(archive_path, e))
tar_file.Extract()
finally:
if tar_file:
tar_file.Close()
else:
try:
subprocess.check_call(['tar', 'xf', archive_path])
except subprocess.CalledProcessError:
raise Error('Error extracting archive: %s' %
archive_path)
logging.info('Changing the directory to %s' % (curpath, ))
os.chdir(curpath)
logging.info('Renaming %s->%s' %
(rename_from_path, rename_to_path))
sdk_update_common.RenameDir(rename_from_path, rename_to_path)
finally:
# Change the directory back so we can remove the update directory.
os.chdir(curpath)
# Clean up the ..._update directory.
try:
sdk_update_common.RemoveDir(extract_path)
except Exception as e:
logging.error('Failed to remove directory \"%s\". %s' %
(extract_path, e))
def ToJson(self):
try:
return json.dumps(self, sort_keys=False, indent=2)
except Exception as e:
raise Error('Json encoding error writing config:\n%s' % e)