def AddRecordedStories(self, stories, upload_to_cloud_storage=False):
if not stories:
os.remove(self.temp_target_wpr_file_path)
return
(target_wpr_file, target_wpr_file_path) = self._NextWprFileName()
for story in stories:
self._SetWprFileForStory(story.display_name, target_wpr_file)
shutil.move(self.temp_target_wpr_file_path, target_wpr_file_path)
# Update the hash file.
target_wpr_file_hash = cloud_storage.CalculateHash(target_wpr_file_path)
with open(target_wpr_file_path + '.sha1', 'wb') as f:
f.write(target_wpr_file_hash)
f.flush()
self._WriteToFile()
self._DeleteAbandonedWprFiles()
# Upload to cloud storage
if upload_to_cloud_storage:
if not self._bucket:
logging.warning('StorySet must have bucket specified to upload '
'stories to cloud storage.')
return
try:
cloud_storage.Insert(self._bucket, target_wpr_file_hash,
target_wpr_file_path)
except cloud_storage.CloudStorageError, e:
logging.warning('Failed to upload wpr file %s to cloud storage. '
'Error:%s' % target_wpr_file_path, e)
def assertCorrectHashFile(self, file_path):
old_ch = cloud_storage.CalculateHash
cloud_storage.CalculateHash = self.overrides.cloud_storage.CalculateHash
try:
self.assertTrue(os.path.exists(file_path + '.sha1'))
with open(file_path + '.sha1', 'rb') as f:
self.assertEquals(cloud_storage.CalculateHash(file_path), f.read())
finally:
cloud_storage.CalculateHash = old_ch
def AddCloudStorageDependencyUpdateJob(
self, dependency, platform, dependency_path, version=None,
execute_job=True):
"""Update the file downloaded from cloud storage for a dependency/platform.
Upload a new file to cloud storage for the given dependency and platform
pair and update the cloud storage hash and the version for the given pair.
Example usage:
The following should update the default platform for 'dep_name':
UpdateCloudStorageDependency('dep_name', 'default', 'path/to/file')
The following should update both the mac and win platforms for 'dep_name',
or neither if either update fails:
UpdateCloudStorageDependency(
'dep_name', 'mac_x86_64', 'path/to/mac/file', execute_job=False)
UpdateCloudStorageDependency(
'dep_name', 'win_AMD64', 'path/to/win/file', execute_job=False)
ExecuteUpdateJobs()
Args:
dependency: The dependency to update.
platform: The platform to update the dependency info for.
dependency_path: Path to the new dependency to be used.
version: Version of the updated dependency, for checking future updates
against.
execute_job: True if the config should be written to disk and the file
should be uploaded to cloud storage after the update. False if
multiple updates should be performed atomically. Must call
ExecuteUpdateJobs after all non-executed jobs are added to complete
the update.
Raises:
ReadWriteError: If the config was not initialized as writable, or if
|execute_job| is True but the config has update jobs still pending
execution.
ValueError: If no information exists in the config for |dependency| on
|platform|.
"""
self._ValidateIsConfigUpdatable(
execute_job=execute_job, dependency=dependency, platform=platform)
cs_hash = cloud_storage.CalculateHash(dependency_path)
if version:
self._SetPlatformData(dependency, platform, 'version_in_cs', version)
self._SetPlatformData(dependency, platform, 'cloud_storage_hash', cs_hash)
cs_base_folder = self._GetPlatformData(
dependency, platform, 'cloud_storage_base_folder')
cs_bucket = self._GetPlatformData(
dependency, platform, 'cloud_storage_bucket')
cs_remote_path = self._CloudStorageRemotePath(
dependency, cs_hash, cs_base_folder)
self._pending_uploads.append(uploader.CloudStorageUploader(
cs_bucket, cs_remote_path, dependency_path))
if execute_job:
self.ExecuteUpdateJobs()
def FetchDataFile(data_file_name):
"""Download the file from the cloud."""
with open(CONFIG_PATH) as f:
config = json.load(f)
remote_path = config[data_file_name]['remote_path']
expected_hash = config[data_file_name]['hash']
filename = posixpath.basename(remote_path)
local_path = os.path.join(LOCAL_STORAGE_FOLDER, filename)
cloud_storage.Get(DATA_BUCKET, remote_path, local_path)
if cloud_storage.CalculateHash(local_path) != expected_hash:
raise RuntimeError('The downloaded data file has wrong hash.')
return local_path
def UploadAndSwitchDataFile(data_file_name, data_file_path, version):
"""Upload the script to the cloud and update config to use the new version."""
filename = os.path.basename(data_file_path)
remote_path = posixpath.join(DATA_CS_FOLDER, data_file_name, version,
filename)
if not cloud_storage.Exists(DATA_BUCKET, remote_path):
cloud_storage.Insert(DATA_BUCKET,
remote_path,
data_file_path,
publicly_readable=False)
with open(CONFIG_PATH) as f:
config = json.load(f)
config[data_file_name]['remote_path'] = remote_path
config[data_file_name]['hash'] = cloud_storage.CalculateHash(
data_file_path)
with open(CONFIG_PATH, 'w') as f:
json.dump(config, f, indent=4, separators=(',', ': '))
def FetchHostBinary(binary_name):
"""Download the binary from the cloud.
This function fetches the binary for the host platform from the cloud.
The cloud path is read from the config.
"""
with open(CONFIG_PATH) as f:
config = json.load(f)
platform = _GetHostPlatform()
remote_path = config[binary_name][platform]['remote_path']
expected_hash = config[binary_name][platform]['hash']
filename = posixpath.basename(remote_path)
local_path = os.path.join(LOCAL_STORAGE_FOLDER, filename)
cloud_storage.Get(BINARY_BUCKET, remote_path, local_path)
if cloud_storage.CalculateHash(local_path) != expected_hash:
raise RuntimeError('The downloaded binary has wrong hash.')
mode = os.stat(local_path).st_mode
os.chmod(local_path, mode | stat.S_IXUSR)
return local_path
def AddRecordedStories(self,
stories,
upload_to_cloud_storage=False,
target_platform=_DEFAULT_PLATFORM):
if not stories:
os.remove(self.temp_target_wpr_file_path)
return
target_wpr_file_hash = cloud_storage.CalculateHash(
self.temp_target_wpr_file_path)
(target_wpr_file,
target_wpr_file_path) = self._NextWprFileName(target_wpr_file_hash)
for story in stories:
# Check to see if the platform has been manually overrided.
if not story.platform_specific:
current_target_platform = _DEFAULT_PLATFORM
else:
current_target_platform = target_platform
self._SetWprFileForStory(story.name, target_wpr_file,
current_target_platform)
shutil.move(self.temp_target_wpr_file_path, target_wpr_file_path)
# Update the hash file.
with open(target_wpr_file_path + '.sha1', 'wb') as f:
f.write(target_wpr_file_hash)
f.flush()
self._WriteToFile()
# Upload to cloud storage
if upload_to_cloud_storage:
if not self._bucket:
logging.warning(
'StorySet must have bucket specified to upload '
'stories to cloud storage.')
return
try:
cloud_storage.Insert(self._bucket, target_wpr_file_hash,
target_wpr_file_path)
except cloud_storage.CloudStorageError as e:
logging.warning(
'Failed to upload wpr file %s to cloud storage. '
'Error:%s' % target_wpr_file_path, e)
def _CalculateHash(remote_path):
with tempfile_ext.NamedTemporaryFile() as f:
f.close()
cloud_storage.Get(BINARY_BUCKET, remote_path, f.name)
return cloud_storage.CalculateHash(f.name)
def _Base64Hash(file_path):
return base64.b64encode(cloud_storage.CalculateHash(file_path))
