def Run(self, args):
dns = self.context['dns_client']
messages = self.context['dns_messages']
resources = self.context['dns_resources']
project_id = properties.VALUES.core.project.Get(required=True)
# Get the managed-zone.
zone_ref = resources.Parse(args.zone, collection='dns.managedZones')
try:
zone = dns.managedZones.Get(zone_ref.Request())
except apitools_base.HttpError as error:
raise exceptions.HttpException(util.GetErrorMessage(error))
# Get all the record-sets.
record_sets = []
for record_set in apitools_base.YieldFromList(
dns.resourceRecordSets,
messages.DnsResourceRecordSetsListRequest(project=project_id,
managedZone=zone_ref.Name()),
field='rrsets'):
record_sets.append(record_set)
# Export the record-sets.
try:
with files.Context(open(args.records_file, 'w')) as export_file:
if args.zone_file_format:
export_util.WriteToZoneFile(export_file, record_sets, zone.dnsName)
else:
export_util.WriteToYamlFile(export_file, record_sets)
except Exception as exp:
msg = 'unable to export record-sets to file [{0}]: {1}'.format(
args.records_file, exp)
raise exceptions.ToolException(msg)
log.status.Print('Exported record-sets to [{0}].'.format(args.records_file))
def WriteEnvYaml(env, output_dir):
"""Writes the given environment values into the output_dir/env.yaml file.
Args:
env: {str: str}, Dictonary of environment values.
output_dir: str, Path of directory to which env.yaml file should be written.
"""
env_file_path = os.path.join(output_dir, 'env.yaml')
with files.Context(open(env_file_path, 'w')) as env_file:
resource_printer.Print([env], print_format='yaml', out=env_file)
def ReadEnvYaml(output_dir):
"""Reads and returns the environment values in output_dir/env.yaml file.
Args:
output_dir: str, Path of directory containing the env.yaml to be read.
Returns:
env: {str: str}
"""
env_file_path = os.path.join(output_dir, 'env.yaml')
with files.Context(open(env_file_path, 'r')) as env_file:
return yaml.safe_load(env_file)
def Run(self, args):
if os.path.isfile(args.transaction_file):
raise exceptions.ToolException(
'transaction already exists at [{0}]'.format(
args.transaction_file))
dns = self.context['dns_client']
messages = self.context['dns_messages']
resources = self.context['dns_resources']
project_id = properties.VALUES.core.project.Get(required=True)
# Get the managed-zone.
zone_ref = resources.Parse(args.zone, collection='dns.managedZones')
try:
zone = dns.managedZones.Get(
dns.MESSAGES_MODULE.DnsManagedZonesGetRequest(
project=zone_ref.project,
managedZone=zone_ref.managedZone))
except apitools_exceptions.HttpError as error:
raise exceptions.HttpException(error)
# Initialize an empty change
change = messages.Change()
# Get the SOA record, there will be one and only one.
# Add addition and deletion for SOA incrementing to change.
records = [
record for record in list_pager.YieldFromList(
dns.resourceRecordSets,
messages.DnsResourceRecordSetsListRequest(
project=project_id,
managedZone=zone_ref.Name(),
name=zone.dnsName,
type='SOA'),
field='rrsets')
]
change.deletions.append(records[0])
change.additions.append(import_util.NextSOARecordSet(records[0]))
# Write change to transaction file
try:
with files.Context(open(args.transaction_file,
'w')) as transaction_file:
transaction_util.WriteToYamlFile(transaction_file, change)
except Exception as exp:
msg = 'unable to write transaction [{0}] because [{1}]'
msg = msg.format(args.transaction_file, exp)
raise exceptions.ToolException(msg)
log.status.Print('Transaction started [{0}].'.format(
args.transaction_file))
def Update(cli):
"""Create or overwrite static completion table.
Args:
cli: Calliope CLI object for generating the completion table.
"""
# Overwrite the completion table file with updated content
with files.Context(open(_TablePath(), 'w')) as table_file:
table = CompletionTableGenerator(cli).Walk(hidden=False)
table_file.write('table=')
pprint(table, table_file)
# _TableDirPath() could contain unicode chars and py_compile chokes on unicode
# paths. Using relative paths from _TableDirPath() works around the problem.
table_dir_path = _TableDirPath()
with files.ChDir(table_dir_path):
# Pre-compile table source to enable fast loading
compileall.compile_dir('.', quiet=True)
def ReadEnvYaml(output_dir):
"""Reads and returns the environment values in output_dir/env.yaml file.
Args:
output_dir: str, Path of directory containing the env.yaml to be read.
Returns:
env: {str: str}
"""
env_file_path = os.path.join(output_dir, 'env.yaml')
try:
with files.Context(open(env_file_path, 'r')) as env_file:
return yaml.safe_load(env_file)
except IOError as err:
if err.errno == errno.ENOENT:
raise NoEnvYamlError(output_dir)
else:
raise err
def BuildTrainingJob(path=None,
module_name=None,
job_name=None,
trainer_uri=None,
region=None,
user_args=None):
"""Builds a GoogleCloudMlV1beta1Job from a config file and/or flag values.
Args:
path: path to a yaml configuration file
module_name: value to set for moduleName field (overrides yaml file)
job_name: value to set for jobName field (overrides yaml file)
trainer_uri: List of values to set for trainerUri field (overrides yaml
file)
region: compute region in which to run the job (overrides yaml file)
user_args: [str]. A list of arguments to pass through to the job.
(overrides yaml file)
Returns:
A constructed GoogleCloudMlV1beta1Job object.
"""
msgs = apis.GetMessagesModule('ml', 'v1beta1')
request_class = msgs.GoogleCloudMlV1beta1Job
obj = request_class()
if path:
with files.Context(open(path)) as config_file:
data = yaml.load(config_file)
if data:
obj = encoding.DictToMessage(data, request_class)
if not obj.trainingInput:
obj.trainingInput = msgs.GoogleCloudMlV1beta1TrainingInput()
if module_name:
obj.trainingInput.pythonModule = module_name
if user_args:
obj.trainingInput.args = user_args
if job_name:
obj.jobId = job_name
if trainer_uri:
obj.trainingInput.packageUris = trainer_uri
if region:
obj.trainingInput.region = region
return obj
def DownloadAndExtractTar(url, download_dir, extract_dir):
"""Download and extract the given tar file.
Args:
url: str, The URL to download.
download_dir: str, The path to put the temporary download file into.
extract_dir: str, The path to extract the tar into.
Returns:
[str], The files that were extracted from the tar file.
Raises:
URLFetchError: If there is a problem fetching the given URL.
"""
for d in [download_dir, extract_dir]:
if not os.path.exists(d):
file_utils.MakeDir(d)
download_file_path = os.path.join(download_dir, os.path.basename(url))
if os.path.exists(download_file_path):
os.remove(download_file_path)
try:
req = ComponentInstaller.MakeRequest(url)
with open(download_file_path, 'wb') as fp:
shutil.copyfileobj(req, fp)
except (urllib2.HTTPError, urllib2.URLError, ssl.SSLError) as e:
raise URLFetchError(e)
with file_utils.Context(tarfile.open(name=download_file_path)) as tar:
tar.extractall(extract_dir)
files = [
item.name + '/' if item.isdir() else item.name
for item in tar.getmembers()
]
os.remove(download_file_path)
return files
def DownloadAndExtractTar(url, download_dir, extract_dir,
progress_callback=None, command_path='unknown'):
"""Download and extract the given tar file.
Args:
url: str, The URL to download.
download_dir: str, The path to put the temporary download file into.
extract_dir: str, The path to extract the tar into.
progress_callback: f(float), A function to call with the fraction of
completeness.
command_path: the command path to include in the User-Agent header if the
URL is HTTP
Returns:
[str], The files that were extracted from the tar file.
Raises:
URLFetchError: If there is a problem fetching the given URL.
"""
for d in [download_dir, extract_dir]:
if not os.path.exists(d):
file_utils.MakeDir(d)
download_file_path = os.path.join(download_dir, os.path.basename(url))
if os.path.exists(download_file_path):
os.remove(download_file_path)
(download_callback, install_callback) = (
console_io.ProgressBar.SplitProgressBar(progress_callback, [1, 1]))
try:
req = ComponentInstaller.MakeRequest(url, command_path)
try:
total_size = float(req.info().getheader('Content-Length', '0'))
# pylint: disable=broad-except, We never want progress bars to block an
# update.
except Exception:
total_size = 0
with open(download_file_path, 'wb') as fp:
# This is the buffer size that shutil.copyfileobj uses.
buf_size = 16*1024
total_written = 0
while True:
buf = req.read(buf_size)
if not buf:
break
fp.write(buf)
total_written += len(buf)
if total_size:
download_callback(total_written / total_size)
download_callback(1)
except (urllib2.HTTPError, urllib2.URLError, ssl.SSLError) as e:
raise URLFetchError(e)
with file_utils.Context(tarfile.open(name=download_file_path)) as tar:
members = tar.getmembers()
total_files = float(len(members))
files = []
for num, member in enumerate(members, start=1):
files.append(member.name + '/' if member.isdir() else member.name)
tar.extract(member, extract_dir)
install_callback(num / total_files)
install_callback(1)
os.remove(download_file_path)
return files
def Run(self, args):
if not os.path.exists(args.records_file):
raise exceptions.ToolException(
'no such file [{0}]'.format(args.records_file))
if os.path.isdir(args.records_file):
raise exceptions.ToolException(
'[{0}] is a directory'.format(args.records_file))
dns = self.context['dns_client']
messages = self.context['dns_messages']
resources = self.context['dns_resources']
project_id = properties.VALUES.core.project.Get(required=True)
# Get the managed-zone.
zone_ref = resources.Parse(args.zone, collection='dns.managedZones')
try:
zone = dns.managedZones.Get(zone_ref.Request())
except apitools_exceptions.HttpError as error:
raise exceptions.HttpException(error)
# Get the current record-sets.
current = {}
for record in list_pager.YieldFromList(
dns.resourceRecordSets,
messages.DnsResourceRecordSetsListRequest(project=project_id,
managedZone=zone_ref.Name()),
field='rrsets'):
current[(record.name, record.type)] = record
# Get the imported record-sets.
try:
with files.Context(open(args.records_file)) as import_file:
if args.zone_file_format:
imported = import_util.RecordSetsFromZoneFile(import_file,
zone.dnsName)
else:
imported = import_util.RecordSetsFromYamlFile(import_file)
except Exception as exp:
msg = (u'unable to read record-sets from specified records-file [{0}] '
u'because [{1}]')
msg = msg.format(args.records_file, exp.message)
raise exceptions.ToolException(msg)
# Get the change resulting from the imported record-sets.
change = import_util.ComputeChange(current, imported,
args.delete_all_existing,
zone.dnsName, args.replace_origin_ns)
if not change:
msg = u'Nothing to do, all the records in [{0}] already exist.'.format(
args.records_file)
log.status.Print(msg)
return None
# Send the change to the service.
result = dns.changes.Create(
messages.DnsChangesCreateRequest(change=change,
managedZone=zone.name,
project=project_id))
change_ref = resources.Create(collection='dns.changes',
project=project_id,
managedZone=zone.name,
changeId=result.id)
msg = u'Imported record-sets from [{0}] into managed-zone [{1}].'.format(
args.records_file, zone_ref.Name())
log.status.Print(msg)
log.CreatedResource(change_ref)
return result
