def get_zotero_fields(self):
z = json.loads(self.zotero_data)
z['itemType'] = utils.type_map['readable'][z['itemType']]
if self.date_information:
date_parts = json.loads(self.date_information)
for part in date_parts:
z[part] = date_parts[part]
if z['creators']:
names = utils.resolve_names(z, 'facets')
z.pop('creators')
output = z.items()
for name in names:
for creator_type, creator_value in name.items():
output.append((creator_type, creator_value))
else:
output = z.items()
return output
def install_openstack(settings_dict,
envs=None,
verbose=None,
url_script=None,
prepare=False,
force=False,
config=None,
use_cobbler=False,
proxy=None):
"""
Install OS with COI on build server
:param settings_dict: settings dictionary for Fabric
:param envs: environment variables to inject when executing job
:param verbose: if to hide all output or print everything
:param url_script: URl of Cisco installer script from Chris
:param force: Use if you don't connect via interface you gonna bridge later
:return: always true
"""
envs = envs or {}
verbose = verbose or []
if settings_dict['user'] != 'root':
use_sudo_flag = True
run_func = sudo
else:
use_sudo_flag = False
run_func = run
with settings(**settings_dict), hide(*verbose), shell_env(**envs):
with cd("/root/"):
if proxy:
warn_if_fail(
put(StringIO(
'Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'
),
"/etc/apt/apt.conf.d/00proxy",
use_sudo=use_sudo_flag))
warn_if_fail(
put(StringIO('Acquire::http::Pipeline-Depth "0";'),
"/etc/apt/apt.conf.d/00no_pipelining",
use_sudo=use_sudo_flag))
run_func("apt-get update")
run_func("apt-get install -y git")
run_func("git config --global user.email '*****@*****.**';"
"git config --global user.name 'Test Node'")
if not force and not prepare:
update_time(run_func)
# avoid grub and other prompts
warn_if_fail(
run_func('DEBIAN_FRONTEND=noninteractive apt-get -y '
'-o Dpkg::Options::="--force-confdef" -o '
'Dpkg::Options::="--force-confold" dist-upgrade'))
# prepare /etc/hosts file
append("/etc/hosts", prepare_hosts(config))
with cd("/root"):
warn_if_fail(
run_func(
"git clone -b icehouse "
"https://github.com/CiscoSystems/puppet_openstack_builder"
))
with cd("puppet_openstack_builder"):
## run the latest, not i.0 release
#run_func('git checkout i.0')
sed("/root/puppet_openstack_builder/install-scripts/cisco.install.sh",
"icehouse/snapshots/i.0",
"icehouse-proposed",
use_sudo=use_sudo_flag)
sed("/root/puppet_openstack_builder/data/hiera_data/vendor/cisco_coi_common.yaml",
"/snapshots/i.0",
"-proposed",
use_sudo=use_sudo_flag)
with cd("install-scripts"):
warn_if_fail(run_func("./install.sh"))
prepare_files(
config,
paths=(
"/etc/puppet/data/hiera_data/user.common.yaml",
"/etc/puppet/data/hiera_data/user.full_ha.yaml",
"/etc/puppet/data/cobbler/cobbler.yaml",
"/etc/puppet/data/role_mappings.yaml",
"/etc/puppet/data/hiera_data/hostname/build_server.yaml"
),
use_sudo_flag=use_sudo_flag)
prepare_new_files(config,
path="/etc/puppet/data/hiera_data/hostname",
use_sudo_flag=use_sudo_flag)
resolve_names(run_func, use_sudo_flag)
result = run_func(
'puppet apply -v /etc/puppet/manifests/site.pp')
tries = 1
if use_cobbler:
cobbler_error = "[cobbler-sync]/returns: unable to connect to cobbler on localhost using cobbler"
while cobbler_error in result and tries <= APPLY_LIMIT:
time.sleep(60)
print >> sys.stderr, "Cobbler is not installed properly, running apply again"
result = run_func(
'puppet apply -v /etc/puppet/manifests/site.pp',
pty=False)
tries += 1
error = "Error:"
while error in result and tries <= APPLY_LIMIT:
time.sleep(60)
print >> sys.stderr, "Some errors found, running apply again"
result = run_func(
'puppet apply -v /etc/puppet/manifests/site.pp',
pty=False)
tries += 1
if exists('/root/openrc'):
get('/root/openrc', "./openrc")
else:
print(red("No openrc file, something went wrong! :("))
print(green("Copying logs and configs"))
collect_logs(
run_func=run_func,
hostname=config["servers"]["build-server"][0]["hostname"],
clean=True)
print(green("Finished!"))
return True
elif not force and prepare:
return True
print(green("Finished!"))
return True
def install_openstack(settings_dict,
envs=None,
verbose=None,
url_script=None,
prepare=False,
force=False,
config=None,
use_cobbler=False,
proxy=None):
"""
Install OS with COI on build server
:param settings_dict: settings dictionary for Fabric
:param envs: environment variables to inject when executing job
:param verbose: if to hide all output or print everything
:param url_script: URl of Cisco installer script from Chris
:param force: Use if you don't connect via interface you gonna bridge later
:return: always true
"""
envs = envs or {}
verbose = verbose or []
if settings_dict['user'] != 'root':
use_sudo_flag = True
run_func = sudo
else:
use_sudo_flag = False
run_func = run
with settings(**settings_dict), hide(*verbose), shell_env(**envs):
with cd("/root/"):
if proxy:
warn_if_fail(put(StringIO('Acquire::http::proxy "http://proxy.esl.cisco.com:8080/";'),
"/etc/apt/apt.conf.d/00proxy",
use_sudo=use_sudo_flag))
warn_if_fail(put(StringIO('Acquire::http::Pipeline-Depth "0";'),
"/etc/apt/apt.conf.d/00no_pipelining",
use_sudo=use_sudo_flag))
run_func("apt-get update")
run_func("apt-get install -y git")
run_func("git config --global user.email '*****@*****.**';"
"git config --global user.name 'Test Node'")
if not force and not prepare:
update_time(run_func)
# avoid grub and other prompts
warn_if_fail(run_func('DEBIAN_FRONTEND=noninteractive apt-get -y '
'-o Dpkg::Options::="--force-confdef" -o '
'Dpkg::Options::="--force-confold" dist-upgrade'))
# prepare /etc/hosts file
append("/etc/hosts", prepare_hosts(config))
with cd("/root"):
warn_if_fail(run_func("git clone -b icehouse "
"https://github.com/CiscoSystems/puppet_openstack_builder"))
with cd("puppet_openstack_builder"):
## run the latest, not i.0 release
#run_func('git checkout i.0')
sed("/root/puppet_openstack_builder/install-scripts/cisco.install.sh",
"icehouse/snapshots/i.0",
"icehouse-proposed", use_sudo=use_sudo_flag)
with cd("install-scripts"):
warn_if_fail(run_func("./install.sh"))
prepare_files(config,
paths=(
"/etc/puppet/data/hiera_data/user.common.yaml",
"/etc/puppet/data/hiera_data/user.full_ha.yaml",
"/etc/puppet/data/cobbler/cobbler.yaml",
"/etc/puppet/data/role_mappings.yaml",
"/etc/puppet/data/hiera_data/hostname/build_server.yaml"
),
use_sudo_flag=use_sudo_flag)
prepare_new_files(
config,
path="/etc/puppet/data/hiera_data/hostname",
use_sudo_flag=use_sudo_flag
)
resolve_names(run_func, use_sudo_flag)
result = run_func('puppet apply -v /etc/puppet/manifests/site.pp')
tries = 1
if use_cobbler:
cobbler_error = "[cobbler-sync]/returns: unable to connect to cobbler on localhost using cobbler"
while cobbler_error in result and tries <= APPLY_LIMIT:
time.sleep(60)
print >> sys.stderr, "Cobbler is not installed properly, running apply again"
result = run_func('puppet apply -v /etc/puppet/manifests/site.pp', pty=False)
tries += 1
error = "Error:"
while error in result and tries <= APPLY_LIMIT:
time.sleep(60)
print >> sys.stderr, "Some errors found, running apply again"
result = run_func('puppet apply -v /etc/puppet/manifests/site.pp', pty=False)
tries += 1
if exists('/root/openrc'):
get('/root/openrc', "./openrc")
else:
print (red("No openrc file, something went wrong! :("))
print (green("Copying logs and configs"))
collect_logs(run_func=run_func, hostname=config["servers"]["build-server"]["hostname"], clean=True)
print (green("Finished!"))
return True
elif not force and prepare:
return True
print (green("Finished!"))
return True