def GetLanguageCatalog(self, language):
'''Returns a dictionary from 'MSG_...' to translated string'''
enthome = E.getEnterpriseHome()
languagelocal = language.replace('-', '_')
languagefile = (
'%s/local/google3/enterprise/i18n/FrontendMessages_%s.xlb' %
(enthome, languagelocal))
try:
msgs = readxlb.Read(languagefile)
except IOError:
logging.error('Missing language file %s' % languagefile)
if language != 'en':
# Try en language so we can at least replace the tags.
return self.GetLanguageCatalog('en')
# Sorry, no language files
msgs = {}
# Handle right-to-left languages
if language in ['ar', 'fa', 'iw', 'ku', 'sd', 'ur', 'yi']:
msgs['DIR'] = 'rtl'
else:
msgs['DIR'] = 'ltr'
# Encode Unicode messages in UTF-8. (Adminrunner expects UTF-8 messages.)
for key in msgs:
msgs[key] = msgs[key].encode('utf-8')
# Stick in the language list inside the advanced search page in
# sorted order.
msgs["INSERT_LANGUAGE_LIST_IN_ADV_SEARCH"] = self.GenerateLanguageList(
msgs)
return msgs
def GetLanguageCatalog(self, language):
'''Returns a dictionary from 'MSG_...' to translated string'''
enthome = E.getEnterpriseHome()
languagelocal = language.replace('-', '_')
languagefile = ('%s/local/google3/enterprise/i18n/FrontendMessages_%s.xlb' %
(enthome, languagelocal))
try:
msgs = readxlb.Read(languagefile)
except IOError:
logging.error('Missing language file %s' % languagefile)
if language != 'en':
# Try en language so we can at least replace the tags.
return self.GetLanguageCatalog('en')
# Sorry, no language files
msgs = {}
# Handle right-to-left languages
if language in ['ar', 'fa', 'iw', 'ku', 'sd', 'ur', 'yi']:
msgs['DIR'] = 'rtl'
else:
msgs['DIR'] = 'ltr'
# Encode Unicode messages in UTF-8. (Adminrunner expects UTF-8 messages.)
for key in msgs:
msgs[key] = msgs[key].encode('utf-8')
# Stick in the language list inside the advanced search page in
# sorted order.
msgs["INSERT_LANGUAGE_LIST_IN_ADV_SEARCH"] = self.GenerateLanguageList(msgs)
return msgs
def GetARClientAndGlobalConfig(master_machine=None):
if master_machine == None:
machine_list = ReadSysconfigParam('MACHINES')
if type(machine_list) == type(""):
machine_list = map(string.strip, string.split(machine_list, ","))
ver = install_utilities.extract_version_from_dir(E.getEnterpriseHome())
master_machine = find_master.FindMasterUsingChubby(ver)
if master_machine is None:
raise "Could not find master."
ar = adminrunner_client.AdminRunnerClient(master_machine, 2100)
if not ar.IsAlive():
raise "AdminRunner on machine %s:%d is not alive" % (master_machine, 2100)
google_config = {}
ok, response = ar.GetAllParamsIntoDict(google_config)
if not ok:
raise "Can not get all global params from %s:%d" % (master_machine, 2100)
return (ar, google_config)
def run(self):
# Stop all services on all machines
ret = None
for service in self.services_to_stop_:
logging.info("STATUS: STOP %s service %s on %s" %
(service, self.version_, self.machine_))
func = lambda: self.stop_service(service)
ret = try_repeatedly(func, success=0)
if ret:
self.msg_ = "Error stopping %s on %s." % (service,
self.machine_)
logging.error("STATUS: %s" % self.msg_)
self.err_ = ret
return
self.stopped_.extend([service])
# Change the STATE file
# TODO(zsyed): Move STATE file to chubby.
logging.info("STATE: version %s on %s: %s" %
(self.version_, self.machine_, self.target_state_))
if not install_utilities.set_install_state(
self.machine_, E.getEnterpriseHome(), self.target_state_):
# Well here is not clear that we want to exit half way ..
logging.error("ERROR changing state on machine %s. " \
"Please rollback and try again" % self.machine_ )
self.err_ = ret
self.msg_ = "Cannot change STATE file."
return
# Start all services to be started
for service in self.services_to_start_:
logging.info("STATUS: START %s service %s on %s" %
(service, self.version_, self.machine_))
func = lambda: self.start_service(service)
ret = try_repeatedly(func, success=0)
if ret:
self.msg_ = "Error starting %s on %s" % (service,
self.machine_)
logging.error("STATUS: %s" % self.msg_)
self.err_ = ret
return
self.started_.extend([service])
def run(self):
# Stop all services on all machines
ret = None
for service in self.services_to_stop_:
logging.info("STATUS: STOP %s service %s on %s" % (
service, self.version_, self.machine_))
func = lambda: self.stop_service(service)
ret = try_repeatedly(func, success=0)
if ret:
self.msg_ = "Error stopping %s on %s." % (service, self.machine_)
logging.error("STATUS: %s" % self.msg_)
self.err_ = ret
return
self.stopped_.extend([service])
# Change the STATE file
# TODO(zsyed): Move STATE file to chubby.
logging.info("STATE: version %s on %s: %s" % (
self.version_, self.machine_, self.target_state_))
if not install_utilities.set_install_state(
self.machine_, E.getEnterpriseHome(), self.target_state_):
# Well here is not clear that we want to exit half way ..
logging.error("ERROR changing state on machine %s. " \
"Please rollback and try again" % self.machine_ )
self.err_ = ret
self.msg_ = "Cannot change STATE file."
return
# Start all services to be started
for service in self.services_to_start_:
logging.info("STATUS: START %s service %s on %s" % (
service, self.version_, self.machine_))
func = lambda: self.start_service(service)
ret = try_repeatedly(func, success=0)
if ret:
self.msg_ = "Error starting %s on %s" % (service, self.machine_)
logging.error("STATUS: %s" % self.msg_)
self.err_ = ret
return
self.started_.extend([service])
from google3.enterprise.legacy.util import C
import threading
from google3.enterprise.core import core_utils
from google3.enterprise.legacy.install import install_utilities
from google3.enterprise.legacy.adminrunner import reset_index
###############################################################################
true = 1
false = 0
##############################################################################
FLAGS = flags.FLAGS
flags.DEFINE_integer("port", 2100, "")
flags.DEFINE_integer("reset_status_cache_timeout", 60, "")
flags.DEFINE_string("enthome", E.getEnterpriseHome(), "")
flags.DEFINE_string("installstate", None, "")
flags.DEFINE_string("box_keys_dir", None, "")
flags.DEFINE_string("license_keys_dir", None, "")
##############################################################################
def StartupWork(cfg, state):
try:
#############################################################################
# check memory-total
logging.info("check memory-total")
if not cfg.CheckMachineMemory():
logging.fatal("failed to check memory-total")
return
def send(cfg, to, problem, subject, msgText, logSubject):
# No mails in install mode
if cfg.getInstallState() == "INSTALL": return
smtpHost = cfg.getGlobalParam("SMTP_SERVER")
fromEmail = cfg.getGlobalParam("OUTGOING_EMAIL_SENDER")
allSubject = None
if not to:
if problem:
to = cfg.getGlobalParam("PROBLEM_EMAIL")
else:
to = cfg.getGlobalParam("NOTIFICATION_EMAIL")
# if to is empty, we still go through the whole process, but bail out right
# before actually sending the email. we want all the side effects to still
# happen (logging, testing hooks, etc..)
# if "*EMAIL" is None, commands.mkarg() will throw a TypeError exception
if not to:
to = ""
else:
to = string.strip(to)
if problem:
allSubject = "%s: %s" % (
cfg.getGlobalParam("ENT_LICENSE_INFORMATION").get("ENT_BOX_ID", ""),
cfg.getGlobalParam("EMAIL_PROBLEM_PREFIX")
)
else:
allSubject = "%s: %s" % (
cfg.getGlobalParam("ENT_LICENSE_INFORMATION").get("ENT_BOX_ID", ""),
cfg.getGlobalParam("EMAIL_NOTIFICATION_PREFIX")
)
allSubject = allSubject + subject
# Log the subject
if logSubject:
logging.info('Sending mail: To: %s; Subject: %s' % (repr(to),
repr(subject)))
cfg.writeAdminRunnerOpMsg(subject)
dateString = time.strftime("%Y/%m/%d %H:%M:%S",
E.getLocaltime(time.time()))
if msgText:
msgText = msgText + "\n"
else:
msgText = ""
ipAddr = cfg.getGlobalParam("EXTERNAL_WEB_IP")
body = "%s [%s @%s from %s]" % (msgText, M.MSG_MAIL_AUTOGENERATED, dateString,
ipAddr)
# if we have a testing hook, call that
if TEST_MAIL_HANDLER_HOOK != None:
logging.info("calling TEST_MAIL_HANDLER_HOOK instead of mailnotify")
to_list = filter(None, map(string.strip, string.split(to, ",")))
TEST_MAIL_HANDLER_HOOK(smtpHost, to_list, fromEmail, allSubject, body)
return
# if to is empty, bail out
if not (to and smtpHost): return
# We execute this and not call the function directly because mailnotify
# uses signal and we might call this from a secondary thread.
E.execute([E.LOCALHOST],
". %s; cd %s/local/google3/enterprise/legacy/util; "\
"./mailnotify.py %s %s %s %s %s" % (
cfg.getGlobalParam('ENTERPRISE_BASHRC'),
E.getEnterpriseHome(),
commands.mkarg(smtpHost),
commands.mkarg(allSubject),
commands.mkarg(fromEmail),
commands.mkarg(to),
commands.mkarg(body)), None, 0)
def send(cfg, to, problem, subject, msgText, logSubject):
# No mails in install mode
if cfg.getInstallState() == "INSTALL": return
smtpHost = cfg.getGlobalParam("SMTP_SERVER")
fromEmail = cfg.getGlobalParam("OUTGOING_EMAIL_SENDER")
allSubject = None
if not to:
if problem:
to = cfg.getGlobalParam("PROBLEM_EMAIL")
else:
to = cfg.getGlobalParam("NOTIFICATION_EMAIL")
# if to is empty, we still go through the whole process, but bail out right
# before actually sending the email. we want all the side effects to still
# happen (logging, testing hooks, etc..)
# if "*EMAIL" is None, commands.mkarg() will throw a TypeError exception
if not to:
to = ""
else:
to = string.strip(to)
if problem:
allSubject = "%s: %s" % (
cfg.getGlobalParam("ENT_LICENSE_INFORMATION").get(
"ENT_BOX_ID", ""), cfg.getGlobalParam("EMAIL_PROBLEM_PREFIX"))
else:
allSubject = "%s: %s" % (
cfg.getGlobalParam("ENT_LICENSE_INFORMATION").get(
"ENT_BOX_ID",
""), cfg.getGlobalParam("EMAIL_NOTIFICATION_PREFIX"))
allSubject = allSubject + subject
# Log the subject
if logSubject:
logging.info('Sending mail: To: %s; Subject: %s' %
(repr(to), repr(subject)))
cfg.writeAdminRunnerOpMsg(subject)
dateString = time.strftime("%Y/%m/%d %H:%M:%S",
E.getLocaltime(time.time()))
if msgText:
msgText = msgText + "\n"
else:
msgText = ""
ipAddr = cfg.getGlobalParam("EXTERNAL_WEB_IP")
body = "%s [%s @%s from %s]" % (msgText, M.MSG_MAIL_AUTOGENERATED,
dateString, ipAddr)
# if we have a testing hook, call that
if TEST_MAIL_HANDLER_HOOK != None:
logging.info("calling TEST_MAIL_HANDLER_HOOK instead of mailnotify")
to_list = filter(None, map(string.strip, string.split(to, ",")))
TEST_MAIL_HANDLER_HOOK(smtpHost, to_list, fromEmail, allSubject, body)
return
# if to is empty, bail out
if not (to and smtpHost): return
# We execute this and not call the function directly because mailnotify
# uses signal and we might call this from a secondary thread.
E.execute([E.LOCALHOST],
". %s; cd %s/local/google3/enterprise/legacy/util; "\
"./mailnotify.py %s %s %s %s %s" % (
cfg.getGlobalParam('ENTERPRISE_BASHRC'),
E.getEnterpriseHome(),
commands.mkarg(smtpHost),
commands.mkarg(allSubject),
commands.mkarg(fromEmail),
commands.mkarg(to),
commands.mkarg(body)), None, 0)
def doReconfigureNet(config, machines=None, i_am_master=1,
force_ntp_reconfig=0):
""" reconfigure serveriron, DNS, NTPs, iptables, timezone as needed.
Force NTP server reconfiguration if force_ntp_reconfig=1
"""
# first we need to reconfigure our external IP address
ret1 = ret2 = ret4 = ret5 = ret6 = ret8 = 0
configType = config.var('ENT_CONFIG_TYPE')
ent_home = E.getEnterpriseHome()
if not machines:
machines = config.var('MACHINES')
# Sanity check
if not config.var('EXTERNAL_WEB_IP'):
logging.error('config file is missing EXTERNAL_WEB_IP, probably corrupt')
raise Exception, 'config file is missing EXTERNAL_WEB_IP, probably corrupt'
if config.var('USE_DHCP') is None:
logging.error('config file is missing USE_DHCP, probably corrupt')
raise Exception, 'config file is missing USE_DHCP, probably corrupt'
if config.var('DNS_DHCP') is None:
logging.error('config file is missing DNS_DHCP, probably corrupt')
raise Exception, 'config file is missing DNS_DHCP, probably corrupt'
tries = 3 # default for CLUSTER and PILE configuration
if configType in ["SUPER", "ONEBOX", "MINI", "LITE", "FULL"]:
tries = 1
# reconfigure eth0 to dhcp mode
if config.var('USE_DHCP') == 1:
ret1 = ExecuteWrapper(
machines,
"%s LOCALNET DHCP %s" % (
C.RECONFIGURE_COMMAND % ent_home,
config.var('ONEBOX_NETCARD_SETTINGS'),
), None, 600, num_tries = tries)
logging.info("reconfigure eth0 IP to dhcp mode: %s" % ret1)
else:
ret1 = ExecuteWrapper(
machines,
"%s LOCALNET %s %s %s %s" % (
C.RECONFIGURE_COMMAND % ent_home,
config.var('EXTERNAL_WEB_IP'),
config.var('EXTERNAL_NETMASK'),
config.var('EXTERNAL_DEFAULT_ROUTE'),
config.var('ONEBOX_NETCARD_SETTINGS'),
), None, 600, num_tries = tries)
logging.info("reconfigure eth0 IP address: %s" % ret1)
elif "CLUSTER" == configType:
# reconfigure serveriron IP address
cmd = "(sleep 5; %s SERVERIRON %s %s %s %s %s %s %s %s %s >&/dev/null " \
"</dev/null)" % (
C.RECONFIGURE_COMMAND % ent_home,
"%s/local/conf/defaults/" % ent_home,
commands.mkarg(config.var("EXTERNAL_SWITCH_IP")),
commands.mkarg(config.var("EXTERNAL_WEB_IP")),
commands.mkarg(config.var("EXTERNAL_CRAWL_IP")),
commands.mkarg(config.var("EXTERNAL_NETMASK")),
commands.mkarg(config.var("EXTERNAL_DEFAULT_ROUTE")),
commands.mkarg(str(config.var("EXTERNAL_WEB_PORT"))),
commands.mkarg(str(config.var("SERVERIRON_AUTONEGOTIATION"))),
commands.mkarg(str(config.var("ENT_ENABLE_EXTERNAL_SSH"))),
)
t = threading.Thread(target=E.execute,
args=([E.getCrtHostName()], cmd, None, 60))
t.start()
logging.info("serveriron IP address reconfigured")
# we don't want to change ANYTHING on PILE clusters
if "PILE" != configType:
# DNS needs to be set everywhere
dns_server = "\"\""
dns_searchpath = "\"\""
if config.var('BOT_DNS_SERVERS') != None:
dns_server = config.var('BOT_DNS_SERVERS')
if config.var('DNS_SEARCH_PATH') != None:
dns_searchpath = config.var('DNS_SEARCH_PATH')
if config.var('DNS_DHCP') != 1:
ret8 = ExecuteWrapper(
machines,
"%s DNSMODE STATIC" % (
C.RECONFIGURE_COMMAND % ent_home
), None, 600, num_tries = tries)
logging.info("setting DNS mode to STATIC: %s" % ret8)
ret2 = ExecuteWrapper(
machines,
"%s DNS %s %s" % (
C.RECONFIGURE_COMMAND % ent_home,
commands.mkarg(dns_server),
commands.mkarg(dns_searchpath),
), None, 600, num_tries = tries)
logging.info("reconfigure DNS: %s" % ret2)
else:
ret8 = ExecuteWrapper(
machines,
"%s DNSMODE DHCP" % (
C.RECONFIGURE_COMMAND % ent_home
), None, 600, num_tries = tries)
logging.info("setting DNS mode to DHCP: %s" % ret8)
# NTP is special: all machines but the master must set their
# NTP server to the master, the master to the external one.
# However, It can take 3 minutes for the stratum level of the master
# node to be set. Before that, non-master nodes using the master
# node to do "ntpdate" may return "no server suitable for synchronization
# found" error.
# To fix the problem, we just use the external ntp servers for all nodes.
# Later, periodic script will set the non-master nodes to use the master
# node. (periodic script will only set it once as long as master does
# not switch)
ntpServers = "\"\""
if config.var('NTP_SERVERS') != None:
ntp_server_list = copy.copy(config.var('NTP_SERVERS'))
AddDefaultNTPServer(ntp_server_list)
ntpServers = string.join(ntp_server_list, ",")
if i_am_master:
ret4 = ExecuteWrapper(
machines,
"%s NTP %d %s" % (
C.RECONFIGURE_COMMAND % ent_home,
force_ntp_reconfig,
commands.mkarg(ntpServers)),
None, 600, num_tries = tries)
logging.info("reconfigure NTP: %s" % ret4)
# whenever we print dates, we want to include the timezone. Since
# the timezone may change on the machine (through config interface),
# and java VM does not pick this change up automatically,
# we keep track of the timezone here
ret5 = ExecuteWrapper(
machines,
"%s TIMEZONE %s" % (
C.RECONFIGURE_COMMAND % ent_home,
commands.mkarg(config.var('TIMEZONE'))),
None, 600, num_tries = tries)
logging.info("reconfigure TIMEZONE: %s" % ret5)
# iptables
if configType in ["SUPER", "ONEBOX", "LITE", "FULL"]:
iptables_file = "%s/local/conf/defaults/iptables.onebox" % ent_home
elif "MINI" == configType:
iptables_file = "%s/local/conf/defaults/iptables.mini" % ent_home
elif "CLUSTER" == configType:
iptables_file = "%s/local/conf/defaults/iptables.cluster" % ent_home
else:
iptables_file = "%s/local/conf/defaults/iptables.open" % ent_home
ret6 = ExecuteWrapper(
machines,
"%s IPTABLES %s %s %s" % (
C.RECONFIGURE_COMMAND % ent_home,
iptables_file,
commands.mkarg(str(config.var("ENT_ENABLE_EXTERNAL_SSH"))),
commands.mkarg(str(config.var("ENT_ENABLE_EXTERNAL_BORGMON"))),
), None, 600, num_tries = tries)
logging.info("reconfigure IPTABLES: %s" % ret6)
# return value from each reconfigurenet call is:
# 0 : no change
# 1 : changed
# 2+: error (but we report no errors)
# _our_ return value is if any of them failed
ret = (ret1 <= 1 and ret2 <= 1 and ret4 <= 1 and
ret5 <= 1 and ret6 <= 1 and ret8 <= 1)
return ret
def main(argv):
""" expects: argv[0] to be a machine name, argv[1] be a google_config file
- copys google_config file from the machine specified into the local machine
(by calling CopyFile() function)
- calls ReplicateConfig() - which """
if len(argv) < 2:
sys.exit(__doc__)
machine = argv[0]
global_file = argv[1]
global_dir = os.path.dirname(global_file)
if E.getCrtHostName() == machine:
logging.info("I do not try to replicate to myself")
sys.exit(0)
config = config_factory.ConfigFactory().CreateConfig(global_file)
if not config.Load():
sys.exit("ERROR: Cannot read file %s " % global_file)
file_to_copy = [ C.ETC_SYSCONFIG,
global_file,
"%s/lic_counter" % global_dir,
"%s/lic_counter.bck" % global_dir,
"%s/passwd" % global_dir,
"/export/hda3/versionmanager/vmanager_passwd",
"%s/server.p12" % global_dir
]
# Sync all the files
for f in file_to_copy:
CopyFile(machine, f, production_tmp)
# sync apache certificate
CopyFileAsRoot(machine, (ssl_cert.CERT_FILENAME % config.var('ENTERPRISE_HOME')),
production_tmp, config)
# sync private key
CopyFileAsRoot(machine, (ssl_cert.KEY_FILENAME % config.var('ENTERPRISE_HOME')),
production_tmp, config)
# sync the support request repository directory
# GRR: ReplicateDirectory(machine, SUPPORTREQUEST_RECORDS_DIR)
# replicate config including per collection/frontend stuff
# some dirs in conf don't need to be rsync'ed
if core_utils.CanRunGSAMaster(E.getCrtHostName()):
exclude_patterns = ['cmr_working/failure/', 'cmr_working/success/',
'cmr_working/statusz/']
else:
exclude_patterns = ['cmr_working/', 'cmr/', 'fixer/', 'fixer_cmr/', 'gems']
ReplicateDirectory(machine, "%s/local/conf/" % config.var('ENTERPRISE_HOME'),
exclude_patterns)
# to replicate per frontend gws stuff (keymatches, gws bad urls)
ReplicateDirectory(machine, "%s/gws/" % config.var('GOOGLEDATA'))
# need to adjust NTP if master has changed
# assumption : this machine is not the master
if 'PILE' != config.var('ENT_CONFIG_TYPE'):
ent_home = E.getEnterpriseHome()
os.system("%s NTP %s" % (C.RECONFIGURE_COMMAND % ent_home, machine))
# also make sure DNS is up to date
os.system("%s DNS %s %s" % (
C.RECONFIGURE_COMMAND % ent_home,
config.var('BOT_DNS_SERVERS'),
config.var('DNS_SEARCH_PATH')))
def main(argv):
""" expects: argv[0] to be a machine name, argv[1] be a google_config file
- copys google_config file from the machine specified into the local machine
(by calling CopyFile() function)
- calls ReplicateConfig() - which """
if len(argv) < 2:
sys.exit(__doc__)
machine = argv[0]
global_file = argv[1]
global_dir = os.path.dirname(global_file)
if E.getCrtHostName() == machine:
logging.info("I do not try to replicate to myself")
sys.exit(0)
config = config_factory.ConfigFactory().CreateConfig(global_file)
if not config.Load():
sys.exit("ERROR: Cannot read file %s " % global_file)
file_to_copy = [
C.ETC_SYSCONFIG, global_file,
"%s/lic_counter" % global_dir,
"%s/lic_counter.bck" % global_dir,
"%s/passwd" % global_dir,
"/export/hda3/versionmanager/vmanager_passwd",
"%s/server.p12" % global_dir
]
# Sync all the files
for f in file_to_copy:
CopyFile(machine, f, production_tmp)
# sync apache certificate
CopyFileAsRoot(machine,
(ssl_cert.CERT_FILENAME % config.var('ENTERPRISE_HOME')),
production_tmp, config)
# sync private key
CopyFileAsRoot(machine,
(ssl_cert.KEY_FILENAME % config.var('ENTERPRISE_HOME')),
production_tmp, config)
# sync the support request repository directory
# GRR: ReplicateDirectory(machine, SUPPORTREQUEST_RECORDS_DIR)
# replicate config including per collection/frontend stuff
# some dirs in conf don't need to be rsync'ed
if core_utils.CanRunGSAMaster(E.getCrtHostName()):
exclude_patterns = [
'cmr_working/failure/', 'cmr_working/success/',
'cmr_working/statusz/'
]
else:
exclude_patterns = [
'cmr_working/', 'cmr/', 'fixer/', 'fixer_cmr/', 'gems'
]
ReplicateDirectory(machine,
"%s/local/conf/" % config.var('ENTERPRISE_HOME'),
exclude_patterns)
# to replicate per frontend gws stuff (keymatches, gws bad urls)
ReplicateDirectory(machine, "%s/gws/" % config.var('GOOGLEDATA'))
# need to adjust NTP if master has changed
# assumption : this machine is not the master
if 'PILE' != config.var('ENT_CONFIG_TYPE'):
ent_home = E.getEnterpriseHome()
os.system("%s NTP %s" % (C.RECONFIGURE_COMMAND % ent_home, machine))
# also make sure DNS is up to date
os.system(
"%s DNS %s %s" %
(C.RECONFIGURE_COMMAND % ent_home, config.var('BOT_DNS_SERVERS'),
config.var('DNS_SEARCH_PATH')))
