def tableau_status_loop(self):
while True:
logger.debug("status-check: About to timeout or " + \
"wait for a new primary to connect")
try:
system_key = SystemKeys.STATUS_REQUEST_INTERVAL
request_interval = self.system[system_key]
except ValueError:
request_interval = self.STATUS_REQUEST_INTERVAL_DEFAULT
new_primary = self.manager.check_status_event.wait(request_interval)
logger.debug("status-check: new_primary: %s", new_primary)
if new_primary:
self.manager.clear_check_status_event()
session = meta.Session()
try:
# Don't do a 'tabadmin status -v' if upgrading
acquired = self.rwlock.read_acquire(blocking=False)
if not acquired:
logger.debug("status-check: Upgrading. Won't run.")
continue
self.check_status()
finally:
if acquired:
self.rwlock.read_release()
session.rollback()
meta.Session.remove()
def get(cls, agentid, **kwargs):
session = meta.Session()
entry = cls.get_by_agentid(agentid)
if not entry:
entry = LicenseEntry(agentid=agentid)
session.add(entry)
if 'interactors' in kwargs and kwargs['interactors'].isdigit():
entry.license_type = LicenseEntry.LICENSE_TYPE_NAMED_USER
entry.interactors = int(kwargs['interactors'])
if 'viewers' in kwargs and kwargs['viewers'].isdigit():
entry.viewers = int(kwargs['viewers'])
entry.cores = 0
entry.core_licenses = 0
return entry
elif 'cores' in kwargs:
entry.license_type = LicenseEntry.LICENSE_TYPE_CORE
entry.cores = kwargs['cores']
entry.core_licenses = kwargs['core-licenses']
entry.interactors = 0
entry.viewers = 0
return entry
# Default setting to let us know the license info has been
# received. It could be 0 interactors and 0 viewers, which
# means "no license".
entry.license_type = LicenseEntry.LICENSE_TYPE_NAMED_USER
entry.cores = 0
entry.core_licenses = 0
return entry
def check_status(self):
logger.setLevel(self.system[SystemKeys.DEBUG_LEVEL])
# FIXME: Tie agent to domain.
agent = self.manager.agent_by_type(AgentManager.AGENT_TYPE_PRIMARY)
if not agent:
logger.debug("status-check: The primary agent is either " + \
"not connected or not enabled.")
return
aconn = agent.connection
if not aconn:
session = meta.Session()
logger.debug("status-check: No primary agent currently connected.")
self.remove_all_status()
session.commit()
return
# Don't do a 'tabadmin status -v' if the user is doing an action.
acquired = aconn.user_action_lock(blocking=False)
if not acquired:
logger.debug("status-check: Primary agent locked for user "
"action. Skipping status check.")
return
# We don't force the user to delay starting their request
# until the 'tabadmin status -v' is finished.
aconn.user_action_unlock()
self.check_status_with_connection(agent)
def check(self, agent):
server = self.server
body = server.cli_cmd('tabadmin license', agent, timeout=60 * 10)
if not 'exit-status' in body or body['exit-status'] != 0:
return body
if not 'stdout' in body:
return body
session = meta.Session()
output = body['stdout']
license_data = LicenseEntry.parse(output)
entry = LicenseEntry.get(agentid=agent.agentid, **license_data)
session.commit()
msg = "License type: %s, " % str(entry.license_type)
if entry.license_type == LicenseEntry.LICENSE_TYPE_NAMED_USER:
msg += "interactors: %s, viewers: %s" % \
(str(entry.interactors), str(entry.viewers))
elif entry.license_type == LicenseEntry.LICENSE_TYPE_CORE:
msg += "cores used: %s of %s" % \
(str(entry.cores), str(entry.core_licenses))
else:
msg += "interactors: %s, viewers: %s, cores used: %s of %s" \
% (str(entry.interactors), str(entry.viewers),
str(entry.cores), str(entry.core_licenses))
notification = self.server.notifications.get("tlicense")
if entry.valid():
if notification.color == 'red':
data = agent.todict()
data['stdout'] = msg
self.server.event_control.gen(EventControl.LICENSE_VALID, data)
# Remember when we sent the notification
notification.modification_time = func.now()
notification.color = 'green'
notification.notified_color = 'green'
notification.description = None
session.commit()
return license_data
else:
# license is invalid
if notification.color != 'red':
# Generate an event
data = agent.todict()
data['error'] = msg
if notification.color != 'red':
self.server.event_control.gen(EventControl.LICENSE_INVALID,
data)
notification.color = 'red'
notification.notified_color = 'red'
# Remember when we sent the notification
notification.modification_time = func.now()
session.commit()
return server.error("License invalid on '%s': %s" % \
(agent.displayname, msg))
def sync(cls, agent):
stmt = 'SELECT * FROM data_connections'
data = agent.odbc.execute(stmt)
if 'error' in data:
return data
if '' not in data:
data['error'] = "Missing '' key in query response."
return data
ids = []
envid = agent.server.environment.envid
session = meta.Session()
for odbcdata in ODBC.load(data):
tid = odbcdata.data['id']
entry = DataConnection.get(envid, tid, default=None)
if not entry:
entry = DataConnection(envid=envid)
session.add(entry)
entry.envid = envid
odbcdata.copyto(entry)
ids.append(entry.dcid)
session.query(DataConnection).\
filter(not_(DataConnection.dcid.in_(ids))).\
delete(synchronize_session='fetch')
session.commit()
d = {u'status': 'OK', u'count': len(data[''])}
return d
def init_firewall_ports(self, agent):
"""Make sure the agent's firewall ports have been initialized."""
session = meta.Session()
rows = session.query(FirewallEntry).\
filter(FirewallEntry.agentid == agent.agentid).\
all()
# Already populated
if rows:
return
# It was empty so add the initial default set.
# First the listen_port
entry = FirewallEntry(agentid=agent.agentid,
name="Palette Agent",
port=agent.listen_port,
color="green")
session.add(entry)
# Add the others
if agent.agent_type == AgentManager.AGENT_TYPE_PRIMARY:
ports = FirewallManager.DEFAULT_PRIMARY_PORTS
else:
ports = FirewallManager.DEFAULT_NON_PRIMARY_PORTS
for port in ports:
entry = FirewallEntry(agentid=agent.agentid,
name=port['name'],
port=port['port'],
color=port['color'])
session.add(entry)
session.commit()
def move_twb_to_db(self):
"""Copy the twb file contents on the controller to the database
and remove the twb files.
"""
path = self.server.config.get('palette', 'workbook_archive_dir')
controller_path = os.path.abspath(path)
session = meta.Session()
rows = session.query(WorkbookUpdateEntry).\
filter(WorkbookUpdateEntry.url != '').\
filter(WorkbookUpdateEntry.url != None).\
filter(WorkbookUpdateEntry.twb == None).\
all()
for row in rows:
twb_path = os.path.join(controller_path, row.url)
try:
with open(twb_path, "r") as fd_twb:
contents = fd_twb.read()
except IOError as err:
logger.error("move_twb_to_db open failed: %s", str(err))
continue
row.twb = contents
session.commit()
twb_path = os.path.join(controller_path, row.url)
os.unlink(twb_path)
def remove(self, fileid):
# envid is technically not required
session = meta.Session()
session.query(FileEntry).\
filter(FileEntry.envid == self.envid).\
filter(FileEntry.fileid == fileid).\
delete()
session.commit()
def add(self, agent, process_name, cpu, memory):
session = meta.Session()
entry = MetricEntry(agentid=agent.agentid,
process_name=process_name,
cpu=cpu,
memory=memory)
session.add(entry)
session.commit()
def __setitem__(self, key, value):
""" Updates the database row for key with 'value'.
NOTE: does *not* commit the database.
"""
# This cast implicity checks if value is correctly typed.
value = cast(key, value)
session = meta.Session()
entry = SystemEntry(envid=self.envid, key=key, value=str(value))
session.merge(entry)
def build(cls, envid, aconn):
"""Create an agent from a new connection."""
body = aconn.auth
session = meta.Session()
uuid = body['uuid']
entry = Agent.get_by_uuid(envid, uuid)
if entry:
# Make a copy of the object
entry = session.merge(entry)
# but points at the same aconn...
else:
entry = Agent(envid=envid, uuid=uuid)
session.add(entry)
entry.conn_id = aconn.conn_id
entry.version = body['version']
entry.os_version = body['os-version']
entry.processor_type = body['processor-type']
entry.processor_count = body['processor-count']
entry.installed_memory = body['installed-memory']
entry.hostname = body['hostname']
entry.fqdn = body['fqdn']
entry.ip_address = body['ip-address']
entry.peername = aconn.peername
entry.listen_port = body['listen-port']
if 'static-hostname' in body:
entry.static_hostname = bool(body['static-hostname'])
else:
entry.static_hostname = False
# Note: Do not set agent_type here since 1) We need to know
# what the agent_type was in the case where the row existed, and
# 2) the agent_type isn't known yet at the time we are called anyway.
entry.username = u'palette' # fixme
entry.password = u'tableau2014'
entry.install_dir = body['install-dir']
# FIXME: make required when all agents are updated.
if 'os-bitness' in body:
entry.bitness = body['os-bitness']
entry.last_connection_time = func.now()
session.commit()
if entry.iswin:
entry.path = ntpath
parts = body['data-dir'].split(':')
entry.data_dir = ntpath.join(parts[0].upper() + ':', parts[1])
else:
entry.path = posixpath
entry.data_dir = body['data-dir']
return entry
def get_tableau_status(self):
try:
return meta.Session().query(TableauProcess).\
join(Agent).\
filter(Agent.envid == self.envid).\
filter(Agent.agent_type == 'primary').\
filter(TableauProcess.name == 'Status').\
one().status
except NoResultFound:
return TableauProcess.STATUS_UNKNOWN
def max(cls, column, filters=None, default=None):
query = meta.Session().query(func.max(getattr(cls, column)))
if filters:
query = cls.apply_filters(query, filters)
# pylint: disable=maybe-no-member
value = query.scalar()
if value is None:
return default
return value
def __setitem__(self, key, value):
""" Update the system table but don't do a database commit """
value = cast(key, value)
if key in self:
if value == self[key]: # always string comparison
return
session = meta.Session()
entry = SystemEntry(envid=self.req.envid, key=key, value=value)
session.merge(entry)
dict.__setitem__(self, key, value)
def _retain_some(self):
"""Retain only the configured number of workbook versions.
Returns:
The number of archive versions removed.
"""
retain_count = self.system[SystemKeys.WORKBOOK_RETAIN_COUNT]
if not retain_count or retain_count == -1:
return 0
removed_count = 0
session = meta.Session()
# List of workbooks that have excess archived versions:
# [(workbookid, total-count), ...]
# Note we select only successfully archived workbook versions
# (url != ''). We don't want want to count unsuccessfully
# archived versions in the count of how many we have.
results = session.query(WorkbookUpdateEntry.workbookid, func.count()).\
filter(WorkbookUpdateEntry.url != '').\
group_by(WorkbookUpdateEntry.workbookid).\
having(func.count() > retain_count).\
all()
# logger.debug("workbooks _retain_some len: %d, results: %s",
# len(results), str(results))
for result in results:
# Get list of old workbook archive entries to delete
rows = session.query(WorkbookUpdateEntry).\
filter(WorkbookUpdateEntry.workbookid == result[0]).\
filter(WorkbookUpdateEntry.url != '').\
order_by(WorkbookUpdateEntry.timestamp.asc()).\
limit(result[1] - retain_count).\
all()
for row in rows:
# We have to remove the WorkbookUpdateEntry first
# due to the foreign key constraint in files pointing to it.
session.query(WorkbookUpdateEntry).\
filter(WorkbookUpdateEntry.wuid == row.wuid).\
delete()
session.commit()
self.server.files.remove_file_by_id(row.fileid)
if row.fileid_twbx:
self.server.files.remove_file_by_id(row.fileid_twbx)
# Fixme: We could increment only if it successfully deleted.
removed_count += 1
return removed_count
def populate(cls):
session = meta.Session()
entry = session.query(cls).first()
if entry:
return
if not cls.defaults_filename is None:
rows = cls.populate_from_file(cls.defaults_filename)
else:
rows = cls.defaults
for d in rows:
obj = cls(**d)
session.add(obj)
session.commit()
def _add(self, agentid, name, pid, status):
"""Note a session is passed. When updating the status table, we
do remove_all_status, then slowly add in the new status before
doing the commit, so the table is not every empty/building if
somebody checks it.
"""
session = meta.Session()
entry = TableauProcess(agentid=agentid, name=name,
pid=pid, status=status)
# We merge instead of add since 'tabadmin status -v' sometimes
# returns duplicate lines.
session.merge(entry)
def sync(cls, agent):
envid = agent.server.environment.envid
stmt = \
'SELECT id, name, status, created_at, updated_at, ' +\
'user_quota, content_admin_mode, storage_quota, metrics_level, '+\
'status_reason, subscriptions_enabled, ' +\
'custom_subscription_footer, custom_subscription_email, '+\
'luid, query_limit, url_namespace ' +\
'FROM sites'
data = agent.odbc.execute(stmt)
if 'error' in data:
return data
if '' not in data:
data['error'] = "Missing '' key in query response."
return data
ids = []
session = meta.Session()
for row in data['']:
entry = Site.get(envid, row[0], default=None)
if not entry:
entry = Site(envid=envid, id=row[0])
session.add(entry)
entry.name = row[1]
entry.status = row[2]
entry.created_at = row[3]
entry.updated_at = row[4]
entry.user_quota = row[5]
entry.content_admin_mode = row[6]
entry.storage_quota = row[7]
entry.metrics_level = row[8]
entry.status_reason = row[9]
entry.subscriptions_enabled = row[10]
entry.custom_subscription_footer = row[11]
entry.custom_subscription_email = row[12]
entry.luid = row[13]
entry.query_limit = row[14]
entry.url_namespace = row[15]
ids.append(entry.siteid)
# FIXME: don't delete
session.query(Site).\
filter(not_(Site.siteid.in_(ids))).\
delete(synchronize_session='fetch')
session.commit()
d = {u'status': 'OK', u'count': len(data[''])}
return d
def _retain_some_obj(self, retain_count, obj_class):
"""Do the removal of excess WorkbookExtractEntry or
DataSourceExtractEntry rows.
Called with:
- How many to retain
- The object class (WorkbookExtractEntry or
DataSourceExtractEntry)
"""
session = meta.Session()
# List of parentids that have excess archived versions:
# [(parentid, total-count), ...]
# Note we select only successfully archived extracts when
# fileid != None. We don't want want to count unsuccessfully
# archived extracts in the count of how many we have.
results = session.query(obj_class.parentid, func.count()).\
filter(obj_class.fileid != None).\
group_by(obj_class.parentid).\
having(func.count() > retain_count).\
all()
logger.debug("refresh _retain_some_obj %s len: %d, results: %s",
obj_class.__tablename__, len(results), str(results))
removed_count = 0
for result in results:
# Get the list of rows to delete
rows = session.query(obj_class).\
filter(obj_class.parentid == result[0]).\
filter(obj_class.fileid != None).\
order_by(obj_class.sid.asc()).\
limit(result[1] - retain_count).\
all()
for row in rows:
# We have to remove the extract row first
# due to the foreign key constraint in the files table
# pointing to it.
session.query(obj_class).\
filter(obj_class.sid == row.sid).\
delete()
session.commit()
self.server.files.remove_file_by_id(row.fileid)
# Fixme: We could increment only if it successfully deleted.
removed_count += 1
return removed_count
def _archive_updates(self, agent, updates):
"""Attempt to archive datasources from DataSourceUpdate rows."""
session = meta.Session()
count = 0
logger.debug("Datasource Archive update count: %d", len(updates))
for update in updates:
if not self.system[SystemKeys.DATASOURCE_RETAIN_COUNT]:
logger.info(
"Datasource Archive disabled during fixup." + \
" Exiting for now.")
break
if not self.server.odbc_ok():
logger.info("Datasource Archive Fixup: Archive build " + \
"stopping due to current state")
break
logger.debug("Datasource Archive update refresh dsid %d",
update.dsid)
session.refresh(update)
try:
self._archive_ds(agent, update)
except ArchiveException as ex:
if ex.value == ArchiveError.BAD_CREDENTIALS:
msg = "datasource _archive_updates: tabcmd failed due " + \
"to bad credentials. " + \
"Skipping any remaining datasource updates now."
logger.info(msg)
break
else:
raise # should never happen
count += 1
# Retain only configured number of versions
if count:
retain_removed_count = self._retain_some()
else:
retain_removed_count = 0
return {
u'status': 'OK',
u'updates-archived': count,
u'retain-removed-count': retain_removed_count
}
def fixup(self, agent):
if not self.system[SystemKeys.WORKBOOK_RETAIN_COUNT]:
logger.debug("Workbook archives are not enabled. Fixup not done.")
return {u'disabled':
'Workbook Archives are not enabled. Fixup not done.'}
session = meta.Session()
# potentially serveral thousand?
updates = session.query(WorkbookUpdateEntry).\
filter(or_(WorkbookUpdateEntry.url == "",
WorkbookUpdateEntry.url == None)).\
all()
return self._archive_updates(agent, updates)
def get_by_envid_type(cls, envid, cloud_type):
session = meta.Session()
filters = {'envid':envid, 'cloud_type':cloud_type}
# pylint: disable=maybe-no-member
subquery = session.query(func.max(cls.modification_time))
subquery = cls.apply_filters(subquery, filters).subquery()
query = session.query(cls).filter(cls.modification_time.in_(subquery))
try:
# There is a *theoretical* possibility that multiple records
# could have the same timestamp so just take the first.
return query.first()
except NoResultFound:
return None
def _remove_dsu(self, update):
"""Remove an update from the datasource_updates table.
We do this if a datasource update entry was listed a new datasource
but then was deleted before we ran 'tabcmd'.
"""
session = meta.Session()
try:
session.query(DataSourceUpdateEntry).\
filter(DataSourceUpdateEntry.dsuid == update.dsuid).\
delete()
except NoResultFound:
logger.error("_remove_dsu: datasource already deleted: %d",
update.dsuid)
return
session.commit()
def _remove_wbu(self, update):
"""Remove an update from the workbook_updates table.
We do this if a workbook update entry listed a new workbook
but then was deleted before we ran 'tabcmd'.
"""
session = meta.Session()
try:
session.query(WorkbookUpdateEntry).\
filter(WorkbookUpdateEntry.wuid == update.wuid).\
delete()
except NoResultFound:
logger.error("_remove_wbu: workbook already deleted: %d",
update.wuid)
return
session.commit()
def _check_existing_unfinished(self, agent, extract_thresholds):
"""Go through our unfinished extracts db to see if any have exceeded
their start delay or duration thresholds.
"""
session = meta.Session()
rows = session.query(ExtractEntry).\
filter(ExtractEntry.progress != 100).\
all()
db_now_utc = agent.odbc.get_db_now_utc()
for row in rows:
data = dict(agent.todict().items() + extract_thresholds.items())
self._process(data, db_now_utc, row)
session.commit() # can move this after proven reliable
def load(self, agent):
envid = self.server.environment.envid
self._prune(agent, envid)
controldata = HttpControlData(self.server)
userdata = self.load_users(agent)
maxid = HttpRequestEntry.maxid(envid)
datadict = agent.odbc.execute(self.get_maxid_statement(maxid))
if 'error' in datadict:
return datadict
if '' not in datadict:
datadict['error'] = "Missing '' key in query response."
return datadict
rows = []
schema = ODBC.schema(datadict)
counter = 0
session = None
for row in datadict['']:
if counter % 1000 == 0:
if session != None:
session.commit()
# Our table was empty so don't test for alerts on the one
# placeholder row we brought in.
if maxid is not None:
for entry in rows:
# print "entry: id", entry.id, "action", entry.action,
# print "vizal_session", entry.vizql_session
self._test_for_alerts(rows, entry, agent,
controldata)
session = meta.Session()
rows = []
counter += 1
odbcdata = ODBCData(schema, row)
entry = HttpRequestEntry()
entry.envid = envid
odbcdata.copyto(entry)
system_user_id = userdata.get(entry.site_id, entry.user_id)
entry.system_user_id = system_user_id
session.add(entry)
rows.append(entry)
if session != None:
session.commit()
return {u'status': 'OK', u'count': len(datadict[''])}
def _archive_updates(self, agent, updates):
"""Attempt to archive workbooks from WorkbookUpdate rows."""
session = meta.Session()
count = 0
data = {}
logger.debug("Workbook archive update count: %d\n", len(updates))
for update in updates:
if not self.system[SystemKeys.WORKBOOK_RETAIN_COUNT]:
logger.info(
"Workbook Archive disabled during fixup." + \
" Exiting for now.")
break
if not self.server.odbc_ok():
logger.info("Workbook Archive Fixup: Archive build " + \
"stopping due to current state")
break
logger.debug("Workbook archive update refresh wid %d",
update.workbookid)
session.refresh(update)
try:
self._archive_wb(agent, update)
except ArchiveException as ex:
if ex.value == ArchiveError.BAD_CREDENTIALS:
msg = "workbook _archive_updates: tabcmd failed due to " + \
"bad credentials. " + \
"Skipping any remaining workbook updates now."
logger.info(msg)
data[u'error'] = msg
break
else:
raise # should never happen
count += 1
# Retain only configured number of versions
if count:
retain_removed_count = self._retain_some()
else:
retain_removed_count = 0
data[u'updates-archived'] = count
data[u'retain-removed-count'] = retain_removed_count
return data
def fixup(self, agent):
if not self.system[SystemKeys.DATASOURCE_RETAIN_COUNT]:
logger.debug("Datasource archives are disabled. Fixup not done.")
return {
u'disabled':
'Datasource Archives are not enabled. Fixup not done.'
}
session = meta.Session()
# potentially serveral thousand?
updates = session.query(DataSourceUpdateEntry).\
filter(or_(DataSourceUpdateEntry.url == "",
DataSourceUpdateEntry.url == None)).\
all()
return self._archive_updates(agent, updates)
def _archive(self, agent, obj_class):
"""Archive extracts for the object type."""
session = meta.Session()
updates = session.query(obj_class).\
filter(obj_class.fileid == None).\
all()
count = 0
for update in updates:
# fixme: When a system table entry is added for extract
# refresh data source, check which one is being archived and
# if it is still enabled, etc.
if not self.system[SystemKeys.EXTRACT_REFRESH_WB_RETAIN_COUNT]:
logger.info("Extract Archiving disabled during archiving."
" Exiting for now.")
break
name = update.parent.name # cache in case it fails and is removed
try:
filename = self._build_extract(agent, update)
except ArchiveException as ex:
if ex.value == ArchiveError.BAD_CREDENTIALS:
msg = "extract archive: tabcmd failed due to " + \
"bad credentials. " + \
"Skipping any remaining extract archives now."
logger.info(msg)
break
else:
raise # should never happen
if not filename:
logger.error(
"Failed to retrieve extract refresh: from %s: %d - %s",
update.__tablename__, update.parentid, name)
continue
# retrieval is a long process, so commit after each.
meta.Session.commit()
count += 1
if count:
self._retain_some()
return count
def _remove_refresh_entry(self, update):
"""Remove a refresh extract entry from the workbook_extracts or
datasource_extracts table."""
session = meta.Session()
try:
session.query(update.__class__).\
filter(update.__class__.sid == update.sid).\
delete()
except NoResultFound:
logger.error(
"_remove_refresh_entry: %s extract already deleted: %d",
update.__tablename__, update.sid)
return
session.commit()
return
