def load(self, dic):
# 获取发布微博的用户信息
uid = 0L
user = dic.get('user',None)
if user is None:
uid = dic.get('uid', dic.get('user_id', None))
if uid is None:
raise ValueError('Data Error: Either User information or user id is expected in the status data!')
else:
self.uid = uid
else:
from profile import UserProfile
uprofile = UserProfile()
uprofile.load(user)
uid = long(uprofile.idstr)
self.uid = uid
# 将用户插入到用户表
self.batches.extend(uprofile.get_batches())
self.mid = long( dic.get('idstr',dic.get('id')) )
# 处理转发微博
r_id = 0
repost = dic.get('retweeted_status')
if repost is None:
pass
else:
from weibo import Repost
# 将转发微博存在repost列族
retweet = Repost()
retweet.load(self.uid, self.mid, repost)
# print retweet
r_id = retweet.id
self.batches.extend(retweet.get_batches())
# 将转发微博对应的原创微博单独存放一份作为原始微博
if repost.get('deleted', 0) == 0: # 如果被转发的微博已经被删除，则不保存为原创微博
original = Status()
original.load(repost)
self.batches.extend(original.get_batches())
self.r_id = long(r_id)
# 处理其他字段信息
for attr in Status.attrs:
v = dic.get(attr)
if attr == 'key':
v = dic.get('id')
if v is None:
if attr in ['reposts_count', 'comments_count', 'attitudes_count']:
v = 0
else:
continue
self.setattr(attr,v)
ustatus = UserStatuses()
ustatus.load(self, self.uid)
self.batches.extend(ustatus.get_batches())
def render_autopass_puzzle(puzzle):
# get passphrase that was submitted with this request, if any:
submitted = request.args.get('pass')
# get current user's passphrase:
profile = UserProfile.get_by_user(request.remote_addr)
# see if they submitted the correct one:
if submitted and (submitted == profile.current_passphrase):
profile.solved_puzzles.append('autopass/' + puzzle)
profile.put()
value = 'correct! '
value += progress('autopass/' + puzzle)
return value
# fallthrough logic - incorrect or no passphrase submitted:
# TODO: add extra output on pass submitted, but incorrect?
# (but then it'd be outside of the manual HTML structure)
# generate a new passphrase:
passphrase = 'default'
with app.open_resource('data/passphrases.json') as f:
passphrases = json.load(f)
passphrase = random.choice(passphrases)
# store it in user's profile:
profile.current_passphrase = passphrase
profile.put()
return render_template('autopass/' + puzzle, passphrase=passphrase)
def get_username_from_system_user_id(self, envid, system_user_id):
# pylint: disable=invalid-name
profile = UserProfile.get_by_system_user_id(envid, system_user_id)
if profile:
return profile.display_name()
else:
# FIXME: probably not correct.
return UNDEFINED
def progress(curr):
profile = UserProfile.get_by_user(request.remote_addr)
nextp = nextPuzzle(curr)
if nextp:
profile.current_puzzle = nextp # this is probably fine.
profile.put()
return '<a href="/' + nextp + '">Next puzzle!</a>'
else:
# no next puzzle
return 'All done!'
def get_batches(self):
#用于存储需要更新的内容，可能涉及多个表、多个列族
batches = []
#根据关系用户，将关系用户的Profile插入到表中
for uid, user in self.users.iteritems():
uprofile = UserProfile()
uprofile.load(user)
batches_uprofile = uprofile.get_batches()
batches.extend(batches_uprofile)
#根据关系情况，把用户的id插入到_user:relation列族中
mutations = []
for uid, relation in self.uids.iteritems():
qualifier = struct.pack('<q', uid)
user = self.users[uid]
gender = user['gender']
vt = user['verified_type']
vt = verified_type_map.get(vt, vt)
v = vt << 3
v |= 4L if gender == 'm' else 0L
v |= relation
v <<= 32
v |= util.now2epoch()
v = struct.pack('>q', v)
m = Mutation(column="%s:%s" % (Relation.column_family, qualifier),
value=v)
mutations.append(m)
key = struct.pack('<q', self.self_uid)
u_relation = {
'tableName': Relation.tableName,
'rowBatches': BatchMutation(row=key, mutations=mutations)
}
batches.append(u_relation)
return batches
def _eventgen(self, key, agent, entry, body=None):
if body is None:
body = {}
body = dict(body.items() +\
agent.todict().items() +\
entry.todict().items())
body['http_status'] = responses[entry.status]
system_user_id = entry.system_user_id
if system_user_id != -1:
profile = UserProfile.get_by_system_user_id(
entry.envid, entry.system_user_id)
if profile:
# This is the browser/viewer user, and not the
# workbook publisher/owner.
body['username'] = profile.display_name()
# An event in the event_control table expects these to exist,
# but if workbook archiving is off, or the workbook isn't
# found they will not be set.
body['owner'] = None
body['workbook'] = None
body['userid'] = None # default
if 'repository_url' in body:
self._translate_workbook(body, entry)
if 'http_referer' in body:
body['http_referer'] = unquote(body['http_referer']).decode('utf8')
if 'http_request_uri' in body:
# Remove query string from the http_request_uri.
urip = urlparse(body['http_request_uri'])
http_request_uri = urip.scheme + "://" + urip.netloc + urip.path
http_request_uri = unquote(http_request_uri)
body['http_request_uri'] = http_request_uri.decode('utf8')
userid = body['userid']
completed_at = utc2local(entry.completed_at)
self.server.event_control.gen(key,
body,
userid=userid,
timestamp=completed_at)
def _eventit(self, agent, data):
"""Send a read-only password event failed/okay event if
the user has done setup with a chance to
configure one and appropriate."""
# user '0', likely 'palette'
entry = UserProfile.get(self.server.environment.envid, 0)
# Potentially send an event only after the user has
# finished with the "Setup" page (the passowrd will then be
# there).
if not entry.hashed_password:
return
notification = self.server.notifications.get("dbreadonly")
if success(data):
if notification.color == 'red':
adata = agent.todict()
self.server.event_control.gen(
EventControl.READONLY_DBPASSWORD_OKAY, adata)
notification.modification_time = func.now()
notification.color = 'green'
notification.description = None
meta.Session.commit()
else:
# Failed
if notification.color != 'red':
if data['error'].find(
"A password is required for this connection.") != -1 or \
data['error'].find(agent.odbc.READONLY_ERROR_TEXT) \
!= -1 or \
data['error'].find("password authentication failed") != -1:
adata = agent.todict()
self.server.event_control.gen(
EventControl.READONLY_DBPASSWORD_FAILED, adata)
notification.modification_time = func.now()
notification.color = 'red'
notification.description = None
meta.Session.commit()
return
def _eventgen(self, key, data, entry):
"""Fill in information into the data dictionary and send the event."""
# Placeholder to be set by the next steps.
entry.system_user_id = -1
item_entry = self._add_info(entry)
if key == EventControl.EXTRACT_OK:
if item_entry:
self.server.extract_archive.add(item_entry, entry)
else:
logger.info("extract OK: Can't archive workbook/datasource "
"row - not found.")
data = dict(data.items() + entry.todict().items())
if entry.completed_at is not None and entry.started_at is not None:
duration = timedelta_total_seconds(entry.completed_at,
entry.started_at)
data['duration'] = duration
data['duration_hms'] = to_hhmmss(duration)
timestamp = utc2local(entry.completed_at)
else:
timestamp = None
envid = self.server.environment.envid
profile = UserProfile.get_by_system_user_id(envid,
data['system_user_id'])
if profile:
data['username'] = profile.display_name()
userid = profile.userid
else:
data['username'] = "******"
userid = None
self.server.event_control.gen(key,
data,
userid=userid,
site_id=data['site_id'],
timestamp=timestamp)
def _translate_workbook(self, body, entry):
envid = self.server.environment.envid
url = body['repository_url']
try:
workbook = WorkbookEntry.get_by_url(envid,
url,
entry.site_id,
default=None)
except MultipleResultsFound:
logger.warning("Multiple rows found for url '%s', site_id: '%s'",
url, entry.site_id)
body['workbook'] = 'Unknown: Duplicate repository url'
body['owner'] = 'Unknown: Duplicate repository url'
return
if not workbook:
logger.warning("repository_url '%s' Not Found.", url)
return
body['workbook'] = workbook.name
# Send this userid on event generation (workbook owner, not viewer).
user = UserProfile.get_by_system_user_id(envid,
workbook.system_user_id)
if user:
body['userid'] = user.userid
body['owner'] = user.display_name()
else:
logger.warning("system user '%d' Not Found.",
workbook.system_user_id)
body['project'] = workbook.project
if workbook.site:
body['site'] = workbook.site
else:
if entry.site_id:
site = Site.get(entry.envid, entry.site_id)
if site:
body['site'] = site.name
def sendevent(self, key, system_user_id, error, data):
"""Send the event."""
if 'embedded' in data:
del data['embedded']
if error:
logger.error(error)
data['error'] = error
profile = UserProfile.get_by_system_user_id(
self.server.environment.envid, system_user_id)
if profile:
username = profile.display_name()
userid = profile.userid
else:
username = None
userid = None
if username and not 'owner' in data:
data['owner'] = username
return self.server.event_control.gen(key, data, userid=userid)
def send(self, event_entry, data, recipient=None, eventid=None):
"""Send an alert.
Arguments:
key: The key to look up.
data: A Dictionary with the event information.
"""
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-return-statements
subject = event_entry.email_subject
if subject == None:
subject = event_entry.subject
if subject.find("%") != -1:
# Use the data dict for template substitution.
try:
subject = subject % data
except (ValueError, KeyError) as ex:
subject = "Email Template subject conversion failure: " + \
str(ex) + \
"subject: " + subject + \
", data: " + str(data)
message = event_entry.email_message
if message:
try:
mako_template = Template(message)
message = mako_template.render(**data)
except StandardError:
message = "Email mako template message conversion failure: " + \
exceptions.text_error_template().render() + \
"\ntemplate: " + message + \
"\ndata: " + str(data)
else:
message = self.make_default_message(event_entry, subject, data)
if not message:
# message is empty, set it to be the subject
message = subject
if recipient:
# Force to only one test recipient.
# It is sent even if alerts are disabled.
to_emails = [recipient]
else:
if not self.system[SystemKeys.ALERTS_ENABLED]:
logger.info("Alerts disabled. Not sending: Subject: %s, %s",
subject, message)
return
to_emails = []
if event_entry.publisher_visibility:
to_emails = self.publisher_email(data)
if event_entry.admin_visibility:
to_emails += self.admin_emails(event_entry)
# Remove any duplicates
to_emails = list(set(to_emails))
bcc = None
if self.system[SystemKeys.ALERTS_ADMIN_ENABLED] and not recipient:
# Get the diagnostics email and bcc it there if it exists.
entry = UserProfile.get(self.envid, 0)
if entry and entry.email != None and entry.email != "":
bcc = [entry.email]
if not to_emails and not bcc:
logger.debug(
"No admin users exist with enabled email addresses. "
"Not sending: Subject: %s, Message: %s", subject, message)
return
# Send only PHONE-HOME related events if their palette license
# has expired.
if event_entry.key not in [
EventControl.PHONE_HOME_FAILED, EventControl.PHONE_HOME_OK,
EventControl.EMAIL_TEST
]:
entry = Domain.getone()
if entry.expiration_time and \
datetime.utcnow() > entry.expiration_time:
logger.debug(
"License expired. " +
"Not sending: Subject: %s, Message: %s", subject, message)
return
if entry.contact_time:
silence_time = (datetime.utcnow() - \
entry.contact_time).total_seconds()
max_silence_time = self.system[SystemKeys.MAX_SILENCE_TIME]
if silence_time > max_silence_time and max_silence_time != -1:
logger.debug(
"Phonehome contact time is %d > %d. " +
"Not sending: Subject: %s, Message: %s", silence_time,
max_silence_time, subject, message)
return
if self.email_limit_manager.email_limit_reached(event_entry, eventid):
return
sendit = True
# print '\n------------event key is', event_entry.key
if self.system[SystemKeys.EMAIL_MUTE_RECONNECT_SECONDS]:
# Potentially mute the connect or reconnect emails.
if event_entry.key == EventControl.AGENT_DISCONNECT:
self._mute_dis_check(data, to_emails, bcc, subject, message)
# If the event is emailed, it is done there,
# after a delay
return
elif event_entry.key in [
EventControl.AGENT_COMMUNICATION,
EventControl.INIT_STATE_STARTED,
EventControl.INIT_STATE_STOPPED,
EventControl.INIT_STATE_DEGRADED
]:
sendit = self._mute_reconn_check(data)
if sendit:
self._do_send(to_emails, bcc, subject, message)
def verify(self, name, password):
return UserProfile.verify(self.server.environment.envid, name,
password)
def load(self, agent, check_odbc_state=True):
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
envid = self.server.environment.envid
if check_odbc_state and not self.server.odbc_ok():
return {"error": "Cannot run command while in state: %s" % \
self.server.state_manager.get_state()}
stmt = \
'SELECT system_users.name, system_users.email, ' +\
' system_users.hashed_password, system_users.salt, ' +\
' system_users.friendly_name, system_users.admin_level, ' +\
' system_users.created_at, system_users.id ' +\
'FROM system_users'
excludes = ['guest', '_system']
data = agent.odbc.execute(stmt)
# Send tableau readonly password-related events if appropriate.
self._eventit(agent, data)
if failed(data):
return data
session = meta.Session()
names = ['palette']
cache = self.load_users(agent)
system_key = SystemKeys.ALERTS_NEW_USER_ENABLED
alerts_new_user_enabled = self.system[system_key]
if alerts_new_user_enabled == 'yes':
default_email_level = 1
else:
default_email_level = 0
user_count = UserProfile.user_count(envid)
if user_count <= 1:
first_load = True
else:
first_load = False
for row in data['']:
name = row[0]
if name.lower() in excludes:
continue
sysid = row[7]
names.append(name)
entry = UserProfile.get_by_name(envid, name)
if not entry:
entry = UserProfile(envid=envid, name=name)
entry.email_level = default_email_level
session.add(entry)
entry.email = row[1]
entry.hashed_password = row[2]
entry.salt = row[3]
entry.friendly_name = row[4]
entry.system_admin_level = row[5]
entry.system_created_at = row[6]
entry.system_user_id = sysid
if sysid in cache:
obj = cache[sysid]
entry.login_at = obj.login_at
entry.user_admin_level = obj.admin_level
entry.licensing_role_id = obj.licensing_role_id
entry.publisher = obj.publisher
# On first user table import, Tableau Server Administrators
# are set to Palette Super Admins.
if first_load and entry.system_admin_level == 10:
entry.roleid = Role.SUPER_ADMIN
session.commit()
# deleted entries no longer found in Tableau are marked inactive.
session.query(UserProfile).\
filter(not_(UserProfile.name.in_(names))).\
update({'active': False}, synchronize_session='fetch')
timestamp = datetime.now().strftime(DATEFMT)
self.system.save(SystemKeys.AUTH_TIMESTAMP, timestamp)
d = {u'status': 'OK', u'count': len(data[''])}
logger.debug("auth load returning: %s", str(d))
return d
def load(self, dic):
# 获取发布微博的用户信息
uid = 0L
user = dic.get('user', None)
if user is None:
uid = dic.get('uid', dic.get('user_id', None))
if uid is None:
raise ValueError(
'Data Error: Either User information or user id is expected in the status data!'
)
else:
self.uid = uid
else:
from profile import UserProfile
uprofile = UserProfile()
uprofile.load(user)
uid = long(uprofile.idstr)
self.uid = uid
# 将用户插入到用户表
self.batches.extend(uprofile.get_batches())
self.mid = long(dic.get('idstr', dic.get('id')))
# 处理转发微博
r_id = 0
repost = dic.get('retweeted_status')
if repost is None:
pass
else:
from weibo import Repost
# 将转发微博存在repost列族
retweet = Repost()
retweet.load(self.uid, self.mid, repost)
# print retweet
r_id = retweet.id
self.batches.extend(retweet.get_batches())
# 将转发微博对应的原创微博单独存放一份作为原始微博
if repost.get('deleted', 0) == 0: # 如果被转发的微博已经被删除，则不保存为原创微博
original = Status()
original.load(repost)
self.batches.extend(original.get_batches())
self.r_id = long(r_id)
# 处理其他字段信息
for attr in Status.attrs:
v = dic.get(attr)
if attr == 'key':
v = dic.get('id')
if v is None:
if attr in [
'reposts_count', 'comments_count', 'attitudes_count'
]:
v = 0
else:
continue
self.setattr(attr, v)
ustatus = UserStatuses()
ustatus.load(self, self.uid)
self.batches.extend(ustatus.get_batches())
def nextStep():
print request.remote_addr
profile = UserProfile.get_by_user(request.remote_addr)
print profile
return redirect(profile.current_puzzle)
def gen(self, key, data=None, userid=None, site_id=None, timestamp=None):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
"""Generate an event.
Arguments:
key: The key to look up.
data: A Dictionary with all of the event information.
Recognized keys:
- From http response:
stdout
stderr
xid
pid
exit-status
run-status
timestamp
- Sometimes added:
error
info
- Available from AgentConnection:
displayname
agent_type
uuid
auth, which has:
hostname
ip-address
version
listen-port
install-dir
"""
if data == None:
data = {}
if 'enabled' in data and not data['enabled']:
logger.debug("Agent is disabled so no event will be " + \
"generated. key: %s, data: %s", key, data)
return
event_entry = self.get_event_control_entry(key)
if event_entry:
subject = event_entry.subject
event_description = event_entry.event_description
else:
logger.error("No such event key: %s. data: %s\n", key, str(data))
return
# add all system table entries to the data dictionary.
data = dict(data.items() + \
self.system.todict(include_defaults=True).items())
logger.debug(key + " DATA: " + str(data))
if 'exit-status' in data:
data['exit_status'] = data['exit-status']
del data['exit-status']
# FIXME: remove when browser-aware timezone support is available.
if timestamp is None:
timestamp = datetime.datetime.now(tz=tz.tzlocal())
logger.debug(key + " timestamp : " + timestamp.strftime(DATEFMT))
data['timestamp'] = timestamp.strftime(DATEFMT)
# The userid for other events is the Palette "userid".
profile = None
if not 'username' in data and userid != None:
profile = UserProfile.get(self.envid, userid)
if not profile is None:
data['username'] = profile.display_name()
data['userid'] = profile.userid
if profile.email:
data['email'] = profile.email
if userid:
# userid passed to us has higher precedence than any
# lookup we did, above.
data['userid'] = userid
if not 'username' in data:
data['username'] = mako.runtime.UNDEFINED
data['event_type'] = event_entry.event_type
data['event_type_label'] = event_entry.event_type_label
data['event_label'] = event_entry.event_label
data['event_label_desc'] = event_entry.event_label_desc
data['admin_visibility'] = event_entry.admin_visibility
data['publisher_visiblity'] = event_entry.publisher_visibility
# set server-url(s)
data['server_url'] = self.system[SystemKeys.SERVER_URL]
url = self.server.public_url()
if url:
data['tableau_server_url'] = url
if not 'environment' in data:
data['environment'] = self.server.environment.name
if 'site_id' in data and 'site' not in data:
site = Site.get_name_by_id(self.envid, data['site_id'])
if not site is None:
data['site'] = site
if 'project_id' in data and 'project' not in data:
project = Project.get_name_by_id(self.envid, data['project_id'])
if not project is None:
data['project'] = project
# Create the row to get the eventid before doing subject/description
# substitution.
session = meta.DBSession()
entry = EventEntry(complete=False, key='incomplete')
# set the timestamp here in case it has tzinfo.
entry.timestamp = timestamp
session.add(entry)
session.commit()
data['eventid'] = entry.eventid
# Use the data dict for template substitution.
try:
subject = subject % data
except (ValueError, KeyError) as ex:
subject = "Template subject conversion failure: " + str(ex) + \
"subject: " + subject + \
", data: " + str(data)
if event_description:
try:
mako_template = Template(event_description,
default_filters=['h'])
event_description = mako_template.render(**data)
except MakoException:
event_description = \
"Mako template message conversion failure: " + \
exceptions.text_error_template().render() + \
"\ntemplate: " + event_description + \
"\ndata: " + str(data)
except TypeError as ex:
event_description = \
"Mako template message conversion failure: " + \
str(ex) + \
"\ntemplate: " + event_description + \
"\ndata: " + str(data)
else:
event_description = self.make_default_description(data)
event_description = re.sub("(\n|\r\n){3,}", "\n\n", event_description)
if not event_description.endswith("\n"):
event_description = event_description + "\n"
if self.server.event_debug:
event_description = event_description + "--------\n" + str(data)
# FIXME: remove when browser-aware timezone support is available.
if timestamp.tzinfo is None:
# if not timezone is specified, assume UTC.
summary = utc2local(timestamp).strftime(DATEFMT)
else:
summary = timestamp.strftime(DATEFMT)
# Log the event to the database
entry.complete = True
entry.key = key
entry.envid = self.envid
entry.title = subject
entry.description = event_description
entry.level = event_entry.level
entry.icon = event_entry.icon
entry.color = event_entry.color
entry.event_type = event_entry.event_type
entry.summary = summary
entry.userid = userid
entry.site_id = site_id
#entry.timestamp = timestamp
session.merge(entry)
session.commit()
if not event_entry.send_email:
return
try:
self.alert_email.send(event_entry, data, eventid=entry.eventid)
except StandardError:
exc_traceback = sys.exc_info()[2]
tback = ''.join(traceback.format_tb(exc_traceback))
report = "Error: %s. Traceback: %s" % (sys.exc_info()[1], tback)
logger.error(
"alert_email: Failed for event '%s', '"
"data '%s'. Will not send email. %s", event_entry.key,
str(data), report)