def GetSerializedItem(cls, key):
"""Returns the deserialized value of a serialized cache."""
entity = cls.MemcacheWrappedGet(key)
if entity and entity.blob_value:
return util.Deserialize(entity.blob_value), entity.mtime
else:
return {}, None
def testDeserializeJson(self):
"""Test Deserialize()."""
with mock.patch.object(util.json, 'loads', return_value='object1'):
self.assertEqual('object1', util.Deserialize('serial1'))
with mock.patch.object(util.json, 'loads', side_effect=ValueError):
self.assertRaises(util.DeserializeError, util.Deserialize,
'serial2')
def _DisplayMsuLogSummary(self):
"""Displays a summary of MSU logs."""
summaries = []
for since_days in None, 7, 1:
key = 'msu_user_summary'
if since_days:
key = '%s_since_%sD_' % (key, since_days)
human_since = '%s day(s)' % since_days
else:
human_since = 'forever'
m = models.ReportsCache.get_by_key_name(key)
if not m or not m.blob_value:
continue
summary = util.Deserialize(m.blob_value)
summary_list = []
keys = summary.keys()
keys.sort(cmp=lambda x, y: cmp(summary[x], summary[y]),
reverse=True)
for x in keys:
summary_list.append({'var': x, 'val': summary[x]})
summaries.append({
'values': summary_list,
'since': human_since,
'mtime': m.mtime
})
# TODO(user): Since the above is a list now, memcache here.
self.Render('msu_log_summary.html', {
'summaries': summaries,
'report_type': 'msu_gui_logs'
})
def testDeserializeUnicode(self):
"""Test Deserialize()."""
ustr = u'Hello there\u2014'
ustr_js = '"Hello there\\u2014"'
self.assertEqual(
ustr,
util.Deserialize(ustr_js, _use_json=True, _use_pickle=False))
def testSet(self, *_):
self.testapp.post(
'/admin/ip_blacklist', {'item_0': '192.168.1.1', 'item_1': 'zerocool'},
status=httplib.FOUND)
self.assertEqual(
{'192.168.1.1': 'zerocool'},
util.Deserialize(models.KeyValueCache.MemcacheWrappedGet(
'client_exit_ip_blocks', 'text_value')))
def testDeserializeFloat(self):
"""Test Deserialize()."""
input = '{"foo": 103.2261}'
output = {'foo': 103.2261}
self.assertEqual(
output,
util.Deserialize(input))
def __set__(self, model_instance, value):
"""Deserializes db.Text values and simply sets other types to self._obj."""
if value is None or value == '':
super(SerializedProperty, self).__set__(model_instance, None)
elif type(value) is db.Text:
# If the value is is db.Text, deserialize it to init _obj.
self._obj = util.Deserialize(value)
else:
# If the incoming value is a not db.Text, it's an obj so just store it.
self._obj = value
def GetMembers(self, group_name):
"""Get a list of members belonging to the group."""
members = models.KeyValueCache.MemcacheWrappedGet(group_name, 'text_value')
member_list = []
if members:
try:
member_list = util.Deserialize(members)
except util.DeserializeError:
pass
return member_list
def IpInList(cls, key_name, ip):
"""Check whether IP is in serialized IP/mask list in key_name.
The KeyValueCache entity at key_name is expected to have a text_value
which is in util.Serialize() form. The deserialized structure looks like
[ "200.0.0.0/24",
"10.0.0.0/8",
etc ...
]
Note that this function is not IPv6 safe and will always return False
if the input ip is IPv6 format.
Args:
key_name: str, like 'auth_bad_ip_blocks'
ip: str, like '127.0.0.1'
Returns:
True if the ip is inside a mask in the list, False if not
"""
if not ip:
return False # lenient response
# TODO(user): Once the underlying util.Ip* methods support ipv6
# this method can go away. Until then, this stops all of the churn
# and exits early.
if ip.find(':') > -1: # ipv6
return False
try:
ip_blocks_str = cls.MemcacheWrappedGet(key_name, 'text_value')
if not ip_blocks_str:
return False
ip_blocks = util.Deserialize(ip_blocks_str)
except (util.DeserializeError, db.Error):
logging.exception('IpInList(%s)', ip)
return False # lenient response
# Note: The method below, parsing a serialized list of networks
# expressed as strings, might seem silly. But the total time to
# deserialize and translate the strings back into IP network/mask
# integers is actually faster than storing them already split, e.g. a
# list of 2 item lists (network,mask). Apparently JSON isn't as
# efficient at parsing ints or nested lists.
#
# (pickle is 2X+ faster but not secure & deprecated inside util module)
ip_int = ipcalc.IpToInt(ip)
for ip_mask_str in ip_blocks:
ip_mask = ipcalc.IpMaskToInts(ip_mask_str)
if (ip_int & ip_mask[1]) == ip_mask[0]:
return True
return False
def testDeserializeJson(self):
"""Test Deserialize()."""
self.mox.StubOutWithMock(util.json, 'loads')
util.json.loads('serial1', parse_float=float).AndReturn('object1')
util.json.loads('serial2', parse_float=float).AndRaise(ValueError)
self.mox.ReplayAll()
self.assertEqual('object1', util.Deserialize('serial1'))
self.assertRaises(util.DeserializeError, util.Deserialize, 'serial2')
self.mox.VerifyAll()
def GetSerializedItem(cls, key):
"""Returns the deserialized value of a serialized cache."""
entity = cls.MemcacheWrappedGet(key)
if entity and entity.blob_value:
data = entity.blob_value
mtime = entity.mtime
i = 1
while entity and len(
entity.blob_value) >= _MEMCACHE_ENTITY_SIZE_LIMIT:
entity = cls.MemcacheWrappedGet('%s_shard_%s' % (key, i))
if entity and entity.blob_value:
data += entity.blob_value
i += 1
return util.Deserialize(data), mtime
else:
return {}, None
def get(self):
"""GET handler."""
if not self.IsAdminUser():
self.error(httplib.FORBIDDEN)
return
ips = {}
try:
ips = util.Deserialize(
models.KeyValueCache.MemcacheWrappedGet('client_exit_ip_blocks',
'text_value'))
except util.DeserializeError:
pass
d = {'report_type': 'ip_blacklist', 'title': 'IP Blacklist', 'columns': 2,
'list': sorted(ips.items()), 'labels': ['IP', 'Comment'],
'regex': ['/%s/' % IP_REGEX, '/^.{0,60}$/'],
'infopanel': 'Subnet format required (e.g. 192.168.1.0/24)'}
self.Render('list_edit.html', d)
def _GetGroupMembers(group_name):
"""Returns a set of email addresses that are members of a given group.
Args:
group_name: str group name to return the members of.
Returns:
List of email addresses of members, or an empty list for unknown groups.
"""
members = getattr(settings, group_name.upper(), [])
try:
json_members = models.KeyValueCache.MemcacheWrappedGet(
group_name, 'text_value')
if json_members:
members.extend(util.Deserialize(json_members))
except (db.Error, util.DeserializeError):
pass
return members
def testDeserializePickle(self):
"""Test Deserialize()."""
self.mox.StubOutWithMock(util.pickle, 'loads')
util.pickle.loads('serial1').AndReturn('object1')
util.pickle.loads('serial2').AndRaise(util.pickle.UnpicklingError)
self.mox.ReplayAll()
self.assertEqual(
'object1',
util.Deserialize('serial1',
_use_json=False,
_use_pickle=True,
_pickle_re=util.re.compile('.')))
self.assertRaises(util.DeserializeError,
util.Deserialize,
'serial2',
_use_json=False,
_use_pickle=True,
_pickle_re=util.re.compile('.'))
self.mox.VerifyAll()
def _DisplayMsuLogSummary(self, since_days=None):
"""Displays a summary of MSU logs."""
key = 'msu_user_summary'
if since_days:
key = '%s_since_%sD_' % (key, since_days)
human_since = '%s day(s)' % since_days
else:
human_since = 'forever'
m = models.ReportsCache.get_by_key_name(key)
summary = util.Deserialize(m.blob_value)
summary_list = []
keys = summary.keys()
keys.sort(cmp=lambda x, y: cmp(summary[x], summary[y]))
map(lambda x: summary_list.append({'var': x, 'val': summary[x]}), keys)
self.response.out.write(
RenderTemplate('templates/stats_msulogsummary.html', {
'summary': summary_list,
'since': human_since,
'mtime': m.mtime,
}))
def _GetUserSettings(self):
"""Returns the user setting dictionary, or None."""
if self._user_settings:
return util.Deserialize(self._user_settings)
else:
return None
def testDeserializeUnicode(self):
"""Test Deserialize()."""
ustr = u'Hello there\u2014'
ustr_js = '"Hello there\\u2014"'
self.assertEqual(ustr, util.Deserialize(ustr_js))
def post(self):
"""Reports get handler.
Returns:
A webapp.Response() response.
"""
session = gaeserver.DoMunkiAuth()
uuid = main_common.SanitizeUUID(session.uuid)
report_type = self.request.get('_report_type')
report_feedback = {}
message = None
details = None
client_id = None
computer = None
if report_type == 'preflight' or report_type == 'postflight':
client_id_str = urllib.unquote(self.request.get('client_id'))
client_id = common.ParseClientId(client_id_str, uuid=uuid)
user_settings_str = self.request.get('user_settings')
user_settings = None
try:
if user_settings_str:
user_settings = util.Deserialize(
urllib.unquote(str(user_settings_str)))
except util.DeserializeError:
logging.warning('Client %s sent broken user_settings: %s',
client_id_str, user_settings_str)
pkgs_to_install = self.request.get_all('pkgs_to_install')
apple_updates_to_install = self.request.get_all(
'apple_updates_to_install')
computer = models.Computer.get_by_key_name(uuid)
ip_address = os.environ.get('REMOTE_ADDR', '')
if report_type == 'preflight':
# we want to get feedback now, before preflight_datetime changes.
client_exit = self.request.get('client_exit', None)
report_feedback = self.GetReportFeedback(
uuid,
report_type,
computer=computer,
ip_address=ip_address,
client_exit=client_exit)
if self.request.get('json') == '1':
self.response.out.write(JSON_PREFIX +
json.dumps(report_feedback))
else:
# For legacy clients that accept a single string, not JSON.
feedback_to_send = 'OK'
for feedback in LEGACY_FEEDBACK_LIST:
if report_feedback.get(feedback.lower()):
feedback_to_send = feedback
self.response.out.write(feedback_to_send)
# if report feedback calls for a client exit, log it.
if report_feedback.get('exit'):
if not client_exit:
# client didn't ask for an exit, which means server decided.
client_exit = 'Connection from defined exit IP address'
common.WriteClientLog(models.PreflightExitLog,
uuid,
computer=computer,
exit_reason=client_exit)
common.LogClientConnection(report_type,
client_id,
user_settings,
pkgs_to_install,
apple_updates_to_install,
computer=computer,
ip_address=ip_address,
report_feedback=report_feedback)
elif report_type == 'install_report':
computer = models.Computer.get_by_key_name(uuid)
self._LogInstalls(self.request.get_all('installs'), computer)
for removal in self.request.get_all('removals'):
common.WriteClientLog(models.ClientLog,
uuid,
computer=computer,
action='removal',
details=removal)
for problem in self.request.get_all('problem_installs'):
common.WriteClientLog(models.ClientLog,
uuid,
computer=computer,
action='install_problem',
details=problem)
elif report_type == 'broken_client':
# Default reason of "objc" to support legacy clients, existing when objc
# was the only broken state ever reported.
reason = self.request.get('reason', 'objc')
details = self.request.get('details')
logging.warning('Broken Munki client (%s): %s', reason, details)
common.WriteBrokenClient(uuid, reason, details)
elif report_type == 'msu_log':
details = {}
for k in ['time', 'user', 'source', 'event', 'desc']:
details[k] = self.request.get(k, None)
common.WriteComputerMSULog(uuid, details)
else:
# unknown report type; log all post params.
params = []
for param in self.request.arguments():
params.append('%s=%s' % (param, self.request.get_all(param)))
common.WriteClientLog(models.ClientLog,
uuid,
action='unknown',
details=str(params))
def post(self):
"""Reports get handler.
Returns:
A webapp.Response() response.
"""
session = gaeserver.DoMunkiAuth()
uuid = main_common.SanitizeUUID(session.uuid)
report_type = self.request.get('_report_type')
feedback_requested = self.request.get('_feedback')
message = None
details = None
client_id = None
computer = None
if report_type == 'preflight' or report_type == 'postflight':
client_id_str = urllib.unquote(self.request.get('client_id'))
client_id = common.ParseClientId(client_id_str, uuid=uuid)
user_settings_str = self.request.get('user_settings')
user_settings = None
try:
if user_settings_str:
user_settings = util.Deserialize(
urllib.unquote(str(user_settings_str)))
except util.DeserializeError:
logging.warning('Client %s sent broken user_settings: %s',
client_id_str, user_settings_str)
pkgs_to_install = self.request.get_all('pkgs_to_install')
apple_updates_to_install = self.request.get_all(
'apple_updates_to_install')
computer = models.Computer.get_by_key_name(uuid)
ip_address = os.environ.get('REMOTE_ADDR', '')
report_feedback = None
if report_type == 'preflight':
# if the UUID is known to be lost/stolen, log this connection.
if models.ComputerLostStolen.IsLostStolen(uuid):
logging.warning('Connection from lost/stolen machine: %s',
uuid)
models.ComputerLostStolen.LogLostStolenConnection(
computer=computer, ip_address=ip_address)
# we want to get feedback now, before preflight_datetime changes.
if feedback_requested:
client_exit = self.request.get('client_exit', None)
report_feedback = self.GetReportFeedback(
uuid,
report_type,
computer=computer,
ip_address=ip_address,
client_exit=client_exit)
self.response.out.write(report_feedback)
# if report feedback calls for a client exit, log it.
if report_feedback == common.ReportFeedback.EXIT:
if not client_exit:
# client didn't ask for an exit, which means server decided.
client_exit = 'Connection from defined exit IP address'
common.WriteClientLog(models.PreflightExitLog,
uuid,
computer=computer,
exit_reason=client_exit)
common.LogClientConnection(report_type,
client_id,
user_settings,
pkgs_to_install,
apple_updates_to_install,
computer=computer,
ip_address=ip_address,
report_feedback=report_feedback)
elif report_type == 'install_report':
computer = models.Computer.get_by_key_name(uuid)
self._LogInstalls(self.request.get_all('installs'), computer)
for removal in self.request.get_all('removals'):
common.WriteClientLog(models.ClientLog,
uuid,
computer=computer,
action='removal',
details=removal)
for problem in self.request.get_all('problem_installs'):
common.WriteClientLog(models.ClientLog,
uuid,
computer=computer,
action='install_problem',
details=problem)
elif report_type == 'preflight_exit':
# NOTE(user): only remains for older clients.
message = self.request.get('message')
computer = common.WriteClientLog(models.PreflightExitLog,
uuid,
exit_reason=message)
elif report_type == 'broken_client':
# Default reason of "objc" to support legacy clients, existing when objc
# was the only broken state ever reported.
reason = self.request.get('reason', 'objc')
details = self.request.get('details')
logging.warning('Broken Munki client (%s): %s', reason, details)
common.WriteBrokenClient(uuid, reason, details)
elif report_type == 'msu_log':
details = {}
for k in ['time', 'user', 'source', 'event', 'desc']:
details[k] = self.request.get(k, None)
common.WriteComputerMSULog(uuid, details)
else:
# unknown report type; log all post params.
params = []
for param in self.request.arguments():
params.append('%s=%s' % (param, self.request.get_all(param)))
common.WriteClientLog(models.ClientLog,
uuid,
action='unknown',
details=str(params))
# If the client asked for feedback, get feedback and respond.
# Skip this if the report_type is preflight, as report feedback was
# retrieved before LogComputerConnection changed preflight_datetime.
if feedback_requested and report_type != 'preflight':
self.response.out.write(
self.GetReportFeedback(
uuid,
report_type,
message=message,
details=details,
computer=computer,
))