def forwards(self, orm):
for row in orm['plugin.PluginData'].objects.all():
try:
# Do nothing if already valid json.
kbjson.loads(row.value)
except ValueError:
print ' ~ {} -> {}'.format(repr(row.value), repr(kbjson.dumps(row.value)))
row.value = kbjson.dumps(row.value)
row.save()
def decode_response(response):
"""Decodes the requests response object as a JSON response.
For normal responses, the return value is the Python JSON-decoded 'result'
field of the response. If the response is an error, a RemoteError exception
is raised.
"""
status_code = response.status_code
try:
response_dict = kbjson.loads(response.text)
except ValueError as e:
raise ServerError('Invalid JSON response from server: %s' % e)
if 'error' in response_dict:
# Response had an error: translate to exception.
err = response_dict['error']
code = err.get('code', status_code)
message = err.get('message', None)
e = ErrorCodeToException(code, message)
raise e
elif 'object' in response_dict or 'objects' in response_dict:
# Response was OK, return the result.
return response_dict
else:
# WTF?
raise ValueError('Invalid response from server: missing result or error')
def read_metadata(zipfile):
for info in zipfile.infolist():
basename = os.path.basename(info.filename)
if basename == METADATA_FILENAME:
meta_file = zipfile.read(info.filename)
return kbjson.loads(meta_file)
return {}
def read_metadata(zipfile):
for info in zipfile.infolist():
basename = os.path.basename(info.filename)
if basename == METADATA_FILENAME:
meta_file = zipfile.read(info.filename)
return kbjson.loads(meta_file)
return {}
def read_metadata(zipfile):
"""Reads and returns metadata from a backup zipfile."""
for info in zipfile.infolist():
basename = os.path.basename(info.filename)
if basename == METADATA_FILENAME:
meta_file = zipfile.read(info.filename)
return kbjson.loads(meta_file)
return {}
def read_metadata(zipfile):
"""Reads and returns metadata from a backup zipfile."""
for info in zipfile.infolist():
basename = os.path.basename(info.filename)
if basename == METADATA_FILENAME:
meta_file = zipfile.read(info.filename)
return kbjson.loads(meta_file)
return {}
def DecodeEvent(msg):
if isinstance(msg, basestring):
msg = kbjson.loads(msg)
event_name = msg.get('event')
if event_name not in EVENT_NAME_TO_CLASS:
raise ValueError, "Unknown event: %s" % event_name
inst = EVENT_NAME_TO_CLASS[event_name]()
for k, v in msg['data'].iteritems():
setattr(inst, k, v)
return inst
def DecodeEvent(msg):
if isinstance(msg, basestring):
msg = kbjson.loads(msg)
event_name = msg.get('event')
if event_name not in EVENT_NAME_TO_CLASS:
raise ValueError, "Unknown event: %s" % event_name
inst = EVENT_NAME_TO_CLASS[event_name]()
for k, v in msg['data'].iteritems():
setattr(inst, k, v)
return inst
def to_python(self, value):
"""Convert our string value to JSON after we load it from the DB"""
if not value:
return {}
elif isinstance(value, basestring):
res = kbjson.loads(value)
assert isinstance(res, dict)
return JSONDict(**res)
else:
return value
def to_python(self, value):
"""Convert our string value to JSON after we load it from the DB"""
if not value:
return {}
elif isinstance(value, basestring):
res = kbjson.loads(value)
assert isinstance(res, dict)
return JSONDict(**res)
else:
return value
def decode_response(response_data):
"""Decodes the string `response_data` as a JSON response.
For normal responses, the return value is the Python JSON-decoded 'result'
field of the response. If the response is an error, a RemoteError exception
is raised.
"""
# Decode JSON.
try:
d = kbjson.loads(response_data)
except ValueError, e:
raise ServerError('Malformed response: %s' % e)
def assertMetadata(self, backup_dir, when=None, site_name='My Kegbot',
num_media_files=0):
when = when or self.now
backup.verify_backup_directory(backup_dir)
metadata_file = os.path.join(backup_dir, backup.METADATA_FILENAME)
metadata_json = kbjson.loads(open(metadata_file).read())
self.assertEqual(when, metadata_json[backup.META_CREATED_TIME])
self.assertEquals(site_name, metadata_json[backup.META_SERVER_NAME])
self.assertEquals(num_media_files, metadata_json[backup.META_NUM_MEDIA_FILES])
self.assertEquals(get_version(), metadata_json[backup.META_SERVER_VERSION])
def decode_response(response_data):
"""Decodes the string `response_data` as a JSON response.
For normal responses, the return value is the Python JSON-decoded 'result'
field of the response. If the response is an error, a RemoteError exception
is raised.
"""
# Decode JSON.
try:
d = kbjson.loads(response_data)
except ValueError, e:
raise ServerError('Malformed response: %s' % e)
def assertMetadata(self, backup_dir, when=None, site_name='My Kegbot',
num_media_files=0):
when = when or self.now
backup.verify_backup_directory(backup_dir)
metadata_file = os.path.join(backup_dir, backup.METADATA_FILENAME)
metadata_json = kbjson.loads(open(metadata_file).read())
self.assertEqual(when, metadata_json[backup.META_CREATED_TIME])
self.assertEquals(site_name, metadata_json[backup.META_SERVER_NAME])
self.assertEquals(num_media_files, metadata_json[backup.META_NUM_MEDIA_FILES])
self.assertEquals(get_version(), metadata_json[backup.META_SERVER_VERSION])
def verify_backup_directory(backup_dir):
metadata_file = os.path.join(backup_dir, METADATA_FILENAME)
if not os.path.exists(metadata_file):
raise InvalidBackup('Metadata file does not exist')
if not os.path.exists(metadata_file):
raise InvalidBackup('SQL dumpfile does not exist')
metadata = kbjson.loads(open(metadata_file, 'r').read())
format = metadata.get(META_BACKUP_FORMAT)
if format != BACKUP_FORMAT:
raise InvalidBackup('Unsupported backup format: {}'.format(format))
return metadata
def verify_backup_directory(backup_dir):
metadata_file = os.path.join(backup_dir, METADATA_FILENAME)
if not os.path.exists(metadata_file):
raise InvalidBackup('Metadata file does not exist')
metadata = kbjson.loads(open(metadata_file, 'r').read())
format = metadata.get(META_BACKUP_FORMAT)
if format != BACKUP_FORMAT:
raise InvalidBackup('Unsupported backup format: {}'.format(format))
for model in get_models_to_restore():
table_json = os.path.join(backup_dir, TABLES_DIRNAME, tbl(model) + '.json')
if not os.path.exists(table_json):
raise InvalidBackup('Backup missing file {}'.format(table_json))
def verify_backup_directory(backup_dir):
metadata_file = os.path.join(backup_dir, METADATA_FILENAME)
if not os.path.exists(metadata_file):
raise InvalidBackup('Metadata file does not exist')
if not os.path.exists(metadata_file):
raise InvalidBackup('SQL dumpfile does not exist')
metadata = kbjson.loads(open(metadata_file, 'r').read())
format = metadata.get(META_BACKUP_FORMAT)
if format != BACKUP_FORMAT:
raise InvalidBackup('Unsupported backup format: {}'.format(format))
return metadata
def verify_backup_directory(backup_dir):
metadata_file = os.path.join(backup_dir, METADATA_FILENAME)
if not os.path.exists(metadata_file):
raise InvalidBackup('Metadata file does not exist')
metadata = kbjson.loads(open(metadata_file, 'r').read())
format = metadata.get(META_BACKUP_FORMAT)
if format != BACKUP_FORMAT:
raise InvalidBackup('Unsupported backup format: {}'.format(format))
for model in get_models_to_restore():
table_json = os.path.join(backup_dir, TABLES_DIRNAME,
tbl(model) + '.json')
if not os.path.exists(table_json):
raise InvalidBackup('Backup missing file {}'.format(table_json))
def found_terminator(self):
strbuf = self._ibuffer.getvalue()
self._ibuffer.seek(0)
self._ibuffer.truncate()
if not strbuf:
self._logger.warning('Received empty message')
return
try:
message_dict = kbjson.loads(strbuf)
except ValueError:
self._logger.warning('Received malformed message, dropping.')
return
# Distinguish between requests, responses, and notifications, using JSON-RPC
# v2.0 rules
# (http://groups.google.com/group/json-rpc/web/json-rpc-1-2-proposal):
self._logger.debug('Received message: %s' % message_dict)
self.HandleNotification(message_dict)
def found_terminator(self):
strbuf = self._ibuffer.getvalue()
self._ibuffer.seek(0)
self._ibuffer.truncate()
if not strbuf:
self._logger.warning('Received empty message')
return
try:
message_dict = kbjson.loads(strbuf)
except ValueError:
self._logger.warning('Received malformed message, dropping.')
return
# Distinguish between requests, responses, and notifications, using JSON-RPC
# v2.0 rules
# (http://groups.google.com/group/json-rpc/web/json-rpc-1-2-proposal):
self._logger.debug('Received message: %s' % message_dict)
self.HandleNotification(message_dict)
def get_logs(self):
for e in self.redis_client.lrange(self.key, 0, -1):
try:
yield json.loads(e)
except ValueError:
continue
def get_user_profile(self, user):
return kbjson.loads(self.datastore.get('user_detail:%s' % user.id, 'null'))
def restore(input_fp, kbsite, log_cb=_no_log):
def _log(obj):
obj_cls = obj.__class__
obj_name = obj_cls.__name__
log_cb(' +++ %s: %s' % (obj_name, obj))
data = kbjson.loads(input_fp.read())
brewer_map = {}
for rec in data.get('bdb_brewers', []):
try:
brewer = bdb_models.Brewer.objects.get(id=rec.id)
except bdb_models.Brewer.DoesNotExist:
brewer = bdb_models.Brewer(id=rec.id)
brewer.name = rec.name
brewer.country = rec.get('country')
brewer.origin_state = rec.get('origin_state')
brewer.origin_city = rec.get('origin_city')
brewer.production = rec.production
brewer.url = rec.get('url')
brewer.description = rec.get('description')
brewer.save()
_log(brewer)
brewer_map[rec.id] = brewer
style_map = {}
for rec in data.get('bdb_styles', []):
try:
style = bdb_models.BeerStyle.objects.get(id=rec.id)
except bdb_models.BeerStyle.DoesNotExist:
style = bdb_models.BeerStyle(id=rec.id)
style.name = rec.name
style.save()
_log(style)
style_map[rec.id] = style
type_map = {}
for rec in data.get('bdb_beertypes', []):
try:
btype = bdb_models.BeerType.objects.get(id=rec.id)
except bdb_models.BeerType.DoesNotExist:
btype = bdb_models.BeerType(id=rec.id)
btype.name = rec.name
btype.brewer = brewer_map[rec.brewer_id]
btype.style = style_map[rec.style_id]
btype.edition = rec.get('edition')
btype.abv = rec.get('abv')
btype.calories_oz = rec.get('calories_oz')
btype.carbs_oz = rec.get('carbs_oz')
btype.save()
_log(btype)
type_map[rec.id] = btype
kbsite.thermosensors.all().delete()
for rec in data.get('thermosensors', []):
sensor = models.ThermoSensor(site=kbsite, seqn=int(rec.id))
sensor.raw_name = rec.sensor_name
sensor.nice_name = rec.nice_name
sensor.save()
_log(sensor)
kbsite.kegs.all().delete()
for rec in data['kegs']:
keg = models.Keg(site=kbsite, seqn=int(rec.id))
keg.type = type_map[rec.type_id]
keg.startdate = rec.started_time
keg.enddate = rec.finished_time
keg.status = rec.status
keg.description = rec.get('description')
keg.spilled_ml = rec.spilled_ml
size, created = models.KegSize.objects.get_or_create(
name=rec.size_name, volume_ml=rec.size_volume_ml)
size.save()
keg.size = size
keg.save()
_log(keg)
kbsite.taps.all().delete()
for rec in data['taps']:
tap = models.KegTap(site=kbsite, seqn=int(rec.id))
tap.name = rec.name
tap.meter_name = rec.meter_name
tap.ml_per_tick = rec.ml_per_tick
tap.description = rec.get('description')
keg_id = int(rec.get('current_keg_id', 0))
if keg_id:
tap.current_keg = models.Keg.objects.get(site=kbsite, seqn=keg_id)
thermo_id = int(rec.get('thermo_sensor_id', 0))
if thermo_id:
tap.temperature_sensor = models.ThermoSensor.objects.get(
site=kbsite, seqn=thermo_id)
tap.save()
_log(tap)
kbsite.sessions.all().delete()
for rec in data.get('sessions', []):
session = models.DrinkingSession(site=kbsite, seqn=int(rec.id))
session.starttime = rec.start_time
session.endtime = rec.end_time
session.volume_ml = rec.volume_ml
session.name = rec.get('name')
session.slug = rec.get('slug')
session.save()
_log(session)
for rec in data.get('thermologs', []):
log = models.Thermolog(site=kbsite, seqn=int(rec.id))
log.sensor = models.ThermoSensor.objects.get(site=kbsite,
seqn=int(rec.sensor_id))
log.temp = rec.temperature_c
log.time = rec.record_time
log.save()
_log(log)
kbsite.thermosummarylogs.all().delete()
for rec in data.get('thermosummarylogs', []):
log = models.ThermoSummaryLog(site=kbsite, seqn=int(rec.sensor_id))
log.site = kbsite
log.seqn = rec.id
log.sensor = models.ThermoSensor.objects.get(site=kbsite,
seqn=int(rec.sensor_id))
log.date = rec.date
log.period = rec.period
log.num_readings = rec.num_readings
log.min_temp = rec.min_temp
log.max_temp = rec.max_temp
log.mean_temp = rec.mean_temp
log.save()
_log(log)
user_map = {}
for rec in data.get('users', []):
user = None
# If there's already a user registered with this e-mail address, use it.
if rec.email:
user_qs = models.User.objects.filter(email=rec.email)
if user_qs.count():
user = user_qs[0]
user_map[rec.username] = user
_log(user)
continue
# Create a new user, creating a new unique username if necessary.
iter = 0
username = rec.username
while True:
username_qs = models.User.objects.filter(username=username)
if not username_qs.count():
break
iter += 1
username = '******' % (rec.username[:30], iter)
user = models.User(username=username)
user.first_name = rec.first_name
user.last_name = rec.last_name
user.email = rec.email
user.password = rec.password
user.is_active = rec.is_active
user.last_login = rec.last_login
user.date_joined = rec.date_joined
# XXX non-prod
user.is_staff = rec.is_staff
user.is_superuser = rec.is_superuser
user.save()
user_map[rec.username] = user
_log(user)
for rec in data.get('profiles', []):
user = user_map.get(rec.username)
if not user:
print 'Warning: profile for non-existant user: %s' % rec.username
continue
profile, created = models.UserProfile.objects.get_or_create(user=user)
profile.gender = rec.gender
profile.weight = rec.weight
profile.save()
_log(profile)
kbsite.tokens.all().delete()
for rec in data.get('tokens', []):
token = models.AuthenticationToken(site=kbsite, seqn=int(rec.id))
token.auth_device = rec.auth_device
token.token_value = rec.token_value
username = rec.get('username')
if username:
token.user = user_map[username]
token.enabled = rec.enabled
token.created = rec.created_time
token.pin = rec.get('pin')
token.save()
_log(token)
kbsite.drinks.all().delete()
for rec in data.get('drinks', []):
drink = models.Drink(site=kbsite, seqn=int(rec.id))
drink.ticks = rec.ticks
drink.volume_ml = rec.volume_ml
drink.session = models.DrinkingSession.objects.get(site=kbsite,
seqn=int(
rec.session_id))
drink.starttime = rec.pour_time
drink.duration = rec.get('duration')
drink.status = rec.status
keg_id = int(rec.get('keg_id', 0))
if keg_id:
drink.keg = models.Keg.objects.get(site=kbsite, seqn=keg_id)
username = rec.get('user_id')
if username:
drink.user = user_map[username]
drink.auth_token = rec.get('auth_token')
drink.save()
_log(drink)
log_cb('Regenerating sessions ...')
_RegenSessions(kbsite)
log_cb('Regenerating stats ...')
_RegenStats(kbsite)
log_cb('Regenerating events ...')
_RegenEvents(kbsite)
def get_venue_detail(self):
return kbjson.loads(self.datastore.get(KEY_VENUE_DETAIL, 'null'))
def get(self, subpath, data={}, follow=False, **extra):
response = self.client.get('/api/%s' % subpath, data=data, follow=follow,
**extra)
return response, kbjson.loads(response.content)
def get(self, subpath, data={}, follow=False, **extra):
response = self.client.get('/api/%s' % subpath,
data=data,
follow=follow,
**extra)
return response, kbjson.loads(response.content)
def _get_profile(self, datastore_key):
s = self.datastore.get(datastore_key)
if s:
return kbjson.loads(s)
return {}
def get_logs(self):
for e in self.redis_client.lrange(self.key, 0, -1):
try:
yield json.loads(e)
except ValueError:
continue
def restore(input_fp, kbsite, log_cb=_no_log):
def _log(obj):
obj_cls = obj.__class__
obj_name = obj_cls.__name__
log_cb(' +++ %s: %s' % (obj_name, obj))
data = kbjson.loads(input_fp.read())
brewer_map = {}
for rec in data.get('brewers', []):
try:
brewer = models.Brewer.objects.get(id=rec.id)
except models.Brewer.DoesNotExist:
brewer = models.Brewer(id=rec.id)
brewer.name = rec.name
brewer.country = rec.get('country')
brewer.origin_state = rec.get('origin_state')
brewer.origin_city = rec.get('origin_city')
brewer.production = rec.production
brewer.url = rec.get('url')
brewer.description = rec.get('description')
brewer.save()
_log(brewer)
brewer_map[rec.id] = brewer
style_map = {}
for rec in data.get('styles', []):
try:
style = models.BeerStyle.objects.get(id=rec.id)
except models.BeerStyle.DoesNotExist:
style = models.BeerStyle(id=rec.id)
style.name = rec.name
style.save()
_log(style)
style_map[rec.id] = style
type_map = {}
for rec in data.get('beertypes', []):
try:
btype = models.BeerType.objects.get(id=rec.id)
except models.BeerType.DoesNotExist:
btype = models.BeerType(id=rec.id)
btype.name = rec.name
btype.brewer = brewer_map[rec.brewer_id]
btype.style = style_map[rec.style_id]
btype.edition = rec.get('edition')
btype.abv = rec.get('abv')
btype.calories_oz = rec.get('calories_oz')
btype.carbs_oz = rec.get('carbs_oz')
btype.save()
_log(btype)
type_map[rec.id] = btype
kbsite.thermosensors.all().delete()
for rec in data.get('thermosensors', []):
sensor = models.ThermoSensor(site=kbsite, id=int(rec.id))
sensor.raw_name = rec.sensor_name
sensor.nice_name = rec.nice_name
sensor.save()
_log(sensor)
kbsite.kegs.all().delete()
for rec in data['kegs']:
keg = models.Keg(site=kbsite, id=int(rec.id))
keg.type = type_map[rec.type_id]
keg.startdate = rec.started_time
keg.enddate = rec.finished_time
keg.status = rec.status
keg.description = rec.get('description')
keg.spilled_ml = rec.spilled_ml
size, created = models.KegSize.objects.get_or_create(
name=rec.size_name, volume_ml=rec.size_volume_ml)
size.save()
keg.size = size
keg.save()
_log(keg)
kbsite.taps.all().delete()
for rec in data['taps']:
tap = models.KegTap(site=kbsite, id=int(rec.id))
tap.name = rec.name
tap.meter_name = rec.meter_name
tap.ml_per_tick = rec.ml_per_tick
tap.description = rec.get('description')
keg_id = int(rec.get('current_keg_id', 0))
if keg_id:
tap.current_keg = models.Keg.objects.get(site=kbsite, id=keg_id)
thermo_id = int(rec.get('thermo_sensor_id', 0))
if thermo_id:
tap.temperature_sensor = models.ThermoSensor.objects.get(site=kbsite, id=thermo_id)
tap.save()
_log(tap)
kbsite.sessions.all().delete()
for rec in data.get('sessions', []):
session = models.DrinkingSession(site=kbsite, id=int(rec.id))
session.starttime = rec.start_time
session.endtime = rec.end_time
session.volume_ml = rec.volume_ml
session.name = rec.get('name')
session.slug = rec.get('slug')
session.save()
_log(session)
for rec in data.get('thermologs', []):
log = models.Thermolog(site=kbsite, id=int(rec.id))
log.sensor = models.ThermoSensor.objects.get(site=kbsite, id=int(rec.sensor_id))
log.temp = rec.temperature_c
log.time = rec.record_time
log.save()
_log(log)
user_map = {}
for rec in data.get('users', []):
user = None
# If there's already a user registered with this e-mail address, use it.
if rec.email:
user_qs = models.User.objects.filter(email=rec.email)
if user_qs.count():
user = user_qs[0]
user_map[rec.username] = user
_log(user)
continue
# Create a new user, creating a new unique username if necessary.
iter = 0
username = rec.username
while True:
username_qs = models.User.objects.filter(username=username)
if not username_qs.count():
break
iter += 1
username = '******' % (rec.username[:30], iter)
user = models.User(username=username)
user.first_name = rec.first_name
user.last_name = rec.last_name
user.email = rec.email
user.password = rec.password
user.is_active = rec.is_active
user.last_login = rec.last_login
user.date_joined = rec.date_joined
# XXX non-prod
user.is_staff = rec.is_staff
user.is_superuser = rec.is_superuser
user.save()
user_map[rec.username] = user
_log(user)
for rec in data.get('profiles', []):
user = user_map.get(rec.username)
if not user:
print 'Warning: profile for non-existant user: %s' % rec.username
continue
profile, created = models.UserProfile.objects.get_or_create(user=user)
profile.save()
_log(profile)
kbsite.tokens.all().delete()
for rec in data.get('tokens', []):
token = models.AuthenticationToken(site=kbsite, id=int(rec.id))
token.auth_device = rec.auth_device
token.token_value = rec.token_value
username = rec.get('username')
if username:
token.user = user_map[username]
token.enabled = rec.enabled
token.created = rec.created_time
token.pin = rec.get('pin')
token.save()
_log(token)
kbsite.drinks.all().delete()
for rec in data.get('drinks', []):
drink = models.Drink(site=kbsite, id=int(rec.id))
drink.ticks = rec.ticks
drink.volume_ml = rec.volume_ml
drink.session = models.DrinkingSession.objects.get(site=kbsite, id=int(rec.session_id))
drink.starttime = rec.pour_time
drink.duration = rec.get('duration')
drink.status = rec.status
keg_id = int(rec.get('keg_id', 0))
if keg_id:
drink.keg = models.Keg.objects.get(site=kbsite, id=keg_id)
username = rec.get('user_id')
if username:
drink.user = user_map[username]
drink.auth_token = rec.get('auth_token')
drink.save()
_log(drink)
log_cb('Regenerating sessions ...')
_RegenSessions(kbsite)
log_cb('Regenerating stats ...')
_RegenStats(kbsite)
log_cb('Regenerating events ...')
_RegenEvents(kbsite)
def _get_profile(self, datastore_key):
s = self.datastore.get(datastore_key)
if s:
return kbjson.loads(s)
return {}
def get_user_profile(self, user):
return kbjson.loads(
self.datastore.get('user_detail:%s' % user.id, 'null'))
def get_venue_detail(self):
return kbjson.loads(self.datastore.get(KEY_VENUE_DETAIL, 'null'))