def BuildResponse(self):
manufacturers = []
for manufacturer in Manufacturer.all():
manufacturers.append({
'name': manufacturer.name,
'id': manufacturer.esta_id
})
return json.dumps({'manufacturers': manufacturers})
def BuildResponse(self):
manufacturers = []
for manufacturer in Manufacturer.all():
manufacturers.append({
'name': manufacturer.name,
'id': manufacturer.esta_id
})
return json.dumps({'manufacturers': manufacturers})
def get(self):
esta_manufacturer = common.GetManufacturer(self.ESTA_ID)
self.response.headers['Content-Type'] = 'text/plain'
# Can be '', 'esta', 'esta-draft', 'manufacturers' or 'manufacturer-names'
pid_selection = self.request.get('pids')
if pid_selection == 'manufacturer-names':
manufacturers = []
query = Manufacturer.all()
query.order('name')
for manufacturer in query:
if manufacturer.esta_id in [self.ESTA_ID, 0xffff]:
continue
self.WriteManufacturer(manufacturer, [])
else:
pids = Pid.all()
if pid_selection == 'esta':
pids.filter('draft =', False)
pids.filter('manufacturer = ', esta_manufacturer)
elif pid_selection == 'esta-draft':
pids.filter('draft =', True)
pids.filter('manufacturer = ', esta_manufacturer)
elif pid_selection == "manufacturers":
pids.filter('manufacturer != ', esta_manufacturer)
manufacturers = {}
esta_pids = []
for pid in pids:
if pid.manufacturer.esta_id == self.ESTA_ID:
esta_pids.append(pid)
else:
# Build the hash of manufacturer pids by manufacturer
manufacturers.setdefault(pid.manufacturer.esta_id, []).append(pid)
esta_pids.sort(key=lambda p: p.pid_id)
for pid in esta_pids:
self.WritePid(pid)
for manufacturer_id in sorted(manufacturers):
manufacturer_pids = manufacturers[manufacturer_id]
manufacturer_pids.sort(key=lambda p: p.pid_id)
self.WriteManufacturer(manufacturers[manufacturer_id][0].manufacturer,
manufacturer_pids)
query = LastUpdateTime.all()
if pid_selection == 'manufacturer-names':
query.filter('name = ', timestamp_keys.MANUFACTURERS)
else:
query.filter('name = ', timestamp_keys.PIDS)
update_time = query.fetch(1)
if update_time:
timestamp = TimestampToInt(update_time[0].update_time)
self.Write('version: %d' % timestamp)
def GetResults(self):
if self._manufacturer_id is not None:
query = Manufacturer.all()
query.filter('esta_id = ', self._manufacturer_id)
for manufacturer in query.fetch(1):
query = manufacturer.pid_set
query.filter('draft = ', False)
query.order('pid_id')
return query
return []
def get(self):
self.response.headers['Content-Type'] = 'text/plain'
self.response.headers['Cache-Control'] = 'public; max-age=300;'
output = []
for manufacturer in Manufacturer.all():
output.append("manufacturer {")
output.append(" name: \"%s\"" % manufacturer.name)
output.append(" id: %d" % manufacturer.esta_id)
output.append("}")
self.response.out.write('\n'.join(output))
def post(self):
json = request.json['manufacturer']
manufacturer = ManufacturerModel(title=json['title'],
description=json['description'],
note=json['note'],
image=json['image'])
manufacturer.save()
model = {
'manufacturer': {
'id': manufacturer.id,
'title': manufacturer.title,
'description': manufacturer.description,
'note': manufacturer.note,
'image': manufacturer.image
}
}
print model
return jsonify(model)
def get(self):
self.response.headers['Content-Type'] = 'text/plain'
self.response.headers['Cache-Control'] = 'public; max-age=300;'
output = []
for manufacturer in Manufacturer.all():
output.append("manufacturer {")
output.append(" name: \"%s\"" % manufacturer.name)
output.append(" id: %d" % manufacturer.esta_id)
output.append("}")
self.response.out.write('\n'.join(output))
def UpdateManufacturers(self):
new_data = {}
for id, name in MANUFACTURER_DATA:
new_data[id] = name
existing_manufacturers = set()
manufacturers_to_delete = []
# invalidate the cache now
memcache.delete(memcache_keys.MANUFACTURER_CACHE_KEY)
memcache.delete(memcache_keys.MANUFACTURER_MODEL_COUNTS)
memcache.delete(memcache_keys.MANUFACTURER_PID_COUNT_KEY)
added = removed = updated = errors = 0
for manufacturer in Manufacturer.all():
id = manufacturer.esta_id
if id in new_data:
existing_manufacturers.add(id)
# update if required
new_name = new_data[id]
if new_name != manufacturer.name:
logging.info('Updating %s -> %s' %
(manufacturer.name, new_name))
manufacturer.name = new_name
manufacturer.put()
updated += 1
else:
manufacturers_to_delete.append(manufacturer)
# add any new manufacturers
manufacturers_to_add = set(new_data.keys()) - existing_manufacturers
for manufacturer_id in sorted(manufacturers_to_add):
try:
manufacturer_name = new_data[manufacturer_id].decode()
except UnicodeDecodeError as e:
logging.error('Failed to add 0x%hx: %s' % (manufacturer_id, e))
errors += 1
continue
logging.info('adding %d (%s)' %
(manufacturer_id, manufacturer_name))
manufacturer = Manufacturer(esta_id=manufacturer_id,
name=manufacturer_name)
manufacturer.put()
added += 1
# remove any extra manufacturers
for manufacturer in manufacturers_to_delete:
logging.info('removing %s' % manufacturer.name)
manufacturer.delete()
removed += 1
logging.info('update complete')
UpdateModificationTime(timestamp_keys.MANUFACTURERS)
return ('Manufacturers: added %d, removed %d, updated %d, errors %d' %
(added, removed, updated, errors))
def UpdateManufacturerLinks(self):
# TODO(Peter): Add ability to remove links if not present in data?
new_data = {}
for id, name in MANUFACTURER_LINKS:
new_data[id] = name
present_manufacturers = set()
# invalidate the cache now
memcache.delete(memcache_keys.MANUFACTURER_CACHE_KEY)
added = updated = missing = errors = 0
for manufacturer in Manufacturer.all():
id = manufacturer.esta_id
if id in new_data:
present_manufacturers.add(id)
new_link = new_data[id]
if not (manufacturer.link) or (new_link != manufacturer.link):
try:
# add/update if required
manufacturer.link = new_link
manufacturer.put()
if manufacturer.link:
logging.info('Updating link for %d (%s) %s -> %s' %
(id, manufacturer.name,
manufacturer.link, new_link))
updated += 1
else:
logging.info('Adding link for %d (%s) - %s' %
(id, manufacturer.name, new_link))
added += 1
except BadValueError as e:
logging.error(
'Failed to add link for 0x%hx (%s) - %s: %s' %
(id, manufacturer.name, new_link, e))
errors += 1
continue
# link no manufacturer
missing_manufacturers = set(new_data.keys()) - present_manufacturers
for manufacturer_id in sorted(missing_manufacturers):
logging.error(
'Failed to add link for 0x%hx as manufacturer missing: %s' %
(manufacturer_id, new_data[manufacturer_id]))
missing += 1
logging.info('update complete')
UpdateModificationTime(timestamp_keys.MANUFACTURERS)
return (
'Manufacturer links: added %d, updated %d, missing %d, errors %d' %
(added, updated, missing, errors))
def GetManufacturer(manufacturer_id):
"""Lookup a manufacturer entity by manufacturer id. The manufacturer id can
be a string in decimal or hex (prepend with 0x), or an int
Returns:
The Manufacturer entity object, or None if not found.
"""
if type(manufacturer_id) not in (int, long):
manufacturer_id = StringToInt(manufacturer_id)
query = Manufacturer.all()
query.filter('esta_id = ', manufacturer_id)
for manufacturer in query.fetch(1):
return manufacturer
return None
def post(self):
json = request.json['manufacturer']
manufacturer = ManufacturerModel(
title=json['title'],
description=json['description'],
note=json['note'],
image=json['image']
)
manufacturer.save()
model = {
'manufacturer': {
'id': manufacturer.id,
'title': manufacturer.title,
'description': manufacturer.description,
'note': manufacturer.note,
'image': manufacturer.image
}
}
print model
return jsonify(model)
def GetTemplateData(self):
manufacturers = []
query = Manufacturer.all()
query.order('name')
for manufacturer in query:
if manufacturer.esta_id in [0, 0xffff]:
continue
manufacturers.append({
'name': manufacturer.name,
'id': manufacturer.esta_id,
})
data = {
'manufacturers': manufacturers,
}
return data
def GetTemplateData(self):
manufacturers = []
query = Manufacturer.all()
query.order('name')
for manufacturer in query:
if manufacturer.esta_id in [0, 0xffff]:
continue
manufacturers.append({
'name': manufacturer.name,
'id': manufacturer.esta_id,
})
data = {
'manufacturers': manufacturers,
}
return data
def UpdateManufacturerLinks(self):
# TODO(Peter): Add ability to remove links if not present in data?
new_data = {}
for id, name in MANUFACTURER_LINKS:
new_data[id] = name
present_manufacturers = set()
# invalidate the cache now
memcache.delete(memcache_keys.MANUFACTURER_CACHE_KEY)
added = updated = missing = errors = 0
for manufacturer in Manufacturer.all():
id = manufacturer.esta_id
if id in new_data:
present_manufacturers.add(id)
new_link = new_data[id]
if not(manufacturer.link) or (new_link != manufacturer.link):
try:
# add/update if required
manufacturer.link = new_link
manufacturer.put()
if manufacturer.link:
logging.info('Updating link for %d (%s) %s -> %s' %
(id, manufacturer.name, manufacturer.link, new_link))
updated += 1
else:
logging.info('Adding link for %d (%s) - %s' %
(id, manufacturer.name, new_link))
added += 1
except BadValueError as e:
logging.error('Failed to add link for 0x%hx (%s) - %s: %s' %
(id, manufacturer.name, new_link, e))
errors += 1
continue
# link no manufacturer
missing_manufacturers = set(new_data.keys()) - present_manufacturers
for manufacturer_id in sorted(missing_manufacturers):
logging.error('Failed to add link for 0x%hx as manufacturer missing: %s' %
(manufacturer_id, new_data[manufacturer_id]))
missing += 1
logging.info('update complete')
UpdateModificationTime(timestamp_keys.MANUFACTURERS)
return ('Manufacturer links: added %d, updated %d, missing %d, errors %d' %
(added, updated, missing, errors))
def UpdateManufacturers(self):
new_data = {}
for id, name in MANUFACTURER_DATA:
new_data[id] = name
existing_manufacturers = set()
manufacturers_to_delete = []
# invalidate the cache now
memcache.delete(memcache_keys.MANUFACTURER_CACHE_KEY)
memcache.delete(memcache_keys.MANUFACTURER_MODEL_COUNTS)
memcache.delete(memcache_keys.MANUFACTURER_PID_COUNT_KEY)
added = removed = updated = errors = 0
for manufacturer in Manufacturer.all():
id = manufacturer.esta_id
if id in new_data:
existing_manufacturers.add(id)
# update if required
new_name = new_data[id]
if new_name != manufacturer.name:
logging.info('Updating %s -> %s' % (manufacturer.name, new_name))
manufacturer.name = new_name
manufacturer.put()
updated += 1
else:
manufacturers_to_delete.append(manufacturer)
# add any new manufacturers
manufacturers_to_add = set(new_data.keys()) - existing_manufacturers
for manufacturer_id in sorted(manufacturers_to_add):
try:
manufacturer_name = new_data[manufacturer_id].decode()
except UnicodeDecodeError as e:
logging.error('Failed to add 0x%hx: %s' % (manufacturer_id, e))
errors += 1
continue
logging.info('adding %d (%s)' % (manufacturer_id, manufacturer_name))
manufacturer = Manufacturer(esta_id=manufacturer_id,
name=manufacturer_name)
manufacturer.put()
added += 1
# remove any extra manufacturers
for manufacturer in manufacturers_to_delete:
logging.info('removing %s' % manufacturer.name)
manufacturer.delete()
removed += 1
logging.info('update complete')
UpdateModificationTime(timestamp_keys.MANUFACTURERS)
return ('Manufacturers: added %d, removed %d, updated %d, errors %d' %
(added, removed, updated, errors))
def get(self, id):
manufacturer = ManufacturerModel.find(id=id)
assets = []
for asset in manufacturer.assets:
assets.append(asset.id)
model = {
'manufacturer': {
'id': manufacturer.id,
'description': manufacturer.description,
'title': manufacturer.title,
'note': manufacturer.note,
'assets': assets,
'image': manufacturer.image
}
}
return jsonify(model)
def GetSearchData(self):
manufacturer_list = memcache.get(memcache_keys.MANUFACTURER_MODEL_COUNTS)
if not manufacturer_list:
manufacturer_list = []
query = Manufacturer.all()
query.order('name')
for manufacturer in query:
responders = manufacturer.responder_set.count()
if responders:
manufacturer_list.append({
'id': manufacturer.esta_id,
'name': manufacturer.name,
'responder_count': responders,
})
memcache.set(memcache_keys.MANUFACTURER_MODEL_COUNTS, manufacturer_list)
return {
'manufacturers': manufacturer_list,
'current_id': self._manufacturer_id,
}
def GetSearchData(self):
manufacturer_list = memcache.get(memcache_keys.MANUFACTURER_PID_COUNTS)
if not manufacturer_list:
manufacturer_list = []
query = Manufacturer.all()
query.order('name')
for manufacturer in query:
pids = manufacturer.pid_set.count()
if pids:
manufacturer_list.append({
'id': manufacturer.esta_id,
'name': manufacturer.name,
'pid_count': pids,
})
memcache.set(memcache_keys.MANUFACTURER_PID_COUNTS,
manufacturer_list)
return {
'manufacturers': manufacturer_list,
'current_id': self._manufacturer_id,
}
def get(self):
manufacturerlist = []
for manufacturer in ManufacturerModel.list():
assets = []
for asset in manufacturer.assets:
assets.append(asset.id)
model = {
'id': manufacturer.id,
'description': manufacturer.description,
'title': manufacturer.title,
'note': manufacturer.note,
'assets': assets,
'image': manufacturer.image
}
manufacturerlist.append(model)
return jsonify({'manufacturers': manufacturerlist})
def get(self):
manufacturerlist = []
for manufacturer in ManufacturerModel.list():
assets = []
for asset in manufacturer.assets:
assets.append(asset.id)
model = {
'id': manufacturer.id,
'description': manufacturer.description,
'title': manufacturer.title,
'note': manufacturer.note,
'assets': assets,
'image': manufacturer.image
}
manufacturerlist.append(model)
return jsonify({'manufacturers': manufacturerlist})
def VerifyAndStoreData(self, data):
"""Check the data look reasonable and if it does, store it.
Returns:
A list of dicts in the form:
[{
'key': key,
}]
"""
if not data:
return []
try:
evaled_data = eval(data, {})
except Exception as e:
logging.info(data)
logging.error(e)
return []
responder_obj_ids = []
for manufacturer_id, responders in evaled_data.iteritems():
try:
manufacturer_id = int(manufacturer_id)
except ValueError:
logging.error('Invalid manufacturer id %s' % manufacturer_id)
continue
# See if we can get the manufacturer name
manufacturer_name = None
manufacturer_query = Manufacturer.all()
manufacturer_query.filter('esta_id = ', manufacturer_id)
results = manufacturer_query.fetch(1)
if results:
manufacturer_name = results[0].name
for responder in responders:
if 'device_model' not in responder:
logging.error('Missing device_model from data')
continue
try:
device_model_id = int(responder['device_model'])
except ValueError:
logging.error('Invalid device model %s' % responder['device_model'])
continue
# ok, that's all the required fields
del responder['device_model']
responder_obj = UploadedResponderInfo(
manufacturer_id=manufacturer_id,
device_model_id=device_model_id,
info=str(responder),
upload_time=datetime.datetime.now()
)
responder_obj.put()
responder_obj_ids.append({
'device_model_id': device_model_id,
'key': str(responder_obj.key()),
'manufacturer': manufacturer_name,
'manufacturer_id': manufacturer_id,
'model_description': responder.get('model_description', ''),
})
return responder_obj_ids
def VerifyAndStoreData(self, data):
"""Check the data look reasonable and if it does, store it.
Returns:
A list of dicts in the form:
[{
'key': key,
}]
"""
if not data:
return []
try:
evaled_data = eval(data, {})
except Exception as e:
logging.info(data)
logging.error(e)
return []
responder_obj_ids = []
for manufacturer_id, responders in evaled_data.iteritems():
try:
manufacturer_id = int(manufacturer_id)
except ValueError:
logging.error('Invalid manufacturer id %s' % manufacturer_id)
continue
# See if we can get the manufacturer name
manufacturer_name = None
manufacturer_query = Manufacturer.all()
manufacturer_query.filter('esta_id = ', manufacturer_id)
results = manufacturer_query.fetch(1)
if results:
manufacturer_name = results[0].name
for responder in responders:
if 'device_model' not in responder:
logging.error('Missing device_model from data')
continue
try:
device_model_id = int(responder['device_model'])
except ValueError:
logging.error('Invalid device model %s' %
responder['device_model'])
continue
# ok, that's all the required fields
del responder['device_model']
responder_obj = UploadedResponderInfo(
manufacturer_id=manufacturer_id,
device_model_id=device_model_id,
info=str(responder),
upload_time=datetime.datetime.now())
responder_obj.put()
responder_obj_ids.append({
'device_model_id':
device_model_id,
'key':
str(responder_obj.key()),
'manufacturer':
manufacturer_name,
'manufacturer_id':
manufacturer_id,
'model_description':
responder.get('model_description', ''),
})
return responder_obj_ids
