def get(self):
self.response.headers['Content-Type'] = 'text/plain'
results = Responder.all()
models = []
for model in results:
model_output = {
'manufacturer_name': model.manufacturer.name,
'manufacturer_id': model.manufacturer.esta_id,
'device_model_id': model.device_model_id,
'model_description': model.model_description,
}
if model.link:
model_output['link'] = model.link
if model.image_url:
model_output['image_url'] = model.image_url
tags = list(model.tag_set)
if tags:
tags = []
for tag in model.tag_set:
tags.append(tag.tag.label)
model_output['tags'] = tags
models.append(model_output)
self.response.out.write(json.dumps({'models': models}))
def GetMissingResponders(self):
responders = []
for responder in Responder.all():
if responder.link and responder.image_url:
continue
responders.append(responder)
return responders
def get(self):
self.response.headers['Content-Type'] = 'text/plain'
results = Responder.all()
models = []
for model in results:
model_output = {
'manufacturer_name': model.manufacturer.name,
'manufacturer_id': model.manufacturer.esta_id,
'device_model_id': model.device_model_id,
'model_description': model.model_description,
'software_versions': [],
}
models.append(model_output)
for software in model.software_version_set:
software_output = {
'id': software.version_id,
'label': software.label,
'personalities': [],
}
for personality in software.personality_set:
personality_output = {
'description': personality.description,
'index': personality.index,
'slot_count': personality.slot_count,
}
software_output['personalities'].append(personality_output)
model_output['software_versions'].append(software_output)
self.response.out.write(json.dumps({'models': models}))
def get(self):
self.response.headers['Content-Type'] = 'text/plain'
results = Responder.all()
results.order('device_model_id')
self.response.out.write(
'Manufacturer ID,Manufacturer Name,Device ID,Model Name,Info Url,'
'Image Url\n')
for model in results:
if model.link and model.image_url:
continue
fields = []
fields.append('0x%hx' % model.manufacturer.esta_id)
fields.append(model.manufacturer.name)
fields.append('0x%hx' % model.device_model_id)
fields.append(model.model_description)
if model.link:
fields.append(model.link)
else:
fields.append('')
if model.image_url:
fields.append(model.image_url)
else:
fields.append('')
self.response.out.write(','.join(fields))
self.response.out.write('\n')
def GarbageCollectBlobs(self):
keys_to_blobs = {}
for blob in BlobInfo.all():
keys_to_blobs[blob.key()] = blob
for responder in Responder.all():
image_blob = responder.image_data
if image_blob:
key = image_blob.key()
if key in keys_to_blobs:
del keys_to_blobs[key]
for product in Product.all():
image_blob = product.image_data
if image_blob:
key = image_blob.key()
if key in keys_to_blobs:
del keys_to_blobs[key]
for key, blob_info in keys_to_blobs.iteritems():
logging.info('deleting %s' % key)
blob_info.delete()
if keys_to_blobs:
return 'Deleted blobs: \n%s' % '\n'.join(
str(k) for k in keys_to_blobs)
else:
return 'No blobs to delete'
def BuildIndex(self):
current_index = self.LoadCurrentIndex()
new_index = {}
for responder in Responder.all():
version = common.GetLatestSoftware(responder)
if not version:
continue
for param in version.supported_parameters:
key = self.KeyFromPID(responder.manufacturer, param)
pid = self._pid_cache.get(key, None)
if not pid:
continue
new_index.setdefault(key, set()).add(responder.key())
# now diff the old and new
for key, responders in new_index.iteritems():
if key in current_index:
if current_index[key] != responders:
logging.info('Responder set changed for %s' % str(key))
pid = self._pid_cache[key]
pid.responders = list(responders)
pid.put()
del current_index[key]
else:
# this should never happen
logging.warn('Missing key for %s' % str(key))
for key, responders in current_index.iteritems():
if responders:
logging.info('Removed %s' % responders)
pid = self._pid_cache[key]
pid.responders = []
pid.put()
def GarbageCollectBlobs(self):
keys_to_blobs = {}
for blob in BlobInfo.all():
keys_to_blobs[blob.key()] = blob
for responder in Responder.all():
image_blob = responder.image_data
if image_blob:
key = image_blob.key()
if key in keys_to_blobs:
del keys_to_blobs[key]
for product in Product.all():
image_blob = product.image_data
if image_blob:
key = image_blob.key()
if key in keys_to_blobs:
del keys_to_blobs[key]
for key, blob_info in keys_to_blobs.iteritems():
logging.info('deleting %s' % key)
blob_info.delete()
if keys_to_blobs:
return 'Deleted blobs: \n%s' % '\n'.join(str(k) for k in keys_to_blobs)
else:
return 'No blobs to delete'
def GetMissingResponders(self):
responders = []
for responder in Responder.all():
if responder.link and responder.image_url:
continue
responders.append(responder)
return responders
def ProductCount(self):
"""Return the number of product."""
product_count = memcache.get(memcache_keys.PRODUCT_COUNT_KEY)
if product_count is None:
product_count = Responder.all().count() + Product.all().count()
if not memcache.add(memcache_keys.PRODUCT_COUNT_KEY, product_count):
logging.error("Memcache set failed.")
return product_count
def _LookupResponder(self, manufacturer_key, model_id):
"""Given a manufacturer key and model_id, lookup the Responder entity."""
models = Responder.all()
models.filter('device_model_id = ', model_id)
models.filter('manufacturer = ', manufacturer_key)
model_data = models.fetch(1)
if not model_data:
return None
return model_data[0]
def _LookupResponder(self, manufacturer_key, model_id):
"""Given a manufacturer key and model_id, lookup the Responder entity."""
models = Responder.all()
models.filter('device_model_id = ', model_id)
models.filter('manufacturer = ', manufacturer_key)
model_data = models.fetch(1)
if not model_data:
return None
return model_data[0]
def GetTemplateData(self):
page = StringToInt(self.request.get('page'), False)
if page is None:
page = 1
# 0 offset
page -= 1
query = Responder.all()
query.order('-score')
total = query.count()
models = query.fetch(limit=self.RESULTS_PER_PAGE,
offset=page * self.RESULTS_PER_PAGE)
rows = []
for model, index in zip(models, range(len(models))):
if index % self.COLUMNS == 0:
rows.append([])
rating_scale = None
if model.rdm_responder_rating is not None:
rating_scale = 2 + int(model.rdm_responder_rating / 10 * 6.5)
output = {
'manufacturer_id': model.manufacturer.esta_id,
'model_id': model.device_model_id,
'name': common.MaybeEncode(model.model_description),
'rating': model.rdm_responder_rating,
'star_width': rating_scale,
}
if hasattr(model.manufacturer, 'name') and model.manufacturer.name:
output['manufacturer_name'] = model.manufacturer.name
if model.image_data:
serving_url = model.image_serving_url
if not serving_url:
serving_url = images.get_serving_url(
model.image_data.key())
model.image_serving_url = serving_url
model.put()
logging.info('saving %s' % serving_url)
output['image_key'] = serving_url
rows[-1].append(output)
start = page * self.RESULTS_PER_PAGE
data = {
'end': start + len(models),
'model_rows': rows,
'page_number': page + 1, # back to 1 offset
'start': start + 1,
'total': total,
}
if page:
data['previous'] = page
if start + len(models) < total:
data['next'] = page + 2
return data
def GetTemplateData(self):
page = StringToInt(self.request.get('page'), False)
if page is None:
page = 1
# 0 offset
page -= 1
query = Responder.all()
query.order('-score')
total = query.count()
models = query.fetch(limit=self.RESULTS_PER_PAGE,
offset=page * self.RESULTS_PER_PAGE)
rows = []
for model, index in zip(models, range(len(models))):
if index % self.COLUMNS == 0:
rows.append([])
rating_scale = None
if model.rdm_responder_rating is not None:
rating_scale = 2 + int(model.rdm_responder_rating / 10 * 6.5)
output = {
'manufacturer_id': model.manufacturer.esta_id,
'model_id': model.device_model_id,
'name': common.MaybeEncode(model.model_description),
'rating': model.rdm_responder_rating,
'star_width': rating_scale,
}
if hasattr(model.manufacturer, 'name') and model.manufacturer.name:
output['manufacturer_name'] = model.manufacturer.name
if model.image_data:
serving_url = model.image_serving_url
if not serving_url:
serving_url = images.get_serving_url(model.image_data.key())
model.image_serving_url = serving_url
model.put()
logging.info('saving %s' % serving_url)
output['image_key'] = serving_url
rows[-1].append(output)
start = page * self.RESULTS_PER_PAGE
data = {
'end': start + len(models),
'model_rows': rows,
'page_number': page + 1, # back to 1 offset
'start': start + 1,
'total': total,
}
if page:
data['previous'] = page
if start + len(models) < total:
data['next'] = page + 2
return data
def BuildResponse(self):
param_counts = {}
responders = 0
max_manufacturer_pids = 0
max_manufacturer_responder = ''
max_pids = 0
max_pids_responder = ''
for responder in Responder.all():
params = []
version = common.GetLatestSoftware(responder)
if version:
params = version.supported_parameters
manufacturer_pids = 0
for param in params:
param_counts.setdefault(param, 0)
param_counts[param] += 1
if param >= 0x8000:
manufacturer_pids += 1
if params:
responders += 1
if manufacturer_pids > max_manufacturer_pids:
max_manufacturer_responder = responder.model_description
max_manufacturer_pids = manufacturer_pids
if len(params) > max_pids:
max_pids_responder = responder.model_description
max_pids = len(params)
pids = []
for param, count in param_counts.iteritems():
param_info = {
'id': param,
'count': count,
}
if param < 0x8000:
query = Pid.all()
query.filter('pid_id = ', param)
pid = query.fetch(1)
if pid:
param_info['name'] = pid[0].name
pids.append(param_info)
output = {
'count':
responders,
'max_manufacturer_pids':
(max_manufacturer_responder, max_manufacturer_pids),
'max_pids': (max_pids_responder, max_pids),
'pids':
pids,
}
return json.dumps(output)
def BuildResponse(self):
param_counts = {}
responders = 0
max_manufacturer_pids = 0
max_manufacturer_responder = ''
max_pids = 0
max_pids_responder = ''
for responder in Responder.all():
params = []
version = common.GetLatestSoftware(responder)
if version:
params = version.supported_parameters
manufacturer_pids = 0
for param in params:
param_counts.setdefault(param, 0)
param_counts[param] += 1
if param >= 0x8000:
manufacturer_pids += 1
if params:
responders += 1
if manufacturer_pids > max_manufacturer_pids:
max_manufacturer_responder = responder.model_description
max_manufacturer_pids = manufacturer_pids
if len(params) > max_pids:
max_pids_responder = responder.model_description
max_pids = len(params)
pids = []
for param, count in param_counts.iteritems():
param_info = {
'id': param,
'count': count,
}
if param < 0x8000:
query = Pid.all()
query.filter('pid_id = ', param)
pid = query.fetch(1)
if pid:
param_info['name'] = pid[0].name
pids.append(param_info)
output = {
'count': responders,
'max_manufacturer_pids': (max_manufacturer_responder,
max_manufacturer_pids),
'max_pids': (max_pids_responder, max_pids),
'pids': pids,
}
return json.dumps(output)
def ClearModels(self):
memcache.delete(memcache_keys.MODEL_COUNT_KEY)
for item in Responder.all():
item.delete()
for item in SoftwareVersion.all():
item.delete()
for item in ResponderTag.all():
item.delete()
for item in ResponderTagRelationship.all():
item.delete()
return ''
def ClearModels(self):
memcache.delete(memcache_keys.MODEL_COUNT_KEY)
for item in Responder.all():
item.delete()
for item in SoftwareVersion.all():
item.delete()
for item in ResponderTag.all():
item.delete()
for item in ResponderTagRelationship.all():
item.delete()
return ''
def get(self):
key = self.request.get('key')
responder = Responder.get(key)
if not responder:
return
if responder.image_url and not responder.image_data:
fetcher = ImageFetcher()
blob_key = fetcher.FetchAndSaveImage(responder.image_url)
if blob_key:
responder.image_data = blob_key
responder.image_serving_url = images.get_serving_url(blob_key)
responder.put()
return
def get(self):
key = self.request.get('key')
responder = Responder.get(key)
if not responder:
return
if responder.image_url and not responder.image_data:
fetcher = ImageFetcher()
blob_key = fetcher.FetchAndSaveImage(responder.image_url)
if blob_key:
responder.image_data = blob_key
responder.image_serving_url = images.get_serving_url(blob_key)
responder.put()
return
def LookupModel(manufacturer, model_id):
"""Lookup a model based on the URL params."""
if type(model_id) not in (int, long):
model_id = StringToInt(model_id)
manufacturer = GetManufacturer(manufacturer)
if manufacturer is None or model_id is None:
return None
models = Responder.all()
models.filter('device_model_id = ', model_id)
models.filter('manufacturer = ', manufacturer.key())
model_data = models.fetch(1)
if not model_data:
return None
return model_data[0]
def InitiateImageFetch(self):
"""Add /fetch_image tasks for all responders missing image data."""
urls = []
for responder in Responder.all():
if responder.image_url and not responder.image_data:
url = '/tasks/fetch_image?key=%s' % responder.key()
task = taskqueue.Task(method='GET', url=url)
task.add()
urls.append(responder.image_url)
for product in Product.all():
if product.image_url and not product.image_data:
url = '/tasks/fetch_product_image?key=%s' % product.key()
task = taskqueue.Task(method='GET', url=url)
task.add()
urls.append(product.image_url)
if urls:
return 'Fetching urls: \n%s' % '\n'.join(urls)
else:
return 'No images to fetch'
def InitiateImageFetch(self):
"""Add /fetch_image tasks for all responders missing image data."""
urls = []
for responder in Responder.all():
if responder.image_url and not responder.image_data:
url = '/tasks/fetch_image?key=%s' % responder.key()
task = taskqueue.Task(method='GET', url=url)
task.add()
urls.append(responder.image_url)
for product in Product.all():
if product.image_url and not product.image_data:
url = '/tasks/fetch_product_image?key=%s' % product.key()
task = taskqueue.Task(method='GET', url=url)
task.add()
urls.append(product.image_url)
if urls:
return 'Fetching urls: \n%s' % '\n'.join(urls)
else:
return 'No images to fetch'
def GetTemplateData(self):
pid = self.LookupPIDFromRequest()
if not pid:
self.error(404)
return
supported_by = []
for responder_key in pid.responders:
responder = Responder.get(responder_key)
if responder:
supported_by.append({
'name': responder.model_description,
'manufacturer': responder.manufacturer.esta_id,
'model': responder.device_model_id,
})
supported_by.sort(key=lambda x: x['name'])
output = {
'link': pid.link,
'manufacturer_name': pid.manufacturer.name,
'manufacturer_id': pid.manufacturer.esta_id,
'notes': pid.notes,
'pid_id': pid.pid_id,
'pid_name': pid.name,
'supported_by': supported_by,
}
if pid.get_command:
command = self.BuildCommand(pid.get_command)
output['get_command'] = command
if pid.discovery_command:
command = self.BuildCommand(pid.discovery_command)
output['discovery_command'] = command
if pid.set_command:
command = self.BuildCommand(pid.set_command)
output['set_command'] = command
return output
def get(self):
for device in Responder.all():
score = 0
if device.image_data is not None:
# 10 point boost for having an image
score += 10
if device.link is not None:
# 1 point boost for having a link
score += 1
if device.software_version_set.count():
# 10 point boost for having version information
score += 10
if device.score_penalty:
score -= device.score_penalty
# up to +20 pts for having test results
if device.rdm_responder_rating is not None:
score += int(device.rdm_responder_rating / 5)
device.score = score
device.put()
return
def get(self):
for device in Responder.all():
score = 0
if device.image_data is not None:
# 10 point boost for having an image
score += 10
if device.link is not None:
# 1 point boost for having a link
score += 1
if device.software_version_set.count():
# 10 point boost for having version information
score += 10
if device.score_penalty:
score -= device.score_penalty
# up to +20 pts for having test results
if device.rdm_responder_rating is not None:
score += int(device.rdm_responder_rating / 5)
device.score = score
device.put()
return
def _AddResponder(self, manufacturer, model_id, model_info):
"""Add a responder to the data store.
Args:
manufacturer:
model_id:
model_info: The dict with the responder information
Returns:
The new Responder entity.
"""
responder = Responder(manufacturer=manufacturer,
device_model_id=model_id,
model_description=self._Encode(
model_info.get('model_description')))
# add product_category if there is one
product_category_id = model_info.get('product_category')
if product_category_id is not None:
category = self._LookupProductCategory(product_category_id)
if category:
responder.product_category = category
else:
logging.info('No product category found for 0x%hx' %
product_category_id)
# add link and image_url if they exist
link_url = model_info.get('link')
if link_url:
responder.link = link_url
image_url = model_info.get('image_url')
if image_url:
responder.image_url = image_url
responder.put()
return responder
def _AddResponder(self, manufacturer, model_id, model_info):
"""Add a responder to the data store.
Args:
manufacturer:
model_id:
model_info: The dict with the responder information
Returns:
The new Responder entity.
"""
responder = Responder(
manufacturer=manufacturer,
device_model_id=model_id,
model_description=self._Encode(model_info.get('model_description')))
# add product_category if there is one
product_category_id = model_info.get('product_category')
if product_category_id is not None:
category = self._LookupProductCategory(product_category_id)
if category:
responder.product_category = category
else:
logging.info('No product category found for 0x%hx' %
product_category_id)
# add link and image_url if they exist
link_url = model_info.get('link')
if link_url:
responder.link = link_url
image_url = model_info.get('image_url')
if image_url:
responder.image_url = image_url
responder.put()
return responder
