def get(self, *args):
url = self.request.uri
qs = urllib.urlsplit(url)
parsed = urllib.parse_qs(qs.query)
"""Placeholder for supporting inheritance.
Inheritance is currently not implemented."""
query = """
SELECT w_id, w_name, p_id, p_category, p_name, p_value FROM {%1}.widget_property wp
JOIN {%1}.widget w ON (wp.wp_widget_id = w.w_id)
JOIN {%1}.property p ON (wp.wp_property_id = p.p_id);
"""
query = query.replace('{%1}', self.schema)
cur = self.db.cursor()
cur.execute(query)
"""Check if the query parameters are valid."""
data = c.parsedata(cur)
categories = py_.pluck(data, 'p_category')
valid_params = py_.intersection(parsed.keys(), categories)
"""Create a hash with widget id as key."""
data_by_id = py_.group_by(data, 'w_id')
"""For each widget, create a new widget object to return."""
widgets = []
for wid in data_by_id.keys():
this = data_by_id[wid]
by_cat = py_.group_by(this, 'p_category')
uniq_cat = py_.uniq(py_.pluck(this, 'p_category'))
widget = {
'w_id': wid,
'w_name': py_.uniq(py_.pluck(this, 'w_name'))[0],
'match': True
}
"""Create a new key, val pair for each property
for the widget."""
for cat in uniq_cat:
widget[cat] = py_.pluck(by_cat[cat], 'p_name')
"""Check if each property associated with the
widget matches the query parameters."""
for key in valid_params:
if key in by_cat.keys():
widget['match'] = widget['match'] and len(py_.intersection(widget[key], parsed[key])) > 0
widgets.append(widget)
"""If query parameters are not provided or invalid,
return all widgets without filtering."""
if len(valid_params) == 0:
ret = widgets
else:
ret = py_.filter(widgets, {'match': True})
self.write(json.dumps(ret, indent=4, separators=(',', ': ')))
def aggregate_entities(
self,
entity_type_value_group: Dict[Tuple[str, Any], List[BaseEntity]],
input_size: int,
) -> List[BaseEntity]:
"""
Reduce entities sharing same type and value.
Entities with same type and value are considered identical even if other metadata is same.
:param entity_type_val_group: A data-structure that groups entities by type and value.
:type entity_type_val_group: Dict[Tuple[str, Any], List[BaseEntity]]
:return: A list of de-duplicated entities.
:rtype: List[BaseEntity]
"""
aggregated_entities = []
for entities in entity_type_value_group.values():
indices = [
entity.alternative_index for entity in entities
if isinstance(entity.alternative_index, int)
]
min_alternative_index = py_.min_(indices) if indices else None
representative = entities[0]
representative.alternative_index = min_alternative_index
representative.alternative_indices = indices
representative.score = entity_scoring(len(py_.uniq(indices)),
input_size)
aggregated_entities.append(representative)
return aggregated_entities
def get_bolded_names(soup):
# other names for the food are bolded in the first paragraph
paragraphs = soup.select('.mw-parser-output p:not(.mw-empty-elt)')
firstParagraph = paragraphs[0]
aliasIndicators = ['known as', 'common name']
if py_.some(aliasIndicators, lambda l: l in firstParagraph.text.lower()):
return filter(
None,
py_.uniq(py_.map(firstParagraph.select('b'), lambda b: b.text)))
def get_ftu_fault_record(xls_path, sheet_name, start_date, end_date):
ret = []
if isinstance(start_date, str) or isinstance(start_date, unicode):
if len(start_date) > 10:
start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d %H:%M:%S')
else:
start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d')
if isinstance(end_date, str) or isinstance(end_date, unicode):
if len(end_date) > 10:
end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d %H:%M:%S')
else:
end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d')
if isinstance(start_date, datetime.datetime) and isinstance(end_date, datetime.datetime):
book = xlrd.open_workbook(xls_path)
startrowidx = 1
colstartrowidx = 0
for sheet in book.sheets():
if sheet.name.lower() == sheet_name.lower():
timestamp_idx = -1
sim_idx = -1
for i in range(100):
if sheet.cell_value(colstartrowidx, i).lower() == 'timestamp':
timestamp_idx = i
if sheet.cell_value(colstartrowidx, i).lower() == 'sim':
sim_idx = i
if timestamp_idx > -1 and sim_idx > -1:
break
for row in range(startrowidx, sheet.nrows):
timestamp = None
if len(sheet.cell_value(row, timestamp_idx)):
timestamp = datetime.datetime.strptime(sheet.cell_value(row, timestamp_idx), "%Y-%m-%d %H:%M:%S")
if timestamp >= start_date and timestamp <= end_date:
sim = ''
try:
sim = str(int(sheet.cell_value(row, sim_idx)))
except:
sim = ''
ret.append({
'sim':sim
})
# ret.append({
# 'sim': sim
# })
# print(len(ret))
# def uniq(value, index, array):
ret = _.uniq(ret, 'sim')
# print(len(ret))
return ret
def resources(self) -> List[Resource]:
"""
Parse list of resources using excel data.
"""
assignments = [[
name.strip() for name in it.split(",")
] for it in self.xlsx_data["Assigned to"].dropna().tolist()]
resources = [
Resource(name=name) for name in py_.uniq(py_.flatten(assignments))
]
# We also add a ghost for unassigned items
resources.append(Resource(name="Ghost"))
return resources
def test6():
def get_line_id(alist, code):
return _.find(alist, lambda x: x['properties'].has_key('func_pos_code') and x['properties']['func_pos_code'] == code)
# return _.matches_property('properties.func_pos_code', code)(alist)
def get_point_id(alist, code):
return _.find(alist, lambda x: x['properties'].has_key('function_pos_code') and x['properties'][
'func_pos_code'] == code)
ret = []
linesmap = {}
with codecs.open(ur'd:\linesmap.json', 'r', 'utf-8-sig') as f:
linesmap = json.loads(f.read())
polyline_dn = mongo_find('kmgd', 'network', {'properties.webgis_type':'polyline_dn'})
# towers = mongo_find('kmgd', 'features', {'properties.webgis_type':'point_tower'})
idx = 0
for k in linesmap.keys():
codes = _.uniq(_.flatten(linesmap[k]))
o = get_line_id(polyline_dn, k)
if o:
# l = mongo_find('kmgd', 'features', {'properties.line_func_code':k})
# ids = _.pluck(l, '_id')
ll = mongo_find('kmgd', 'features', {'properties.function_pos_code':{'$in':codes}})
if len(ll):
lll = _.pluck(ll, '_id')
o['properties']['nodes'] = lll
# o = add_mongo_id(o)
ret.append(o)
idx += 1
# if idx > 10:
# break
mongo_action('kmgd', 'network', 'save', ret)
