def getInputFields():
address = Database.find()
df = pd.DataFrame()
for var in address:
if var is not None:
count = 0
content = var['url_content']
url = Search.from_content(content)['url']
distance = Search.from_url(url)['distance_from_root']
soup = BeautifulSoup(content, "html.parser")
password = soup.findAll('input', {'type': 'password'})
for input in password:
count = count + 1
text = soup.findAll('input', {'type': 'text'})
for input in text:
count = count + 1
df = df.append(
{
'url': url,
'input_count': count,
'distance': distance
},
ignore_index=True)
print(df)
def search(img=None, page=1):
results = None
if not img:
return render_template('search.html', img=img, results=results)
try:
# image_filename = os.path.join(config.UPLOAD_FOLDER, img)
# r = Recognizer()
# image_label = r.recognize(image_filename) # Returns a string of image label
image_label = 'technology'
_search = Search()
results = _search.search(image_label, page)
except Exception as e:
flash('ERROR: {}'.format(e))
return render_template('search.html', img=img, page=page, results=results)
def create(cls, user, name, private):
# PRECONDITION: user is a teacher
if not is_title(name):
return 'name', 'Empty Course Name' # client side error
name = make_title(name)
if cls.all().filter('name =', name).get() is not None:
return 'name', 'Course name is already taken'
if private:
code = os.urandom(8).encode('hex')
else:
code = None
course = cls(name=name, teacher=user, code=code)
course.put()
Search.add_words(course.name, course.key().id(), COURSE_TABLE)
return course
def search_doctor(self, doctor, depart, names):
try:
## 保留搜尋條件、condition
reserved = Reserve().doc_reserved(doctor, depart, names)
## 取得醫師、科別、醫療機構名稱的condition
doctor_condition = Search().search_doctor(doctor)
depart_condition = Search().search_depart(depart)
name_condition = Search().search_name(names)
## 串接所有condition
conditions = [depart_condition, name_condition, doctor_condition]
sql_where = Form().form_sqlwhere(conditions)
return Select().select_info(sql_where, reserved)
except BaseException as e:
print('search_doctor Exception' + e)
return render_template('search.html')
def search_all(self, disease_id, county, township, types, names, star, indexes):
try:
## 保留搜尋條件
reserved = Reserve().disease_reserved(disease_id, county, township, names, types, star)
## 取得地區、名稱、層級、星等的condition
area_condition = Search().search_area(county, township)
name_condition = Search().search_name(names)
type_condition = Search().search_type(types)
star_condition = Search().search_star(star)
## 串接所有condition
conditions = [area_condition, name_condition, type_condition, star_condition]
sql_where = Form().form_sqlwhere(conditions)
return Select().exclude_none_data(indexes, sql_where, reserved)
except BaseException as e:
print('search_all Exception' + e)
return render_template('search.html')
def comp_hosp(self, county, township, names, types, star):
try:
## 保留搜尋條件
reserved = Reserve().hosp_reserved(county, township, names, types, star)
## 取得地區、名稱、層級、星等的condition
area_condition = Search().search_area(county, township)
name_condition = Search().search_name(names)
type_condition = Search().search_type(types)
reviews_condition = Search().search_star(star)
## 串接所有condition
conditions = [area_condition, name_condition, type_condition, reviews_condition]
sql_where = Form().form_sqlwhere(conditions)
return Select().select_info(sql_where, reserved)
except BaseException as e:
print('search_all Exception' + e)
return render_template('search.hml')
def populate() -> bool:
""" populate database from all dumps
:return:
"""
file_storage: str = current_app.config.get('FILE_STORAGE')
file_parser = FileParser(storage_path=file_storage)
parsed_results: dict = file_parser.parse_storage()
for filename, search in parsed_results.items():
products: list = search.pop('products', [])
new_search = Search(filename=filename, **search)
db.session.add(new_search)
db.session.commit()
print(new_search)
for product in products:
flights: list = product.pop('flights', [])
new_product = Product(search_id=new_search.id, **product)
db.session.add(new_product)
db.session.commit()
for flight in flights:
new_flight = Flight(product_id=new_product.id, **flight)
db.session.add(new_flight)
db.session.commit()
return True
def do_GET(self):
queries = parse_qs(urlparse(self.path).query)
search = Search(queries)
limit = 10
if 'limit' in queries and queries['limit'] and queries['limit'][
0].isdigit():
limit = int(queries['limit'][0])
offset = 0
if 'offset' in queries and queries['offset'] and queries['offset'][
0].isdigit():
offset = int(queries['offset'][0])
result = ElasticsearchQueryer().postings().filter(search).limit(
limit).offset(offset).execute(include_source=True)
# TODO: detect possible key errors
resp_body = {
"total": result["hits"]["total"]["value"],
"hits": [hit["_source"] for hit in result["hits"]["hits"]],
}
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.send_header('Access-Control-Allow-Origin', '*')
self.end_headers()
self.wfile.write(json.dumps(resp_body).encode('utf-8'))
return
def process(self, posting, log: Optional[LogSegment]):
search = Search(
queries={},
title=[posting.title],
company_name=[posting.company_name],
link=[posting.link],
)
query_result = ElasticsearchQueryer(
should_log=False).postings().filter(search).limit(1).execute()
if query_result["hits"]["total"]["value"] > 0:
print('Skipping duplicate')
return
data = posting.to_json()
resp = self.ingestor.ingest(data)
if 'result' not in resp or resp['result'] != 'created':
if not log:
log = LogSegment()
log.data = data
log.status = 'failure'
log.errors.append('Ingest failed')
if log and '_id' in resp:
log.doc_id = resp['_id']
def do_GET(self):
queries = parse_qs(urlparse(self.path).query)
search = Search(queries)
result = ElasticsearchQueryer().postings().filter(search).limit(
0).aggregation(
'is_remote',
'location',
'base_location',
'seniority',
'industry',
'employment_type',
'job_functions',
'skills',
'words',
).execute()
resp_body = result["aggregations"]
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.send_header('Access-Control-Allow-Origin', '*')
self.end_headers()
self.wfile.write(json.dumps(resp_body).encode('utf-8'))
return
def find(fields, tags):
search = Search(
measurement="event",
time={"start": "1770-06-19T20:07:36.356Z"},
fields=fields,
tags=tags,
)
tables = search.get()
result = dict()
for table in tables:
for record in table.records:
lifecycle_enum = dict([(1.0, "stop"), (0.0, "start")])
if record.values["_value"] == 2.0:
event = {}
event["name"] = record.values["name"].replace("_", " ")
event["effect"] = record.values["effect"].replace("_", " ")
event["time"] = record.values["_time"]
if "event_id" in record.values:
event["event_id"] = record.values["event_id"]
result[record.values["name"]] = event
elif record.values["_value"] < 2.0:
event = {}
if record.values["name"] in result:
event = result.get(record.values["name"])
event["name"] = record.values["name"].replace("_", " ")
event["effect"] = record.values["effect"].replace("_", " ")
if not "time" in event:
event["time"] = {}
event["time"][lifecycle_enum.get(
record.values["_value"])] = record.values["_time"]
result[record.values["name"]] = event
return list(result.values())
def clean_up(days=2):
""" Clean up searches by anonymous users
"""
stale = datetime.today() - timedelta(days=days)
for search in Search.objects():
if not search.user and search.created <= stale:
print "deleting {} for {}-{}".format(search.pk, search.make,
search.model)
search.delete()
def _save_search(self, param: str, identity: dict) -> SearchModel:
"""
Save the identity in the app DB.
:param param str: Search param.
:param dict identity: identity information.
"""
# Check the DB to know if the record was already saved.
search = SearchModel.query.filter(SearchModel.param == param).first()
if not search:
# Create a new record.
search = SearchModel()
# Save the information.
search.param = param
search.create_datetime = datetime.utcnow()
search.identity = identity
db.session.add(search)
db.session.commit()
# Return the search model.
return search
def search_depart(self, depart_id, indexes, county, township, types,
names):
try:
## 保留搜尋條件
reserved = Reserve().depart_reserved(depart_id, county, township,
types, names)
## 取得科別、地區、名稱、層級的condition
depart_condition = Search().search_depart(depart_id)
area_condition = Search().search_area(county, township)
name_condition = Search().search_name(names)
type_condition = Search().search_type(types)
## 串接所有condition
conditions = [
depart_condition, area_condition, name_condition,
type_condition
]
sql_where = Form().form_sqlwhere(conditions)
return Select().select_info(indexes, sql_where, reserved)
except BaseException as e:
print('search_subj Exception' + e)
return render_template('search.html')
def landing(search=None, page=0):
""" Listings page """
# If there is no search, get the first on off the list
if current_user.is_anonymous():
searches = []
else:
searches = Search.related(current_user.pk)
if not search:
if len(searches) > 0:
for i in xrange(len(searches)):
if searches[i].make:
display = searches[i]
break
else:
return redirect(url_for('landing'))
else:
return redirect(url_for('landing'))
else:
display = Search.objects.get(pk=search).select_related()
try:
make = Car.objects.get(make=display.make)
except Car.DoesNotExist:
pass
models = [model.model for model in make.models]
models.insert(0, 'Any')
model = make.get_model(display.model)
trims = ['Any']
if model:
for trim in model.trims:
trims.append(trim)
# Show all the finds for a search
finds = display.get_finds(page, 100)
context = dict(display=display,
searches=searches,
finds=finds,
page=page,
pages=len(display.finds) / 100,
makes=MAKES,
models=models,
trims=trims)
return render_template('listings.html', **context)
def search_obj(self, hospital_id, indexes):
try:
## 取得醫療機構資訊
hosp_info = SelectData().hosp_info(hospital_id)
## 用共用程式取得客觀指標資訊，首參數用來分辨資訊(v為指標值、d為就醫人數、l為指標值等級)
value = SelectData().hosp_obj_data('v', indexes, hospital_id)
deno = SelectData().hosp_obj_data('d', indexes, hospital_id)
level = SelectData().hosp_obj_data('l', indexes, hospital_id)
## 將指標值、就醫人數、指標值等級包裝成zip
z_data = zip(value, deno, level)
return Result().get_obj_column_name(indexes, hosp_info, z_data,
hospital_id)
except:
alert = "抱歉，找不到您要的資料訊息。"
hosp_info = Search().search_hosp(hospital_id)
return render_template('hospObjResult.html',
hosp_info=hosp_info,
alert=alert)
def parse_query( data, lati=None, longi=None, user=None ):
""" parse the data string and figure out what is wanted
data - user string query
"""
# Create a search record
search = Search()
search.search = data
search.distance = 50
if lati and longi:
search.geo = [float(lati),float(longi)]
try:
address,zipcode = reverse_geocode(lati,longi)
except Exception, e:
log( "Geosearch error {}".format( str(e) ) )
else:
search.zip = zipcode
def get_obj_column_name(self, indexes, hosp_info, z_data, hospital_id):
try:
## 建立columns[]，存入從資料庫中取得的欄位名稱(縮寫)、正負向指標註記、指標說明
columns = []
for index in indexes:
sqlstr = (
"SELECT abbreviation, P**N, description FROM column_name WHERE name = '{}' "
.format(index))
columns.append(self.cursor.execute(sqlstr).fetchall()[0])
## 供前端使用之參數：選取的指標數量
col_len = len(columns)
return render_template('hospObjResult.html',
scroll='indexes',
hosp_info=hosp_info,
z_data=z_data,
columns=columns,
col_len=col_len)
except:
alert = "抱歉，找不到您要的資料訊息。"
hosp_info = Search().search_hosp(hospital_id)
return render_template('hospObjResult.html',
hosp_info=hosp_info,
alert=alert)