def results(year: hug.types.text, firstName: hug.types.text,
lastName: hug.types.text):
"""Returns the results for a given candidate for a given year"""
engine = create_engine('postgresql://%s:%s@%s/%s' % (user, pwd, ip, user),
client_encoding='utf8',
echo=False)
conn = engine.connect()
Base = declarative_base()
query = "SELECT * FROM names WHERE election_year = '%s' AND candidate_first_name = '%s' AND candidate_last_name = '%s'" % (
str(year), firstName.upper(), lastName.upper())
df = pd.read_sql(query, conn)
candidateId = df['candidate_id'].tolist()[0]
resultQuery = "SELECT * FROM votes WHERE candidate_id = '%s';" % (
str(candidateId))
result = pd.read_sql(resultQuery, conn)
officeId, districtId = result['office_code'].tolist(
)[0], result['district_code'].tolist()[0]
totalQuery = "Select office_code, district_code, county_code, city_code, ward_number, precinct_number, SUM(precinct_votes) AS total_votes FROM votes WHERE office_code = '%s' AND district_code = '%s' AND election_year = '%s' GROUP BY 1,2,3,4,5,6" % (
str(officeId), str(districtId), str(year))
totalTable = pd.read_sql(totalQuery, conn)
output = pd.merge(result,
totalTable,
on=[
'office_code', 'district_code', 'county_code',
'city_code', 'ward_number', 'precinct_number'
],
how="inner")
output['candidate_percentage'] = 100 * output['precinct_votes'] / output[
'total_votes']
conn.close()
engine.dispose()
return output.reset_index().to_json(orient="records")
def get_goat_movies(genres: hug.types.text, api_key: hug.types.text, hug_timer=5):
"""
Get a list of the most upvoted (GOAT) movies.
Input: gg_id, genres, api_key
URL: http://HOST:PORT/get_single_training_movie?genres=none&api_key=API_KEY
"""
if (check_api_token(api_key) == True):
# API KEY VALID
if ((genres == ' ') | (genres == '%20')):
result = db.movie.find().sort([('goat_upvotes', -1)])[:10] #
print("a")
return build_goat_json(result, hug_timer)
else:
print("b")
genres.replace('%20', ' ') # Browser pushes ' ', NodeJS pushes '%20'
if (' ' in genres):
genres = genres.split(' ') # Splitting the genre string into a list of genres!
genre_list_check = isinstance(genres, list) # Check if the genres variable is a list (or a single genre string)
print("is list? {}".format(genre_list_check))
print("genres: {}".format(genres))
if (genre_list_check == True):
# Multiple genres detected! Count the quantity of movies w/ all of these genres!
movie_count = db.movie.find({"genre": {'$all': genres}}).sort([('goat_upvotes', -1)]).count()
else:
# Single genre detected! Count how many movies there are w/ this genre
movie_count = db.movie.find({"genre": genres}).sort([('goat_upvotes', -1)]).count()
print("movie count: {}".format(movie_count))
if (movie_count > 0):
print("greater than 0")
# Results found!
if (movie_count >= 10):
# More than 10 movies found? Let's limit top-k to 10!
k_limit = 10
else:
# Less than 10 movies found? Let's reduce the top-k limit!
k_limit = movie_count
if (genre_list_check == True):
# Multiple genre 'all' search
result = db.movie.find({"genre": {'$all': genres}}).sort([('goat_upvotes', -1)])[:k_limit]
else:
# Single genre search
result = db.movie.find({"genre": genres}).sort([('goat_upvotes', -1)])[:k_limit]
return build_goat_json(result, hug_timer)
else:
# No results, provide json result for nodejs to detect!
return {'success': False,
'valid_key': True,
'took': float(hug_timer)}
else:
# API KEY INVALID!
return {'success': False,
'valid_key': False,
'took': float(hug_timer)}
def ObtenerFotos(imagen: hug.types.text):
directorio = imagen.strip().split("-")[0]
pathito = f"./Reportes/{directorio}/"
foto = os.path.join(pathito, imagen.strip())
return foto
def bulk_search(
self,
fasta: hug.types.text,
threshold: hug.types.float_number = 1.0,
config: hug.types.text = None,
score: hug.types.smart_boolean = False,
format: hug.types.one_of(["json", "csv"]) = "json",
stream: hug.types.smart_boolean = False,
):
config = get_config_from_file(config)
fasta = Fasta(fasta)
if not stream:
_config = copy.copy(config)
_config["nproc"] = 1
csv_combined = ""
nproc = config.get("nproc", 1)
with multiprocessing.Pool(processes=nproc) as pool:
args = [(_config, str(seq), threshold, score)
for seq in fasta.values()]
dd = pool.map_async(search_bigsi_parallel,
chunks(args, math.ceil(len(args) /
nproc))).get()
dd = [item for sublist in dd for item in sublist]
if format == "csv":
return "\n".join([d_to_csv(d, False, False) for d in dd])
else:
return json.dumps(dd, indent=4)
else:
bigsi = BIGSI(config)
csv_combined = ""
for i, seq in enumerate(fasta.values()):
seq = str(seq)
d = {
"query": seq,
"threshold": threshold,
"results": bigsi.search(seq, threshold, score),
"citation": "http://dx.doi.org/10.1038/s41587-018-0010-1",
}
if format == "csv":
if i == 0:
with_header = True
carriage_return = False
elif i == len(fasta) - 1:
carriage_return = True
else:
with_header = False
carriage_return = False
csv_result = d_to_csv(d, with_header, carriage_return)
csv_combined += csv_result
if stream:
print(csv_result)
else:
if stream:
print(json.dumps(d))
def get_single_training_movie(gg_id: hug.types.text, genres: hug.types.text, actors: hug.types.text, api_key: hug.types.text, hug_timer=5):
"""
Get a single movie for the training bot section.
Retrieves a list of movies the user has previously voted for, used as a filter!
URL: http://HOST:PORT/get_single_training_movie?gg_id=anonymous_google_id&genres=none&actors=none&api_key=API_KEY
"""
if (check_api_token(api_key) == True):
# API KEY VALID
user_id = get_user_id_by_gg_id(gg_id) # Get the user's movie rating user ID (incremental)
if user_id is not None:
imdb_list = get_voted_movie_by_user_id(user_id) # Produce a list of the user's previously rated movies.
if ((genres == ' ') | (genres == '%20')):
remaining_count = db.movie.find({'imdbID': {'$nin': imdb_list}}).count()
result = db.movie.find({'imdbID': {'$nin': imdb_list}})[0] # Filter out the list of previously voted for movies, sorted by imdb vote data (rating, quantity votes), takes top 1.
return build_training_response(result, hug_timer, remaining_count)
else:
genres.replace('%20', ' ') # Browser pushes ' ', NodeJS pushes '%20'
if (' ' in genres):
#blah!
genres = genres.split(' ')
#print(genres)
genre_list_check = isinstance(genres, list)
if (genre_list_check == True):
remaining_count = db.movie.find({'imdbID': {'$nin': imdb_list}, "genre": {'$all': genres}}).count()
else:
remaining_count = db.movie.find({'imdbID': {'$nin': imdb_list}, "genre": genres}).count()
if (remaining_count > 0):
# Results found! Return 1 result to the user.
if (genre_list_check == True):
result = db.movie.find({'imdbID': {'$nin': imdb_list}, "genre": {'$all': genres}})[0]
else:
result = db.movie.find({'imdbID': {'$nin': imdb_list}, "genre": genres})[0]
return build_training_response(result, hug_timer, remaining_count)
else:
# No results, provide json result for nodejs to detect!
return {'success': False,
'valid_key': True,
'took': float(hug_timer)}
else:
# Invalid GG_ID
return {'success': False,
'valid_key': True,
'took': float(hug_timer)}
else:
# API KEY INVALID!
return {'success': False,
'valid_key': False,
'took': float(hug_timer)}
def formatData(date: hug.types.text,
time: hug.types.text,
location1: hug.types.text,
location2: hug.types.text,
hug_timer=3):
"""Changing the data types"""
print(date)
dateString = date[0:4] + "/" + date[4:6] + "/" + date[6:8]
timeString = time[0:2] + ":" + time[3:6]
location1String = location1.replace(".", ":")
location2String = location2.replace(".", ":")
return runAstroScript(dateString, timeString, location1String,
location2String)
def bulk_search(
self,
fasta: hug.types.text,
threshold: hug.types.float_number = 1.0,
config: hug.types.text = None,
score: hug.types.smart_boolean = False,
format: hug.types.one_of(["json", "csv"]) = "json",
stream: hug.types.smart_boolean = False,
):
config = get_config_from_file(config)
bigsi = BIGSI(config)
fasta = Fasta(fasta)
if not stream:
csv_combined = ""
nproc = config.get("nproc", 1)
with ThreadPool(processes=nproc) as pool:
args = [(bigsi, str(seq), threshold, score)
for seq in fasta.values()]
dd = pool.starmap(search_bigsi, args)
if format == "csv":
return "\n".join([d_to_csv(d, False, False) for d in dd])
else:
return json.dumps(dd, indent=4)
else:
dd = []
csv_combined = ""
for i, seq in enumerate(fasta.values()):
seq = str(seq)
d = {
"query": seq,
"threshold": threshold,
"results": bigsi.search(seq, threshold, score),
"citation": "http://dx.doi.org/10.1038/s41587-018-0010-1",
}
dd.append(d)
if format == "csv":
if i == 0:
with_header = True
carriage_return = False
elif i == len(fasta) - 1:
carriage_return = True
else:
with_header = False
carriage_return = False
csv_result = d_to_csv(d, with_header, carriage_return)
csv_combined += csv_result
if stream:
print(csv_result)
else:
if stream:
print(json.dumps(d))
def set_frontend_config(db: directives.PeeweeSession, instance_name: hug.types.text, long_instance_name: hug.types.text,
contact_info_bookings: hug.types.text, contact_info_appointments: hug.types.text = None,
form_fields: hug.types.text = "base,address,dayOfBirth,reason",
for_real: hug.types.smart_boolean = False):
with db.atomic():
if not contact_info_appointments:
appointments_contact = contact_info_bookings
else:
appointments_contact = contact_info_appointments
template = {
"instanceName": f"{instance_name}",
"longInstanceName": f"{long_instance_name}",
"contactInfoCoupons": f"{contact_info_bookings}",
"contactInfoAppointment": f"{appointments_contact}",
"formFields": form_fields.split(","),
}
if not for_real:
print(f"This would update the config with '{json.dumps(template, indent=2)}'. Run with --for_real if you "
f"are sure.")
sys.exit(1)
else:
print(
f"Updating the config with '{json.dumps(template, indent=2)}'.")
try:
config = FrontendConfig.get()
config.config = template
except FrontendConfig.DoesNotExist:
config = FrontendConfig.create(config=template)
config.save()
print("Done.")
def build(
self,
bloomfilters: hug.types.multiple,
samples: hug.types.multiple = [],
config: hug.types.text = None,
):
config = get_config_from_file(config)
if samples:
assert len(samples) == len(bloomfilters)
else:
samples = bloomfilters
if config.get("max_build_mem_bytes"):
max_memory_bytes = humanfriendly.parse_size(
config["max_build_mem_bytes"])
else:
max_memory_bytes = None
return build(
config=config,
bloomfilter_filepaths=bloomfilters,
samples=samples,
max_memory=max_memory_bytes,
)
def suggestion(text: hug.types.text):
"""
For a given text, return possible terms from a suggest list in elastic search index
example query: http://localhost:8888/api/suggestion?text=department
"""
suggest_body = {
"suggest": {
"field-suggest": {
"prefix": text,
"completion": {
"field": "suggest"
}
}
}
}
responses = es.search(index='penn-events', body=suggest_body)
# return all possible full term from suggest list
suggest_terms = []
for response in responses['suggest']['field-suggest'][0]['options']:
for s in response['_source']['suggest']:
if text.lower() in s.lower():
suggest_terms.append(s)
return list(pd.unique(suggest_terms))
def get_token(login: hug.types.text, password: hug.types.text):
try:
user = User.get(login=login,
password=hashlib.sha1(password.encode()).hexdigest())
return {'token': user.token.token}
except User.DoesNotExist:
raise HTTPError(HTTP_404)
def city(year: hug.types.text, city: hug.types.text):
"""Returns the results for a given candidate for a given year"""
engine = create_engine('postgresql://%s:%s@%s/%s' % (user, pwd, ip, user),
client_encoding='utf8',
echo=False)
conn = engine.connect()
Base = declarative_base()
query = "SELECT * FROM cities WHERE election_year = '%s' AND city_description = '%s CITY' OR city_description = '%s TOWNSHIP'" % (
str(year), city.upper(), city.upper())
df = pd.read_sql(query, conn)
cityId = df['city_code'].tolist()[0]
resultQuery = "SELECT * FROM votes WHERE city_code = '%s';" % (str(cityId))
result = pd.read_sql(resultQuery, conn)
conn.close()
engine.dispose()
return result.reset_index().to_json(orient="records")
def get_occurrences(url: hug.types.text,
key_word: hug.types.text,
case_sensitive=True):
"""
Returns JSON containing number of occurrences of "key_word" in website
given by "url" with case sensitivity defined by optional argument
case_sensitive (True by default)
"""
# get and decode web content
content = requests.get(url).content
html_text = content.decode('utf-8')
# remove Javascript, CSS and HTML
from bs4 import BeautifulSoup
soup = BeautifulSoup(html_text, "lxml")
for script in soup(["script", "style"]):
script.extract()
text = soup.get_text()
# convert text and keyword to lowercase if case_sensitive = False
if not case_sensitive:
text = text.lower()
key_word = key_word.lower()
# split text into iterable list of words without punctuation
text = re.split(r",|;|:|\W", text)
# count keyword occurrences
counter = 0
for word in text:
if word == key_word:
counter += 1
return ('{{"{0}": "{1}"}}'.format("Occurrences", counter))
def log_this(username: hug.types.text, compname: hug.types.text,stat: hug.types.text,time: hug.types.text, hug_timer=3):
data = {'username':'******'.format(username),'compname':'{0}'.format(compname),'stat':'{0}'.format(stat),'time':'{0}'.format(time)}
data2 = (username.strip(),compname,stat,time)
try:
c.execute("INSERT INTO users VALUES "+str(data2))
except sqlite3.OperationalError:
makedb()
c.execute("INSERT INTO users VALUES "+str(data2))
conn.commit()
return data
def _add_one_user(db: directives.PeeweeSession, username: hug.types.text, password: hug.types.text = None,
role: hug.types.one_of(UserRoles.user_roles()) = UserRoles.USER,
coupons: hug.types.number = 10):
with db.atomic():
name = username.lower()
salt = get_random_string(2)
secret_password = password or get_random_string(12)
hashed_password = hash_pw(name, salt, secret_password)
user = User.create(user_name=name, role=role, salt=salt,
password=hashed_password, coupons=coupons)
user.save()
return {"name": user.user_name, "password": secret_password}
def get_sim_score(seq1: hug.types.text,
seq2: hug.types.text,
method: hug.types.text = 'levenshtein',
response=None):
""" Compare Similarity between sequences
Args:
Two Sequences for which similarity score needs to be calculated,
and the method used to calculate the score. Available methods are
levenshtein (default), jaccard, jaro-winkler, hamming and sequencer-matcher
Returns:
Scaled Score between 0.0 and 1.0 with
0.0 - Sequences are not similar at all
1.0 - Sequences are completely similar (case-insensitive)
"""
logger.info("Method: %s", method)
logger.info('Seq1: %s Seq2: %s', seq1, seq2)
text = [seq1, seq2]
method = method.lower()
simscore = SimilarityMetric(text)
if method == 'levenshtein':
similarity = simscore.levenshtein()
return {'sim_score': round(similarity, 4)}
elif method == 'jaccard':
similarity = simscore.jaccard()
return {'sim_score': round(similarity, 4)}
elif method == 'jaro-winkler':
similarity = simscore.jaro_winkler()
return {'sim_score': round(similarity, 4)}
elif method == 'hamming':
similarity = simscore.hamming()
return {'sim_score': round(similarity, 4)}
elif method == 'sequencer-matcher':
similarity = simscore.sequencer_matcher()
return {'sim_score': round(similarity, 4)}
else:
response.status = falcon.HTTP_400
return {
'error':
'Unsupported method. Supported method types are Levenshtein, Jaccard, Jaro-Winkler, Hamming, Sequence-Matcher'
}
def create_user(username: hug.types.text, password: hug.types.text, response):
print(password, type(password))
session = Session()
existing_user = session.query(User).filter(
User.username.is_(username)).first()
if existing_user:
response.status = HTTP_400
return {'success': False,
'errors': ['User with this username already exists']}
password = bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt())
session.add(User(username, password))
session.commit()
response.status = HTTP_201
return {'success': True}
def registration(login: hug.types.text, password: hug.types.text):
if len(password) >= 6:
password = hashlib.sha1(password.encode()).hexdigest()
else:
raise HTTPError(HTTP_422)
token = uuid.uuid4().hex
token = Token.create(token=token)
try:
User.create(**locals())
except peewee.IntegrityError:
token.delete()
raise HTTPError(HTTP_409)
return {'status': 'ok'}
def change_user_pw(db: directives.PeeweeSession, username: hug.types.text, password: hug.types.text, for_real: hug.types.smart_boolean = False):
if not for_real:
print(
f"this would change {username}'s pw to {password}. Run with --for_real if you're sure.")
sys.exit(1)
with db.atomic():
name = username.lower()
salt = get_random_string(2)
secret_password = password
hashed_password = hash_pw(name, salt, secret_password)
user = User.get(User.user_name == username)
user.salt = salt
user.password = hashed_password
user.save()
print(f"{user.user_name}'s pw successfully changed.")
def token_verify(self, token: hug.types.text):
log.info('try to verify this token: ' + str(token))
for word in token.split(' '):
try:
validPayloadFromToken = jwt.decode(jwt=word,
key=self.__apikey__,
verify=True,
algorithms='HS256')
log.info('validPayloadFromToken: ' +
str(validPayloadFromToken))
return validPayloadFromToken
except Exception as e:
log.debug(
"This word in 'Authorization' field of the HTTP-Header is not a valid JWT token: '"
+ word + "'")
return False
def AltaUsuario(nombre: hug.types.text, apellidoPaterno: hug.types.text,
apellidoMaterno: hug.types.text, email: hug.types.text,
FeNa: hug.types.text, Password: hug.types.text,
CURP: hug.types.text, Tipo: hug.types.number, body):
global USUARIOLOG
if (Tipo == 1):
Delegacion = body.get("Delegacion")
if Delegacion == None:
return {
"status": 404,
"message": "El Usuario Gestor necesita una Delegacion asociada"
}
if nombre == "" or apellidoPaterno == "" or apellidoMaterno == "" or email == "" or FeNa == "" or Password == "" or CURP == "":
return {
"status": 404,
"message": "El Usuario necesita todos los parametro"
}
today = datetime.now()
year = today.year
yearBorn = FeNa.strip().split("-")[0]
if (int(year) - int(yearBorn)) < 16:
return {
"status": 404,
"message": "El Usuario necesita tener mas de 16"
}
statusBase = base.insertarUsuario(
nombre=nombre,
apellidoPaterno=apellidoPaterno,
apellidoMaterno=apellidoMaterno,
email=email,
FeNa=FeNa,
Password=base.hash_string(Password),
CURP=CURP,
Tipo=Tipo,
Delegacion="" if Tipo == 0 else Delegacion)
if statusBase == 201:
USUARIOLOG = email
return {
"status":
statusBase,
"message":
"Usuario registrado con Exito"
if statusBase == 201 else "Error al registrar Usuario"
}
def put_user(db: PeeweeSession, newUserName: hug.types.text, newUserPassword: hug.types.text,
newUserPasswordConfirm: hug.types.text):
if newUserPassword != newUserPasswordConfirm:
raise hug.HTTPBadRequest
with db.atomic():
try:
name = newUserName.lower()
salt = get_random_string(2)
secret_password = newUserPassword
hashed_password = hash_pw(name, salt, secret_password)
user = User.create(user_name=name, role=UserRoles.USER, salt=salt, password=hashed_password, coupons=10)
user.save()
return {
"username": user.user_name
}
except IntegrityError:
raise hug.HTTPConflict('User already exists.')
def create_api_key(username: hug.types.text, password: hug.types.text,
response):
session = Session()
user = session.query(User).filter(User.username.is_(username)).first()
if not user:
response.status = HTTP_404
return
password_is_correct = bcrypt.checkpw(password.encode('utf-8'),
user.password)
if not password_is_correct:
response.status = 401
return
api_key = ApiKey(owner_id=user.id)
session.add(api_key)
session.commit()
session.status = HTTP_201
return {'success': True, 'key': api_key.serialize()}
def build(
self,
bloomfilters: hug.types.multiple = [],
samples: hug.types.multiple = [],
from_file: hug.types.text = None,
config: hug.types.text = None,
):
config = get_config_from_file(config)
if from_file and bloomfilters:
raise ValueError(
"You can only specify blooms via from_file or bloomfilters, but not both"
)
elif from_file:
samples = []
bloomfilters = []
with open(from_file, "r") as tsvfile:
reader = csv.reader(tsvfile, delimiter="\t")
for row in reader:
bloomfilters.append(row[0])
samples.append(row[1])
if samples:
assert len(samples) == len(bloomfilters)
else:
samples = bloomfilters
if config.get("max_build_mem_bytes"):
max_memory_bytes = humanfriendly.parse_size(
config["max_build_mem_bytes"])
else:
max_memory_bytes = None
return build(
config=config,
bloomfilter_filepaths=bloomfilters,
samples=samples,
max_memory=max_memory_bytes,
)
def activities(activity_type: hug.types.text, number: hug.types.number):
"""
Gets the number of longest activities by a given type.
Args:
activity_type: string
number: integer
Returns: list of dictionaries
"""
activity_by_type = sorted([
activity for activity in client.activities
if activity.type == activity_type.capitalize()
],
key=attrgetter('distance.num'),
reverse=True)
longest_activities = []
for activity in activity_by_type[:number]:
_activities = {}
_activities[activity.name] = activity.distance.num
longest_activities.append(_activities)
return longest_activities
def sms(msisdn: hug.types.number, to: hug.types.number, text: hug.types.text,
**kwargs):
""" Looks for messages in the format <phone num> <from name> <message body>"""
for key, val in kwargs.items():
print(f"{key}: {val}")
if int(msisdn) not in allowed_nums:
print("Phone number not allowed.")
return ('', 200)
try:
to_num, from_name, message = text.split(" ", 2)
except ValueError:
raise UserError(
"The message needs to be: "
"<number> <from> <message> where the number is a UK mobile number"
"and the <from> is up to 9 letters no spaces.")
to_num = _parse_number(to_num)
from_name = _parse_from_name(from_name)
if (len(message) > 480):
raise UserError(
f"The message is too long ({len(message)}/480 characters)")
client.send_message({
'from': from_name,
'to': to_num,
'text': message,
})
client.send_message({
'from': from_name,
'to': os.environ.get("ALEX_PHONE_NUM"),
'text': message,
})
return ('', 200)
def synthesize_data(query: hug.types.text, method: hug.types.text):
if query_ok(query):
parsed = sqlparse.parse(query)[0]
order_found = False
order_clauses = []
limit_found = False
if parsed.get_type() == 'SELECT':
for t in parsed.tokens:
if(t.is_whitespace):
continue
if (t.is_keyword and t.normalized == 'ORDER'):
order_found = True
continue
if order_found:
if t.is_keyword and t.normalized != 'BY':
break
elif isinstance(t, (sqlparse.sql.Identifier, sqlparse.sql.IdentifierList)):
order_clauses.append(str(t))
for t in parsed.tokens:
if (t.is_keyword and t.normalized == 'LIMIT'):
limit_found = True
# replace order by clauses with random()
# as order by doesn't do anything once synthesis occurs
fixed_query = query
if order_found:
i = query.rfind(order_clauses[0])
fixed_query = fixed_query[:i] + "random()" + fixed_query[i + len(order_clauses[0]):]
for o in order_clauses[1:]:
i = fixed_query.rfind(o)
fixed_query = fixed_query[:i] + fixed_query[i + len(o):]
# some cleanup
fixed_query = re.sub('random\(\),', 'random()', fixed_query, flags=re.M)
fixed_query = re.sub('^\s+,', '', fixed_query, flags=re.M)
# if no order by statement present, add it
else:
if limit_found:
i = fixed_query.lower().rfind('limit')
fixed_query = fixed_query[:i] + "\norder by random()\n" + fixed_query[i:]
else:
fixed_query += '\norder by random()'
try:
if method is not None:
for m in kfpd.synthesis_methods:
if method.lower() == m.lower():
kfpd.plugin = globals()[m + 'Plugin']()
df = kfpd.read_sql(fixed_query, db_conn)
# if any order by clauses were present, re-apply them
if len(order_clauses) > 0:
sort_by = []
asc_flags = []
orig_columns = df.columns
df.columns = df.columns.str.lower()
for o in order_clauses:
sub_o = o.split(',')
# If you don’t specify the ASC or DESC keyword, SQLite uses ASC or ascending order by default.
for s in sub_o:
if s.lower().find(' desc') != -1:
asc_flags.append(False)
else:
asc_flags.append(True)
sort_by.append(re.sub('\s+asc|\s+desc', '', s, flags=re.IGNORECASE).strip().lower())
df.sort_values(sort_by, ascending=asc_flags, inplace=True)
df.columns = orig_columns
df_html = (
df.style
.hide_index()
.set_table_attributes("class='table table-hover'")
.set_uuid('_')
.render()
)
# pandas generated html has a lot of stuff we don't want returned
# chuck it!
df_html = re.sub(' id="T__row\d+_col\d+"', '', df_html)
df_html = re.sub(' class="data row\d+ col\d+" ', '', df_html)
return {
'message': 'success',
'query': '{0}'.format(query),
'executed_query': fixed_query,
'response': df_html,
'csv': df.to_csv(index=False)}
except Exception as e:
print('web-service.synthesize_data() caught exception', str(e))
return {
'message': 'error',
'query': '{0}'.format(query),
'response': str(e)}
else:
return {
'message': 'error',
'query': '{0}'.format(query),
'response': 'Invalid query provided.'}
def fixed_simplify(formula: hug.types.text):
return sp.simplify(formula.replace(' ', '+'))
def maintainers_by_category(category: hug.types.text):
"""Retrieve all maintainers by category"""
category = category.lower()
return FeedsAlchemy.db_maintainers_by_category(category)
def search(conn: directive.connection,
tables: directive.tables,
locale: directive.locale,
query: hug.types.text,
limit: hug.types.in_range(1, 100) = 20,
page: hug.types.in_range(1, 10) = 1):
""" Search a route by name. `query` contains the string to search for.
_limit_ ist the maximum number of results to return. _page_ the batch
number of results to return, i.e. the requests returns results
`[(page - 1) * limit, page * limit[`.
"""
maxresults = page * limit
res = RouteList(query=query, page=page)
r = tables.routes.data
base = sa.select(RouteItem.make_selectables(r))
# First try: exact match of ref
sql = base.where(
sa.func.lower(r.c.ref) == query.lower()).limit(maxresults + 1)
res.set_items(conn.execute(sql), locale)
# If that did not work and the search term is a number, maybe a relation
# number?
if len(res) == 0 and len(query) > 3 and query.isdigit():
sql = base.where(r.c.id == int(query))
res.set_items(conn.execute(sql), locale)
if len(res) > 0:
return res
# Second try: fuzzy matching of text
if len(res) <= maxresults:
remain = maxresults - len(res)
# Preselect matches by doing a word match on name and intnames.
primary_sim = r.c.name + sa.func.jsonb_path_query_array(
r.c.intnames, '$.*', type_=sa.Text)
primary_sim = primary_sim.op('<->>>', return_type=sa.Float)(query)
primary_sim = primary_sim.label('sim')
# Rerank by full match against main name
second_sim = r.c.name.op('<->', return_type=sa.Float)(query)
second_sim = second_sim.label('secsim')
inner = base.add_columns(primary_sim, second_sim)\
.order_by(primary_sim)\
.limit(min(1100, remain * 10))\
.alias('inner')
# Rerank by full match against main name
rematch_sim = (inner.c.sim + inner.c.secsim).label('finsim')
sql = sa.select(inner.c)\
.add_columns(rematch_sim)\
.order_by(rematch_sim)\
.limit(remain)
minsim = None
for o in conn.execute(sql):
if minsim is None:
minsim = o['finsim']
elif o['finsim'] - 0.3 > minsim:
break
res.add_item(o, locale)
if page > 1:
res.drop_leading_results((page - 1) * limit)
return res
def ip_bulk_by_category(category: hug.types.text):
"""Retrieve all IP addresses that are in feeds by feed category"""
category_lower = category.lower()
return FeedsAlchemy.db_ip_bulk_by_category(category_lower)