def mms_parse_packet(self, packet):
"""Parses the MMS packet."""
# packet.dump()
start = packet.get_bytes(4)[::-1]
cmd_length = utils.str2num(packet.get_bytes(4)[::-1])
proto_type = packet.get_bytes(4)
if start != self.MMS_COMMAND_START or proto_type != self.MMS_PROTO:
return None
packet.get_bytes(4) # skip first LEN_to_END
packet.get_bytes(4) # skip second LEN_to_END
mms_cmd_type = packet.get_bytes(2)[::-1]
mms_direction = packet.get_bytes(2)[::-1]
logger.debug("Got command %s in direction %s", repr(
mms_cmd_type), repr(mms_direction))
# Only care about connect info and request file command sent to server
if mms_direction != self.MMS_DIRECTION_TO_SERVER or \
mms_cmd_type not in [
self.MMS_COMMAND_CONNECT_INFO,
self.MMS_COMMAND_REQUEST_SERVER_FILE]:
logger.debug("Irrelative command %s in direction %s, skipping",
repr(mms_cmd_type), repr(mms_direction))
return None
if cmd_length <= 32:
return None
mms_cmd = MMSCommand()
data = ''
if mms_cmd_type == self.MMS_COMMAND_CONNECT_INFO:
mms_cmd.name = 'connect_info'
packet.get_bytes(4) # skip 4 bytes, unknown
while packet.have_bytes():
data += unichr(
utils.str2num(packet.get_bytes(2)[::-1])).encode('utf-8')
data = data.rstrip('\x00')
data = data.split('; ')
for i in data:
if 'Host: ' in i:
mms_cmd.args['host'] = i[i.find('Host: ') + 6:]
elif mms_cmd_type == self.MMS_COMMAND_REQUEST_SERVER_FILE:
mms_cmd.name = 'request_server_file'
packet.get_bytes(8) # skip 8 bytes, usually 8 zeros
while packet.have_bytes():
data += unichr(
utils.str2num(packet.get_bytes(2)[::-1])).encode('utf-8')
mms_cmd.args['server_file'] = data.rstrip('\x00')
logger.debug("Got data %s", repr(mms_cmd.args))
return mms_cmd
def onPatchStrideEditChanged(self):
stride = utils.str2num(self.__ui.edtPatchStride.text())
if stride > 0:
self.__imgpatcher.setStride(stride)
[fn, size, stride, img_shape, img_size, img_ndim] = self.__imgpatcher.info()
print('file = %s，size = %d, stride = %d' % (fn, size, stride))
print(img_shape, img_size, img_ndim)
def onEditIndexFinished(self):
# 分state处理
if self.__state == "graphics":
index = utils.str2num(self.ui.edtIndex.text())
if (index != False) and (index < self.__total_image) and (index >=
0):
self.__current_image = index
self.loadGraphicsThenShow(self.__current_image)
elif self.__state == "animation":
[ani_str, act_str] = self.ui.edtIndex.text().split(
' ', 1) # FIXME: 这里并未对动画编号和动作编号的有效性进行验证，会造成后续操作异常终止
self.__current_animation = utils.str2num(ani_str)
self.__current_action = utils.str2num(act_str)
self.loadAnimationThenShow(self.__current_animation,
self.__current_action)
else:
pass
def main():
result = np.load(os.path.join(result_dir, 'result.npy'))
pig_list = np.load(os.path.join(test_name_dir, 'test_A_name_resnet50.npy'))
row_result = 0 # the row of result handled now
all_result = [] # save all the handled date
for pig in result:
for predict in range(30):
tmp_array = [pig_list[row_result][0], predict + 1, pig[predict]]
tmp_array = u.str2num(tmp_array)
all_result.append(tmp_array)
row_result += 1
output_dir = os.path.join(result_dir, 'tidy_out.csv')
u.write_csv(all_result, output_dir)
print(all_result)
def go():
"""Display prediction of lenders"""
if request.method == "POST":
# save user input
cityEntry = request.form["cityEntry"]
stateDropdown = request.form["stateDropdown"]
mfDropdown = request.form.getlist("mfDropdown")
yearBuilt = request.form["yearBuilt"]
numberUnits = request.form["numberUnits"]
originalLoan = request.form["originalLoan"]
noteRate = request.form["noteRate"]
loanTerm = request.form["loanTerm"]
appraisedValue = request.form["appraisedValue"]
# retrieve lat and lng based on city and state
address = f"{cityEntry}, {stateDropdown}"
api_results = address2api(address)
lat, lng = api2latlng(api_results)
# convert user input to dataframe
user_input_df = pd.DataFrame(columns=features)
user_input_df['Built'] = [str2num(yearBuilt)]
user_input_df['Units'] = [str2num(numberUnits)]
user_input_df['Original Loan'] = [str2num(originalLoan)]
user_input_df['Note Rate'] = [str2num(noteRate)]
user_input_df['Loan Term (Original)'] = [str2num(loanTerm)]
user_input_df['Appraised Value'] = [str2num(appraisedValue)]
user_input_df['lat'] = [lat]
user_input_df['lng'] = [lng]
for mf_col in [col for col in features if "MF" in col]:
if mf_col.lstrip("MF_") in mfDropdown:
user_input_df[mf_col] = [1]
else:
user_input_df[mf_col] = [0]
# in case of missing values, impute by median
user_input_df[columns_to_impute] = missing_imputer.transform(
user_input_df[columns_to_impute])
# apply feature scaling (excluding binary features)
to_scale = [col for col in features if ('MF' not in col)]
user_input_df[to_scale] = scaler.transform(user_input_df[to_scale])
# use model to predict top 3 most likely lenders
probabilities = model.predict_proba(user_input_df.iloc[0:1])[0]
classes = model.classes_
top3 = pd.Series(data=probabilities,
index=classes).sort_values(ascending=False).iloc[:3]
top3_lenders = top3.index.tolist()
top3_prob = top3.values.tolist()
# import pdb; pdb.set_trace()
# render the go.html
return render_template('go.html',
top3_lenders=top3_lenders,
top3_prob=top3_prob)
return render_template("master.html")
def scrap_tiktok(mydb, mycursor, options, query):
print('Scrapping: ' + query)
driver = webdriver.Chrome('./chromedriver', options=options)
#Acceder a perfil de TikTok
driver.get(url_base + query)
sleep(0.5)
#Captcha check
try:
#El captcha es mover una pieza de rompecabezas a cierta posición, no pude completarlo desde selenium
captcha = driver.find_element(By.CLASS_NAME, captcha_check)
print('ERROR: Captcha encontrado!!')
return
except:
print('No hay captcha ✓✓')
#Verificar si el usuario existe
try:
usuario_non = driver.find_element(By.CLASS_NAME, usuario_check)
print('ERROR: El usuario NO existe!!')
return
except:
print('Usuario existe ✓✓')
#Check version de TikTok [Feed o Cuadrícula]
try:
#Si está en cuadrícula, se puede leer el numero de vistas de los videos
flagTiktokView = driver.find_element(By.CLASS_NAME, clase_video_count)
print('Se encuentra en modo cuadrícula')
except:
#Si esta en feed, se pone en modo cuadrícula
driver.find_element(By.XPATH, xpath_cuad).click()
print('Cambiado a modo cuadrícula')
sleep(1)
#Se obtiene el momento en el que se obtuvieron los datos
fechahoy = datetime.today()
diahoy = date.today()
#Atributos del perfil
img_url = driver.find_element(By.XPATH, img_xpath).get_attribute('src')
descripcion = driver.find_element(By.CLASS_NAME, desc_cuenta)
id_cuenta = driver.find_element(By.CLASS_NAME, clase_id_cuenta)
nombre_cuenta = driver.find_element(By.CLASS_NAME, clase_nombre_cuenta)
numeros = driver.find_elements(By.CLASS_NAME, clase_metrica_numeros)
links = driver.find_element(By.CLASS_NAME, clase_link_cuenta)
#Si la cuenta no comparte links, se deja una cadena vacía
links_cuenta = ""
if (links.text):
links_cuenta = links.text
#Cantidad de: Siguiendo, Seguidores y Me Gusta
metricas = []
for numero in numeros:
datos = numero.text.split("\n")
metricas.append(str2num(datos[0]))
#Quitar mensaje en esquina inferior derecha que tapa videos
try:
driver.find_element(By.XPATH, xpath_cerrar_overlay).click()
#print("Overlay de 'Obtener TikTok' cerrado ✓✓")
except:
print("No hay overlay de 'Obtener TikTok' sobre la página ✓✓")
#Verificar si el perfil ya existe en la base de datos
queryIF = "SELECT idPerfil FROM perfiles WHERE username='******';"
mycursor.execute(queryIF)
perfilExistente = mycursor.fetchall()
#Si no se encuentra, se inserta
if not perfilExistente:
sqlPerfil = (
"INSERT INTO perfiles "
"(username, url, nombre_cuenta, descripcion, links) "
"VALUES (%(username)s, %(url)s, %(nombre_cuenta)s, %(descripcion)s, %(links)s)"
)
perfil_dict = {
'username': query,
'url': url_base + id_cuenta.text,
'nombre_cuenta': nombre_cuenta.text,
'descripcion': descripcion.text,
'links': links_cuenta
}
mycursor.execute(sqlPerfil, perfil_dict)
mydb.commit()
id_perfil = mycursor.lastrowid
#Si existe, se obtiene el idPerfil para ser usado mas adelante
else:
id_perfil = perfilExistente[0][0]
#Se insertan los datos que cambian del perfil
sqlPerfilD = (
"INSERT INTO data_perfil "
"(idPerfil, date, dia, siguiendo, seguidores, me_gusta, videos, img_url, ready, loopVideo) "
"VALUES (%(idPerfil)s, %(date)s, %(dia)s, %(siguiendo)s, %(seguidores)s, %(me_gusta)s, %(videos)s, %(img_url)s, %(ready)s, %(loopVideo)s)"
)
perfilD_dict = {
'idPerfil': id_perfil,
'date': fechahoy,
'dia': diahoy,
'siguiendo': metricas[0],
'seguidores': metricas[1],
'me_gusta': metricas[2],
'videos': 0, #len(videos),
'img_url': img_url,
'ready': 1,
'loopVideo': 0
}
mycursor.execute(sqlPerfilD, perfilD_dict)
mydb.commit()
#Scroll infinito para obtener todos los videos de la cuenta
loops = 50 #Puede cambiar la cantidad de loops realizados aqui
for i in range(loops):
scroll_page(driver)
print('Scroll N°: ' + str(i + 1) + '/' + str(loops))
sleep(1)
#Obtener todos los elementos de video que existen en el perfil de TikTok
videos = driver.find_elements(By.CLASS_NAME, clase_videos)
print('N° Videos: ' + str(len(videos)))
#Update status a Preparandose
mycursor.execute('UPDATE data_perfil SET ready=2, videos=' +
str(len(videos)) + ' WHERE idPerfil=' + str(id_perfil) +
' ORDER BY idData DESC LIMIT 1')
mydb.commit()
#Se obtienen los links y la cantidad de vistas para todos los videos en la cuenta
videosPerf = []
for video in videos:
driver.execute_script("arguments[0].scrollIntoView();", video)
urlVideo = video.get_attribute('href')
video_count = video.find_element(By.CLASS_NAME, clase_video_count).text
objVideo = {'url': urlVideo, 'vistas': video_count}
videosPerf.append(objVideo)
#Se prepara el loop para obtener los datos de cada video en la cuenta
count = 1
#Scrapping a los videos
try:
for video in videosPerf:
print(str(count) + ' - get: ' + video['url'])
driver.get(video['url'])
try:
video_musica = driver.find_element(By.XPATH,
xpath_video_music).text
except:
video_musica = "None"
try:
video_desc_meta = driver.find_element(By.XPATH,
xpath_video_desc).text
video_mg = str2num(
driver.find_element(By.XPATH, xpath_video_mg).text)
video_com = str2num(
driver.find_element(By.XPATH, xpath_video_com).text)
except:
# El link del video es correcto, pero la página carga como si el video no existiera
print('^ LINK ERROR ^')
video_musica = 'LINK ERROR'
video_desc_meta = 'LINK ERROR'
video_mg = 0
video_com = 0
video_link = video['url']
video_count = str2num(video['vistas'])
#Con el id de la url del video, se puede obtener la fecha en la que se publicó el video
video_fecha = id2date(video_link.split('/')[-1])
#Engagement
if metricas[1] == 0 or isinstance(video_mg, str):
engagement = 0
else:
engagement = ((video_mg + video_com) / metricas[1]) * 100
if engagement > 100:
engagement = 100
sqlVideo = (
"INSERT INTO videos "
"(date, dia, username, idPerfil, video_n, url, vistas, descripcion, fecha, musica, me_gusta, comentarios, engagement) "
"VALUES (%(date)s, %(dia)s, %(username)s, %(idPerfil)s, %(video_n)s, %(url)s, %(vistas)s, %(descripcion)s, %(fecha)s, %(musica)s, %(me_gusta)s, %(comentarios)s, %(engagement)s)"
)
video_dict = {
'date': fechahoy,
'dia': diahoy,
'username': query,
'idPerfil': id_perfil,
'video_n': count,
'url': video_link,
'vistas': video_count,
'descripcion': video_desc_meta,
'fecha': video_fecha,
'musica': video_musica,
'me_gusta': video_mg,
'comentarios': video_com,
'engagement': engagement
}
mycursor.execute(sqlVideo,
video_dict) #Update status a Preparandose
mycursor.execute('UPDATE data_perfil SET loopVideo=' + str(count) +
' WHERE idPerfil=' + str(id_perfil) +
' ORDER BY idData DESC LIMIT 1')
mydb.commit()
count += 1
except Exception as e:
########################
# UPDATE READY A ERROR #
########################
mycursor.execute('UPDATE data_perfil SET ready=3 WHERE idPerfil=' +
str(id_perfil) + ' ORDER BY idData DESC LIMIT 1')
mydb.commit()
print(e)
print('Error: ' + query)
return
########################
# UPDATE READY A LISTO #
########################
mycursor.execute('UPDATE data_perfil SET ready=0 WHERE idPerfil=' +
str(id_perfil) + ' ORDER BY idData DESC LIMIT 1')
mydb.commit()
driver.quit()
print('Listo: ' + query)