def _checkOutput(process, websocketConnection):
"""
Checks the output of stdout and stderr to send it to the WebSocket client
"""
while process.poll() is None: # while the process isn't exited
try:
output = process.stdout.read(
) # Read the stdout PIPE (which contains stdout and stderr)
except:
output = None
if output:
websocketConnection.send(
json.dumps({
"message": output.decode("utf-8"),
"code": 0
})) # Send the new output
time.sleep(0.1) # Wait a lil bit to avoid confusion from the computer
### EXITED THE LOOP ### (process exited)
try:
websocketConnection.send(
json.dumps({
"message":
f"ErinaConsole: The process has exited with code {str(process.returncode)}",
"code": process.returncode
})) # Send a disconnection message
except:
log("ErinaAdmin", "Unable to send an exit message to ErinaConsole")
async def on_ready():
'''
When the bot is ready
'''
await client.change_presence(activity=discord.Game(
name='.erina help | Ready to give you the sauce!')) # GAME ACTIVITY
log("ErinaDiscord", "Erina is ready")
def verify_manami_adb(force=False):
"""
Checks for a new version of the database on GitHub
"""
## Checking if new week
current_release_week = currentReleaseFile.read().replace(" ", '').replace("\n", '')
iso_calendar = datetime.date.today().isocalendar()
current_week = str(iso_calendar[0]) + '-' + str(iso_calendar[1])
if current_release_week != current_week or force: # If new week
log("ErinaDB", "Updating Manami Title Vector DB...")
manami_adb = json.loads(requests.get('https://raw.githubusercontent.com/manami-project/anime-offline-database/master/anime-offline-database.json').text)
data = {}
for anime in manami_adb["data"]:
link = None
anilist_id = None
for sourceLink in anime["sources"]:
if sourceLink.find("anilist.co") != -1:
link = sourceLink
if link is not None: # parser the data
anilist_id = convert_to_int(link[link.rfind("/"):])
data[anilist_id] = [ str(anime["title"]).lower().replace(" ", '') ]
data[anilist_id].extend([str(title).lower().replace(" ", '') for title in anime["synonyms"]])
else:
continue
# write out the data
JSONFile(manami_database_path + 'manami_database_data.json', separators=(',', ':'), indent=None).write(data)
currentReleaseFile.write(current_week)
Database.updateData(data)
def tweet(self, message, replyID=None, imageURL=None):
"""
Tweets something
"""
twitterImage = None
if imageURL is not None:
image = BytesIO(requests.get(str(imageURL)).content)
twitterImage = self.api.media_upload(
file=image, filename="ErinaSauce — trace.moe Image Preview")
if replyID is not None:
log("ErinaTwitter", "Replying to " + str(replyID))
if twitterImage is not None:
return self.api.update_status(
status=str(message)[:280],
in_reply_to_status_id=replyID,
auto_populate_reply_metadata=True,
media_ids=[twitterImage.media_id])
else:
return self.api.update_status(
status=str(message)[:280],
in_reply_to_status_id=replyID,
auto_populate_reply_metadata=True)
else:
log("ErinaTwitter", "Sending a tweet...")
if twitterImage is not None:
return self.api.update_status(
status=str(message)[:280],
media_ids=[twitterImage.media_id])
else:
return self.api.update_status(status=str(message)[:280])
def anilist_search_caching(query):
'''
Caches the first search result from the given query (from AniList API)\n
Returns the new cache's filename\n
Project Erina
© Anime no Sekai - 2020
'''
try:
log("ErinaCaches", f'Caching {str(query)} from AniList Search API...')
try:
apiResponse = anilist.anilist_api_search(query)
except:
return CachingError("ANILIST_SEARCH_API_RESPONSE", f"An error occured while retrieving AniList Search API Data ({str(query)})")
if "errors" in apiResponse:
if apiResponse["errors"][0]["status"] == 404:
return CachingError("ANILIST_NOT_FOUND", str(query) + " has not been found")
else:
return CachingError("ANILIST_SERVER_ERROR", f"An error occured with the AniList API: {apiResponse['errors'][0]['message']}")
try:
cache = anilist.anilist_json_to_cache(apiResponse)
except:
return CachingError("ERINA_CONVERSION", f"An error occured while converting AniList's Search API Data to a caching format ({str(query)})")
try:
TextFile(anilist_cache_path + cache['filename'], blocking=False).write(cache["content"])
except:
return CachingError("FILE_WRITE", f"An error occured while writing out the cache data to a file ({str(query)})")
return anilist_parser.AnilistCache(cache["content"])
except:
return CachingError("UNKNOWN_ERROR", f"An unknown error occured while caching AniList Search API Data ({str(query)})")
def __init__(self, type, message) -> None:
self.type = str(type)
self.message = str(message)
self.timestamp = time()
self.datetime = datetime.fromtimestamp(self.timestamp)
self.formatted_timestamp = f"{str(self.datetime.year)}-{str(self.datetime.month)}-{str(self.datetime.day)} at {str(self.datetime.hour)}:{str(self.datetime.minute)}:{str(self.datetime.second)}"
log("ErinaLine", self.type + ": " + self.message, error=True)
StatsAppend(erina.errorsCount, "ErinaLine")
def searchAnime(query):
"""
Searches an anime by its title
Erina Project — 2020\n
© Anime no Sekai
"""
log("ErinaSearch", "Searching for " + str(query) + "... (title search)")
StatsAppend(SearchStats.titleSearchCount, query)
return title_search.searchAnime(query)
def runServer():
global ErinaWSGIServer
## RUNNING ErinaServer
log("Erina", "Running ErinaServer...")
wsgiEnv = {
'SERVER_SOFTWARE': 'ErinaServer ' + erina_version,
'wsgi.multithread': True,
'wsgi.run_once': False
}
ErinaWSGIServer = pywsgi.WSGIServer((ServerConfig.host, ServerConfig.port), ErinaServer, handler_class=WebSocketHandler, environ=wsgiEnv)
ErinaWSGIServer.serve_forever()
def anilistIDSearch(anilistID):
"""
Searches an anime from AniList Caches or AniList API
Erina Project — 2020\n
© Anime no Sekai
"""
log("ErinaSearch",
"Searching for " + str(anilistID) + "... (anilist id search)")
StatsAppend(SearchStats.anilistIDSearchCount, anilistID)
return anilist_id_search.search_anime_by_anilist_id(anilistID)
def imageSearch(image):
"""
Searches an anime from an image (anime scene for example)
image: Can be an URL, a path, a base64 encoded string or a PIL.Image.Image instance
Erina Project — 2020\n
© Anime no Sekai
"""
log("ErinaSearch", "Searching for an image...")
StatsAppend(SearchStats.imageSearchCount, None)
return hash_search.search_anime_by_hash(erinahash.hash_image(image))
def base64_from_image(image_path):
"""
Converts an image to base64
Erina Project — 2020\n
© Anime no Sekai
"""
log("ErinaHash", "Converting " + str(image_path) + " to base64...")
image_content = BinaryFile(image_path).read()
result = base64.b64encode(image_content).decode("utf-8")
StatsAppend(ErinaHashStats.createdBase64String,
f"New Base64 String (length: {str(len(result))})")
return result
def shutdownErinaServer(num, info):
"""
SIGTERM, SIGQUIT, SIGINT signals handler --> Shutdowns Erina
"""
try:
for handler in psutil.Process(os.getpid()).open_files():
os.close(handler.fd)
except:
pass
TextFile(erina_dir + "/launch.erina").write("0")
ErinaWSGIServer.stop()
ErinaWSGIServer.close()
logFile.blocking = True
log("Erina", "Goodbye!")
def checkImages():
'''
Timeout checking function.
'''
global current_images_dict
number_of_deleted_files = 0 # logging purposes
for entry in current_images_dict:
if time.time() - current_images_dict[entry] > LineConfig.images_timeout:
if filecenter.delete(images_path + entry + '.erina_image') == 0:
current_images_dict.pop(entry, None)
number_of_deleted_files += 1 # logging purposes
### LOGGING
if number_of_deleted_files > 0:
if number_of_deleted_files == 1:
log("ErinaLine", "[Image Checker] Deleted 1 entry")
else:
log("ErinaLine", f'[Image Checker] Deleted {str(number_of_deleted_files)} entries')
StatsAppend(LineStats.storedImages, len(filecenter.files_in_dir(images_path)))
def erina_caching(image_hash, database_path, similarity, anilist_id):
'''
Caches an ErinaDatabase path according to the image_hash\n
Project Erina
© Anime no Sekai - 2020
'''
try:
log("ErinaCaches", f'Caching {str(image_hash)} Erina Database data...')
try:
cache = erina.erina_from_data(str(image_hash), database_path, similarity, anilist_id)
except:
return CachingError("ERINA_CONVERSION", f"An error occured while converting Erina Database Data to a caching format ({str(database_path)})")
try:
TextFile(erina_cache_path + str(image_hash) + '.erina', blocking=False).write(cache)
except:
return CachingError("FILE_WRITE", f"An error occured while writing out the cache data to a file {str(database_path)}")
return erina_parser.ErinaCache(cache)
except:
return CachingError("UNKNOWN", "An unknown error occured while caching Erina Database Data")
def dm(self, message, recipientID, imageURL=None):
"""
DMs someone
"""
twitterImage = None
if imageURL is not None:
image = BytesIO(requests.get(str(imageURL)).content)
filename = imageURL[imageURL.rfind("/"):]
twitterImage = self.api.media_upload(filename=filename, file=image)
log("ErinaTwitter", "Sending a direct message to " + str(recipientID))
if twitterImage is not None:
return self.api.send_direct_message(
recipientID,
str(message)[:10000],
attachment_media_id=twitterImage.media_id)
else:
return self.api.send_direct_message(recipientID,
str(message)[:10000])
def restartErinaServer(num, info):
"""
SIGUSR1 signal handler --> Restarts Erina
"""
try:
for handler in psutil.Process(os.getpid()).open_files():
os.close(handler.fd)
except:
pass
from ErinaTwitter.utils.Stream import lastDM
from ErinaTwitter.utils.Stream import sinceID
TextFile(erina_dir + "/ErinaTwitter/lastDM.erina").write(str(lastDM))
TextFile(erina_dir + "/ErinaTwitter/lastStatusID.erina").write(str(sinceID))
TextFile(erina_dir + "/launch.erina").write("0")
ErinaWSGIServer.stop()
ErinaWSGIServer.close()
logFile.blocking = True
log("Erina", "Restarting...")
os.execl(sys.executable, sys.executable, __file__, *sys.argv[1:])
def tracemoe_caching(image_hash):
'''
Caches the given Trace.moe API response\n
Project Erina
© Anime no Sekai - 2020
'''
try:
log("ErinaCaches", f'Caching {str(image_hash)} trace.moe data...')
try:
if image_hash.has_url is not None:
if str(config.Caches.keys.tracemoe).replace(" ", "") in ["None", ""]:
requestResponse = json.loads(requests.get('https://trace.moe/api/search?url=' + image_hash.url).text)
else:
requestResponse = json.loads(requests.get('https://trace.moe/api/search?url=' + image_hash.url + '&token=' + str(config.Caches.keys.tracemoe)).text)
else:
if str(config.Caches.keys.tracemoe).replace(" ", "") in ["None", ""]:
requestResponse = json.loads(requests.post('https://trace.moe/api/search', json={'image': image_hash.base64}))
else:
requestResponse = json.loads(requests.post('https://trace.moe/api/search?token=' + str(config.Caches.keys.tracemoe), json={'image': image_hash.base64}))
except:
return CachingError("TRACEMOE_API_RESPONSE", "An error occured while retrieving information from the trace.moe API")
StatsAppend(ExternalStats.tracemoeAPICalls)
try:
cache = tracemoe.erina_from_json(requestResponse)
except:
print(exc_info()[0])
print(exc_info()[1])
print(traceback.print_exception(*exc_info()))
return CachingError("ERINA_CONVERSION", f"An error occured while converting trace.moe API Data to a caching format ({str(image_hash)})")
try:
TextFile(tracemoe_cache_path + str(image_hash) + '.erina', blocking=False).write(cache)
except:
return CachingError("FILE_WRITE", f"An error occured while writing out the cache data to a file ({str(image_hash)})")
return tracemoe_parser.TraceMOECache(cache)
except:
return CachingError("UNKNOWN_ERROR", f"An unknown error occured while caching trace.moe API Data ({str(image_hash)})")
def on_direct_message(message):
"""
DM Receiving
"""
directMessagesHistory.append(message)
log("ErinaTwitter",
"New direct message from @" + str(message.message_create['sender_id']))
if Twitter.dmAskingForSauce(message):
StatsAppend(TwitterStats.directMessagingHit,
str(message.message_create['sender_id']))
image = Twitter.getDirectMedia(message)
if image is not None:
searchResult = imageSearch(image)
ErinaTwitter.dm(makeImageResponse(searchResult),
message.message_create['sender_id'])
elif isAnError(image):
ErinaTwitter.dm(
"An error occured while retrieving information on the anime",
message.message_create['sender_id'])
else:
ErinaTwitter.dm(
"You did not send any image along with your message",
message.message_create['sender_id'])
def saucenao_caching(image_hash):
'''
Caches the result from the given url\n
Project Erina\n
© Anime no Sekai - 2020
'''
try:
log("ErinaCaches", f"Caching {str(image_hash)} SauceNAO data...")
if str(config.Caches.keys.saucenao).replace(" ", "") not in ["None", ""]:
saucenao_api = SauceNao(api_key=config.Caches.keys.saucenao, numres=1)
else:
saucenao_api = SauceNao(numres=1)
if image_hash.has_url:
try:
api_results = saucenao_api.from_url(image_hash.url)[0]
except:
return CachingError("SAUCENAO_API_RESPONSE", "An error occured while retrieving SauceNAO API Data")
else:
try:
api_results = saucenao_api.from_file(image_hash.ImageIO)[0]
except:
return CachingError("SAUCENAO_API_RESPONSE", "An error occured while retrieving SauceNAO API Data")
StatsAppend(ExternalStats.saucenaoAPICalls)
try:
cache = saucenao.erina_from_api(api_results)
except:
traceback.print_exc()
return CachingError("ERINA_CONVERSION", "An error occured while converting SauceNAO API Data to a caching format")
try:
TextFile(saucenao_cache_path + str(image_hash) + '.erina', blocking=False).write(cache)
except:
return CachingError("FILE_WRITE", "An error occured while writing out the cache data to a file")
return saucenao_parser.SauceNAOCache(cache)
except:
return CachingError("UNKNOWN", "An unknown error occured while caching SauceNAO API Data")
def runClients():
import asyncio
from Erina import config
import ErinaLine.erina_linebot # Already runs the LINE Client
log("Erina", "Starting to check for timeouts and updates...")
from Erina import setInterval # Already runs the checkers
if config.Twitter.run:
log("Erina", "Running the ErinaTwitter Client...")
from ErinaTwitter.utils import Stream as twitterClient
Thread(target=twitterClient.startStream, daemon=True).start()
if config.Discord.run:
log("Erina", "Running the ErinaDiscord Client...")
from ErinaDiscord.erina_discordbot import startDiscord
startDiscord()
if config.Line.run:
initHandler()
if __name__ == '__main__' or TextFile(erina_dir + "/launch.erina").read().replace(" ", "") in ["", "0"]:
TextFile(erina_dir + "/launch.erina").read() == "1"
#### INITIALIZING ERINASERVER --> Manages the whole server
print("[Erina]", "Initializing ErinaServer")
import requests
import requests.packages.urllib3.contrib.pyopenssl as requestsPyOpenSSL
requestsPyOpenSSL.extract_from_urllib3()
from ErinaServer.Server import ErinaServer
from ErinaServer import WebSockets
from gevent import pywsgi
from geventwebsocket.handler import WebSocketHandler
from Erina.erina_log import log, logFile
log("Erina", "Initializing Erina configuration...")
from Erina.config import Server as ServerConfig
from Erina.env_information import erina_version
## RECORDING Endpoints
log("Erina", "---> Initializing Static File Endpoints")
from ErinaServer.Erina import Static
log("Erina", "---> Initializing API")
from ErinaServer.Erina.api import API
log("Erina", "---> Initializing ErinaAdmin")
from ErinaServer.Erina.admin import Admin
from ErinaServer.Erina.auth import Auth
from ErinaServer.Erina.admin import Config
log("Erina", "---> Initializing Custom Endpoints")
from ErinaServer import Custom
log("Erina", "---> Initializing ErinaConsole")
async def on_message(message):
'''
When the bot receives a message
'''
#await message.add_reaction(roger_reaction) # REACT TO SHOW THAT THE BOT HAS UNDESTAND HIS COMMAND
if message.author.id == client.user.id:
return
if message.content.startswith('.erina'): # ERINA SPECIFIC COMMAND
userCommand = utils.removeSpaceBefore(str(message.content)[6:])
commandLength = len(userCommand.split(" ")[0])
command, commandSimilarity = searchCommand(
userCommand.split(" ")[0].lower())
if commandSimilarity < 0.75:
await message.channel.send("Sorry, this command is not available.")
return
else:
if command == "search":
query = utils.removeSpaceBefore(userCommand[commandLength:])
log(
"ErinaDiscord", "New info hit from @" +
str(message.author) + " (asking for " + str(query) + ")")
StatsAppend(DiscordStats.infoHit,
f"{str(query)} >>> {str(message.author)}")
anime, thumbnail, discordResponse = Parser.makeInfoResponse(
erinasearch.searchAnime(query))
if discordResponse is not None:
newEmbed = discord.Embed(title='Anime Info',
colour=discord.Colour.blue())
newEmbed.add_field(name=anime.capitalize(),
value=discordResponse)
if thumbnail is not None:
newEmbed.set_thumbnail(url=thumbnail)
await message.channel.send(embed=newEmbed)
else:
await message.channel.send(
"An error occured while searching for your anime: " +
query)
elif command == "description":
query = utils.removeSpaceBefore(userCommand[commandLength:])
log(
"ErinaDiscord", "New description hit from @" +
str(message.author) + " (asking for " + str(query) + ")")
StatsAppend(DiscordStats.descriptionHit,
f"{str(query)} >>> {str(message.author)}")
anime, thumbnail, discordResponse = Parser.makeDescriptionResponse(
erinasearch.searchAnime(query))
if discordResponse is not None:
newEmbed = discord.Embed(
title=f'Anime Description: {str(anime)}',
colour=discord.Colour.blue())
newEmbed.add_field(name=anime.capitalize(),
value=discordResponse)
if thumbnail is not None:
newEmbed.set_thumbnail(url=thumbnail)
await message.channel.send(embed=newEmbed)
else:
await message.channel.send(
"An error occured while searching for your anime: " +
query)
elif command == "dev":
await StaticResponse.erinadev(message.channel)
elif command == "donate":
await StaticResponse.erinadonate(message.channel)
elif command == "help":
await StaticResponse.erinahelp(message.channel, message.author)
elif command == "stats":
await StaticResponse.erinastats(message.channel, client)
elif command == "invite":
await StaticResponse.erinainvite(message.channel)
else:
if any([
flag in str(message.content).lower()
for flag in (config.Discord.flags if str(config.Discord.flags).
replace(" ", "") not in
["None", "", "[]"] else config.Erina.flags)
]):
listOfResults = []
#await message.add_reaction(roger_reaction) # REACT TO SHOW THAT THE BOT HAS UNDESTAND HIS COMMAND
log("ErinaDiscord",
"New image search from @" + str(message.author))
StatsAppend(DiscordStats.imageSearchHit, f"{str(message.author)}")
for file in message.attachments:
if filecenter.type_from_extension(
filecenter.extension_from_base(file.filename)
) == 'Image': # If the file is an image
current_anime = Parser.makeImageResponse(
erinasearch.imageSearch(
file.url)) # Get infos about the anime
listOfResults.append(
current_anime) # Append to the results list
else:
message_history = await message.channel.history(
limit=5
).flatten() # Search from the last 3 messages a picture
for message in message_history:
for file in message.attachments:
if filecenter.type_from_extension(
filecenter.extension_from_base(file.filename)
) == 'Image': # If the file is an image
current_anime = Parser.makeImageResponse(
erinasearch.imageSearch(
file.url)) # Get infos about the anime
listOfResults.append(
current_anime) # Append to the results list
if len(listOfResults) == 0:
await message.channel.send("Sorry, I couldn't find anything..."
)
elif len(listOfResults) == 1:
title, thumbnail, reply = listOfResults[0]
if reply is None:
await message.channel.send(
"An error occured while retrieving information on the anime..."
)
await message.channel.send(f"It seems to be {title}!")
newEmbed = discord.Embed(title='Anime Info',
colour=discord.Colour.blue())
newEmbed.add_field(name=title.capitalize(), value=reply)
if thumbnail is not None:
newEmbed.set_thumbnail(url=thumbnail)
await message.channel.send(embed=newEmbed)
await asyncio.sleep(1)
else:
for iteration, result in enumerate(listOfResults):
number = ''
if iteration == 0:
number = '1st'
elif iteration == 1:
number = '2nd'
elif iteration == 2:
number = '3rd'
else:
number = f'{str(iteration)}th'
title, thumbnail, reply = result
if reply is None:
await message.channel.send(
f"An error occured while searching for the {number} anime"
)
await message.channel.send(
f"The {number} anime seems to be {title}!")
newEmbed = discord.Embed(title='Anime Info',
colour=discord.Colour.blue())
newEmbed.add_field(name=title.capitalize(), value=reply)
if thumbnail is not None:
newEmbed.set_thumbnail(url=thumbnail)
await message.channel.send(embed=newEmbed)
await asyncio.sleep(1)
return
def ErinaServer_Endpoint_Admin_Console_ErinaConsole(ws):
currentProcess = None
message = ws.receive()
try:
data = json.loads(message) # retrieve a message from the client
if "token" in data:
tokenVerification = authManagement.verifyToken(data)
if tokenVerification.success:
log("ErinaAdmin", "> New ErinaConsole connection!")
bash = False
if os.path.isfile("/bin/bash"):
bash = True
currentProcess = subprocess.Popen(
shlex.split("/bin/bash"),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) # Open a bash prompt
else:
currentProcess = subprocess.Popen(
shlex.split("/bin/sh"),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) # Open a shell prompt
fcntl.fcntl(
currentProcess.stdout.fileno(), fcntl.F_SETFL,
os.O_NONBLOCK) # Non blocking stdout and stderr reading
threading.Thread(
target=_checkOutput,
args=[currentProcess, ws],
daemon=True).start(
) # Start checking for new text in stdout and stderr
if bash:
ws.send(
json.dumps({
"message":
f"ErinaConsole: Connection successfully established (bash) with PID: {str(currentProcess.pid)}",
"code": 0
})
) # Send a message to notifiy that the process has started
else:
ws.send(
json.dumps({
"message":
f"ErinaConsole: Connection successfully established (sh) with PID: {str(currentProcess.pid)}",
"code": 0
})
) # Send a message to notifiy that the process has started
else:
ws.send(
json.dumps({
"message":
f"ErinaConsole: Authentification Error: {str(tokenVerification)}",
"code": 400
}))
else:
ws.send(
json.dumps({
"message":
f"ErinaConsole: Your connection could not be authentificated",
"code": 400
}))
except:
try:
ws.send(
json.dumps({
"message":
f"ErinaConsole: An error occured ({str(sys.exc_info()[0])}) while creating a new ErinaConsole instance",
"code": -1
}))
except:
log("ErinaAdmin", "ErinaConsole >>> Failed to send a message")
try:
while not ws.closed:
try:
message = ws.receive()
try:
data = json.loads(
message) # retrieve a message from the client
tokenVerification = authManagement.verifyToken(data)
if tokenVerification.success:
if "input" in data:
userInput = str(data["input"])
#if userInput != "exit":
currentProcess.stdin.write(
str(userInput + "\n").encode("utf-8")
) # Write user input (ws client) to stdin
currentProcess.stdin.flush() # Run the command
log(
"ErinaAdmin",
"ErinaConsole >> Admin ran command > " +
str(userInput))
"""
else:
currentProcess.terminate() # If "exit" sent by client, terminate the process
"""
else:
ws.send(
json.dumps({
"message":
f"ErinaConsole: Authentification Error: {str(tokenVerification)}",
"code": 400
}))
except:
try:
ws.send(
json.dumps({
"message":
f"ErinaConsole: An error occured ({str(sys.exc_info()[0])})",
"code": -1
}))
except:
log("ErinaAdmin",
"ErinaConsole >>> Failed to send a message")
except:
log("ErinaAdmin", "< ErinaConsole disconnected")
except:
log("ErinaAdmin", "< ErinaConsole disconnected")
if currentProcess is not None:
if currentProcess.poll() is None:
currentProcess.terminate()
log("ErinaAdmin", "< ErinaConsole disconnected")
def hash_image(image, algorithm=None):
"""
Hashes a given image
image: Can be an URL, a path, a base64 encoded string or a PIL.Image.Image instance
Erina Project — 2020\n
© Anime no Sekai
"""
result = None
has_url = False
url = None
log("ErinaHash", "Hashing an image...")
# Needs to be a PIL instance
if isfile(str(image)):
image = Image.open(image)
elif isinstance(image, Image.Image):
image = image
else:
try:
if base64.b64decode(str(image), validate=True):
image = Image.open(BytesIO(base64.b64decode(str(image))))
else:
raise ValueError("b64decode returned an empty string")
except:
try:
url = image
image = Image.open(
BytesIO(requests.get(str(image)).content)
) # Open the downloaded image as a PIL Image instance
has_url = True
except:
return HashingError(
"INVALID_IMAGE_TYPE",
"We couldn't convert the given image to a PIL.Image.Image instance"
)
if algorithm is None:
algorithm = str(config.Hash.algorithm)
algorithm = str(algorithm).lower().replace(" ", "")
if algorithm in ['ahash', 'a', 'averagehash', 'average']:
result = imagehash.average_hash(image)
elif algorithm in ['chash', 'c']:
result = imagehash.colorhash(image)
elif algorithm in ['dhash', 'd']:
result = imagehash.dhash(image)
elif algorithm in ['phash', 'p', 'perceptual', 'perceptualhash']:
result = imagehash.phash(image)
elif algorithm in ['wHash', 'w']:
result = imagehash.whash(image)
else:
algorithm = algorithm.replace("_", "")
if algorithm in [
'dhashvertical', 'dvertical', 'dvert', 'verticald',
'verticaldhash'
]:
result = imagehash.dhash_vertical(image)
elif algorithm in [
'phashsimple', 'psimple', 'perceptualsimple',
'simpleperceptual', 'simplep', 'simplephash',
'simpleperceptualhas'
]:
result = imagehash.phash_simple(image)
else:
return HashingError(
"INVALID_ALGORITHM",
"We couldn't determine the hashing algorithm you wanted to use."
)
if has_url:
return HashObject(result, image, url)
else:
return HashObject(result, image)
def on_status(self, tweet, force=False):
"""
Tweet Receiving
"""
global sinceID
StatsAppend(TwitterStats.streamHit)
if TwitterConfig.ignore_rt and Twitter.isRetweet(tweet):
return
try:
if Twitter.isReplyingToErina(
tweet
): # If replying, analyze if it is a positive or a negative feedback
responseSentiment = sentiment(tweet.text)[0]
StatsAppend(TwitterStats.responsePolarity, responseSentiment)
latestResponses.append({
"timestamp":
time(),
"user":
tweet.user.screen_name,
"text":
tweet.text,
"sentiment":
responseSentiment,
"url":
"https://twitter.com/twitter/statuses/" + str(tweet.id),
})
except:
traceback.print_exc()
if isinstance(
TwitterConfig.monitoring.accounts,
(list, tuple)) and len(TwitterConfig.monitoring.accounts) > 0:
if TwitterConfig.monitoring.check_replies and Twitter.isReplyingToErina(
tweet): # Monitor Mode ON, Check Replies to Monitored ON
log("ErinaTwitter",
"New monitoring hit from @" + str(tweet.user.screen_name))
StatsAppend(TwitterStats.askingHit,
str(tweet.user.screen_name))
imageURL = Twitter.findImage(tweet)
if imageURL is None:
imageURL = Twitter.findParentImage(tweet)
if imageURL is not None:
searchResult = imageSearch(imageURL)
tweetResponse = makeTweet(searchResult)
if tweetResponse is not None:
StatsAppend(TwitterStats.responses)
ErinaTwitter.tweet(tweetResponse, replyID=tweet.id)
elif tweet.user.screen_name in TwitterConfig.monitoring.accounts: # Monitor Mode ON, Check Replies to Monitored OFF
log("ErinaTwitter", "New monitoring hit")
StatsAppend(TwitterStats.askingHitstr(tweet.user.screen_name))
imageURL = Twitter.findImage(tweet)
if imageURL is not None:
searchResult = imageSearch(imageURL)
tweetResponse = makeTweet(searchResult)
if tweetResponse is not None:
StatsAppend(TwitterStats.responses)
ErinaTwitter.tweet(tweetResponse, replyID=tweet.id)
else: # Monitor Mode OFF, Public Account
imageURL = Twitter.findImage(tweet)
if imageURL is None:
imageURL = Twitter.findParentImage(tweet)
if imageURL is not None and Twitter.isAskingForSauce(
tweet) or force:
log("ErinaTwitter",
"New asking hit from @" + str(tweet.user.screen_name))
StatsAppend(TwitterStats.askingHit,
str(tweet.user.screen_name))
searchResult = imageSearch(imageURL)
tweetResponse = makeTweet(searchResult)
if tweetResponse is not None:
StatsAppend(TwitterStats.responses)
responseImageURL = None
if isinstance(searchResult.detectionResult, TraceMOECache):
if TwitterConfig.image_preview:
if not searchResult.detectionResult.hentai:
responseImageURL = f"https://trace.moe/thumbnail.php?anilist_id={str(searchResult.detectionResult.anilist_id)}&file={str(searchResult.detectionResult.filename)}&t={str(searchResult.detectionResult.timing.at)}&token={str(searchResult.detectionResult.tokenthumb)}"
ErinaTwitter.tweet(tweetResponse,
replyID=tweet.id,
imageURL=responseImageURL)
elif Twitter.isMention(tweet):
ErinaTwitter.tweet(
"Sorry, I searched everywhere but coudln't find it...",
replyID=tweet.id)
TextFile(erina_dir + "/ErinaTwitter/lastStatusID.erina").write(
str(tweet.id))
sinceID = tweet.id
return
def on_connect(self):
"""
Connection
"""
log("ErinaTwitter", "ErinaTwitter is connected to the Twitter API")
def startStream():
global sinceID
global lastDM
Thread(target=_startStream, daemon=True).start()
while True:
if TwitterConfig.check_mentions:
try:
if sinceID is not None and sinceID != "":
for message in tweepy.Cursor(
ErinaTwitter.api.mentions_timeline,
since_id=sinceID,
count=200,
include_entities=True).items():
try:
ErinaStreamListener.on_status(message)
except:
log(
"ErinaTwitter",
f"Error while reading a mention {str(sys.exc_info()[0])}: {str(sys.exc_info()[1])}",
True)
else:
for message in tweepy.Cursor(
ErinaTwitter.api.mentions_timeline,
count=200,
include_entities=True).items():
try:
ErinaStreamListener.on_status(message)
except:
log(
"ErinaTwitter",
f"Error while reading a mention {str(sys.exc_info()[0])}",
True)
except:
log("ErinaTwitter",
f"Error while reading mentions {str(sys.exc_info()[0])}",
True)
if str(sys.exc_info()[0]).replace(
" ",
"").lower() == "<class'tweepy.error.ratelimiterror'>":
sleep(3600)
if TwitterConfig.check_dm:
try:
for message in tweepy.Cursor(
ErinaTwitter.api.list_direct_messages,
count=50).items():
try:
timestamp = convert_to_int(message.created_timestamp)
if message not in directMessagesHistory and timestamp > lastDM:
on_direct_message(message)
lastDM = timestamp
except:
log(
"ErinaTwitter",
f"Error while reading a DM {str(sys.exc_info()[0])}",
True)
except:
log("ErinaTwitter",
f"Error while reading DMs {str(sys.exc_info()[0])}", True)
if str(sys.exc_info()[0]).replace(
" ",
"").lower() == "<class'tweepy.error.ratelimiterror'>":
sleep(3600)
sleep(60)
def iqdb_caching(image_hash):
"""
Searches and caches IQDB for anime/manga related images.
Erina Project - 2020\n
© Anime no Sekai
"""
try:
log("ErinaCaches", 'Searching for IQDB Data...')
### If a file is given, send the file to iqdb.
try:
if image_hash.has_url:
IQDBresponse = requests.get(f'https://iqdb.org/?url={image_hash.url}')
StatsAppend(ExternalStats.iqdbCalls, "New Call")
else:
IQDBresponse = requests.post('https://iqdb.org/', files={'file': ('image_to_search', image_hash.ImageIO) })
StatsAppend(ExternalStats.iqdbCalls, "New Call")
except:
return CachingError("IQDB_RESPONSE", "An error occured while retrieving IQDB Data")
### If the image format is not supported by IQDB
if 'Not an image or image format not supported' in IQDBresponse.text:
return CachingError("IQDB_FORMAT_NOT_SUPPORTED", "The given image's format is not supported by IQDB")
###### IQDB SCRAPING
try:
iqdb = BeautifulSoup(IQDBresponse.text, 'html.parser')
##### Search for the IQDB result
try:
tables = iqdb.find_all('table')
search_result = tables[1].findChildren("th")[0].get_text()
except:
return CachingError("IQDB_CLIENT_ERROR", f"An error occured while searching for the results: {exc_info()[0]}")
##### Verify if the result is relevant or not
iqdb_tags = []
if search_result == 'No relevant matches':
return CachingError("IQDB_NO_RELEVANT_MATCH", "No relevant matches was found with IQDB", no_log=True)
else:
try:
### Getting the tags from IQDB
alt_string = tables[1].findChildren("img")[0]['alt']
iqdb_tags = alt_string.split('Tags: ')[1].split(' ')
except:
iqdb_tags = []
#### Getting the Database URL from IQDB
try:
url = tables[1].find_all('td', attrs={'class': 'image'})[0].findChildren('a')[0]['href']
url = 'https://' + url.split('//')[1]
except:
url = 'No URL'
#### Getting the result image size
try:
size = tables[1].find_all('tr')[3].get_text().split(' [')[0]
except:
size = 'Unknown'
#### Getting the image rating (if it is NSFW or not)
if tables[1].find_all('tr')[3].get_text().split()[1].replace('[', '').replace(']', '').replace(' ', '') == 'Safe':
is_safe = True
else:
is_safe = False
#### Getting the similarity
try:
similarity = tables[1].find_all('tr')[4].get_text().replace('% similarity', '')
except:
similarity = '0'
############ FUNCTION DEFINITION FOR RESULTS SCRAPING
database = "Unknown"
if url.find('gelbooru.') != -1:
database = 'Gelbooru'
elif url.find('danbooru.') != -1:
database = 'Danbooru'
elif url.find('zerochan.') != -1:
database = 'Zerochan'
elif url.find('konachan.') != -1:
database = 'Konachan'
elif url.find('yande.re') != -1:
database = 'Yande.re'
elif url.find('anime-pictures.') != -1:
database = 'Anime-Pictures'
elif url.find('e-shuushuu') != -1:
database = 'E-Shuushuu'
title = "Unknown"
try:
databaseWebsiteData = requests.get(url).text
databaseWebsite = BeautifulSoup(databaseWebsiteData.text, 'html.parser')
title = databaseWebsite.find("title").get_text()
except:
title = "Unkown"
except:
return CachingError("IQDB_PARSING", "An error occured while parsing the data from IQDB")
try:
#### Adding the results to the main result variable
newCacheFile = TextFile(erina_dir + "/ErinaCaches/IQDB_Cache/" + str(image_hash) + ".erina")
newCacheFile.append(" --- IQDB CACHE --- \n")
newCacheFile.append('\n')
newCacheFile.append('IQDB Tags: ' + ":::".join(iqdb_tags) + "\n")
newCacheFile.append('URL: ' + str(url) + "\n")
newCacheFile.append('Title: ' + str(title) + "\n")
newCacheFile.append('Size: ' + str(size) + "\n")
newCacheFile.append('isSafe: ' + str(is_safe) + "\n")
newCacheFile.append('Similarity: ' + str(similarity) + "\n")
newCacheFile.append('Database: ' + str(database) + "\n")
return iqdb_parser.IQDBCache(newCacheFile.read())
except:
return CachingError("FILE_WRITE", f"An error occured while writing out the cache data to a file")
except:
return CachingError("UNKNOWN", "An unknown error occured while caching IQDB Data")
