def _checkOutput(process, websocketConnection): """ Checks the output of stdout and stderr to send it to the WebSocket client """ while process.poll() is None: # while the process isn't exited try: output = process.stdout.read( ) # Read the stdout PIPE (which contains stdout and stderr) except: output = None if output: websocketConnection.send( json.dumps({ "message": output.decode("utf-8"), "code": 0 })) # Send the new output time.sleep(0.1) # Wait a lil bit to avoid confusion from the computer ### EXITED THE LOOP ### (process exited) try: websocketConnection.send( json.dumps({ "message": f"ErinaConsole: The process has exited with code {str(process.returncode)}", "code": process.returncode })) # Send a disconnection message except: log("ErinaAdmin", "Unable to send an exit message to ErinaConsole")
async def on_ready(): ''' When the bot is ready ''' await client.change_presence(activity=discord.Game( name='.erina help | Ready to give you the sauce!')) # GAME ACTIVITY log("ErinaDiscord", "Erina is ready")
def verify_manami_adb(force=False): """ Checks for a new version of the database on GitHub """ ## Checking if new week current_release_week = currentReleaseFile.read().replace(" ", '').replace("\n", '') iso_calendar = datetime.date.today().isocalendar() current_week = str(iso_calendar[0]) + '-' + str(iso_calendar[1]) if current_release_week != current_week or force: # If new week log("ErinaDB", "Updating Manami Title Vector DB...") manami_adb = json.loads(requests.get('https://raw.githubusercontent.com/manami-project/anime-offline-database/master/anime-offline-database.json').text) data = {} for anime in manami_adb["data"]: link = None anilist_id = None for sourceLink in anime["sources"]: if sourceLink.find("anilist.co") != -1: link = sourceLink if link is not None: # parser the data anilist_id = convert_to_int(link[link.rfind("/"):]) data[anilist_id] = [ str(anime["title"]).lower().replace(" ", '') ] data[anilist_id].extend([str(title).lower().replace(" ", '') for title in anime["synonyms"]]) else: continue # write out the data JSONFile(manami_database_path + 'manami_database_data.json', separators=(',', ':'), indent=None).write(data) currentReleaseFile.write(current_week) Database.updateData(data)
def tweet(self, message, replyID=None, imageURL=None): """ Tweets something """ twitterImage = None if imageURL is not None: image = BytesIO(requests.get(str(imageURL)).content) twitterImage = self.api.media_upload( file=image, filename="ErinaSauce — trace.moe Image Preview") if replyID is not None: log("ErinaTwitter", "Replying to " + str(replyID)) if twitterImage is not None: return self.api.update_status( status=str(message)[:280], in_reply_to_status_id=replyID, auto_populate_reply_metadata=True, media_ids=[twitterImage.media_id]) else: return self.api.update_status( status=str(message)[:280], in_reply_to_status_id=replyID, auto_populate_reply_metadata=True) else: log("ErinaTwitter", "Sending a tweet...") if twitterImage is not None: return self.api.update_status( status=str(message)[:280], media_ids=[twitterImage.media_id]) else: return self.api.update_status(status=str(message)[:280])
def anilist_search_caching(query): ''' Caches the first search result from the given query (from AniList API)\n Returns the new cache's filename\n Project Erina © Anime no Sekai - 2020 ''' try: log("ErinaCaches", f'Caching {str(query)} from AniList Search API...') try: apiResponse = anilist.anilist_api_search(query) except: return CachingError("ANILIST_SEARCH_API_RESPONSE", f"An error occured while retrieving AniList Search API Data ({str(query)})") if "errors" in apiResponse: if apiResponse["errors"][0]["status"] == 404: return CachingError("ANILIST_NOT_FOUND", str(query) + " has not been found") else: return CachingError("ANILIST_SERVER_ERROR", f"An error occured with the AniList API: {apiResponse['errors'][0]['message']}") try: cache = anilist.anilist_json_to_cache(apiResponse) except: return CachingError("ERINA_CONVERSION", f"An error occured while converting AniList's Search API Data to a caching format ({str(query)})") try: TextFile(anilist_cache_path + cache['filename'], blocking=False).write(cache["content"]) except: return CachingError("FILE_WRITE", f"An error occured while writing out the cache data to a file ({str(query)})") return anilist_parser.AnilistCache(cache["content"]) except: return CachingError("UNKNOWN_ERROR", f"An unknown error occured while caching AniList Search API Data ({str(query)})")
def __init__(self, type, message) -> None: self.type = str(type) self.message = str(message) self.timestamp = time() self.datetime = datetime.fromtimestamp(self.timestamp) self.formatted_timestamp = f"{str(self.datetime.year)}-{str(self.datetime.month)}-{str(self.datetime.day)} at {str(self.datetime.hour)}:{str(self.datetime.minute)}:{str(self.datetime.second)}" log("ErinaLine", self.type + ": " + self.message, error=True) StatsAppend(erina.errorsCount, "ErinaLine")
def searchAnime(query): """ Searches an anime by its title Erina Project — 2020\n © Anime no Sekai """ log("ErinaSearch", "Searching for " + str(query) + "... (title search)") StatsAppend(SearchStats.titleSearchCount, query) return title_search.searchAnime(query)
def runServer(): global ErinaWSGIServer ## RUNNING ErinaServer log("Erina", "Running ErinaServer...") wsgiEnv = { 'SERVER_SOFTWARE': 'ErinaServer ' + erina_version, 'wsgi.multithread': True, 'wsgi.run_once': False } ErinaWSGIServer = pywsgi.WSGIServer((ServerConfig.host, ServerConfig.port), ErinaServer, handler_class=WebSocketHandler, environ=wsgiEnv) ErinaWSGIServer.serve_forever()
def anilistIDSearch(anilistID): """ Searches an anime from AniList Caches or AniList API Erina Project — 2020\n © Anime no Sekai """ log("ErinaSearch", "Searching for " + str(anilistID) + "... (anilist id search)") StatsAppend(SearchStats.anilistIDSearchCount, anilistID) return anilist_id_search.search_anime_by_anilist_id(anilistID)
def imageSearch(image): """ Searches an anime from an image (anime scene for example) image: Can be an URL, a path, a base64 encoded string or a PIL.Image.Image instance Erina Project — 2020\n © Anime no Sekai """ log("ErinaSearch", "Searching for an image...") StatsAppend(SearchStats.imageSearchCount, None) return hash_search.search_anime_by_hash(erinahash.hash_image(image))
def base64_from_image(image_path): """ Converts an image to base64 Erina Project — 2020\n © Anime no Sekai """ log("ErinaHash", "Converting " + str(image_path) + " to base64...") image_content = BinaryFile(image_path).read() result = base64.b64encode(image_content).decode("utf-8") StatsAppend(ErinaHashStats.createdBase64String, f"New Base64 String (length: {str(len(result))})") return result
def shutdownErinaServer(num, info): """ SIGTERM, SIGQUIT, SIGINT signals handler --> Shutdowns Erina """ try: for handler in psutil.Process(os.getpid()).open_files(): os.close(handler.fd) except: pass TextFile(erina_dir + "/launch.erina").write("0") ErinaWSGIServer.stop() ErinaWSGIServer.close() logFile.blocking = True log("Erina", "Goodbye!")
def checkImages(): ''' Timeout checking function. ''' global current_images_dict number_of_deleted_files = 0 # logging purposes for entry in current_images_dict: if time.time() - current_images_dict[entry] > LineConfig.images_timeout: if filecenter.delete(images_path + entry + '.erina_image') == 0: current_images_dict.pop(entry, None) number_of_deleted_files += 1 # logging purposes ### LOGGING if number_of_deleted_files > 0: if number_of_deleted_files == 1: log("ErinaLine", "[Image Checker] Deleted 1 entry") else: log("ErinaLine", f'[Image Checker] Deleted {str(number_of_deleted_files)} entries') StatsAppend(LineStats.storedImages, len(filecenter.files_in_dir(images_path)))
def erina_caching(image_hash, database_path, similarity, anilist_id): ''' Caches an ErinaDatabase path according to the image_hash\n Project Erina © Anime no Sekai - 2020 ''' try: log("ErinaCaches", f'Caching {str(image_hash)} Erina Database data...') try: cache = erina.erina_from_data(str(image_hash), database_path, similarity, anilist_id) except: return CachingError("ERINA_CONVERSION", f"An error occured while converting Erina Database Data to a caching format ({str(database_path)})") try: TextFile(erina_cache_path + str(image_hash) + '.erina', blocking=False).write(cache) except: return CachingError("FILE_WRITE", f"An error occured while writing out the cache data to a file {str(database_path)}") return erina_parser.ErinaCache(cache) except: return CachingError("UNKNOWN", "An unknown error occured while caching Erina Database Data")
def dm(self, message, recipientID, imageURL=None): """ DMs someone """ twitterImage = None if imageURL is not None: image = BytesIO(requests.get(str(imageURL)).content) filename = imageURL[imageURL.rfind("/"):] twitterImage = self.api.media_upload(filename=filename, file=image) log("ErinaTwitter", "Sending a direct message to " + str(recipientID)) if twitterImage is not None: return self.api.send_direct_message( recipientID, str(message)[:10000], attachment_media_id=twitterImage.media_id) else: return self.api.send_direct_message(recipientID, str(message)[:10000])
def restartErinaServer(num, info): """ SIGUSR1 signal handler --> Restarts Erina """ try: for handler in psutil.Process(os.getpid()).open_files(): os.close(handler.fd) except: pass from ErinaTwitter.utils.Stream import lastDM from ErinaTwitter.utils.Stream import sinceID TextFile(erina_dir + "/ErinaTwitter/lastDM.erina").write(str(lastDM)) TextFile(erina_dir + "/ErinaTwitter/lastStatusID.erina").write(str(sinceID)) TextFile(erina_dir + "/launch.erina").write("0") ErinaWSGIServer.stop() ErinaWSGIServer.close() logFile.blocking = True log("Erina", "Restarting...") os.execl(sys.executable, sys.executable, __file__, *sys.argv[1:])
def tracemoe_caching(image_hash): ''' Caches the given Trace.moe API response\n Project Erina © Anime no Sekai - 2020 ''' try: log("ErinaCaches", f'Caching {str(image_hash)} trace.moe data...') try: if image_hash.has_url is not None: if str(config.Caches.keys.tracemoe).replace(" ", "") in ["None", ""]: requestResponse = json.loads(requests.get('https://trace.moe/api/search?url=' + image_hash.url).text) else: requestResponse = json.loads(requests.get('https://trace.moe/api/search?url=' + image_hash.url + '&token=' + str(config.Caches.keys.tracemoe)).text) else: if str(config.Caches.keys.tracemoe).replace(" ", "") in ["None", ""]: requestResponse = json.loads(requests.post('https://trace.moe/api/search', json={'image': image_hash.base64})) else: requestResponse = json.loads(requests.post('https://trace.moe/api/search?token=' + str(config.Caches.keys.tracemoe), json={'image': image_hash.base64})) except: return CachingError("TRACEMOE_API_RESPONSE", "An error occured while retrieving information from the trace.moe API") StatsAppend(ExternalStats.tracemoeAPICalls) try: cache = tracemoe.erina_from_json(requestResponse) except: print(exc_info()[0]) print(exc_info()[1]) print(traceback.print_exception(*exc_info())) return CachingError("ERINA_CONVERSION", f"An error occured while converting trace.moe API Data to a caching format ({str(image_hash)})") try: TextFile(tracemoe_cache_path + str(image_hash) + '.erina', blocking=False).write(cache) except: return CachingError("FILE_WRITE", f"An error occured while writing out the cache data to a file ({str(image_hash)})") return tracemoe_parser.TraceMOECache(cache) except: return CachingError("UNKNOWN_ERROR", f"An unknown error occured while caching trace.moe API Data ({str(image_hash)})")
def on_direct_message(message): """ DM Receiving """ directMessagesHistory.append(message) log("ErinaTwitter", "New direct message from @" + str(message.message_create['sender_id'])) if Twitter.dmAskingForSauce(message): StatsAppend(TwitterStats.directMessagingHit, str(message.message_create['sender_id'])) image = Twitter.getDirectMedia(message) if image is not None: searchResult = imageSearch(image) ErinaTwitter.dm(makeImageResponse(searchResult), message.message_create['sender_id']) elif isAnError(image): ErinaTwitter.dm( "An error occured while retrieving information on the anime", message.message_create['sender_id']) else: ErinaTwitter.dm( "You did not send any image along with your message", message.message_create['sender_id'])
def saucenao_caching(image_hash): ''' Caches the result from the given url\n Project Erina\n © Anime no Sekai - 2020 ''' try: log("ErinaCaches", f"Caching {str(image_hash)} SauceNAO data...") if str(config.Caches.keys.saucenao).replace(" ", "") not in ["None", ""]: saucenao_api = SauceNao(api_key=config.Caches.keys.saucenao, numres=1) else: saucenao_api = SauceNao(numres=1) if image_hash.has_url: try: api_results = saucenao_api.from_url(image_hash.url)[0] except: return CachingError("SAUCENAO_API_RESPONSE", "An error occured while retrieving SauceNAO API Data") else: try: api_results = saucenao_api.from_file(image_hash.ImageIO)[0] except: return CachingError("SAUCENAO_API_RESPONSE", "An error occured while retrieving SauceNAO API Data") StatsAppend(ExternalStats.saucenaoAPICalls) try: cache = saucenao.erina_from_api(api_results) except: traceback.print_exc() return CachingError("ERINA_CONVERSION", "An error occured while converting SauceNAO API Data to a caching format") try: TextFile(saucenao_cache_path + str(image_hash) + '.erina', blocking=False).write(cache) except: return CachingError("FILE_WRITE", "An error occured while writing out the cache data to a file") return saucenao_parser.SauceNAOCache(cache) except: return CachingError("UNKNOWN", "An unknown error occured while caching SauceNAO API Data")
def runClients(): import asyncio from Erina import config import ErinaLine.erina_linebot # Already runs the LINE Client log("Erina", "Starting to check for timeouts and updates...") from Erina import setInterval # Already runs the checkers if config.Twitter.run: log("Erina", "Running the ErinaTwitter Client...") from ErinaTwitter.utils import Stream as twitterClient Thread(target=twitterClient.startStream, daemon=True).start() if config.Discord.run: log("Erina", "Running the ErinaDiscord Client...") from ErinaDiscord.erina_discordbot import startDiscord startDiscord() if config.Line.run: initHandler()
if __name__ == '__main__' or TextFile(erina_dir + "/launch.erina").read().replace(" ", "") in ["", "0"]: TextFile(erina_dir + "/launch.erina").read() == "1" #### INITIALIZING ERINASERVER --> Manages the whole server print("[Erina]", "Initializing ErinaServer") import requests import requests.packages.urllib3.contrib.pyopenssl as requestsPyOpenSSL requestsPyOpenSSL.extract_from_urllib3() from ErinaServer.Server import ErinaServer from ErinaServer import WebSockets from gevent import pywsgi from geventwebsocket.handler import WebSocketHandler from Erina.erina_log import log, logFile log("Erina", "Initializing Erina configuration...") from Erina.config import Server as ServerConfig from Erina.env_information import erina_version ## RECORDING Endpoints log("Erina", "---> Initializing Static File Endpoints") from ErinaServer.Erina import Static log("Erina", "---> Initializing API") from ErinaServer.Erina.api import API log("Erina", "---> Initializing ErinaAdmin") from ErinaServer.Erina.admin import Admin from ErinaServer.Erina.auth import Auth from ErinaServer.Erina.admin import Config log("Erina", "---> Initializing Custom Endpoints") from ErinaServer import Custom log("Erina", "---> Initializing ErinaConsole")
async def on_message(message): ''' When the bot receives a message ''' #await message.add_reaction(roger_reaction) # REACT TO SHOW THAT THE BOT HAS UNDESTAND HIS COMMAND if message.author.id == client.user.id: return if message.content.startswith('.erina'): # ERINA SPECIFIC COMMAND userCommand = utils.removeSpaceBefore(str(message.content)[6:]) commandLength = len(userCommand.split(" ")[0]) command, commandSimilarity = searchCommand( userCommand.split(" ")[0].lower()) if commandSimilarity < 0.75: await message.channel.send("Sorry, this command is not available.") return else: if command == "search": query = utils.removeSpaceBefore(userCommand[commandLength:]) log( "ErinaDiscord", "New info hit from @" + str(message.author) + " (asking for " + str(query) + ")") StatsAppend(DiscordStats.infoHit, f"{str(query)} >>> {str(message.author)}") anime, thumbnail, discordResponse = Parser.makeInfoResponse( erinasearch.searchAnime(query)) if discordResponse is not None: newEmbed = discord.Embed(title='Anime Info', colour=discord.Colour.blue()) newEmbed.add_field(name=anime.capitalize(), value=discordResponse) if thumbnail is not None: newEmbed.set_thumbnail(url=thumbnail) await message.channel.send(embed=newEmbed) else: await message.channel.send( "An error occured while searching for your anime: " + query) elif command == "description": query = utils.removeSpaceBefore(userCommand[commandLength:]) log( "ErinaDiscord", "New description hit from @" + str(message.author) + " (asking for " + str(query) + ")") StatsAppend(DiscordStats.descriptionHit, f"{str(query)} >>> {str(message.author)}") anime, thumbnail, discordResponse = Parser.makeDescriptionResponse( erinasearch.searchAnime(query)) if discordResponse is not None: newEmbed = discord.Embed( title=f'Anime Description: {str(anime)}', colour=discord.Colour.blue()) newEmbed.add_field(name=anime.capitalize(), value=discordResponse) if thumbnail is not None: newEmbed.set_thumbnail(url=thumbnail) await message.channel.send(embed=newEmbed) else: await message.channel.send( "An error occured while searching for your anime: " + query) elif command == "dev": await StaticResponse.erinadev(message.channel) elif command == "donate": await StaticResponse.erinadonate(message.channel) elif command == "help": await StaticResponse.erinahelp(message.channel, message.author) elif command == "stats": await StaticResponse.erinastats(message.channel, client) elif command == "invite": await StaticResponse.erinainvite(message.channel) else: if any([ flag in str(message.content).lower() for flag in (config.Discord.flags if str(config.Discord.flags). replace(" ", "") not in ["None", "", "[]"] else config.Erina.flags) ]): listOfResults = [] #await message.add_reaction(roger_reaction) # REACT TO SHOW THAT THE BOT HAS UNDESTAND HIS COMMAND log("ErinaDiscord", "New image search from @" + str(message.author)) StatsAppend(DiscordStats.imageSearchHit, f"{str(message.author)}") for file in message.attachments: if filecenter.type_from_extension( filecenter.extension_from_base(file.filename) ) == 'Image': # If the file is an image current_anime = Parser.makeImageResponse( erinasearch.imageSearch( file.url)) # Get infos about the anime listOfResults.append( current_anime) # Append to the results list else: message_history = await message.channel.history( limit=5 ).flatten() # Search from the last 3 messages a picture for message in message_history: for file in message.attachments: if filecenter.type_from_extension( filecenter.extension_from_base(file.filename) ) == 'Image': # If the file is an image current_anime = Parser.makeImageResponse( erinasearch.imageSearch( file.url)) # Get infos about the anime listOfResults.append( current_anime) # Append to the results list if len(listOfResults) == 0: await message.channel.send("Sorry, I couldn't find anything..." ) elif len(listOfResults) == 1: title, thumbnail, reply = listOfResults[0] if reply is None: await message.channel.send( "An error occured while retrieving information on the anime..." ) await message.channel.send(f"It seems to be {title}!") newEmbed = discord.Embed(title='Anime Info', colour=discord.Colour.blue()) newEmbed.add_field(name=title.capitalize(), value=reply) if thumbnail is not None: newEmbed.set_thumbnail(url=thumbnail) await message.channel.send(embed=newEmbed) await asyncio.sleep(1) else: for iteration, result in enumerate(listOfResults): number = '' if iteration == 0: number = '1st' elif iteration == 1: number = '2nd' elif iteration == 2: number = '3rd' else: number = f'{str(iteration)}th' title, thumbnail, reply = result if reply is None: await message.channel.send( f"An error occured while searching for the {number} anime" ) await message.channel.send( f"The {number} anime seems to be {title}!") newEmbed = discord.Embed(title='Anime Info', colour=discord.Colour.blue()) newEmbed.add_field(name=title.capitalize(), value=reply) if thumbnail is not None: newEmbed.set_thumbnail(url=thumbnail) await message.channel.send(embed=newEmbed) await asyncio.sleep(1) return
def ErinaServer_Endpoint_Admin_Console_ErinaConsole(ws): currentProcess = None message = ws.receive() try: data = json.loads(message) # retrieve a message from the client if "token" in data: tokenVerification = authManagement.verifyToken(data) if tokenVerification.success: log("ErinaAdmin", "> New ErinaConsole connection!") bash = False if os.path.isfile("/bin/bash"): bash = True currentProcess = subprocess.Popen( shlex.split("/bin/bash"), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) # Open a bash prompt else: currentProcess = subprocess.Popen( shlex.split("/bin/sh"), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) # Open a shell prompt fcntl.fcntl( currentProcess.stdout.fileno(), fcntl.F_SETFL, os.O_NONBLOCK) # Non blocking stdout and stderr reading threading.Thread( target=_checkOutput, args=[currentProcess, ws], daemon=True).start( ) # Start checking for new text in stdout and stderr if bash: ws.send( json.dumps({ "message": f"ErinaConsole: Connection successfully established (bash) with PID: {str(currentProcess.pid)}", "code": 0 }) ) # Send a message to notifiy that the process has started else: ws.send( json.dumps({ "message": f"ErinaConsole: Connection successfully established (sh) with PID: {str(currentProcess.pid)}", "code": 0 }) ) # Send a message to notifiy that the process has started else: ws.send( json.dumps({ "message": f"ErinaConsole: Authentification Error: {str(tokenVerification)}", "code": 400 })) else: ws.send( json.dumps({ "message": f"ErinaConsole: Your connection could not be authentificated", "code": 400 })) except: try: ws.send( json.dumps({ "message": f"ErinaConsole: An error occured ({str(sys.exc_info()[0])}) while creating a new ErinaConsole instance", "code": -1 })) except: log("ErinaAdmin", "ErinaConsole >>> Failed to send a message") try: while not ws.closed: try: message = ws.receive() try: data = json.loads( message) # retrieve a message from the client tokenVerification = authManagement.verifyToken(data) if tokenVerification.success: if "input" in data: userInput = str(data["input"]) #if userInput != "exit": currentProcess.stdin.write( str(userInput + "\n").encode("utf-8") ) # Write user input (ws client) to stdin currentProcess.stdin.flush() # Run the command log( "ErinaAdmin", "ErinaConsole >> Admin ran command > " + str(userInput)) """ else: currentProcess.terminate() # If "exit" sent by client, terminate the process """ else: ws.send( json.dumps({ "message": f"ErinaConsole: Authentification Error: {str(tokenVerification)}", "code": 400 })) except: try: ws.send( json.dumps({ "message": f"ErinaConsole: An error occured ({str(sys.exc_info()[0])})", "code": -1 })) except: log("ErinaAdmin", "ErinaConsole >>> Failed to send a message") except: log("ErinaAdmin", "< ErinaConsole disconnected") except: log("ErinaAdmin", "< ErinaConsole disconnected") if currentProcess is not None: if currentProcess.poll() is None: currentProcess.terminate() log("ErinaAdmin", "< ErinaConsole disconnected")
def hash_image(image, algorithm=None): """ Hashes a given image image: Can be an URL, a path, a base64 encoded string or a PIL.Image.Image instance Erina Project — 2020\n © Anime no Sekai """ result = None has_url = False url = None log("ErinaHash", "Hashing an image...") # Needs to be a PIL instance if isfile(str(image)): image = Image.open(image) elif isinstance(image, Image.Image): image = image else: try: if base64.b64decode(str(image), validate=True): image = Image.open(BytesIO(base64.b64decode(str(image)))) else: raise ValueError("b64decode returned an empty string") except: try: url = image image = Image.open( BytesIO(requests.get(str(image)).content) ) # Open the downloaded image as a PIL Image instance has_url = True except: return HashingError( "INVALID_IMAGE_TYPE", "We couldn't convert the given image to a PIL.Image.Image instance" ) if algorithm is None: algorithm = str(config.Hash.algorithm) algorithm = str(algorithm).lower().replace(" ", "") if algorithm in ['ahash', 'a', 'averagehash', 'average']: result = imagehash.average_hash(image) elif algorithm in ['chash', 'c']: result = imagehash.colorhash(image) elif algorithm in ['dhash', 'd']: result = imagehash.dhash(image) elif algorithm in ['phash', 'p', 'perceptual', 'perceptualhash']: result = imagehash.phash(image) elif algorithm in ['wHash', 'w']: result = imagehash.whash(image) else: algorithm = algorithm.replace("_", "") if algorithm in [ 'dhashvertical', 'dvertical', 'dvert', 'verticald', 'verticaldhash' ]: result = imagehash.dhash_vertical(image) elif algorithm in [ 'phashsimple', 'psimple', 'perceptualsimple', 'simpleperceptual', 'simplep', 'simplephash', 'simpleperceptualhas' ]: result = imagehash.phash_simple(image) else: return HashingError( "INVALID_ALGORITHM", "We couldn't determine the hashing algorithm you wanted to use." ) if has_url: return HashObject(result, image, url) else: return HashObject(result, image)
def on_status(self, tweet, force=False): """ Tweet Receiving """ global sinceID StatsAppend(TwitterStats.streamHit) if TwitterConfig.ignore_rt and Twitter.isRetweet(tweet): return try: if Twitter.isReplyingToErina( tweet ): # If replying, analyze if it is a positive or a negative feedback responseSentiment = sentiment(tweet.text)[0] StatsAppend(TwitterStats.responsePolarity, responseSentiment) latestResponses.append({ "timestamp": time(), "user": tweet.user.screen_name, "text": tweet.text, "sentiment": responseSentiment, "url": "https://twitter.com/twitter/statuses/" + str(tweet.id), }) except: traceback.print_exc() if isinstance( TwitterConfig.monitoring.accounts, (list, tuple)) and len(TwitterConfig.monitoring.accounts) > 0: if TwitterConfig.monitoring.check_replies and Twitter.isReplyingToErina( tweet): # Monitor Mode ON, Check Replies to Monitored ON log("ErinaTwitter", "New monitoring hit from @" + str(tweet.user.screen_name)) StatsAppend(TwitterStats.askingHit, str(tweet.user.screen_name)) imageURL = Twitter.findImage(tweet) if imageURL is None: imageURL = Twitter.findParentImage(tweet) if imageURL is not None: searchResult = imageSearch(imageURL) tweetResponse = makeTweet(searchResult) if tweetResponse is not None: StatsAppend(TwitterStats.responses) ErinaTwitter.tweet(tweetResponse, replyID=tweet.id) elif tweet.user.screen_name in TwitterConfig.monitoring.accounts: # Monitor Mode ON, Check Replies to Monitored OFF log("ErinaTwitter", "New monitoring hit") StatsAppend(TwitterStats.askingHitstr(tweet.user.screen_name)) imageURL = Twitter.findImage(tweet) if imageURL is not None: searchResult = imageSearch(imageURL) tweetResponse = makeTweet(searchResult) if tweetResponse is not None: StatsAppend(TwitterStats.responses) ErinaTwitter.tweet(tweetResponse, replyID=tweet.id) else: # Monitor Mode OFF, Public Account imageURL = Twitter.findImage(tweet) if imageURL is None: imageURL = Twitter.findParentImage(tweet) if imageURL is not None and Twitter.isAskingForSauce( tweet) or force: log("ErinaTwitter", "New asking hit from @" + str(tweet.user.screen_name)) StatsAppend(TwitterStats.askingHit, str(tweet.user.screen_name)) searchResult = imageSearch(imageURL) tweetResponse = makeTweet(searchResult) if tweetResponse is not None: StatsAppend(TwitterStats.responses) responseImageURL = None if isinstance(searchResult.detectionResult, TraceMOECache): if TwitterConfig.image_preview: if not searchResult.detectionResult.hentai: responseImageURL = f"https://trace.moe/thumbnail.php?anilist_id={str(searchResult.detectionResult.anilist_id)}&file={str(searchResult.detectionResult.filename)}&t={str(searchResult.detectionResult.timing.at)}&token={str(searchResult.detectionResult.tokenthumb)}" ErinaTwitter.tweet(tweetResponse, replyID=tweet.id, imageURL=responseImageURL) elif Twitter.isMention(tweet): ErinaTwitter.tweet( "Sorry, I searched everywhere but coudln't find it...", replyID=tweet.id) TextFile(erina_dir + "/ErinaTwitter/lastStatusID.erina").write( str(tweet.id)) sinceID = tweet.id return
def on_connect(self): """ Connection """ log("ErinaTwitter", "ErinaTwitter is connected to the Twitter API")
def startStream(): global sinceID global lastDM Thread(target=_startStream, daemon=True).start() while True: if TwitterConfig.check_mentions: try: if sinceID is not None and sinceID != "": for message in tweepy.Cursor( ErinaTwitter.api.mentions_timeline, since_id=sinceID, count=200, include_entities=True).items(): try: ErinaStreamListener.on_status(message) except: log( "ErinaTwitter", f"Error while reading a mention {str(sys.exc_info()[0])}: {str(sys.exc_info()[1])}", True) else: for message in tweepy.Cursor( ErinaTwitter.api.mentions_timeline, count=200, include_entities=True).items(): try: ErinaStreamListener.on_status(message) except: log( "ErinaTwitter", f"Error while reading a mention {str(sys.exc_info()[0])}", True) except: log("ErinaTwitter", f"Error while reading mentions {str(sys.exc_info()[0])}", True) if str(sys.exc_info()[0]).replace( " ", "").lower() == "<class'tweepy.error.ratelimiterror'>": sleep(3600) if TwitterConfig.check_dm: try: for message in tweepy.Cursor( ErinaTwitter.api.list_direct_messages, count=50).items(): try: timestamp = convert_to_int(message.created_timestamp) if message not in directMessagesHistory and timestamp > lastDM: on_direct_message(message) lastDM = timestamp except: log( "ErinaTwitter", f"Error while reading a DM {str(sys.exc_info()[0])}", True) except: log("ErinaTwitter", f"Error while reading DMs {str(sys.exc_info()[0])}", True) if str(sys.exc_info()[0]).replace( " ", "").lower() == "<class'tweepy.error.ratelimiterror'>": sleep(3600) sleep(60)
def iqdb_caching(image_hash): """ Searches and caches IQDB for anime/manga related images. Erina Project - 2020\n © Anime no Sekai """ try: log("ErinaCaches", 'Searching for IQDB Data...') ### If a file is given, send the file to iqdb. try: if image_hash.has_url: IQDBresponse = requests.get(f'https://iqdb.org/?url={image_hash.url}') StatsAppend(ExternalStats.iqdbCalls, "New Call") else: IQDBresponse = requests.post('https://iqdb.org/', files={'file': ('image_to_search', image_hash.ImageIO) }) StatsAppend(ExternalStats.iqdbCalls, "New Call") except: return CachingError("IQDB_RESPONSE", "An error occured while retrieving IQDB Data") ### If the image format is not supported by IQDB if 'Not an image or image format not supported' in IQDBresponse.text: return CachingError("IQDB_FORMAT_NOT_SUPPORTED", "The given image's format is not supported by IQDB") ###### IQDB SCRAPING try: iqdb = BeautifulSoup(IQDBresponse.text, 'html.parser') ##### Search for the IQDB result try: tables = iqdb.find_all('table') search_result = tables[1].findChildren("th")[0].get_text() except: return CachingError("IQDB_CLIENT_ERROR", f"An error occured while searching for the results: {exc_info()[0]}") ##### Verify if the result is relevant or not iqdb_tags = [] if search_result == 'No relevant matches': return CachingError("IQDB_NO_RELEVANT_MATCH", "No relevant matches was found with IQDB", no_log=True) else: try: ### Getting the tags from IQDB alt_string = tables[1].findChildren("img")[0]['alt'] iqdb_tags = alt_string.split('Tags: ')[1].split(' ') except: iqdb_tags = [] #### Getting the Database URL from IQDB try: url = tables[1].find_all('td', attrs={'class': 'image'})[0].findChildren('a')[0]['href'] url = 'https://' + url.split('//')[1] except: url = 'No URL' #### Getting the result image size try: size = tables[1].find_all('tr')[3].get_text().split(' [')[0] except: size = 'Unknown' #### Getting the image rating (if it is NSFW or not) if tables[1].find_all('tr')[3].get_text().split()[1].replace('[', '').replace(']', '').replace(' ', '') == 'Safe': is_safe = True else: is_safe = False #### Getting the similarity try: similarity = tables[1].find_all('tr')[4].get_text().replace('% similarity', '') except: similarity = '0' ############ FUNCTION DEFINITION FOR RESULTS SCRAPING database = "Unknown" if url.find('gelbooru.') != -1: database = 'Gelbooru' elif url.find('danbooru.') != -1: database = 'Danbooru' elif url.find('zerochan.') != -1: database = 'Zerochan' elif url.find('konachan.') != -1: database = 'Konachan' elif url.find('yande.re') != -1: database = 'Yande.re' elif url.find('anime-pictures.') != -1: database = 'Anime-Pictures' elif url.find('e-shuushuu') != -1: database = 'E-Shuushuu' title = "Unknown" try: databaseWebsiteData = requests.get(url).text databaseWebsite = BeautifulSoup(databaseWebsiteData.text, 'html.parser') title = databaseWebsite.find("title").get_text() except: title = "Unkown" except: return CachingError("IQDB_PARSING", "An error occured while parsing the data from IQDB") try: #### Adding the results to the main result variable newCacheFile = TextFile(erina_dir + "/ErinaCaches/IQDB_Cache/" + str(image_hash) + ".erina") newCacheFile.append(" --- IQDB CACHE --- \n") newCacheFile.append('\n') newCacheFile.append('IQDB Tags: ' + ":::".join(iqdb_tags) + "\n") newCacheFile.append('URL: ' + str(url) + "\n") newCacheFile.append('Title: ' + str(title) + "\n") newCacheFile.append('Size: ' + str(size) + "\n") newCacheFile.append('isSafe: ' + str(is_safe) + "\n") newCacheFile.append('Similarity: ' + str(similarity) + "\n") newCacheFile.append('Database: ' + str(database) + "\n") return iqdb_parser.IQDBCache(newCacheFile.read()) except: return CachingError("FILE_WRITE", f"An error occured while writing out the cache data to a file") except: return CachingError("UNKNOWN", "An unknown error occured while caching IQDB Data")