def addDownloadTask(self, url, current=None):
'''
Processes a download task from an url
'''
print('[DOWNLOADER] Received download task: ' + url)
try:
audio = download_mp3(url)
except:
raise DownloadError('[DOWNLOADER] Error in audio download')
print('[DOWNLOADER] The file ' +
os.path.basename(audio) + ' has been downloaded')
file_info = TrawlNet.FileInfo()
file_info.name = os.path.basename(audio)
file_info.hash = hashlib.sha256(file_info.name.encode()).hexdigest()
if not self.updater:
raise DownloadError('[ÐOWNLOADER] Error getting publisher')
self.updater.newFile(file_info)
return file_info
def getPeerInfo(self, transfer, file_name):
"""Obtenemos el objeto PeerInfo"""
peer_info = TrawlNet.PeerInfo()
peer_info.transfer = transfer
peer_info.fileName = file_name
return peer_info
def downloadTask(self, url, current=None):
'''
Sends a download task to a downloader
'''
video_hash = hashlib.sha256(get_title(url).encode()).hexdigest()
if video_hash in self.orchestrator.files:
print('[ORCHESTRATOR] The file has been previously downloaded')
file_info = TrawlNet.FileInfo()
file_info.name = self.orchestrator.files[video_hash]
file_info.hash = video_hash
return file_info
else:
downloader = self.downloader_factory.create()
if not downloader:
raise DownloadError(
'[ORCHESTRATOR] Error: error creating downloader')
file_info = downloader.addDownloadTask(url)
downloader.destroy()
# Inform to the rest of orchestrators
for other in self.orchestrator.orchestrators:
other_list = other.getFileList()
if file_info not in other_list:
other_list.append(file_info)
return file_info
def transfer_request(orchestrator, file_name):
"""
Download file from a host
@param orchestrator: server
@param file_name: file to download
@return:
"""
# pylint: disable=C0103
BLOCK_SIZE = 1024
try:
transfer = orchestrator.getFile(file_name)
except TrawlNet.TransferError as e:
raise TrawlNet.DownloadError(e)
with open(os.path.join('./', file_name), 'wb') as file_:
remote_EOF = False
while not remote_EOF:
data = transfer.recv(BLOCK_SIZE)
if len(data) > 1:
data = data[1:]
data = binascii.a2b_base64(data)
remote_EOF = len(data) < BLOCK_SIZE
if data:
file_.write(data)
transfer.close()
transfer.destroy()
print(Color.GREEN + Color.BOLD + 'Transfer finished!' + Color.END)
def downloadTask(self, url, current):
print('\n[Orchestrator]Peticion de descarga, url: %s' % url)
hash = ''
name = ''
from youtube_dl import YoutubeDL
with YoutubeDL({'quiet': True}) as youtube:
info = youtube.extract_info(url, download=False)
hash = info.get("id", None)
name = info.get("title", None)
if utiles.isInList(hash):
file_info = TrawlNet.FileInfo()
file_info.name = name
file_info.hash = hash
print('\n[Orchestrator] ERROR*** Cancion ya descargada.')
return file_info
else:
downloader = self.downloader_factory.create() # Crear el downloader
if downloader is not None:
print(f'\nDescargando cancion.\nDownloader: {downloader}')
file_info = downloader.addDownloadTask(url)
self.addToList(file_info=file_info)
downloader.destroy()
return file_info
else:
print('***ERROR. No se ha encontrado ningun downloader')
def addDownloadTask(self, url, current=None): # pylint: disable=C0103, R0201, W0613
'''
Downloader
'''
try:
file_to_download = download_mp3(url)
except:
raise TrawlNet.DownloadError()
fileInfo = TrawlNet.FileInfo()
fileInfo.name = os.path.basename(file_to_download)
fileInfo.hash = calculate_hash(fileInfo.name)
if self.publisher is not None:
self.publisher.newFile(fileInfo)
return fileInfo
def __init__(self, file_path):
"""
@param file_path: path of the file
"""
try:
self.file_ = open(file_path, 'rb')
except Exception as msg_exception:
raise TrawlNet.TransferError(str(msg_exception))
def get_file(self, name):
"""
transfer for get a file
@return: the transfer
"""
try:
return self.transfer_factory.create(name)
except:
raise TrawlNet.TransferError("Transfer error")
def getFileList(self, current=None):
fileList = []
for fileHash in self.files:
fileInfo = TrawlNet.FileInfo()
fileInfo.hash = fileHash
fileInfo.name = self.files[fileHash]
fileList.append(fileInfo)
return fileList
def addDownloadTask(self, url, current=None):
"""
Download the video from url
@param url: link to the video
@param current:
@return: file info
"""
try:
out_file, file_id = download_mp3_with_id(
url, destination=self.downloads_directory)
except Exception as msg_exception:
raise TrawlNet.DownloadError(str(msg_exception))
file = TrawlNet.FileInfo()
file.name = os.path.basename(out_file)
file.hash = file_id
self.publisher.newFile(file)
return file
def create(self, fileName, current=None):
try:
prueba = open("./" + self.path + fileName, 'rb').readline()
except IOError:
print("El archivo \"" + fileName + "\" no existe.")
raise TrawlNet.FileDoesNotExistError("El archivo \"" + fileName +
"\" no existe.")
else:
servant = SenderI(self.path, fileName)
proxy = current.adapter.addWithUUID(servant)
return TrawlNet.SenderPrx.checkedCast(proxy)
def create(self, fileName, current=None):
ruta=os.getcwd()+"/files/"+fileName
if not os.path.isfile(ruta):
raise TrawlNet.FileDoesNotExistError("Error al buscar archivo: {0}".format(fileName))
servant=SenderI(fileName)
proxyS=current.adapter.addWithUUID(servant)
#print("Sender Created, file: {0}".format(fileName))
sys.stdout.flush()
return TrawlNet.SenderPrx.checkedCast(proxyS)
def get_files(self):
"""
Get the list of files already downloaded
@return: the list
"""
files = []
for file_id in self.files_dic:
file = TrawlNet.FileInfo()
file.hash = file_id
file.name = self.files_dic[file_id]
files.append(file)
return files
def send_download_task(self, url):
"""
Send an download request
@param url: link to video
@return: the information file
"""
if not supported(url):
raise ValueError(Color.RED + 'Incorrect URL' + Color.END)
file_id = generate_id(url)
if file_id not in self.files_dic:
try:
downloader = self.downloader_factory.create()
return downloader.addDownloadTask(url)
except TrawlNet.DownloadError as msg_exception:
raise TrawlNet.DownloadError(msg_exception)
else:
file = TrawlNet.FileInfo()
file.hash = file_id
file.name = self.files_dic[file_id] + " (Downloaded previously)"
return file
def getFileList(self, current=None):
print("\n[Orchestrator]Peticion de informacion de archivos disponibles")
list_file = []
archivo = utiles.jsonRead()
if not archivo:
return archivo
for k, v in archivo.items():
file_info = TrawlNet.FileInfo()
file_info.name = v
file_info.hash = k
list_file.append(file_info)
return list_file
def recv(self, size, current):
"""
For recieve date within transfer
@param size:
@param current:
@return:
"""
try:
return str(
binascii.b2a_base64(self.file_.read(size), newline=False))
except Exception as msg_exception:
raise TrawlNet.TransferError(str(msg_exception))
def addDownloadTask(self, link, current=None):
try:
print("Download request: %s" % link)
sys.stdout.flush()
fileInfo = TrawlNet.FileInfo()
fileInfo.name = download_mp3(link)
fileInfo.hash = computeHash(fileInfo.name)
self.publisher.newFile(fileInfo)
return fileInfo
except TrawlNet.DownloadError:
print("Download failed")
return 1
def create(self, file_name, current=None):
"""Devolvera un objeto sender al que se le ha asignado el fichero fileName si existe"""
try:
file_path = os.path.join(FILES_DIRECTORY, file_name)
servant = SenderI(file_path)
proxy = current.adapter.addWithUUID(servant)
print("# New sender for {} #".format(file_path))
except Exception as err:
print("Error. Sender failed {}".format(err))
raise TrawlNet.FileDoesNotExistError(str(err))
return TrawlNet.SenderPrx.checkedCast(proxy)
def hello(self, me, current=None):
if me in self.orchestrators:
return
print("Hello! My proxy is:", me)
self.orchestrators.append(me)
for fileHash in self.files:
fileInfo = TrawlNet.FileInfo()
fileInfo.hash = fileHash
fileInfo.name = self.files[fileHash]
self.publisher.newFile(fileInfo)
other = TrawlNet.OrchestratorPrx.uncheckedCast(self.proxy)
me.announce(other)
def createFileInfo(self, url):
file_info = TrawlNet.FileInfo()
file_info.name = ''
file_info.hash = ''
try:
from youtube_dl import YoutubeDL
with YoutubeDL() as youtube:
info = youtube.extract_info(url, download=False)
file_info.hash = info.get("id", None)
file_info.name = info.get('title', None)
return file_info
except Exception:
print('ERROR: Hubo un error creando el objeto FileInfo.')
sys.exit(1)
def getFileList(self, current=None):
'''
Returns the list of available files
'''
file_list = []
for file_hash in self.orchestrator.files:
file_info = TrawlNet.FileInfo()
file_info.hash = file_hash
file_info.name = self.orchestrator.files[file_hash]
file_list.append(file_info)
return file_list
def create(self, fileName, current=None):
#Realizamos una comprobación de la existencia del fichero en el directorio
if not os.path.isfile(os.path.join(FILE_DIR, fileName)):
raise TrawlNet.FileDoesNotExistError(
'ERROR: The file \'%s\' does not exist' % file)
servant = SenderI(fileName)
proxy = current.adapter.add(
servant,
current.adapter.getCommunicator().stringToIdentity(fileName))
print('Creado sender para descarga del archivo \'%s\'' % fileName)
sys.stdout.flush()
return TrawlNet.SenderPrx.checkedCast(proxy)
def createPeers(self, files, current=None):
"""Crea una pareja receiver-sender por cada fichero que se desea transferir"""
try:
receiver_list = list()
self.transfer = TrawlNet.TransferPrx.checkedCast(self.proxy)
for i in files:
sender = self.sender_factory.create(i)
receiver = self.receiver_factory.create(
i, sender, self.transfer)
self.dict_peers["../files/" + i] = [sender, receiver]
receiver_list.append(receiver)
except Exception as err:
print("Error. Transfer create peers failed {}".format(err))
raise TrawlNet.FileDoesNotExistError(str(err))
return receiver_list
def start(self, current=None):
remote_EOF = False
BLOCK_SIZE = 1024
#Transferencia del archivo.
print("Transferiendo el archivo " + self.fileName + ".")
with open(os.path.join("./downloads/", self.fileName), "wb") as file_:
remote_EOF = False
while not remote_EOF:
data = self.sender.receive(BLOCK_SIZE)
if len(data) > 1:
data = data[1:]
data = binascii.a2b_base64(data)
remote_EOF = len(data) < BLOCK_SIZE
if data:
file_.write(data)
self.sender.close()
print("Transferencia del archivo " + self.fileName + " completada.")
#Crear PeerInfo(Transfer, filename)
peerInfo = TrawlNet.PeerInfo()
peerInfo.transfer = self.transfer
peerInfo.fileName = self.fileName
#Llamar a peerFinished
self.peerEvent.peerFinished(peerInfo)
def start(self, current=None):
print('Iniciando transferencia de fichero \'%s\'' % self.fileName)
#Solicitamos la transferencia de un chunk del archivo con el tamaño BLOCK_SIZE
#Si se quiere recibir una mayor cantidad de archivo haría falta modificar a un numero mayor
data = binascii.a2b_base64(self.sender.receive(BLOCK_SIZE)[1:])
#Realizamos la escritura en el directorio de descargas con los datos que hemos recibido del sender
with open(os.path.join(DOWNLOAD_DIR,self.fileName), "wb") as file:
file.write(data)
#Solicitamos el cierre del archivo cuando ya se ha recibido el archivo
self.sender.close()
print('Finalizada transferencia de fichero.')
#Preparamos todo lo necesario para utilizar el canal de eventos con el topico PeerEvent
key = 'IceStorm.TopicManager.Proxy'
proxy = current.adapter.getCommunicator().propertyToProxy(key)
if proxy is None:
raise RuntimeError('La propiedad \'%s\' no está definida' % key)
topic_manager = IceStorm.TopicManagerPrx.checkedCast(proxy)
if not topic_manager:
raise RuntimeError('El proxy \'%s\' no es válido.' % topic_manager)
topic_name = "PeerEvent"
try:
topic = topic_manager.retrieve(topic_name)
except IceStorm.NoSuchTopic:
topic = topic_manager.create(topic_name)
publisher = topic.getPublisher()
event = TrawlNet.PeerEventPrx.uncheckedCast(publisher)
#Publicamos en el canal de eventos de PeerEvent que ha terminado esta peer
event.peerFinished(TrawlNet.PeerInfo(transfer=self.transfer, fileName=self.fileName))
def createPeers(self, files, current=None):
TrawlNet.receiversList = []
#Comprobamos que no se ha repetido el nombre de ningun archivo
if set([file for file in files if files.count(file) > 1]):
raise RuntimeError(
'Se está intentando descargar varias veces un mismo archivo.')
#Comprobamos que todos los archivos que se han indicado existen en el directorio
for file in files:
if not os.path.isfile(os.path.join(FILE_DIR, file)):
raise TrawlNet.FileDoesNotExistError(
'El archivo \'%s\' no existe en el directorio' % file)
#Creamos la pareja sender-receiver para cada archivo
for file in files:
sender = self.senderFactory.create(file)
receiver = self.receiverFactory.create(file, sender, self.transfer)
#Almacenamos los receivers en la lista definida en TrawlNet
TrawlNet.receiversList.append(receiver)
print('Creadas parejas sender-receiver')
return TrawlNet.receiversList
def addDownloadTask(self, message, current=None):
fileInfo = TrawlNet.FileInfo()
fileInfo.name = download_mp3(message)
fileInfo.hash = video_id(message)
self.updateEventsPublisher.newFile(fileInfo)
return fileInfo