def __init__(self, filename, torrent_dir, download_dir):
self.id = self.generate_id()
tracker_url, info = self.parse_meta(torrent_dir + '/' + filename)
self.info = info
self.info_hash = self.get_info_hash()
self.name = filename[:-8]
self.files = Files(info, download_dir + '/' + self.name)
self.tracker = Tracker(tracker_url)
self.uploaded = 0
self.downloaded = self.files.get_downloaded()
self.speed = 0
self.started = False
self.active = False
self.completed = False
self.connections = {}
self.connections_updated = None
self.lock = Lock()
def __init__(self, argv):
self.word_list = []
self.thread_pool = ThreadPool(4)
self.verbose = False
self.version = '0.0.1'
self.errors = []
self.website = 'https://github.com/m4l1c3/url-check'
self.urls = []
self.out_file = ''
self.presentation = Presentation()
self.files = Files()
self.source_word_list = argv.wordlist
self.url_validation = re.compile(
r'^(?:http|ftp)s?://' # http:// or https://
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
r'(?::\d+)?' # optional port
r'(?:/?|[/?]\S+)$',
re.IGNORECASE)
self.parse_arguments(argv)
self.run()
class Torrent:
def __init__(self, filename, torrent_dir, download_dir):
self.id = self.generate_id()
tracker_url, info = self.parse_meta(torrent_dir + '/' + filename)
self.info = info
self.info_hash = self.get_info_hash()
self.name = filename[:-8]
self.files = Files(info, download_dir + '/' + self.name)
self.tracker = Tracker(tracker_url)
self.uploaded = 0
self.downloaded = self.files.get_downloaded()
self.speed = 0
self.started = False
self.active = False
self.completed = False
self.connections = {}
self.connections_updated = None
self.lock = Lock()
def generate_id(self):
unique_part = ''.join(random.choices(string.digits, k=12))
return '-{}-{}'.format(ID_PREFIX, unique_part)
def parse_meta(self, filename):
raw_meta = bencode.decode(open(filename, 'rb').read())[0]
return raw_meta[b'announce'].decode(), raw_meta[b'info']
def get_info_hash(self):
return sha1(bencode.encode(self.info)).digest()
def get_num_of_active_peers(self):
return len([c for c in self.connections.values()
if not c.broken])
def start(self, files_indices):
self.set_files_status(files_indices)
self.update_connections()
self.started = True
self.active = True
while self.downloaded < self.files.total_length and self.active:
self.update_connections()
self.distribute_requests()
self.collect_pieces()
time.sleep(LOOP_TIME)
self.speed = 0
if self.downloaded == self.files.total_length:
self.completed = True
def set_files_status(self, files_indices):
if not files_indices:
self.files.download_files()
else:
for index in files_indices:
if index < 0 or index >= len(self.files.files):
continue
self.files.download_file(index)
def save_state(self):
with self.lock:
self.files.close_files()
self.files.save_bitfield()
def get_download_info(self):
return '{} of {} KB on {} kbps'.format(self.downloaded,
self.files.total_length,
self.speed/1000)
def update_connections(self):
if (self.connections_updated and
time.time() - self.connections_updated <
self.tracker.interval):
return
for peer in self.request_peers():
if peer['ip'] in self.connections:
continue
conn = Connection(self.id, self.info_hash,
self.files, Lock())
self.connections[peer['ip']] = conn
t = Thread(target=conn.initiate, args=(peer['ip'],
peer['port']))
t.daemon = True
t.start()
self.peers_updated = time.time()
def request_peers(self):
def get_query():
def get_event():
return 'started' if not self.started else ''
params = {'info_hash': parse.quote(self.info_hash),
'peer_id': self.id,
'port': PORT,
'uploaded': self.uploaded,
'downloaded': self.downloaded,
'left': self.get_left()}
event = get_event()
if event:
params['event'] = event
return '?{}'.format('&'.join(['{}={}'.format(k, v)
for k, v in params.items()]))
return self.tracker.announce(get_query())
def get_left(self):
return self.files.total_length - self.downloaded
def distribute_requests(self):
def should_request(index, piece):
return (not self.files.piece_skip(index) and
not piece['have'] and (not piece['requested'] or
time.time() - piece['requested'] > REQUEST_TIMEOUT))
def get_max_requests_num():
return max(2, 2 * self.speed / self.files.piece_length)
requested = 0
conn_list = list(self.connections.values())
for index, piece in enumerate(self.files.pieces):
if should_request(index, piece):
random.shuffle(conn_list)
for conn in conn_list:
if index in conn.has_pieces and not conn.broken:
conn.request_piece(index)
piece['requested'] = time.time()
requested += 1
break
if requested == get_max_requests_num():
break
def collect_pieces(self):
data_length = 0
for conn in self.connections.values():
with conn.lock:
taken_pieces = []
for index, data in conn.completed_pieces.items():
hash_ = sha1(data).digest()
if self.files.hash_is_correct(index, hash_):
with self.lock:
self.files.write_piece(index, data)
data_length += len(data)
taken_pieces.append(index)
for index in taken_pieces:
conn.completed_pieces.pop(index)
self.downloaded += data_length
self.speed = data_length / LOOP_TIME
def __init__(self):
self.constants = cts()
self.files = Files()
class UrlChecker(object):
def __init__(self, argv):
self.word_list = []
self.thread_pool = ThreadPool(4)
self.verbose = False
self.version = '0.0.1'
self.errors = []
self.website = 'https://github.com/m4l1c3/url-check'
self.urls = []
self.out_file = ''
self.presentation = Presentation()
self.files = Files()
self.source_word_list = argv.wordlist
self.url_validation = re.compile(
r'^(?:http|ftp)s?://' # http:// or https://
r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # domain...
r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|' # ...or ipv4
r'\[?[A-F0-9]*:[A-F0-9:]+\]?)' # ...or ipv6
r'(?::\d+)?' # optional port
r'(?:/?|[/?]\S+)$',
re.IGNORECASE)
self.parse_arguments(argv)
self.run()
def main(self):
if self.validate_wordlist():
self.thread_pool.map(self.request, self.word_list)
if self.out_file is not '':
self.files.save_output(self.out_file, urls)
def run(self):
self.setup_wordlist()
self.presentation.print_header(self.version)
self.main()
self.presentation.print_footer()
def parse_arguments(self, argv):
if len(sys.argv) < 1:
sys.exit("Not enough args")
if argv.output is not None:
self.out_file = argv.output
if argv.threads is not None and str.isdigit(argv.threads):
self.thread_pool = ThreadPool(int(argv.threads))
if argv.url is not None and self.url_validation.match(argv.url):
self.word_list.append(argv.url)
def validate_wordlist(self):
if not self.word_list:
print(colored('No URLs specified', 'red'))
return False
else:
print(colored('Beginning URL Check', 'magenta'))
return True
def setup_wordlist(self):
if self.source_word_list is not None and os.path.isfile(
self.source_word_list):
try:
with open(self.source_word_list, 'r') as word_list:
self.word_list = word_list.read().splitlines()
except IOError as error:
self.errors.append(error)
print(error)
@staticmethod
def request(url):
try:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
http = urllib3.PoolManager()
response = http.request('GET', url)
print(
colored('* ' + url + ' ' + str(response.status),
get_response_color(response)))
urls.append(url + ' ' + str(response.status))
except Exception as ex:
response = ex
import os
import sys
import time
from datetime import datetime
from datetime import timedelta
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from modules.tasksManager import TaskManager
from modules.files import Files
from modules.constants import Constants as cts
from modules.db_session import session
if __name__ == '__main__':
fls = Files()
cts = cts()
n=0
while True:
task = TaskManager.next()
if not task:
print('---No tasks to execute---')
time.sleep(5)
else:
try:
exitValue = fls.fn_move_file(cts.ORIGIN_PATH, cts.DESTINY_PATH, task.file_name)
except Exception as ex:
task.failed(ex)
if (task.error is None or task.error == ' ') and (exitValue == cts.OK):
n+=1
import os
import sys
import random
from os import path
import time
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from modules.tasksManager import TaskManager
from modules.files import Files
from modules.constants import Constants as cts
from modules.db_session import session
if __name__ == '__main__':
cts = cts()
files = Files()
iters = cts.ITER
doneList = list()
while True:
try:
filesList = files.fn_get_list_path(cts.ORIGIN_PATH)[0]
fileName = random.choice(filesList)
id = fileName.split('.')[0]
if id not in doneList:
taskManager = TaskManager(id=id,
file_name=fileName,
fails=False,
processing=False,
error=' ')
session.add(taskManager)