def upload(name):
"""
This page allows a user to upload a text or image file.
"""
error = valid_user(name)
if error is None:
if request.method == 'POST':
file = request.files['file']
if file and valid_file(file.filename):
# Sanitize the filename, save the file to the uploads
# folder, and add the file and owner info to the file database.
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
file_instance = Upload(name, filename)
flash('File was uploaded successfully.')
return redirect(url_for('files', name=name))
else:
flash("Invalid filename or file type.")
return render_template('upload.html')
# If an error occurs, display the error and
# redirect to the appropriate page.
display(error)
if 'logged_in' in session:
return redirect(url_for('upload', name=session['logged_in']))
else:
return redirect(url_for('login'))
def delete_entry(name, id):
"""
This page will delete an entry from the database.
"""
# Check if the user is logged in before allowing to delete files.
error = valid_user(name)
if error is None:
entry_instance = Entry.query.filter_by(id=id, userid=name).first()
if entry_instance and entry_instance.userid == name:
# Delete the entry from the database if it exists.
edb.session.delete(entry_instance)
edb.session.commit()
flash('Entry was deleted successfully.')
return redirect(url_for('entries', name=name))
else:
error = "Specified entry does not exist."
# If an error occurs, display the error and
# redirect to the appropriate page.
display(error)
if 'logged_in' in session:
return redirect(url_for('entries', name=session['logged_in']))
else:
return redirect(url_for('login'))
def upload(name):
"""
This page allows a user to upload a text or image file.
"""
error = valid_user(name)
if error is None:
if request.method == 'POST':
file = request.files['file']
if file and valid_file(file.filename):
# Sanitize the filename, save the file to the uploads
# folder, and add the file and owner info to the file database.
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
file_instance = Upload(name, filename)
flash('File was uploaded successfully.')
return redirect(url_for('files', name=name))
else:
flash("Invalid filename or file type.")
return render_template('upload.html')
# If an error occurs, display the error and
# redirect to the appropriate page.
display(error)
if 'logged_in' in session:
return redirect(url_for('upload', name=session['logged_in']))
else:
return redirect(url_for('login'))
def change_theme(name):
"""
This page will allow the user to change the appearance of their blog.
"""
# Check if the user is logged in before allowing to change theme.
error = valid_user(name)
if error is None:
if request.method == 'POST':
new_theme = request.form['theme']
user_instance = get_user(name)
# Change the user's theme, change the theme in browser and
# store the changed theme in the user database.
user_instance.theme = session['theme'] = new_theme
udb.session.commit()
flash('Theme changed to %s.' % new_theme.lower())
return redirect(url_for('change_theme', name=name))
return render_template('theme.html', username=name, theme=session['theme'])
# If an error occurs, display the error and
# redirect to the appropriate page.
display(error)
if 'logged_in' in session:
return redirect(url_for('upload', name=session['logged_in']))
else:
return redirect(url_for('login'))
def get_profile(profile):
tools.display(preset.message["retrieve_user"])
user_data = tools.get_chrome_element(profile, preset.preferences)
if not user_data:
tools.display(preset.message["invalid_profile"])
exit(1)
return tools.get_user(user_data)
def delete_file(name, filename):
"""
This page will delete a file from the database and uploads folder.
"""
# Check if the user is logged in before allowing to delete files.
error = valid_user(name)
if error is None:
file = Upload.query.filter_by(filename=filename).first()
if file and file.userid == name:
# Delete the file from the upload folder if it exists.
filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
if os.path.isfile(filepath):
os.remove(filepath)
# Delete the upload object from the database.
fdb.session.delete(file)
fdb.session.commit()
flash('File was deleted successfully.')
return redirect(url_for('entries', name=name))
else:
error = "Specified file does not exist."
# If an error occurs, display the error and
# redirect to the appropriate page.
display(error)
if 'logged_in' in session:
return redirect(url_for('entries', name=session['logged_in']))
else:
return redirect(url_for('login'))
def create():
"""
Allows a user to create a new account.
"""
error = None
if request.method == 'POST':
# Get submitted values from fields.
username = request.form['username']
password = request.form['password-a']
# Check if there is a user by the same username in the database.
user_instance = get_user(username)
if user_instance is not None:
error = 'Username is already taken.'
elif password != request.form['password-b']:
error = 'Passwords do not match.'
else:
# Add user to the database and log user in.
user_instance = User(username, password)
flash('Account created!')
session['logged_in'] = username
return redirect(url_for('files', name=username))
display(error)
return render_template('create.html')
def login():
"""
Allows a user to log in to their account.
Note that the index page redirects here by default.
"""
error = None
if 'logged_in' not in session:
session['theme'] = app.config['DEFAULT_THEME']
if request.method == 'POST':
username = request.form['username']
# Check to see if the user exists.
user_instance = get_user(username)
if user_instance is None:
error = 'Invalid username.'
elif not user_instance.check_pw(request.form['password']):
error = 'Invalid password.'
else:
session['logged_in'], session['posting_enabled'], session['theme'] = \
username, user_instance.posting_enabled, user_instance.theme
flash('Successfully logged in.')
return redirect(url_for('entries', name=username))
else:
return render_template('login.html', theme=session['theme'])
display(error)
return render_template('login.html', theme=session['theme'])
def entries(name):
"""
This page presents a user's blog posts and file uploads,
allowing visitors to download the files individually.
"""
if get_user(name) is not None:
session['theme'] = get_user(name).theme
entries = [e for e in Entry.query.all()]
entries.reverse()
uploads = [dict(userid=f.userid, filename=f.filename, filetype=f.filetype) \
for f in Upload.query.all()]
if request.method == "POST":
title, text = request.form['title'], request.form['text']
text = urlify(text)
entry_instance = Entry(name, title, text)
edb.session.add(entry_instance)
edb.session.commit()
flash('Post successful!')
return redirect(url_for('entries', name=name))
else:
return render_template('entries.html', username=name,
uploads=uploads, entries=entries, theme=session['theme'])
else:
display("User does not exist.")
return redirect(url_for('login'))
def create():
"""
Allows a user to create a new account.
"""
error = None
if request.method == 'POST':
# Get submitted values from fields.
username = request.form['username']
password = request.form['password-a']
# Check if there is a user by the same username in the database.
user_instance = get_user(username)
if user_instance is not None:
error = 'Username is already taken.'
elif password != request.form['password-b']:
error = 'Passwords do not match.'
else:
# Add user to the database and log user in.
user_instance = User(username, password)
flash('Account created!')
session['logged_in'] = username
return redirect(url_for('files', name=username))
display(error)
return render_template('create.html')
def append_data_table(data_table, url_list):
tools.display(preset.message["appending_data_table"])
for url_item in url_list:
head = preset.Header()
head.Hostname = htmlSupport.parse_url(url_item)[2]
head.URL_Clean = htmlSupport.clean_url(url_item)
head.URL = url_item
data_table.append(head.to_tuple())
return data_table
def import_text_file(text_file=preset.text_filename):
tools.underline()
tools.display(preset.message["import_text_file"] + " [" + text_file + "]")
tools.overline()
url_list = []
with open(text_file, encoding='utf-8') as text_file:
for url_item in text_file:
url_list.append(url_item.strip())
return url_list
def upload(name):
"""
This page allows a user to upload a text or image file.
"""
# Refuse access if posting is disabled for the user.
if "posting_enabled" in session and session['posting_enabled'] == False:
error = "Access denied."
display(error)
if 'logged_in' in session:
return redirect(url_for('entries', name=session['logged_in']))
else:
return redirect(url_for('login'))
# Check if the user is logged in before allowing to upload files.
error = valid_user(name)
if error is None:
if request.method == 'POST':
file = request.files['file']
if file and valid_file(file.filename):
# Sanitize the filename, save the file to the uploads
# folder, and add the file and owner info to the file database.
old_filename = filename = secure_filename(file.filename)
filetype = filename.rsplit('.', 1)[1].lower()
# Prevents duplicate filenames by appending (1), (2), etc.
# e.g. if two "images.jpg" are uploaded, the second one would
# become "images(1).jpg".
x = 0
while os.path.isfile(os.path.join(app.config['UPLOAD_FOLDER'], filename)):
x += 1
filename = ("%s(%s).%s" % (old_filename.rsplit('.', 1)[0], x, filetype))
# Save the file to the uploads folder.
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
file_instance = Upload(name, filename, filetype)
# Insert the upload object into the database.
fdb.session.add(file_instance)
fdb.session.commit()
flash('File was uploaded successfully.')
return redirect(url_for('entries', name=name))
else:
flash("Invalid filename or file type.")
return render_template('upload.html', username=name, theme=session['theme'])
# If an error occurs, display the error and
# redirect to the appropriate page.
display(error)
if 'logged_in' in session:
return redirect(url_for('upload', name=session['logged_in']))
else:
return redirect(url_for('login'))
def on_open_account(self, event):
profile_chooser_dialog = ProfileChooser(self, -1)
button_pressed = profile_chooser_dialog.ShowModal()
if button_pressed == wx.ID_OK:
tools.display(preset.message["loading_bookmarks"])
data_table = bookMarks.generate_data(
bookMarks.generate_bookmarks(self.selected_account))
self.main_url_panel.update_list(data_table)
self.main_url_panel.Update()
set_total_items(self)
else:
self.selected_account = -1
def get_terminal_size():
current_os = platform.system()
tuple_xy = None
if current_os == 'Windows':
tuple_xy = _get_terminal_size_windows()
if tuple_xy is None:
tuple_xy = _get_terminal_size_tput()
if current_os in ['Linux', 'Darwin'] or current_os.startswith('CYGWIN'):
tuple_xy = _get_terminal_size_linux()
if tuple_xy is None:
tools.display("default")
tuple_xy = (80, 25)
return tuple_xy
def view_users(name):
"""
This page will allow the administrator to view the blogs of all registered users.
"""
# Check if the logged in user is the administrator.
if 'logged_in' in session and session['logged_in'] == "admin":
users = [u for u in User.query.all()]
return render_template('all.html', username=name, users=users, theme=session['theme'])
else:
error = "Must be logged in as the administrator to access this page."
display(error)
return redirect(url_for('login'))
def run_chrome(profile, output, refresh, undupe, clean, import_txt,
get_hostname, output_name, list_profile, x_org_gui):
tools.debug("profile [", str(profile), "]\noutput [",
str(output), "]\nrefresh [", str(refresh),
"]\nundupe [", str(undupe), "]\nclean [",
str(clean), "]\nimport_txt [", str(import_txt),
"]\nget_hostname [", str(get_hostname), "]\noutput_name [",
str(output_name),
"]\nlist_profiles [" + str(list_profile) + "]")
if x_org_gui:
import chromeExport
chromeExport.main()
elif list_profile:
if not list_profile.isdigit():
list_profile = preset.all_profiles
tools.list_profiles(list_profile)
else:
if import_txt == preset.none and profile == preset.none:
tools.underline()
tools.display(preset.message["missing_parameter"])
tools.overline()
else:
tools.display(preset.new_line + preset.new_line)
tools.underline()
tools.display(preset.message["starting_export"])
if profile:
email, full, name = get_profile(profile)
bookmarks = bookMarks.generate_bookmarks(profile)
tools.underline()
tools.display(preset.message["process_user"] + ": {", full,
"} [" + email + "]")
tools.overline()
bookmarks_data = bookMarks.generate_data(bookmarks)
else:
bookmarks_data = []
if import_txt:
bookmarks_data = append_data_table(
bookmarks_data, import_text_file(import_txt))
if not output_name:
if output == "xlsx":
output_name = preset.xlsx_filename
else:
output_name = preset.html_filename
refresh = normalize(refresh)
undupe = normalize(undupe)
clean = normalize(clean)
get_hostname = normalize(get_hostname)
if output == "xlsx":
generate_work_book(output_name, bookmarks_data, refresh,
undupe, clean, get_hostname)
else:
generate_web_page(output_name, bookmarks_data, refresh, undupe,
clean, get_hostname)
def files(name):
"""
This page presents a user's uploaded files, and allows
the user to download them individually.
"""
error = valid_user(name)
if error is None:
uploads = [dict(userid=f.userid, filename=f.filename) \
for f in Upload.query.all()]
return render_template('files.html', username=name, uploads=uploads)
# If an error occurs, display the error and
# redirect to the appropriate page.
display(error)
if 'logged_in' in session:
return redirect(url_for('files', name=session['logged_in']))
else:
return redirect(url_for('login'))
def write_html(webpage_filename, folder):
with open(webpage_filename, 'w', encoding='utf-8') as html_file:
html_file.write(header)
for item in folder:
if isinstance(item, tools.Folder):
html_file.write(
open_folder(item.add_date, item.modify_date,
item.folder_name))
for url in item.children:
if isinstance(url, tools.Urls):
html_file.write(
add_url(url.url, url.add_date, url.title))
elif isinstance(url, tools.Folder):
tools.display(preset.message["new_folder"])
html_file.write(close_folder())
html_file.write(tail)
def files(name):
"""
This page presents a user's uploaded files, and allows
the user to download them individually.
"""
error = valid_user(name)
if error is None:
uploads = [dict(userid=f.userid, filename=f.filename) \
for f in Upload.query.all()]
return render_template('files.html', username=name, uploads=uploads)
# If an error occurs, display the error and
# redirect to the appropriate page.
display(error)
if 'logged_in' in session:
return redirect(url_for('files', name=session['logged_in']))
else:
return redirect(url_for('login'))
def uploaded_file(name, filename):
"""
This page will fetch a given file from the uploads folder,
provided the user has privileges to access the file.
"""
error = valid_user(name)
if error is None:
if has_file_access(session['logged_in'], filename):
return send_from_directory(app.config['UPLOAD_FOLDER'], filename)
else:
error = "Access denied."
# If an error occurs, display the error and
# redirect to the appropriate page.
display(error)
if 'logged_in' in session:
return redirect(url_for('files', name=session['logged_in']))
else:
return redirect(url_for('login'))
def uploaded_file(name, filename):
"""
This page will fetch a given file from the uploads folder,
provided the user has privileges to access the file.
"""
error = valid_user(name)
if error is None:
if has_file_access(session['logged_in'], filename):
return send_from_directory(app.config['UPLOAD_FOLDER'], filename)
else:
error = "Access denied."
# If an error occurs, display the error and
# redirect to the appropriate page.
display(error)
if 'logged_in' in session:
return redirect(url_for('files', name=session['logged_in']))
else:
return redirect(url_for('login'))
def login():
"""
Allows a user to log in to their account.
Note that the index page redirects here by default.
"""
error = None
if request.method == 'POST':
username = request.form['username']
# Check to see if the user exists.
user_instance = get_user(username)
if user_instance is None:
error = 'Invalid username.'
elif not user_instance.check_pw(request.form['password']):
error = 'Invalid password.'
else:
session['logged_in'] = username
flash('Successfully logged in.')
return redirect(url_for('files', name=username))
display(error)
return render_template('login.html')
def login():
"""
Allows a user to log in to their account.
Note that the index page redirects here by default.
"""
error = None
if request.method == 'POST':
username = request.form['username']
# Check to see if the user exists.
user_instance = get_user(username)
if user_instance is None:
error = 'Invalid username.'
elif not user_instance.check_pw(request.form['password']):
error = 'Invalid password.'
else:
session['logged_in'] = username
flash('Successfully logged in.')
return redirect(url_for('files', name=username))
display(error)
return render_template('login.html')
def create():
"""
Allows a user to create a new account.
"""
error = None
if 'logged_in' not in session:
session['theme'] = app.config['DEFAULT_THEME']
if request.method == 'POST':
# Get submitted values from fields.
username, password = request.form['username'], request.form['password-a']
# Check if there is a user by the same username in the database.
user_instance = get_user(username)
if user_instance is not None:
error = 'Username is already taken.'
elif re.search(r'[^_a-zA-Z0-9]', username):
error = 'Username must only contain letters, numbers, and underscores.'
elif password == '':
error = 'Please enter a password.'
elif password != request.form['password-b']:
error = 'Passwords do not match.'
else:
# Add user to the database and log user in.
user_instance = User(username, password)
user_instance.theme = app.config['DEFAULT_THEME']
udb.session.add(user_instance)
udb.session.commit()
flash('Account created!')
session['logged_in'], session['posting_enabled'], session['theme'] = \
username, user_instance.posting_enabled, user_instance.theme
return redirect(url_for('entries', name=username))
display(error)
return render_template('create.html', theme=session['theme'])
def generate_bookmarks(profile):
tools.display(preset.message["generating_bookmarks"])
bookmarks_file = tools.get_chrome_element(profile, preset.bookmarks)
if bookmarks_file:
return Bookmarks(bookmarks_file)
return None
def generate_work_book(spreadsheet_filename, data_table, reload_url_title,
remove_duplicated_urls, remove_tracking_from_url,
get_hostname_title):
tools.debug("filename [", str(spreadsheet_filename), "]\nreload_title[",
str(reload_url_title), "]\nremove_dupes[",
str(remove_duplicated_urls), "]\nremove_track[",
str(remove_tracking_from_url), "]\nget_hostname[",
str(get_hostname_title), "]")
tools.display(preset.message["generating_workbook"] + " [" +
spreadsheet_filename + "]")
excel_workbook = Workbook()
excel_worksheet = excel_workbook.active
excel_worksheet.title = preset.message["chrome_urls"]
reload_url_title_disabled = True
if reload_url_title:
tools.underline()
tools.display(preset.message["get_url_status"])
tools.overline()
reload_url_title_disabled = False
if get_hostname_title:
tools.underline()
tools.display(preset.message["resolving_hostnames"])
tools.overline()
total_items = len(data_table)
disabled = False
if preset.gui_progress_dialog:
disabled = True
with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
temporary_table = []
utils.update_progress(preset.message["resolving_hostnames"], -1,
total_items)
for index, data_row in enumerate(data_table):
temporary_table.append(
utils.update_tuple(
data_row,
htmlSupport.get_title(preset.protocol +
data_row[22])[1], 2))
progress_bar.update(1)
if utils.update_progress(preset.message["resolving_hostnames"],
index, total_items):
break
data_table = temporary_table
visited_url_address = set()
data_table_without_duplicates = []
tools.display(preset.message["find_duplicated_lines"])
total_items = len(data_table)
disabled = False
if reload_url_title_disabled:
disabled = True
elif preset.gui_progress_dialog:
disabled = True
with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
utils.update_progress(preset.message["find_duplicated_lines"], -1,
total_items)
for index, data_row in enumerate(data_table):
if utils.update_progress(preset.message["find_duplicated_lines"],
index, total_items):
break
header = preset.Header()
header.set_data(data_row)
if remove_tracking_from_url:
url_address = header.get_name(preset.url_clean_attr)
else:
url_address = header.get_name(preset.url_attr)
url_name = header.get_name(preset.url_name_attr)
if url_address not in visited_url_address:
visited_url_address.add(url_address)
data_table_without_duplicates.append(
("MAIN",
get_title_conditional(progress_bar,
reload_url_title_disabled, url_name,
url_address)) + data_row)
elif not remove_duplicated_urls:
data_table_without_duplicates.append(
("DUPE",
get_title_conditional(progress_bar,
reload_url_title_disabled, url_name,
url_address)) + data_row)
tools.display(preset.message["writing_spreadsheet"])
tools.overline()
total_items = len(data_table_without_duplicates)
disabled = False
if preset.gui_progress_dialog:
disabled = True
with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
utils.update_progress(preset.message["writing_spreadsheet"], -1,
total_items)
data_row_header = ["DUPE", "Site Name"]
for item in preset.label_dictionary:
data_row_header.append(preset.label_dictionary[item])
excel_worksheet.append(tuple(data_row_header))
for index, data_row in enumerate(data_table_without_duplicates):
progress_bar.update(1)
if utils.update_progress(preset.message["writing_spreadsheet"],
index, total_items):
break
excel_worksheet.append(data_row)
excel_worksheet.freeze_panes = "A2"
excel_worksheet.auto_filter.ref = "A1:AU30000"
tools.underline()
tools.display(preset.message["format_columns"])
tools.overline()
font_columns = [
'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'AA', 'AB', 'AC', 'AD', 'AE', 'AF',
'AG', 'AH', 'AI', 'AJ', 'AK', 'AL', 'AM', 'AN', 'AO', 'AP', 'AQ', 'AR',
'AS', 'AT', 'AU'
]
total_items = (len(font_columns) * len(excel_worksheet['T']))
disabled = False
if preset.gui_progress_dialog:
disabled = True
with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
utils.update_progress(preset.message["format_columns"], -1,
total_items)
for font_column in font_columns:
for index, worksheet_column in enumerate(
excel_worksheet[font_column]):
progress_bar.update(1)
if utils.update_progress(preset.message["format_columns"],
index, total_items):
break
worksheet_column.font = Font(size=10, name='Courier New')
tools.underline()
tools.display(preset.message["format_dates"])
tools.overline()
date_columns = ['G', 'H', 'I', 'P', 'Q', 'R']
total_items = (len(date_columns) * len(excel_worksheet['G']))
disabled = False
if preset.gui_progress_dialog:
disabled = True
with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
utils.update_progress(preset.message["format_dates"], -1, total_items)
for date_column in date_columns:
excel_worksheet.column_dimensions[date_column].width = 18
for index, worksheet_column in enumerate(
excel_worksheet[date_column]):
progress_bar.update(1)
if utils.update_progress(preset.message["format_dates"], index,
total_items):
break
worksheet_column.number_format = preset.number_format
tools.underline()
tools.display(preset.message["hide_columns"])
hidden_columns = [
'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'AA', 'AB', 'AC',
'AD'
]
for h in hidden_columns:
excel_worksheet.column_dimensions[h].width = 9
excel_worksheet.column_dimensions[h].hidden = True
tools.display(preset.message["format_header"])
for cell in excel_worksheet["1:1"]:
cell.font = Font(bold=True)
tools.display(preset.message["sizing_columns"])
excel_worksheet.column_dimensions['S'].width = 30
excel_worksheet.column_dimensions['T'].width = 85
tools.display(preset.message["saving_workbook"])
excel_workbook.save(spreadsheet_filename)
tools.display(preset.message["done"])
tools.overline()
def generate_from_txt(url_list):
tools.display(preset.message["generating_from_text"])
txt_header = []
return append_data_table(txt_header, url_list)
def generate_web_page(web_page_filename, data_table, reload_url_title,
remove_duplicated_urls, remove_tracking_from_url,
get_hostname_title):
tools.debug("filename [", str(web_page_filename), "]\nreload_title[",
str(reload_url_title), "]\nremove_dupes[",
str(remove_duplicated_urls), "]\nremove_track[",
str(remove_tracking_from_url), "]\nget_hostname[",
str(get_hostname_title), "]")
tools.display(preset.message["generating_html"] + " [" +
web_page_filename + "]")
visited_hostname_title = set()
visited_url_address = set()
folder_list = []
data_table_without_duplicates = []
if remove_duplicated_urls:
tools.underline()
tools.display(preset.message["removing_duplicates"])
tools.overline()
total_items = len(data_table)
disabled = False
if preset.gui_progress_dialog:
disabled = True
with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
utils.update_progress(preset.message["removing_duplicates"], -1,
total_items)
for index, data_row in enumerate(data_table):
header = preset.Header()
header.set_data(data_row)
progress_bar.update(1)
if utils.update_progress(preset.message["removing_duplicates"],
index, total_items):
break
if remove_tracking_from_url:
url_address = header.get_name(preset.url_clean_attr)
else:
url_address = header.get_name(preset.url_attr)
if url_address not in visited_url_address:
visited_url_address.add(url_address)
data_table_without_duplicates.append(data_row)
else:
data_table_without_duplicates = data_table
tools.underline()
tools.display(preset.message["writing_html"])
tools.overline()
total_items = len(data_table_without_duplicates)
disabled = False
if preset.gui_progress_dialog:
disabled = True
with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
utils.update_progress(preset.message["writing_html"], -1, total_items)
for index, data_row in enumerate(data_table_without_duplicates):
header = preset.Header()
header.set_data(data_row)
progress_bar.update(1)
if utils.update_progress(preset.message["writing_html"], index,
total_items):
break
url_title = header.get_name(preset.url_name_attr)
if remove_tracking_from_url:
url_address = header.get_name(preset.url_clean_attr)
else:
url_address = header.get_name(preset.url_attr)
hostname_title = header.get_name(preset.hostname_attr)
original_hostname = header.get_name(preset.hostname_attr)
if reload_url_title:
status_number, url_title = htmlSupport.get_title(url_address)
if status_number != 0:
url_title = "[ " + url_title + " " + str(
status_number) + " - " + header.get_name(
preset.url_name_attr) + " ]"
if get_hostname_title:
status_number, hostname_title = htmlSupport.get_title(
preset.protocol + original_hostname)
if status_number != 0:
hostname_title = "[ " + hostname_title + " " + str(
status_number) + " - " + original_hostname + " ]"
if hostname_title not in visited_hostname_title:
url_data = tools.Urls(url_address,
header.get_name(preset.url_added_attr),
url_title)
visited_hostname_title.add(hostname_title)
folder_list.append(
tools.Folder(header.get_name(preset.folder_added_attr),
header.get_name(preset.folder_modified_attr),
hostname_title, [url_data]))
else:
url_data = tools.Urls(url_address,
header.get_name(preset.url_added_attr),
url_title)
for folder in folder_list:
if folder.folder_name == hostname_title:
folder.add_url(url_data)
tools.underline()
tools.display(preset.message["saving_html"])
tools.overline()
htmlExport.write_html(web_page_filename, folder_list)
tools.display(preset.message["done"])
tools.overline()
