Example #1
0
def upload(name):
    """
	This page allows a user to upload a text or image file.
	"""
    error = valid_user(name)
    if error is None:
        if request.method == 'POST':
            file = request.files['file']
            if file and valid_file(file.filename):
                # Sanitize the filename, save the file to the uploads
                # folder, and add the file and owner info to the file database.
                filename = secure_filename(file.filename)
                file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
                file_instance = Upload(name, filename)

                flash('File was uploaded successfully.')
                return redirect(url_for('files', name=name))
            else:
                flash("Invalid filename or file type.")
        return render_template('upload.html')

    # If an error occurs, display the error and
    # redirect to the appropriate page.
    display(error)
    if 'logged_in' in session:
        return redirect(url_for('upload', name=session['logged_in']))
    else:
        return redirect(url_for('login'))
Example #2
0
def delete_entry(name, id):
    """
    This page will delete an entry from the database.
    """

    # Check if the user is logged in before allowing to delete files.
    error = valid_user(name)
    if error is None:
        entry_instance = Entry.query.filter_by(id=id, userid=name).first()
        if entry_instance and entry_instance.userid == name:
            # Delete the entry from the database if it exists.
            edb.session.delete(entry_instance)
            edb.session.commit()

            flash('Entry was deleted successfully.')
            return redirect(url_for('entries', name=name))
        else:
            error = "Specified entry does not exist."

    # If an error occurs, display the error and
    # redirect to the appropriate page.
    display(error)
    if 'logged_in' in session:
        return redirect(url_for('entries', name=session['logged_in']))
    else:
        return redirect(url_for('login'))
Example #3
0
def upload(name):
	"""
	This page allows a user to upload a text or image file.
	"""
	error = valid_user(name)
	if error is None:
		if request.method == 'POST':
			file = request.files['file']
			if file and valid_file(file.filename):
				# Sanitize the filename, save the file to the uploads
				# folder, and add the file and owner info to the file database.
				filename = secure_filename(file.filename)
				file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
				file_instance = Upload(name, filename)

				flash('File was uploaded successfully.')
				return redirect(url_for('files', name=name))
			else:
				flash("Invalid filename or file type.")
		return render_template('upload.html')

	# If an error occurs, display the error and
	# redirect to the appropriate page.
	display(error)
	if 'logged_in' in session:
		return redirect(url_for('upload', name=session['logged_in']))
	else:
		return redirect(url_for('login'))
Example #4
0
def change_theme(name):
    """
    This page will allow the user to change the appearance of their blog.
    """

    # Check if the user is logged in before allowing to change theme.
    error = valid_user(name)
    if error is None:
        if request.method == 'POST':
            new_theme = request.form['theme']
            user_instance = get_user(name)

            # Change the user's theme, change the theme in browser and
            # store the changed theme in the user database.
            user_instance.theme = session['theme'] = new_theme
            udb.session.commit()

            flash('Theme changed to %s.' % new_theme.lower())
            return redirect(url_for('change_theme', name=name))

        return render_template('theme.html', username=name, theme=session['theme'])

    # If an error occurs, display the error and
    # redirect to the appropriate page.
    display(error)
    if 'logged_in' in session:
        return redirect(url_for('upload', name=session['logged_in']))
    else:
        return redirect(url_for('login'))
Example #5
0
def get_profile(profile):
    tools.display(preset.message["retrieve_user"])
    user_data = tools.get_chrome_element(profile, preset.preferences)
    if not user_data:
        tools.display(preset.message["invalid_profile"])
        exit(1)
    return tools.get_user(user_data)
Example #6
0
def delete_file(name, filename):
    """
    This page will delete a file from the database and uploads folder.
    """

    # Check if the user is logged in before allowing to delete files.
    error = valid_user(name)
    if error is None:
        file = Upload.query.filter_by(filename=filename).first()
        if file and file.userid == name:
            # Delete the file from the upload folder if it exists.
            filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
            if os.path.isfile(filepath):
                os.remove(filepath)

            # Delete the upload object from the database.
            fdb.session.delete(file)
            fdb.session.commit()

            flash('File was deleted successfully.')
            return redirect(url_for('entries', name=name))
        else:
            error = "Specified file does not exist."

    # If an error occurs, display the error and
    # redirect to the appropriate page.
    display(error)
    if 'logged_in' in session:
        return redirect(url_for('entries', name=session['logged_in']))
    else:
        return redirect(url_for('login'))
Example #7
0
def create():
	"""
	Allows a user to create a new account.
	"""
	error = None
	if request.method == 'POST':
		# Get submitted values from fields.
		username = request.form['username']
		password = request.form['password-a']

		# Check if there is a user by the same username in the database.
		user_instance = get_user(username)
		if user_instance is not None:
			error = 'Username is already taken.'
		elif password != request.form['password-b']:
			error = 'Passwords do not match.'
		else:
			# Add user to the database and log user in.
			user_instance = User(username, password)
			flash('Account created!')
			session['logged_in'] = username
			return redirect(url_for('files', name=username))

	display(error)
	return render_template('create.html')
Example #8
0
def login():
    """
    Allows a user to log in to their account.
    Note that the index page redirects here by default.
    """

    error = None
    if 'logged_in' not in session:
        session['theme'] = app.config['DEFAULT_THEME']

    if request.method == 'POST':
        username = request.form['username']

        # Check to see if the user exists.
        user_instance = get_user(username)
        if user_instance is None:
            error = 'Invalid username.'
        elif not user_instance.check_pw(request.form['password']):
            error = 'Invalid password.'
        else:
            session['logged_in'], session['posting_enabled'], session['theme'] = \
                username, user_instance.posting_enabled, user_instance.theme
            flash('Successfully logged in.')
            return redirect(url_for('entries', name=username))
    else:
        return render_template('login.html', theme=session['theme'])

    display(error)
    return render_template('login.html', theme=session['theme'])
Example #9
0
def entries(name):
    """
    This page presents a user's blog posts and file uploads,
    allowing visitors to download the files individually.
    """

    if get_user(name) is not None:
        session['theme'] = get_user(name).theme
        entries = [e for e in Entry.query.all()]
        entries.reverse()
        uploads = [dict(userid=f.userid, filename=f.filename, filetype=f.filetype) \
                   for f in Upload.query.all()]

        if request.method == "POST":
            title, text = request.form['title'], request.form['text']
            text = urlify(text)

            entry_instance = Entry(name, title, text)
            edb.session.add(entry_instance)
            edb.session.commit()

            flash('Post successful!')
            return redirect(url_for('entries', name=name))
        else:
            return render_template('entries.html', username=name,
                                   uploads=uploads, entries=entries, theme=session['theme'])
    else:
        display("User does not exist.")
        return redirect(url_for('login'))
Example #10
0
def create():
    """
	Allows a user to create a new account.
	"""
    error = None
    if request.method == 'POST':
        # Get submitted values from fields.
        username = request.form['username']
        password = request.form['password-a']

        # Check if there is a user by the same username in the database.
        user_instance = get_user(username)
        if user_instance is not None:
            error = 'Username is already taken.'
        elif password != request.form['password-b']:
            error = 'Passwords do not match.'
        else:
            # Add user to the database and log user in.
            user_instance = User(username, password)
            flash('Account created!')
            session['logged_in'] = username
            return redirect(url_for('files', name=username))

    display(error)
    return render_template('create.html')
Example #11
0
def append_data_table(data_table, url_list):
    tools.display(preset.message["appending_data_table"])
    for url_item in url_list:
        head = preset.Header()
        head.Hostname = htmlSupport.parse_url(url_item)[2]
        head.URL_Clean = htmlSupport.clean_url(url_item)
        head.URL = url_item
        data_table.append(head.to_tuple())
    return data_table
Example #12
0
def import_text_file(text_file=preset.text_filename):
    tools.underline()
    tools.display(preset.message["import_text_file"] + " [" + text_file + "]")
    tools.overline()
    url_list = []
    with open(text_file, encoding='utf-8') as text_file:
        for url_item in text_file:
            url_list.append(url_item.strip())
    return url_list
Example #13
0
def upload(name):
    """
    This page allows a user to upload a text or image file.
    """

    # Refuse access if posting is disabled for the user.
    if "posting_enabled" in session and session['posting_enabled'] == False:
        error = "Access denied."
        display(error)
        if 'logged_in' in session:
            return redirect(url_for('entries', name=session['logged_in']))
        else:
            return redirect(url_for('login'))

    # Check if the user is logged in before allowing to upload files.
    error = valid_user(name)
    if error is None:
        if request.method == 'POST':
            file = request.files['file']
            if file and valid_file(file.filename):
                # Sanitize the filename, save the file to the uploads
                # folder, and add the file and owner info to the file database.
                old_filename = filename = secure_filename(file.filename)
                filetype = filename.rsplit('.', 1)[1].lower()

                # Prevents duplicate filenames by appending (1), (2), etc.
                # e.g. if two "images.jpg" are uploaded, the second one would
                # become "images(1).jpg".
                x = 0
                while os.path.isfile(os.path.join(app.config['UPLOAD_FOLDER'], filename)):
                    x += 1
                    filename = ("%s(%s).%s" % (old_filename.rsplit('.', 1)[0], x, filetype))

                # Save the file to the uploads folder.
                file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
                file_instance = Upload(name, filename, filetype)

                # Insert the upload object into the database.
                fdb.session.add(file_instance)
                fdb.session.commit()

                flash('File was uploaded successfully.')
                return redirect(url_for('entries', name=name))
            else:
                flash("Invalid filename or file type.")
        return render_template('upload.html', username=name, theme=session['theme'])

    # If an error occurs, display the error and
    # redirect to the appropriate page.
    display(error)
    if 'logged_in' in session:
        return redirect(url_for('upload', name=session['logged_in']))
    else:
        return redirect(url_for('login'))
Example #14
0
 def on_open_account(self, event):
     profile_chooser_dialog = ProfileChooser(self, -1)
     button_pressed = profile_chooser_dialog.ShowModal()
     if button_pressed == wx.ID_OK:
         tools.display(preset.message["loading_bookmarks"])
         data_table = bookMarks.generate_data(
             bookMarks.generate_bookmarks(self.selected_account))
         self.main_url_panel.update_list(data_table)
         self.main_url_panel.Update()
         set_total_items(self)
     else:
         self.selected_account = -1
Example #15
0
def get_terminal_size():
    current_os = platform.system()
    tuple_xy = None
    if current_os == 'Windows':
        tuple_xy = _get_terminal_size_windows()
        if tuple_xy is None:
            tuple_xy = _get_terminal_size_tput()
    if current_os in ['Linux', 'Darwin'] or current_os.startswith('CYGWIN'):
        tuple_xy = _get_terminal_size_linux()
    if tuple_xy is None:
        tools.display("default")
        tuple_xy = (80, 25)
    return tuple_xy
Example #16
0
def view_users(name):
    """
    This page will allow the administrator to view the blogs of all registered users.
    """

    # Check if the logged in user is the administrator.
    if 'logged_in' in session and session['logged_in'] == "admin":
        users = [u for u in User.query.all()]
        return render_template('all.html', username=name, users=users, theme=session['theme'])
    else:
        error = "Must be logged in as the administrator to access this page."
        display(error)
        return redirect(url_for('login'))
Example #17
0
def run_chrome(profile, output, refresh, undupe, clean, import_txt,
               get_hostname, output_name, list_profile, x_org_gui):
    tools.debug("profile       [", str(profile), "]\noutput        [",
                str(output), "]\nrefresh       [", str(refresh),
                "]\nundupe        [", str(undupe), "]\nclean         [",
                str(clean), "]\nimport_txt    [", str(import_txt),
                "]\nget_hostname  [", str(get_hostname), "]\noutput_name   [",
                str(output_name),
                "]\nlist_profiles [" + str(list_profile) + "]")
    if x_org_gui:
        import chromeExport
        chromeExport.main()
    elif list_profile:
        if not list_profile.isdigit():
            list_profile = preset.all_profiles
        tools.list_profiles(list_profile)
    else:
        if import_txt == preset.none and profile == preset.none:
            tools.underline()
            tools.display(preset.message["missing_parameter"])
            tools.overline()
        else:
            tools.display(preset.new_line + preset.new_line)
            tools.underline()
            tools.display(preset.message["starting_export"])

            if profile:
                email, full, name = get_profile(profile)
                bookmarks = bookMarks.generate_bookmarks(profile)
                tools.underline()
                tools.display(preset.message["process_user"] + ": {", full,
                              "} [" + email + "]")
                tools.overline()
                bookmarks_data = bookMarks.generate_data(bookmarks)
            else:
                bookmarks_data = []

            if import_txt:
                bookmarks_data = append_data_table(
                    bookmarks_data, import_text_file(import_txt))

            if not output_name:
                if output == "xlsx":
                    output_name = preset.xlsx_filename
                else:
                    output_name = preset.html_filename

            refresh = normalize(refresh)
            undupe = normalize(undupe)
            clean = normalize(clean)
            get_hostname = normalize(get_hostname)

            if output == "xlsx":
                generate_work_book(output_name, bookmarks_data, refresh,
                                   undupe, clean, get_hostname)
            else:
                generate_web_page(output_name, bookmarks_data, refresh, undupe,
                                  clean, get_hostname)
Example #18
0
def files(name):
    """
	This page presents a user's uploaded files, and allows
	the user to download them individually.
	"""
    error = valid_user(name)
    if error is None:
        uploads = [dict(userid=f.userid, filename=f.filename) \
         for f in Upload.query.all()]
        return render_template('files.html', username=name, uploads=uploads)

    # If an error occurs, display the error and
    # redirect to the appropriate page.
    display(error)
    if 'logged_in' in session:
        return redirect(url_for('files', name=session['logged_in']))
    else:
        return redirect(url_for('login'))
Example #19
0
def write_html(webpage_filename, folder):
    with open(webpage_filename, 'w', encoding='utf-8') as html_file:
        html_file.write(header)

        for item in folder:
            if isinstance(item, tools.Folder):
                html_file.write(
                    open_folder(item.add_date, item.modify_date,
                                item.folder_name))
                for url in item.children:
                    if isinstance(url, tools.Urls):
                        html_file.write(
                            add_url(url.url, url.add_date, url.title))
                    elif isinstance(url, tools.Folder):
                        tools.display(preset.message["new_folder"])
                html_file.write(close_folder())

        html_file.write(tail)
Example #20
0
def files(name):
	"""
	This page presents a user's uploaded files, and allows
	the user to download them individually.
	"""
	error = valid_user(name)
	if error is None:
		uploads = [dict(userid=f.userid, filename=f.filename) \
			for f in Upload.query.all()]
		return render_template('files.html', username=name, uploads=uploads)

	# If an error occurs, display the error and
	# redirect to the appropriate page.
	display(error)
	if 'logged_in' in session:
		return redirect(url_for('files', name=session['logged_in']))
	else:
		return redirect(url_for('login'))
Example #21
0
def uploaded_file(name, filename):
	"""
	This page will fetch a given file from the uploads folder,
	provided the user has privileges to access the file.
	"""
	error = valid_user(name)
	if error is None:
		if has_file_access(session['logged_in'], filename):
			return send_from_directory(app.config['UPLOAD_FOLDER'], filename)
		else:
			error = "Access denied."

	# If an error occurs, display the error and
	# redirect to the appropriate page.
	display(error)
	if 'logged_in' in session:
		return redirect(url_for('files', name=session['logged_in']))
	else:
		return redirect(url_for('login'))
Example #22
0
def uploaded_file(name, filename):
    """
	This page will fetch a given file from the uploads folder,
	provided the user has privileges to access the file.
	"""
    error = valid_user(name)
    if error is None:
        if has_file_access(session['logged_in'], filename):
            return send_from_directory(app.config['UPLOAD_FOLDER'], filename)
        else:
            error = "Access denied."

    # If an error occurs, display the error and
    # redirect to the appropriate page.
    display(error)
    if 'logged_in' in session:
        return redirect(url_for('files', name=session['logged_in']))
    else:
        return redirect(url_for('login'))
Example #23
0
def login():
	"""
	Allows a user to log in to their account.
	Note that the index page redirects here by default.
	"""
	error = None
	if request.method == 'POST':
		username = request.form['username']

		# Check to see if the user exists.
		user_instance = get_user(username)
		if user_instance is None:
			error = 'Invalid username.'
		elif not user_instance.check_pw(request.form['password']):
			error = 'Invalid password.'
		else:
			session['logged_in'] = username
			flash('Successfully logged in.')
			return redirect(url_for('files', name=username))

	display(error)
	return render_template('login.html')
Example #24
0
def login():
    """
	Allows a user to log in to their account.
	Note that the index page redirects here by default.
	"""
    error = None
    if request.method == 'POST':
        username = request.form['username']

        # Check to see if the user exists.
        user_instance = get_user(username)
        if user_instance is None:
            error = 'Invalid username.'
        elif not user_instance.check_pw(request.form['password']):
            error = 'Invalid password.'
        else:
            session['logged_in'] = username
            flash('Successfully logged in.')
            return redirect(url_for('files', name=username))

    display(error)
    return render_template('login.html')
Example #25
0
def create():
    """
    Allows a user to create a new account.
    """
    error = None
    if 'logged_in' not in session:
        session['theme'] = app.config['DEFAULT_THEME']

    if request.method == 'POST':
        # Get submitted values from fields.
        username, password = request.form['username'], request.form['password-a']

        # Check if there is a user by the same username in the database.
        user_instance = get_user(username)
        if user_instance is not None:
            error = 'Username is already taken.'
        elif re.search(r'[^_a-zA-Z0-9]', username):
            error = 'Username must only contain letters, numbers, and underscores.'
        elif password == '':
            error = 'Please enter a password.'
        elif password != request.form['password-b']:
            error = 'Passwords do not match.'
        else:
            # Add user to the database and log user in.
            user_instance = User(username, password)
            user_instance.theme = app.config['DEFAULT_THEME']
            udb.session.add(user_instance)
            udb.session.commit()

            flash('Account created!')
            session['logged_in'], session['posting_enabled'], session['theme'] = \
                username, user_instance.posting_enabled, user_instance.theme
            return redirect(url_for('entries', name=username))

    display(error)
    return render_template('create.html', theme=session['theme'])
Example #26
0
def generate_bookmarks(profile):
    tools.display(preset.message["generating_bookmarks"])
    bookmarks_file = tools.get_chrome_element(profile, preset.bookmarks)
    if bookmarks_file:
        return Bookmarks(bookmarks_file)
    return None
Example #27
0
def generate_work_book(spreadsheet_filename, data_table, reload_url_title,
                       remove_duplicated_urls, remove_tracking_from_url,
                       get_hostname_title):
    tools.debug("filename    [", str(spreadsheet_filename), "]\nreload_title[",
                str(reload_url_title), "]\nremove_dupes[",
                str(remove_duplicated_urls), "]\nremove_track[",
                str(remove_tracking_from_url), "]\nget_hostname[",
                str(get_hostname_title), "]")
    tools.display(preset.message["generating_workbook"] + " [" +
                  spreadsheet_filename + "]")
    excel_workbook = Workbook()
    excel_worksheet = excel_workbook.active
    excel_worksheet.title = preset.message["chrome_urls"]

    reload_url_title_disabled = True
    if reload_url_title:
        tools.underline()
        tools.display(preset.message["get_url_status"])
        tools.overline()
        reload_url_title_disabled = False

    if get_hostname_title:
        tools.underline()
        tools.display(preset.message["resolving_hostnames"])
        tools.overline()
        total_items = len(data_table)
        disabled = False
        if preset.gui_progress_dialog:
            disabled = True
        with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
            temporary_table = []
            utils.update_progress(preset.message["resolving_hostnames"], -1,
                                  total_items)
            for index, data_row in enumerate(data_table):
                temporary_table.append(
                    utils.update_tuple(
                        data_row,
                        htmlSupport.get_title(preset.protocol +
                                              data_row[22])[1], 2))
                progress_bar.update(1)
                if utils.update_progress(preset.message["resolving_hostnames"],
                                         index, total_items):
                    break
            data_table = temporary_table

    visited_url_address = set()
    data_table_without_duplicates = []
    tools.display(preset.message["find_duplicated_lines"])
    total_items = len(data_table)
    disabled = False
    if reload_url_title_disabled:
        disabled = True
    elif preset.gui_progress_dialog:
        disabled = True
    with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
        utils.update_progress(preset.message["find_duplicated_lines"], -1,
                              total_items)
        for index, data_row in enumerate(data_table):
            if utils.update_progress(preset.message["find_duplicated_lines"],
                                     index, total_items):
                break

            header = preset.Header()
            header.set_data(data_row)
            if remove_tracking_from_url:
                url_address = header.get_name(preset.url_clean_attr)
            else:
                url_address = header.get_name(preset.url_attr)

            url_name = header.get_name(preset.url_name_attr)

            if url_address not in visited_url_address:
                visited_url_address.add(url_address)
                data_table_without_duplicates.append(
                    ("MAIN",
                     get_title_conditional(progress_bar,
                                           reload_url_title_disabled, url_name,
                                           url_address)) + data_row)
            elif not remove_duplicated_urls:
                data_table_without_duplicates.append(
                    ("DUPE",
                     get_title_conditional(progress_bar,
                                           reload_url_title_disabled, url_name,
                                           url_address)) + data_row)

    tools.display(preset.message["writing_spreadsheet"])
    tools.overline()
    total_items = len(data_table_without_duplicates)
    disabled = False
    if preset.gui_progress_dialog:
        disabled = True
    with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
        utils.update_progress(preset.message["writing_spreadsheet"], -1,
                              total_items)
        data_row_header = ["DUPE", "Site Name"]
        for item in preset.label_dictionary:
            data_row_header.append(preset.label_dictionary[item])
        excel_worksheet.append(tuple(data_row_header))
        for index, data_row in enumerate(data_table_without_duplicates):
            progress_bar.update(1)
            if utils.update_progress(preset.message["writing_spreadsheet"],
                                     index, total_items):
                break
            excel_worksheet.append(data_row)

    excel_worksheet.freeze_panes = "A2"
    excel_worksheet.auto_filter.ref = "A1:AU30000"

    tools.underline()
    tools.display(preset.message["format_columns"])
    tools.overline()
    font_columns = [
        'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'AA', 'AB', 'AC', 'AD', 'AE', 'AF',
        'AG', 'AH', 'AI', 'AJ', 'AK', 'AL', 'AM', 'AN', 'AO', 'AP', 'AQ', 'AR',
        'AS', 'AT', 'AU'
    ]
    total_items = (len(font_columns) * len(excel_worksheet['T']))
    disabled = False
    if preset.gui_progress_dialog:
        disabled = True
    with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
        utils.update_progress(preset.message["format_columns"], -1,
                              total_items)
        for font_column in font_columns:
            for index, worksheet_column in enumerate(
                    excel_worksheet[font_column]):
                progress_bar.update(1)
                if utils.update_progress(preset.message["format_columns"],
                                         index, total_items):
                    break
                worksheet_column.font = Font(size=10, name='Courier New')

    tools.underline()
    tools.display(preset.message["format_dates"])
    tools.overline()
    date_columns = ['G', 'H', 'I', 'P', 'Q', 'R']
    total_items = (len(date_columns) * len(excel_worksheet['G']))
    disabled = False
    if preset.gui_progress_dialog:
        disabled = True
    with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
        utils.update_progress(preset.message["format_dates"], -1, total_items)
        for date_column in date_columns:
            excel_worksheet.column_dimensions[date_column].width = 18
            for index, worksheet_column in enumerate(
                    excel_worksheet[date_column]):
                progress_bar.update(1)
                if utils.update_progress(preset.message["format_dates"], index,
                                         total_items):
                    break
                worksheet_column.number_format = preset.number_format

    tools.underline()
    tools.display(preset.message["hide_columns"])
    hidden_columns = [
        'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'AA', 'AB', 'AC',
        'AD'
    ]
    for h in hidden_columns:
        excel_worksheet.column_dimensions[h].width = 9
        excel_worksheet.column_dimensions[h].hidden = True

    tools.display(preset.message["format_header"])
    for cell in excel_worksheet["1:1"]:
        cell.font = Font(bold=True)

    tools.display(preset.message["sizing_columns"])
    excel_worksheet.column_dimensions['S'].width = 30
    excel_worksheet.column_dimensions['T'].width = 85

    tools.display(preset.message["saving_workbook"])
    excel_workbook.save(spreadsheet_filename)
    tools.display(preset.message["done"])
    tools.overline()
Example #28
0
def generate_from_txt(url_list):
    tools.display(preset.message["generating_from_text"])
    txt_header = []
    return append_data_table(txt_header, url_list)
Example #29
0
def generate_web_page(web_page_filename, data_table, reload_url_title,
                      remove_duplicated_urls, remove_tracking_from_url,
                      get_hostname_title):
    tools.debug("filename    [", str(web_page_filename), "]\nreload_title[",
                str(reload_url_title), "]\nremove_dupes[",
                str(remove_duplicated_urls), "]\nremove_track[",
                str(remove_tracking_from_url), "]\nget_hostname[",
                str(get_hostname_title), "]")
    tools.display(preset.message["generating_html"] + " [" +
                  web_page_filename + "]")

    visited_hostname_title = set()
    visited_url_address = set()
    folder_list = []
    data_table_without_duplicates = []
    if remove_duplicated_urls:
        tools.underline()
        tools.display(preset.message["removing_duplicates"])
        tools.overline()
        total_items = len(data_table)
        disabled = False
        if preset.gui_progress_dialog:
            disabled = True
        with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
            utils.update_progress(preset.message["removing_duplicates"], -1,
                                  total_items)
            for index, data_row in enumerate(data_table):

                header = preset.Header()
                header.set_data(data_row)

                progress_bar.update(1)
                if utils.update_progress(preset.message["removing_duplicates"],
                                         index, total_items):
                    break

                if remove_tracking_from_url:
                    url_address = header.get_name(preset.url_clean_attr)
                else:
                    url_address = header.get_name(preset.url_attr)
                if url_address not in visited_url_address:
                    visited_url_address.add(url_address)
                    data_table_without_duplicates.append(data_row)
    else:
        data_table_without_duplicates = data_table

    tools.underline()
    tools.display(preset.message["writing_html"])
    tools.overline()
    total_items = len(data_table_without_duplicates)
    disabled = False
    if preset.gui_progress_dialog:
        disabled = True
    with tqdm.tqdm(total=total_items, disable=disabled) as progress_bar:
        utils.update_progress(preset.message["writing_html"], -1, total_items)
        for index, data_row in enumerate(data_table_without_duplicates):

            header = preset.Header()
            header.set_data(data_row)

            progress_bar.update(1)
            if utils.update_progress(preset.message["writing_html"], index,
                                     total_items):
                break

            url_title = header.get_name(preset.url_name_attr)

            if remove_tracking_from_url:
                url_address = header.get_name(preset.url_clean_attr)
            else:
                url_address = header.get_name(preset.url_attr)

            hostname_title = header.get_name(preset.hostname_attr)
            original_hostname = header.get_name(preset.hostname_attr)

            if reload_url_title:
                status_number, url_title = htmlSupport.get_title(url_address)
                if status_number != 0:
                    url_title = "[ " + url_title + " " + str(
                        status_number) + " - " + header.get_name(
                            preset.url_name_attr) + " ]"

            if get_hostname_title:
                status_number, hostname_title = htmlSupport.get_title(
                    preset.protocol + original_hostname)
                if status_number != 0:
                    hostname_title = "[ " + hostname_title + " " + str(
                        status_number) + " - " + original_hostname + " ]"

            if hostname_title not in visited_hostname_title:
                url_data = tools.Urls(url_address,
                                      header.get_name(preset.url_added_attr),
                                      url_title)
                visited_hostname_title.add(hostname_title)
                folder_list.append(
                    tools.Folder(header.get_name(preset.folder_added_attr),
                                 header.get_name(preset.folder_modified_attr),
                                 hostname_title, [url_data]))
            else:
                url_data = tools.Urls(url_address,
                                      header.get_name(preset.url_added_attr),
                                      url_title)
                for folder in folder_list:
                    if folder.folder_name == hostname_title:
                        folder.add_url(url_data)

    tools.underline()
    tools.display(preset.message["saving_html"])
    tools.overline()
    htmlExport.write_html(web_page_filename, folder_list)
    tools.display(preset.message["done"])
    tools.overline()