dark=False)
# Footer docs:
# dbc.Container, dbc.Row, dbc.Col: https://dash-bootstrap-components.opensource.faculty.ai/l/components/layout
# html.P: https://dash.plot.ly/dash-html-components
# fa (font awesome) : https://fontawesome.com/icons/github-square?style=brands
# mr (margin right) : https://getbootstrap.com/docs/4.3/utilities/spacing/
# className='lead' : https://getbootstrap.com/docs/4.3/content/typography/#lead
footer = dbc.Container(
dbc.Row(
dbc.Col(
html.P([
html.Span('Brendan Hoss', className='Ds-19'),
html.A(html.I(className='fas fa-envelope-square mr-1'),
href='mailto:<you>@<provider>.com'),
html.A(html.I(className='fab fa-github-square mr-1'),
href='https://github.com/Bh0ss/bw3template'),
html.A(html.I(className='fab fa-twitter-square mr-1'),
href='https://twitter.com/bhoss13'),
],
className='lead'))))
image_file = 'assets\lambda.png'
encoded_image = base64.b64encode(open(image_file, 'rb').read())
# Layout docs:
# html.Div: https://dash.plot.ly/getting-started
# dcc.Location: https://dash.plot.ly/dash-core-components/location
# dbc.Container: https://dash-bootstrap-components.opensource.faculty.ai/l/components/layout
app.layout = html.Div([
dcc.Location(id='url', refresh=False), navbar,
# Footer docs:
# dbc.Container, dbc.Row, dbc.Col: https://dash-bootstrap-components.opensource.faculty.ai/l/components/layout
# html.P: https://dash.plot.ly/dash-html-components
# fa (font awesome) : https://fontawesome.com/icons/github-square?style=brands
# mr (margin right) : https://getbootstrap.com/docs/4.3/utilities/spacing/
# className='lead' : https://getbootstrap.com/docs/4.3/content/typography/#lead
footer = dbc.Container(
dbc.Row(
dbc.Col(
html.P(
[
html.Span('Victoria Debebe', className='mr-2'),
html.A(html.I(className='fas fa-envelope-square mr-1'),
href='*****@*****.**'),
html.
A(html.I(className='fab fa-github-square mr-1'),
href='https://github.com/vdeb/Build-Gender-Bias-Insights'
),
#html.A(html.I(className='fab fa-linkedin mr-1'), href='https://www.linkedin.com/in/<you>/'),
#html.A(html.I(className='fab fa-twitter-square mr-1'), href='https://twitter.com/<you>'),
],
className='lead'))))
# Layout docs:
# html.Div: https://dash.plot.ly/getting-started
# dcc.Location: https://dash.plot.ly/dash-core-components/location
# dbc.Container: https://dash-bootstrap-components.opensource.faculty.ai/l/components/layout
app.layout = html.Div([
dcc.Location(id='url', refresh=False), navbar,
dbc.Container(id='page-content', className='mt-4'),
html.Hr(), footer
import dash_core_components as dcc
import dash_html_components as html
from utils.charts import make_question_pie, make_bar
from utils.summary_table import make_summary_table
from utils.playground import make_playground_header
footercontent = [
html.Div(
children=[
html.Div(
children=[
dbc.Row(
children=[
dbc.Col(children=[
html.P("2020 © LEAD at Krea University"),
],
className="col-md-6 paddzero"),
dbc.Col(children=[
html.A("Privacy Policy",
href="/privacy",
target="_blank"),
],
className="col-md-6 paddzero text_rgt"),
],
className="",
),
],
className="container paddzero",
),
],
className="footer",
The layout of your app should be built as a series of rows of columns.
We set md=4 indicating that on a 'medium' sized or larger screen each column
should take up a third of the width. Since we don't specify behaviour on
smaller size screens Bootstrap will allow the rows to wrap so as not to squash
the content.
"""
column1 = dbc.Col(
[
dcc.Markdown("""
## Predict the Salary of the NBA Players
What if you were a professional basketball player, How much you will be making?
Use this Sport app to see how much you will be making, depends on your position,
height, weight, age and college.
"""),
dcc.Link(dbc.Button('Predict', color='primary'), href='/predictions')
],
md=4,
)
# gapminder = px.data.gapminder()
# fig = px.scatter(gapminder.query("year==2007"), x="gdpPercap", y="lifeExp", size="pop", color="continent",
# hover_name="country", log_x=True, size_max=60)
column2 = dbc.Col([
# dcc.Graph(figure=fig),
html.Img(src='assets/athlete-ball-basketball-163423.jpg',
app.layout = html.Div([
dbc.Row([
dbc.Button(
f"Open {graph}",
id=f"button-{graph}",
className="mb-3",
color="primary",
) for graph in ["pie", "hist", "strip"]
],
justify="center"),
dbc.Row([
dbc.Col(dbc.Collapse(dcc.Graph(
id="pie_chart",
config={'displayModeBar': False},
figure=px.pie(
df, names="Status Mission",
values="Mission Cost").update_traces(showlegend=False)),
id="collapse_pie",
is_open=False),
width=4),
dbc.Col(dbc.Collapse(dcc.Graph(id="hist_chart",
config={'displayModeBar': False},
figure=px.histogram(df,
x="Mission Cost",
range_x=[0, 500])),
id="collapse_hist",
is_open=False),
width=4),
dbc.Col(dbc.Collapse(dcc.Graph(id="strip_chart",
config={'displayModeBar': False},
figure=px.strip(df,
Row, and Col.
The layout of your app should be built as a series of rows of columns.
We set md=4 indicating that on a 'medium' sized or larger screen each column
should take up a third of the width. Since we don't specify behaviour on
smaller size screens Bootstrap will allow the rows to wrap so as not to squash
the content.
"""
column1 = dbc.Col(
[
dcc.Markdown("""
## Adjusted Ratings Apps
This is the Data Science portion for an Application that uses Machine Learning to build a Yelp Adjusted Ratings App based on User's text review which is then put through a rigorous Sentiment Analysis.
"""),
dcc.Link(dbc.Button('Predict Yelp Adjusted Ratings', color='danger'),
href='/predictions')
],
md=4,
)
column2 = dbc.Col([
# dcc.Graph(figure=fig),
html.Img(src='assets/chef.jpeg', className='img-fluid')
])
layout = dbc.Row([column1, column2])
className="m-4",
)
alerts = html.Div([
dbc.Alert("This is a primary alert", color="primary"),
dbc.Alert("This is a secondary alert", color="secondary"),
dbc.Alert("This is a success alert! Well done!", color="success"),
dbc.Alert("This is a warning alert... be careful...", color="warning"),
])
"""
===============================================================================
Layout
"""
app.layout = dbc.Container(
dbc.Row([
dbc.Col(["Select Theme", dropdown], width=3),
dbc.Col([buttons, alerts]),
html.Div(id="blank_output"),
], ),
className="m-4",
fluid=True,
)
# Using 2 stylesheets with the delay reduces the annoying flicker when the theme changes
app.clientside_callback(
"""
function(url) {
// Select the FIRST stylesheet only.
var stylesheets = document.querySelectorAll('link[rel=stylesheet][href^="https://stackpath"]')
// Update the url of the main stylesheet.
stylesheets[stylesheets.length - 1].href = url
dbc.Col(width={'size': 2},
children=[
html.Br(),
dcc.Dropdown(id='hist_dropdown',
options=[{
'label': 'Price',
'value': 'price'
}, {
'label': 'Rooms',
'value': 'rooms'
}, {
'label': 'Residence Size',
'value': 'residence_size'
}],
value='price',
clearable=False),
html.Br(),
html.P('Sold date'),
dcc.RangeSlider(id='hist_range_slider',
min=1992,
max=2020,
step=1,
value=[1992, 2020],
tooltip={
'always_visible': True,
'placement': 'bottom'
},
allowCross=False),
html.Br(),
html.Br(),
dcc.Checklist(id='hist_checklist',
options=[{
'label': 'Jylland',
'value': 'jylland'
}, {
'label': 'Fyn',
'value': 'fyn'
}, {
'label': 'Sjælland',
'value': 'sjaelland'
}],
value=['jylland', 'fyn', 'sjaelland'],
labelStyle={'display': 'block'})
]),
import json
import dash_table
from dash.exceptions import PreventUpdate
external_stylesheets = [dbc.themes.BOOTSTRAP]
app = dash.Dash(__name__, external_stylesheets = external_stylesheets)
#logo crime.data
img_logo = "assets/Logo - (5).png"
encoded_image = base64.b64encode(open(img_logo, 'rb').read())
navbar = dbc.Navbar(
children=[
html.A(
dbc.Row(
[
dbc.Col(html.Img(src='data:image/png;base64,{}'.format(encoded_image.decode()),height="75px")),
dbc.Col(html.P("Carlos Eduardo, Gerônimo, Lucas Natanael, Raíssa e Sérgio", style={'font-size':'10px','font-weight': 'bold'}),align='center',md=12)
],
),
)
],
color='#F7F7F7',
sticky="top",
#className='container'
)
##### DROPDOWNS PARA QUANTIDADES #####
#dropdown com os anos no campo de quantidade de crimes por ano
dropdown1_crimes_ano = dbc.FormGroup(
[
html.P('Ano', style = {
'textAlign': 'left',
dbc.Row(
[
dbc.Col(
dbc.Row(
[
dbc.Col(
html.A(
html.Img(
src=app.get_asset_url(
"dash-logo-new.png"),
height="30px",
),
href="https://plotly.com/dash/",
),
style={"width": "min-content"},
),
dbc.Col(
html.Div(
[
html.H3("Covid X-Ray app"),
html.
P("Exploration and annotation of CT images"
),
],
id="app_title",
)),
],
align="center",
style={"display": "inline-flex"},
)),
dbc.Col([
dbc.NavbarToggler(id="navbar-toggler"),
href="https://www.linkedin.com/in/edison-yepes/",
target="_blank",
),
]),
],
style={
"width": "18rem",
"textAlign": "center"
},
)
layout = html.Div(
[
dbc.Row(
[
dbc.Col(card_camilo, width=3),
dbc.Col(card_diana, width=3),
dbc.Col(card_wbeimar, width=3),
],
justify="center",
),
dbc.Row([
dbc.Col(card_julian, width=3),
dbc.Col(card_edison, width=3),
],
justify="center",
style={'margin-top': '30px'})
],
className="ds4a-body",
) # justify="start", "center", "end", "between", "around"
cards,
dbc.Row([
dbc.Col([
dbc.Row(
html.H2("Company Objective",
style={"text-decoration": "underline"},
className='mt-5 mb-5')),
dbc.Row(
html.Ul([
html.Li("Improve company conversion"),
html.Li("Reduce CAC"),
html.Li(html.B("Make 'offers' to undecided* clients")),
])),
dbc.Row(
html.H2("Model Objective",
style={"text-decoration": "underline"},
className='mt-5 mb-5')),
dbc.Row(
html.Ul([
html.Li(
[html.B("Main: "), "Calculate purchase probability"]),
html.Li(
[html.B("Secondary: "), "Predict binary class (0,1)"]),
html.Li(html.B("Make 'offers' to undecided* clients")),
])),
dbc.Row(
html.H6("* Undecided = Medium probability to purchase",
className='mt-5 mb-5')),
],
width=6),
dbc.Col([
dbc.Row(
def create_vulnstab(vulndata, projname, vername):
global col_data_vulns
return [
dcc.Interval(
id='vulns_interval',
disabled=True,
interval=1 * 6000, # in milliseconds
n_intervals=0,
max_intervals=400
),
dbc.Row(
dbc.Col(html.H2("Vulnerabilities")),
),
dbc.Row(
[
dbc.Col(html.H5("Project: " + projname + " - Version: " + vername), width=7),
dbc.Col(
dcc.Dropdown(
id="sel_vuln_action",
# DUPLICATE, IGNORED, MITIGATED, NEEDS_REVIEW, NEW, PATCHED, REMEDIATION_COMPLETE,
# REMEDIATION_COMPLETE
options=[
{'label': 'Select Remediation ...', 'value': 'NOTHING'},
{'label': 'New', 'value': 'NEW'},
{'label': 'Duplicate', 'value': 'DUPLICATE'},
{'label': 'Ignored', 'value': 'IGNORED'},
{'label': 'Mitigated', 'value': 'MITIGATED'},
{'label': 'Needs Review', 'value': 'NEEDS_REVIEW'},
{'label': 'Patched', 'value': 'PATCHED'},
{'label': 'Remediation Required', 'value': 'REMEDIATION_REQUIRED'},
{'label': 'Remediation Complete', 'value': 'REMEDIATION_COMPLETE'},
],
multi=False,
placeholder='Select Remediation ...'
), width=2,
align='center',
),
dbc.Col(dbc.Button("Selected Rows", id="button_vuln_selected",
className="mr-2", size='sm'), width=1),
dbc.Col(dbc.Button("All Filtered Rows", id="button_vuln_all",
className="mr-2", size='sm'), width=1),
dbc.Col(dbc.Button("Reload", id="button_vuln_reload",
className="mr-2", size='sm'), width=1),
]
),
dbc.Row(
dbc.Col(
dash_table.DataTable(
id='vulnstable',
columns=col_data_vulns,
style_cell={
'overflow': 'hidden',
'textOverflow': 'ellipsis',
'maxWidth': 0,
'font_size': '12px',
},
data=vulndata.to_dict('records'),
page_size=30, sort_action='native',
filter_action='native',
row_selectable="multi",
cell_selectable=False,
style_header={'backgroundColor': 'rgb(30, 30, 30)', 'color': 'white'},
tooltip_data=[
{
column: {'value': str(value), 'type': 'markdown'}
for column, value in row.items()
} for row in vulndata.to_dict('records')
],
tooltip_duration=None,
style_data_conditional=[
{
'if': {
'filter_query': '{ignored} eq "True"',
'column_id': 'ignored'
},
'display': 'none',
},
{
'if': {'column_id': 'componentName'},
'width': '15%'
},
{
'if': {'column_id': 'componentVersionName'},
'width': '5%'
},
{
'if': {'column_id': 'vulnerabilityWithRemediation.vulnerabilityName'},
'width': '8%'
},
{
'if': {'column_id': 'vulnerabilityWithRemediation.relatedVulnerability'},
'width': '5%'
},
{
'if': {'column_id': 'vulnerabilityWithRemediation.description'},
'width': '15%'
},
{
'if': {'column_id': 'vulnerabilityWithRemediation.vulnerabilityPublishedDate'},
'width': '5%'
},
{
'if': {'column_id': 'vulnerabilityWithRemediation.vulnerabilityUpdatedDate'},
'width': '5%'
},
{
'if': {'column_id': 'vulnerabilityWithRemediation.overallScore'},
'width': '5%'
},
{
'if': {'column_id': 'vulnerabilityWithRemediation.exploitabilitySubscore'},
'width': '5%'
},
{
'if': {'column_id': 'vulnerabilityWithRemediation.impactSubscore'},
'width': '5%'
},
{
'if': {'column_id': 'vulnerabilityWithRemediation.severity'},
'width': '5%'
},
{
'if': {'column_id': 'vulnerabilityWithRemediation.remediationStatus'},
'width': '5%'
},
{
'if': {'column_id': 'vulnerabilityWithRemediation.remediationUpdatedAt'},
'width': '5%'
},
{
'if': {'column_id': 'vulnerabilityWithRemediation.cweId'},
'width': '5%'
},
{
'if': {
'filter_query': '{vulnerabilityWithRemediation.severity} = "CRITICAL"',
'column_id': 'vulnerabilityWithRemediation.severity'
},
'backgroundColor': 'maroon',
'color': 'white'
},
{
'if': {
'filter_query': '{vulnerabilityWithRemediation.severity} = "HIGH"',
'column_id': 'vulnerabilityWithRemediation.severity'
},
'backgroundColor': 'crimson',
'color': 'black'
},
{
'if': {
'filter_query': '{vulnerabilityWithRemediation.severity} = "MEDIUM"',
'column_id': 'vulnerabilityWithRemediation.severity'
},
'backgroundColor': 'coral',
'color': 'black'
},
{
'if': {
'filter_query': '{vulnerabilityWithRemediation.severity} = "LOW"',
'column_id': 'vulnerabilityWithRemediation.severity'
},
'backgroundColor': 'gold',
'color': 'black'
},
{
'if': {
'filter_query': '{vulnerabilityWithRemediation.remediationStatus} = "IGNORED"',
'column_id': 'vulnerabilityWithRemediation.remediationStatus'
},
'backgroundColor': 'grey',
'color': 'white'
},
{
'if': {
'filter_query': '{vulnerabilityWithRemediation.remediationStatus} = "PATCHED"',
'column_id': 'vulnerabilityWithRemediation.remediationStatus'
},
'backgroundColor': 'grey',
'color': 'white'
},
{
'if': {
'filter_query': '{vulnerabilityWithRemediation.remediationStatus} = "MITIGATED"',
'column_id': 'vulnerabilityWithRemediation.remediationStatus'
},
'backgroundColor': 'grey',
'color': 'white'
},
{
'if': {
'filter_query': '{vulnerabilityWithRemediation.remediationStatus} = "DUPLICATE"',
'column_id': 'vulnerabilityWithRemediation.remediationStatus'
},
'backgroundColor': 'grey',
'color': 'white'
},
{
'if': {
'filter_query':
'{vulnerabilityWithRemediation.remediationStatus} = "REMEDIATION_COMPLETE"',
'column_id': 'vulnerabilityWithRemediation.remediationStatus'
},
'backgroundColor': 'grey',
'color': 'white'
},
],
sort_by=[{'column_id': 'vulnerabilityWithRemediation.overallScore', 'direction': 'desc'}],
# merge_duplicate_headers=True
),
width=12
),
),
]
x=1,
xanchor="right",
orientation="h",
font=dict(size=10)))
Pgry = dict(data=trace6, layout=layout6)
######################
#external_stylesheets = [dbc.themes.PULSE]
#app = dash.Dash(__name__, external_stylesheets=external_stylesheets)
tab5 = dbc.Card(
dbc.CardBody([
html.Div([
dbc.Container([
dbc.Row(
[dbc.Col(html.H1("WMG Student Data"), className="mb-2")]),
dbc.Row([
dbc.Col(dbc.Card(html.H3(
children="Postgraduate Research | PGR",
className="text-center text-light bg-primary"),
body=True,
color="primary"),
className="mb-4")
]),
dbc.Row([
dbc.Col(html.H5("Total PGR Students including PhD"),
className="mb-4")
]),
html.Label("◆ Select the calculated method:"),
dcc.Dropdown(id="c-m6",
options=[dict(label=i, value=i) for i in methods],
def create_app_ui():
# Create the UI of the Webpage here
main_layout = html.Div([
dbc.Container([
html.Img(src=b64_image(image_filename),
style={
'height': '50%',
'width': '50%'
})
],
style={'textAlign': 'center'}),
html.
H1('Welcome!',
style={
'color':
'black',
'text-align':
'center',
'background-image':
'url(https://s3.amazonaws.com/blog4.0/blog/wp-content/uploads/2018/04/customer-sentiment-analysis-tools.png)'
}),
html.H1(children='Sentiments Analysis Meter',
id='Main_title',
style={
'color': 'red',
'text-align': 'center'
}),
dbc.Row([html.H3('Recent sentiment analysis summary vizualisation')],
justify="center",
align="center"),
dbc.Container(dcc.Loading(
dcc.Graph(
figure={
'data': [
go.Pie(labels=labels,
values=values,
textinfo='label+percent',
insidetextorientation='radial')
],
'layout':
go.Layout(height=600, width=1000, autosize=False),
'hoverinfo':
'label+percent',
'textinfo':
'value',
'textfont_size':
'20',
'marker':
[dict(colors=colors, line=dict(color='#000000', width=2))]
})),
className='d-flex justify-content-center'),
dbc.Row([html.H3('Check the service quality with our sample input ')],
justify="center",
align="center"),
dbc.InputGroup(
[
dbc.InputGroupAddon("Look how good your review is!",
addon_type="prepend"),
dbc.Textarea(id='textarea_review',
placeholder='Enter the review here...',
value='Enter review',
style={
'width': '100%',
'height': 100
}),
],
className="mb-3",
),
dbc.Row([html.H3('Check the service quality with our sample input ')],
justify="center",
align="center"),
dbc.Container(
[
dcc.Dropdown(
id='dropdown',
placeholder='Select a Review',
options=[{
'label': i,
'value': i
} for i in dff.Reviews.sample(81)],
className='select_box',
# {'label': 'New York City', 'value': 'NYC'},
# {'label': 'Montreal', 'value': 'MTL'},
# {'label': 'San Francisco', 'value': 'SF'}
value=dff.Reviews[0],
style={'margin-bottom': '30px'})
],
style={
'padding-left': '50px',
'padding-right': '50px'
}),
#butten
dbc.Row(
[
dbc.Col(html.Div(
dbc.Button(children='Show Review',
id='show_review',
n_clicks=0,
style={
'textAlign': 'center',
'color': colors['text']
})),
align="center"),
],
style={
'margin-bottom': '10px',
'textAlign': 'center',
'width': '220px',
'margin': 'auto'
}),
#text for review
dbc.Row([html.H3(children=None, id='result')],
justify="center",
align="center"),
dbc.Row([
html.H3(children=None, id='result1'),
],
justify="center",
align="center"),
html.Div(
[
dbc.Button("Click me to see who is behind all this",
id="alert-toggle-auto",
className="mr-1"),
html.Hr(),
dbc.Alert(
"Hello! I am Vidura,AI enthusiastic from little country called SriLanka!",
id="alert-auto",
is_open=True,
duration=4000,
),
],
style={
'margin-bottom': '10px',
'textAlign': 'center',
'width': '220px',
'margin': 'auto'
}),
dbc.Row([dbc.Col(card_main, width=3)], justify="around"),
html.Div(
[
dbc.Button("Submit",
type='submit',
outline=True,
color="info",
className="mr-1"),
],
style={
'margin-bottom': '10px',
'textAlign': 'center',
'width': '220px',
'margin': 'auto'
})
])
return main_layout
column1 = dbc.Col(
[
dcc.Markdown(
"""
## Process
********
To build this model, two datasets with similar labels were combined to form a dataset with 102,840 observations.
I would like to thank the research team behind [this study](https://arxiv.org/pdf/1802.00393.pdf), as they promptly gave me access to their data, which was labeled through Crowdflower.
This model builds largely on their work, as well as that of [this previous study](https://aaai.org/ocs/index.php/ICWSM/ICWSM17/paper/view/15665).
After gaining access to both datasets, I proceeded to retrieve the corresponding tweet text for all IDs in the second set (as it was not provided) via Twitter's API.
This was [the code](https://stackoverflow.com/questions/44581647/retrieving-a-list-of-tweets-using-tweet-id-in-tweepy) I used to retrieve the text, without exceeding the rate limit.
"""
),
html.Iframe(src='data:text/html;charset=utf-8,%3Cbody%3E%3Cscript%20src%3D%22https%3A%2F%2Fgist.github.com%2Fnchibana%2F20d6d9f8ae62a6cc36b773d37dd7dc70.js%22%3E%3C%2Fscript%3E%3C%2Fbody%3E', style=dict(border=0, padding=40), height=780, width=1000),
dcc.Markdown(
"""
After that, I proceeded to combine the datasets and eliminate all duplicate tweets. I also defined a baseline accuracy score of 56%, which is the percent accuracy the model would achieve
if it predicted the majority class for all tweets.
Using some of the processes followed by the studies mentioned above, I also continued to preprocess the data by eliminating excess spaces, removing punctuation and retrieving the stem words of terms
used in tweets.
Next, I used Scikit-learn's [TfidVectorizer](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html) to convert tweet text into a matrix of
TF-IDF features, which is a statistic that calculates how important a word is to a document or collection of words.
"""
),
html.Iframe(src='data:text/html;charset=utf-8,%3Cbody%3E%3Cscript%20src%3D%22https://gist.github.com/nchibana/c15cbc4a1d97af02fa62fff5868bc36e.js%22%3E%3C%2Fscript%3E%3C%2Fbody%3E', style=dict(border=0, padding=40), height=460, width=1000),
dcc.Markdown(
"""
To increase the accuracy of the model, additional features were engineered, such as the number of syllables per word, the total number of characters, the number of words, the number of unique
terms, as well as readability and sentiment scores for each tweet.
Additionally, the number of mentions, hashtags and links in each tweet were also counted. For this study, images or any other type of media content were not analyzed.
"""
),
html.Iframe(src='data:text/html;charset=utf-8,%3Cbody%3E%3Cscript%20src%3D%22https%3A%2F%2Fgist.github.com%2Fnchibana%2F5cebfbfa700974edcd9f5fa6e43cc513.js%22%3E%3C%2Fscript%3E%3C%2Fbody%3E', style=dict(border=0, padding=40), height=600, width=1000),
dcc.Markdown(
"""
After testing several models such as Linear SVC, I finally settled on a logistic regression model which I trained on the data and used for the final model and app.
I also used grid search to find the optimal parameters for this logistic regression model.
Finally, I computed all accuracy scores and proceeded to plot visualizations to help me get a deeper understanding of the model, such as a confusion matrix to visualize misclassified tweets.
"""
),
html.Iframe(src='data:text/html;charset=utf-8,%3Cbody%3E%3Cscript%20src%3D%22https%3A%2F%2Fgist.github.com%2Fnchibana%2F0cc0c44c9b5a991adbc2690c97023d0c.js%22%3E%3C%2Fscript%3E%3C%2Fbody%3E', style=dict(border=0, padding=40), height=300, width=1000),
dcc.Markdown(
"""
## Sources
********
1. Automated Hate Speech Detection and the Problem of Offensive Language
Davidson, Thomas and Warmsley, Dana and Macy, Michael and Weber, Ingmar
Proceedings of the 11th International AAAI Conference on Web and Social Media p. 512-515. 2017
2. Large Scale Crowdsourcing and Characterization of Twitter Abusive Behavior
Founta, Antigoni-Maria and Djouvas, Constantinos and Chatzakou, Despoina and Leontiadis, Ilias and Blackburn, Jeremy and Stringhini, Gianluca and Vakali, Athena and Sirivianos, Michael and Kourtellis, Nicolas
11th International Conference on Web and Social Media, ICWSM 2018 2018
"""
),
],
md=12,
)
'Wheel Base [m]:', 'Trail [m]:', 'Steer Axis Tilt [degrees]:',
'Gravity [N/kg]:'
]
app = dash.Dash(__name__, external_stylesheets=[dbc.themes.CYBORG])
app.title = 'Bicycle Dynamics Analysis App'
server = app.server # needed for heroku
app.layout = html.Div([
dbc.Container(
fluid=False,
children=[
dbc.Row([
dbc.Col(dbc.NavbarSimple(brand='Bicycle Dynamics Analysis App',
dark=True,
color='info'),
width='auto',
className='my-2')
],
justify='center'),
dbc.Row([
dbc.Col(dcc.Loading(id='geometry-load',
type='dot',
children=[
html.Div(id='geometry-bin',
children=[
html.Img(
src='',
id='geometry-plot',
className='img-fluid')
])
xlabel = figures_labels[k][1]
else:
title = k.replace('_', ' ')
title = title[0].upper() + title[1:].lower()
ylabel = title
xlabel = title
figures_hist[k] = [
ylabel + ' (per utterance)',
plot_histogram(data, k, xlabel)
]
if metrics_available:
figure_word_acc = plot_word_accuracy(vocabulary)
stats_layout = [
dbc.Row(dbc.Col(html.H5(children='Global Statistics'),
className='text-secondary'),
className='mt-3'),
dbc.Row(
[
dbc.Col(html.Div('Number of hours', className='text-secondary'),
width=3,
className='border-right'),
dbc.Col(html.Div('Number of utterances',
className='text-secondary'),
width=3,
className='border-right'),
dbc.Col(html.Div('Vocabulary size', className='text-secondary'),
width=3,
className='border-right'),
dbc.Col(html.Div('Alphabet size', className='text-secondary'),
width=3),
"padding": "2rem 1rem",
}
#Websiten-Aufbau
layout = html.Div(
children=[
#region Menüleiste
dbc.Row([
dbc.Col(html.Div(
style={
"margin-top": "20px",
},
children=[
html.H1(children="Kennziffern im aktuellen Jahr"),
dcc.Tabs(id="tabs_kpi",
value='Gewinn',
children=[
dcc.Tab(label='Gewinn', value='Gewinn'),
dcc.Tab(label='Anzahl', value='Anzahl'),
dcc.Tab(label="Kaufwahrscheinlichkeit",
value="Kaufbereitschaft")
])
]),
width=12),
],
justify="center"),
#endregion
#region Chart-Reihe 1
dbc.Row([
dbc.Col(html.Div(id="Produktplot_1"), width=7),
dbc.Col(html.Div(style={
### The function update_stats_cards() is relocated from here
### The function update_case_description() is relocated from here
### We relocated the plot_map() function from here
#######################################
### We define the appearance of the dashboard, using syntax like HTML
#######################################
app.layout = html.Div(children=[
dbc.Row([
dbc.Col([
html.Img(src=app.get_asset_url(r'fti_logo.png'),
id='fti_logo',
style={"height": '60px'})
],
width=3),
dbc.Col([
html.H1('COVID-19 Hong Kong Dashboard',
style={'text-align': 'center'}),
],
width=6),
dbc.Col(
id='live-update-time', ### Challenging Exercise
width=3)
]),
dbc.Row(id='live-update-stats'
), ### We changed here from calling the function to using the id.
dbc.Row([
dbc.Col(
dbc.NavLink("About", href="/page-4", id="page-4-link"),
],
vertical=True,
pills=True,
),
],
style=SIDEBAR_STYLE,
)
tab_dashboard = html.Div(
[
dbc.Col([
html.Div([
dcc.Graph(id='live-update-heatmap-graph',
style={'height': 700}),
dcc.Interval(
id='interval-component',
interval=10 * 1000, # in milliseconds
n_intervals=0),
])
]),
dbc.Col([
html.
P('This where we could show a line graph of the time series of a specific sensor through a drop down menu or some other line graph.'
)
])
],
style={'columnCount': 2})
tab_annomalies = html.Div([
dbc.Row([
dbc.Col([
#Import Packages
import dash_bootstrap_components as dbc
import dash_core_components as dcc
import dash_html_components as html
from collections import deque
#Navbar Items
dashboard_item = dbc.NavItem(dbc.NavLink("Dashboard", href="/"))
#Navbar
navigator = dbc.Navbar(
dbc.Container([
html.A(
dbc.Row(
[dbc.Col(dbc.NavbarBrand("COVID-19 Guelph eABM"))],
align="left",
),
),
dbc.NavbarToggler(id="navbar-toggler"),
dbc.Collapse(
dbc.Nav(
navbar=True
),
id="navbar-collapse", navbar=True,
),
], className="margin-left-none"),
className="mb-5",
)
#Button Container
buttons = html.Div([
),
# This can be replaced with other summary stacked-bar chart
dbc.Card(
[
html.H2(f"{attrition_rate*100:.2f}%", className="card-title"),
html.P("Overall Attrition Rate", className="card-text"),
],
body=True,
color="light",
),
]
app.layout = dbc.Container([
dbc.Row(
dbc.Col([
html.H1(
dcc.Markdown("""# **Key Factors for Employee Attrition**"""))
]),
align="auto",
),
# Add three widgets to the sidebar
html.Div(
[
html.Br(),
html.Br(),
html.H2("Dashboard Filters", className="display-10"),
html.Hr(),
html.Br(),
dcc.Markdown("""_Department_"""),
dcc.Dropdown(
id="depart-widget",
value=["Sales", "Human Resources", "Research & Development"],
},
name=i))
fig = {'data': traces, 'layout': graph_settings["layout"]}
title = "Release year of top-200 songs"
content = [
dbc.Row([
dbc.Col([
dbc.FormGroup([
dbc.Label("Country:"),
dbc.Select(id='isDutch-dropdown-outro',
options=[{
'label': key,
'value': key
} for key in isDutch_list],
value='All',
disabled=True)
])
],
width=4)
]),
dbc.Row([
dbc.Col([
dcc.Graph(id='scatter-outro',
figure=fig,
config=graph_settings["config"])
],
width=12)
]),
'padding-left': '10px',
'border': '0px'
},
className='lead')
layout = html.Div(
[
dcc.Location(id='url', refresh=False, pathname=TABS[0]['url']),
# header,
html.Div([
dbc.Row(
[
dbc.Col(html.H1(PROJECT_TITLE,
style={
'text-align': 'center',
'padding': '5px 20px 0px 30px',
'margin': '0px'
}),
width='auto'),
dbc.Col(tabs),
],
no_gutters=True,
),
],
className='pt-2 text-light bg-dark',
style={
'border-bottom-style': 'solid',
'border-width': '1px'
}),
dbc.Container(
[], id='tab-container', fluid=True, style={'padding-top': '3vh'}),
# sets the layout of the page
app.layout = html.Div(
children=[
header.getHeader(),
#
html.Div(children=[
dbc.Row(children=[
dbc.Col(children=[
createCard(
cardTitle="Top Countries by Vaccinations per Capita",
cardText="The safest places to vist "
"in the world...",
cardButtonText="View",
cardLink="/data_view1",
cardImage=
"./assets/pexels-karolina-grabowska-4047186.jpg",
cardSubtitle="Photo by Karolina Grabowska from Pexels")
],
align="center",
width=4),
dbc.Col(children=[
createCard(
cardTitle="Worst Countries by Vaccinations per Capita",
cardText="Is it time to cancel your ticket?",
cardButtonText="View",
cardLink="/data_view2",
cardImage="./assets/pexels-pavel-danilyuk.jpg",
cardSubtitle="Photo by Pavel Danilyuk from Pexels")
],
column1 = dbc.Col([
dcc.Markdown("""
# **Process**
---
""",
style={'textAlign': 'center'}),
dbc.Row(
dcc.Markdown('''
The Dataset that I chose (from https://www.kaggle.com/wendykan/lending-club-loan-data#loan.csv)
has data on over 2.2 million individual loan applications and 145 unique features. After some initial data exploration I decided to only
look at loans that had been finalized (either payed off, charged off, or defaulted). For this project I could have included open loans as
the target is return on investment and it is possible (and probable) to have made your money back while the loan is still open, however,
when doing so the return is often extremely low due to brand new loans that have not had time to generate any income.
So my first step was to seperate out all finalized loans.
```
df = pd.read_csv(datapath + '\\loan.csv', low_memory=False)
df.shape
(2260668, 145)
loans_usable = ['Fully Paid', 'Charged Off', 'Default']
df1 = df[df['loan_status'].isin(loans_usable)]
df1.shape
(1303638, 145)
```
Next I noticed that there were around 58 columns that had over 500,000+ null values. Considering
that none of these features would end up in my final model I decided to drop them. I then exported a new .csv file with 1,303,638 loans and
87 features as working with multiple large dataframes slows the whole process down. Next I engineered the ROI feature:
```
# First we'll drop the rows where funded_amnt_inv == 0 (1. There will be no return if there is no investment 2. Can't divide by 0 later!)
df = df[df['funded_amnt_inv'] != 0]
df['Return'] = df['total_pymnt_inv'] - df['funded_amnt_inv']
df['ROI'] = ((df['Return']/df['funded_amnt_inv'])*100).round(4)
```
I did this by creating a `Return` feature which was the difference between amount invested and
total amount paid back to investors. Next, create the `ROI` feature by dividing `Return` by total amount invested and multiplying by 100 to
get a percent.\n\n
Next, I took a look at which features would be valid for use in the model. Many of the features
included are generated well AFTER the loan has been accepted, and are thus unuseable in the model (Time travel is still science fiction!).
After narrowing it down I defined a function to filter features, fill NaN values, convert date time features, and ordinally encode categorical
features (this also included my earlier `Return` and `ROI` creation).
```
def wrangle(dataframe):
feats = ['loan_amnt', 'term', 'int_rate', 'installment', 'sub_grade', 'emp_title', 'emp_length', 'home_ownership',
'annual_inc', 'verification_status', 'issue_d', 'purpose', 'title', 'zip_code', 'addr_state', 'dti', 'delinq_2yrs',
'earliest_cr_line', 'inq_last_6mths', 'open_acc', 'pub_rec', 'revol_bal', 'revol_util', 'total_acc', 'total_bal_ex_mort',
'total_bc_limit', 'mo_sin_old_il_acct','mo_sin_old_rev_tl_op', 'num_accts_ever_120_pd', 'num_sats', 'total_pymnt_inv', 'funded_amnt_inv']
# Features to fillna with 0
na_fill = ['emp_length', 'mo_sin_old_il_acct', 'mo_sin_old_rev_tl_op', 'num_accts_ever_120_pd', 'num_sats',
'revol_util', 'dti', 'inq_last_6mths', 'total_bal_ex_mort', 'total_bc_limit']
# DF to work with
x = dataframe[feats].copy()
# Remove loans that had no investments
x = x[x['funded_amnt_inv'] != 0]
# Create ROI and Return features
x['Return'] = x['total_pymnt_inv'] - x['funded_amnt_inv']
x['ROI'] = (x['Return']/x['funded_amnt_inv']*100)
# Drop income outliers
x = x[(x['annual_inc'] > 10000) & (x['annual_inc'] < 500000)]
# Fix date time
x['days_since_earliest_cr_line'] = (pd.to_datetime(x['issue_d']) - pd.to_datetime(x['earliest_cr_line'])).dt.days
# Fill nans
x[na_fill] = x[na_fill].fillna(0)
# Cat encode categorical features and remove other columns
encoder = ce.OrdinalEncoder()
x = encoder.fit_transform(x.drop(columns=['issue_d', 'earliest_cr_line', 'total_pymnt_inv', 'funded_amnt_inv']))
return x
```
Unfortunately for me, I ended up using only 6 of these features in my final model, however,
many of these features were necessary to do much of the data exploration discussed in the [Insights](insights) section.
Next I made a pipeline including sklearn's `StandardScaler` and `RandomForestRegressor` and
used a `RandomizedSearchCV` to fit an initial model that I would use to get feature importances for 28 of the features. The results
were...a little disappointing to say the least.
```
RFR Val MAE: 19.46089566971158
Baseline MAE: 20.001037505776097
Improvement over Baseline for Random Forest Regression: 0.5401418360645174
```
My Mean Absolute Error had only improved by .54 over the baseline but I was able to gather many
[insights](insights) from the outcome.\n
At this point I had decided to only use 'front facing' features so I created a new pipeline again
using `StandardScaler` but this time with `LGBMRegressor` from lightgbm and using a very large GridSearchCV. Even with the feature reduction
this change gave me a much better result:
```
LGBM Test MAE: 16.685289066414494
Baseline MAE for Test data: 20.223643467120777
Improvement over Baseline for Light GBM Regression: 3.5383544007062824
```
As I stated before, this is in no way perfect, or necessarily even THAT good, but for only considering
a few features it is a great improvement over the baseline. I believe that including many of the hidden or backend features included in the data
set would allow someone to much more accurately predict the Return on Investment.
'''))
], )
sidebar = dbc.Col(
children=[
html.H1("World Happiness Report Explorer", className="display-5"),
html.Hr(),
dbc.Collapse(
html.P(
"""
This dashboard helps you find the overall happiness of all the
different countries globally and the details of the contributing factors to their happiness score.""",
style={
"color": "black",
"width": "100%"
},
),
id="collapse",
),
dbc.Col([collapse]),
html.Hr(),
html.H2("Features", className="display-6"),
html.Hr(),
dbc.Checklist(
id="feature-select-1",
options=[{
"label": k,
"value": v
} for k, v in feature_dict.items()],
value=[v for k, v in feature_dict.items()],
),
html.Hr(),
html.H3("Year Range", className="display-6"),
dcc.RangeSlider(
id="year-select-1",
min=min(summary_df.year),
max=max(summary_df.year),
step=1,
marks={
int(x): {
"label": str(x),
"style": {
"transform": "rotate(45deg)"
}
}
for x in list(summary_df.year.unique())
},
value=[2015, 2019],
pushable=1,
),
html.Hr(),
html.H3(
"Countries",
className="display-6",
style={
"width": "50%",
"display": "inline-block"
},
),
dbc.Button(
"?",
id="country-help",
color="info",
outline=True,
size="sm",
style={"float": "right"},
),
dbc.Col(
dcc.Dropdown(
id="country-select-1",
multi=True,
options=[{
"label": x,
"value": x
} for x in summary_df.country.unique()],
value=["Canada", "Switzerland", "China"],
),
width=12,
style={
"padding": "10px 10px 10px 0px",
},
),
# Add a help box for showing how to select countries
dbc.Popover(
[
dbc.PopoverHeader("Country Selection"),
dbc.PopoverBody(
"Select countries to evaluate here, or you can click on them on the map"
),
],
id="popover",
is_open=False,
target="country-help",
),
],
style={"background-color": "#f8f9fa"},
md=3,
)
def index(responses, summary_table):
business_recovery_data_plot = make_question_pie(
responses,
"How confident are you about recovery in a post-lockdown scenario?")
cash_recovery_plot = make_question_pie(
responses, "How much cash reserves do you currently have?")
employment_plot = make_question_pie(responses, "Were employees laid off?")
household_plot = make_bar(
responses,
"What household challenges did you face during the lockdown?")
row_style = {}
return [
dbc.Row(make_jumbotron("Explore", "/playground")),
html.Div(
children=[
dbc.Row(style=row_style,
children=[
dbc.Col(children=[
html.H3("Overview"),
make_summary_table(responses, summary_table),
html.Br(),
html.Br(),
],
className="col-md-12")
],
className=""),
dbc.Row(
style=row_style,
children=[
dbc.Col(children=[
html.H3(
dcc.Link("Business Recovery",
href="business")),
html.
P("""Despite most businesses remaining closed during lockdown, 81% of the respondents suggested that they are confident of business recovery in a post COVID-29 scenario.
""",
className="overview_txt"),
html.
P(""" (Click on the title to dive deeper into the section)""",
className="overview_italic"),
business_recovery_data_plot,
],
className="col-xs-12 col-sm-12 col-lg-6"),
dbc.Col(children=[
html.H3(
dcc.Link("Credit/Loans/Financial Status",
href="credit")),
html.
P("""57% reported not having any cash reserves to survive. Further, 40% tried to borrow money to cover expenses. Only 14% of the overall borrowing was from formal borrowing sources.""",
className="overview_txt"),
html.
P(""" (Click on the title to dive deeper into the section)""",
className="overview_italic"),
cash_recovery_plot,
],
className="col-xs-12 col-sm-12 col-lg-6")
],
className="",
),
# html.Hr(className="my-2"),
dbc.Row(
style=row_style,
children=[
dbc.Col(children=[
html.H3(dcc.Link("Employment", href="employment")),
html.
P("""Interestingly, most of the respondents during our Month 1 Survey suggested that they have not laid off employees. 79% of the respondents said they did not lay off, while 13% suggested that they are not laying off but are unable to pay salaries.
""",
className="overview_txt"),
html.
P(""" (Click on the title to dive deeper into the section)""",
className="overview_italic"),
employment_plot,
],
className="col-xs-12 col-sm-12 col-lg-6"),
dbc.Col(children=[
html.H3(
dcc.Link("Household Challenges",
href="household")),
html.
P("""Across all respondents, Female business owners faced more household challenges than men. 70% female business owners suggested inter house conflicts as the biggest challenge while 53% men responded to the same. Women also faced more unaffordable expenses than men, with 46% suggesting the same.
""",
className="overview_txt"),
html.
P(""" (Click on the title to dive deeper into the section)""",
className="overview_italic"),
household_plot,
],
className="col-xs-12 col-sm-12 col-lg-6")
],
),
],
className="container overviewpage",
),
html.Div(children=[
html.Div(children=[
dbc.Row(
style=row_style,
children=[
dbc.Col(children=[
html.H3(
"Survey Methodology", className="text-center"),
html.
P("""Our stratified, convenience sample was drawn from various sub-industries in manufacturing, services and trade to provide a sectoral representation of microbusinesses. The sample was selected from lists provided by partner organizations. The following regions and states will be covered in the survey: North India (Delhi, Haryana, Punjab, Uttar Pradesh), South India (Tamil Nadu), West India (Gujarat, Maharashtra, Rajasthan). The surveys were conducted telephonically in the area’s local language, and each survey took 18-25 minutes to administer.""",
className="survey_txt"),
],
className="paddzero"),
],
className=""),
],
className="container paddzero"),
],
className="overviewpage"),
] + footercontent
dcc.Dropdown(
id="file-header",
options=[{'label':"True", 'value':1}, {"label": "False", "value": 0}],
value=None,
multi=False),
dbc.Label("Separator"),
dbc.Input(id="file-separator", placeholder="Separator", type="text", step=1),
html.Br(),
dbc.Button("Apply", color="primary", id = 'file-apply-properties'),
],
style = {'padding': '10px'})
])
csv_properties_div = html.Div([
dbc.Row([
dbc.Col(csv_properties, md=4)
],
align="center")
],
style = {'margin': '10px', 'font-size': '16px'})
@app.callback(
Output("file-properties", "children"),
[Input('file-apply-properties', 'n_clicks')]
)
def apply_file_properties(n):
file = db.get("file")
format = db.get("format")
sep = db.get("file_separator")
header = db.get("file_header")
div = None
