def create(self, patch: Patch, spoiler: Spoiler, settings: Settings) -> str:
if not self.enabled:
raise EnvironmentError("Database not enabled")
if not spoiler is None:
entry = Entry(settings.seed, patch.version, patch.patch, patch.patchName, spoiler.spoiler, spoiler.spoilerName, settings)
else:
entry = Entry(settings.seed, patch.version, patch.patch, patch.patchName, None, None, settings)
key = self.collection.insert_one(entry.__dict__)
return str(key.inserted_id)
def get_all_entries():
with sqlite3.connect("./daily-journal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute("""
SELECT
e.id,
e.concept,
e.entry,
e.date,
e.moodId,
m.id,
m.label
FROM entries e
JOIN moods m
ON m.id = e.moodId
""")
entries = []
dataset = db_cursor.fetchall()
for row in dataset:
entry = Entry(row['id'], row['concept'], row['entry'], row['date'],
row['moodId'])
mood = Mood(row['id'], row['label'])
entry.mood = mood.__dict__
entries.append(entry.__dict__)
return json.dumps(entries)
def entries_query(query):
my_query = '%{}%'.format(query)
with sqlite3.connect('./daily_journal.db') as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute(
"""
SELECT
e.id,
e.concept,
e.entry,
e.date,
e.moodId
FROM entries e
WHERE e.entry LIKE ?
""", (my_query, ))
entries = []
dataset = db_cursor.fetchall()
for row in dataset:
entry = Entry(row['id'], row['concept'], row['entry'], row['date'],
row['moodId'])
entries.append(entry.__dict__)
return json.dumps(entries)
def get_single_entry(id):
with sqlite3.connect("./daily-journal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute(
"""
SELECT
e.id,
e.concept,
e.entry,
e.date,
e.moodId,
m.id,
m.label
FROM entries e
JOIN moods m
ON m.id = e.moodId
WHERE e.id = ?
""", (id, ))
data = db_cursor.fetchone()
entry = Entry(data['id'], data['concept'], data['entry'], data['date'],
data['moodId'])
mood = Mood(data['id'], data['label'])
entry.mood = mood.__dict__
return json.dumps(entry.__dict__)
def search_entry(entry):
with sqlite3.connect('./daily-journal.db') as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute(f"""
SELECT
e.id,
e.concept,
e.entry,
e.date,
e.moodId
FROM entries e
WHERE e.entry LIKE '%{entry}%'
""")
entries = []
dataset = db_cursor.fetchall()
for row in dataset:
entry = Entry(
row['id'],
row['concept'],
row['entry'],
row['date'],
row['moodId'],
)
entries.append(entry.__dict__)
return json.dumps(entries)
def get_data(session,
limit,
offset=None,
order_descending=False,
entry_id=False,
entry_type=False,
handled_utc=False,
original_url=False,
canonical_url=False,
note=False):
log.info(f"Getting data of type {entry_type} from {Table.__tablename__},"
f" limit = {limit}, order_descending = {order_descending}")
# Store the values in a dict
filter_options = {
Table.entry_id: entry_id,
Table.entry_type: entry_type,
Table.handled_utc: handled_utc,
Table.original_url: original_url,
Table.canonical_url: canonical_url,
Table.note: note
}
# Create a new query
q = session.query(Table)
# Loop through the dict and add it to the query if a value was specified
for attr, value in filter_options.items():
log.debug(f"attr= {attr}, value={value}")
if value is True:
q = q.filter(attr.isnot(None))
elif value is not False:
q = q.filter(attr == value)
# Sort descending (returns most recent rows)
if order_descending:
q = q.order_by(Table.entry_id.desc())
if offset:
q = q.offset(offset)
# Set a limit
q = q.limit(limit)
log.info(q)
log.info(f"Received data, amount of rows returned: {q.count()}")
# Generate entry instance for each returned row, add these to a list
entries = []
for entry in q:
entries.append(
Entry(entry_id=entry.entry_id,
entry_type=entry.entry_type,
handled_utc=entry.handled_utc,
original_url=entry.original_url,
canonical_url=entry.canonical_url,
note=entry.note))
log.info("Generated entry instances for each row")
return entries
def get_single_entry(id):
with sqlite3.connect("./dailyjournal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
# Use a ? parameter to inject a variable's value
# into the SQL statement.
db_cursor.execute(
"""
SELECT
a.id,
a.concept,
a.entry,
a.date,
a.moods_id
FROM JournalEntries a
WHERE a.id = ?
""", (id, ))
# Load the single result into memory
data = db_cursor.fetchone()
# Create an entry instance from the current row
entry = Entry(data['id'], data['concept'], data['entry'], data['date'],
data['moods_id'])
return json.dumps(entry.__dict__)
def get_all_entries():
with sqlite3.connect("./daily_journal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute("""
SELECT
e.id,
e.concept,
e.entry,
e.date,
e.mood_id
FROM entries e
""")
entries = []
dataset = db_cursor.fetchall()
for row in dataset:
entry = Entry(row['id'], row['concept'], row['entry'], row['date'],
row['mood_id'])
entries.append(entry._dict_)
return json.dumps(entries)
def search_for_entry(search_term):
with sqlite3.connect("./dailyjournal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute(
"""
SELECT
a.id,
a.concept,
a.entry,
a.date,
a.moodId
FROM entries a
WHERE a.entry LIKE ?
""", ('%' + search_term + '%', ))
entries = []
dataset = db_cursor.fetchall()
for row in dataset:
entry = Entry(row['id'], row['concept'], row['entry'], row['date'],
row['moodId'])
entries.append(entry.__dict__)
return json.dumps(entries)
def get_entries(self):
"""Gets film entries as list comprehension
Returns:
list comprehension of film entries
"""
return [Entry(item) for item in self.data["entries"]]
def parse_entry(self, entry):
return Entry(
id=entry.find('id').text,
title=entry.find('title').text,
link=entry.find('link')['href'],
updated=entry.find('updated').text,
summary=entry.find('summary').text,
content=entry.find('content').text.replace('\r',
'').replace('\n', ''),
author=Author(name=entry.find('author').find('name').text,
uri=entry.find('author').find('uri').text,
email=entry.find('author').find('email').text))
def __init__(self, name: str, body: dict):
if body['type'] != TYPE_DIRECTORY:
raise TypeError('cant parse non-directory object')
self.name = name
self.created_at = body['created_at']
self.updated_at = body['updated_at']
self._content = {}
for k in body['content'].keys():
k_object = body['content'][k]
if k_object['type'] == TYPE_DIRECTORY:
self._content[k] = Directory(k, k_object)
elif k_object['type'] == TYPE_ENTRY:
self._content[k] = Entry(k, k_object)
def parse_entries(self, soup):
results = []
entries = soup.find_all(
id=re.compile('MAQA_Search_gvResults_ctl00__\d+'))
for entry in entries:
heading = entry.find('a')
body = entry.find(id=re.compile(
'MAQA_Search_gvResults_ctl00_ctl\d+_pnl(Motion|Question)(?<!Header)$'
))
if body == None:
continue
# body = body.text
link = self.entry_link.format(
re.search('\w{3}-\w{5}', heading.text).group())
entry_obj = Entry(heading, body, link)
results.append(entry_obj)
return results
def insert_entries(base_url, db_path, event_ids_dict):
current_file = os.path.basename(__file__)
current_file_name = os.path.splitext(current_file)[0]
for key in event_ids_dict:
for event_id in event_ids_dict[key]:
url = base_url + "/" + current_file_name + "/" + str(
event_id) + "/"
try:
print(url)
response = requests.get(url)
except requests.exceptions.RequestException as e:
print(e)
sys.exit(1)
if response.status_code == 200:
doc = pq(response.text)
connection = sqlite3.connect(db_path)
try:
# Entries
startlist = doc("table.results")
startlist('td.entry-sct > span.text-danger').parents(
'tr').remove() # Remove course cars
for tr in startlist('tr').items():
entry = Entry(event_id, tr)
if entry.driver_id:
connection.execute(
'''INSERT INTO entries
(event_id,car_number,driver_id,codriver_id,team,car,plate,tyres,category,startlist_m,championship,created_at,updated_at,deleted_at)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', entry.get_tuple())
connection.commit()
except Exception as e:
connection.rollback()
raise e
finally:
connection.close()
def database_entry():
path = request.args['path']
entry_name = request.args['entry_name']
if request.method == 'POST':
try:
e = Entry(
entry_name,
dict(type=TYPE_ENTRY,
created_at=get_current_date(),
updated_at=get_current_date(),
content=request.json))
return created(session_global.database.decrypted.new_entry(
path, e))
except KeyError as k:
return internal(k)
elif request.method == 'GET':
try:
return retrieved(
session_global.database.decrypted.get_entry(path, entry_name))
except KeyError as k:
return not_found(k)
except TypeError as te:
return internal(te)
elif request.method == 'PUT':
try:
return edited(
session_global.database.decrypted.update_entry(
path, entry_name, request.json))
except KeyError as k:
return not_found(k)
elif request.method == 'DELETE':
try:
return deleted(
session_global.database.decrypted.delete_entry(
path, entry_name))
except KeyError as k:
return not_found(k)
except TypeError as te:
return internal(te)
def test_get_canonical_from_database_by_url(self, use_database=False):
amount_of_correct_retrievals = 0
session = get_engine_session()
# Use data from the database
if use_database:
old_entries = get_data(session=session,
limit=100,
offset=5000,
order_descending=True,
canonical_url=True)
# Or use a single entry as specified below
else:
old_entries = [
Entry(
original_url=
"https://www.mynbc5.com/amp/article/emily-ferlazzo-joseph-bolton-vermont-missing-update/38004866",
canonical_url=
"https://abc3340.com/news/inside-your-world/the-federal-government-spends-billions-each-year-maintaining-empty-buildings-nationwide"
)
]
for old_entry in old_entries:
log.info("OLD")
log.info(old_entry.entry_id)
log.info(old_entry.canonical_url)
found_entry = get_entry_by_original_url(old_entry.original_url,
session)
if found_entry:
log.info("NEW")
log.info(found_entry.entry_id)
log.info(old_entry.canonical_url)
if old_entry.entry_id == found_entry.entry_id:
amount_of_correct_retrievals += 1
else:
log.warning("No entry found!")
self.assertEqual(amount_of_correct_retrievals, old_entries.len)
def get_all_entries():
# Open a connection to the database
with sqlite3.connect("./dailyjournal.db") as conn:
# Just use these. It's a Black Box.
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
# Write the SQL query to get the information you want
db_cursor.execute("""
SELECT
a.id,
a.concept,
a.entry,
a.date,
a.moods_id
FROM JournalEntries a
""")
# Initialize an empty list to hold all entry representations
entries = []
# Convert rows of data into a Python list
dataset = db_cursor.fetchall()
# Iterate list of data returned from database
for row in dataset:
# Create an entry instance from the current row.
# Note that the database fields are specified in
# exact order of the parameters defined in the
# entry class above.
entry = Entry(row['id'], row['concept'], row['entry'], row['date'],
row['moods_id'])
entries.append(entry.__dict__)
# Use `json` package to properly serialize list as JSON
return json.dumps(entries)
def get_single_entry(id):
with sqlite3.connect("./daily_journal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute(
"""
SELECT
e.id,
e.concept,
e.entry,
e.date,
e.mood_id
FROM entries e
WHERE e.id = ?
""", (id, ))
data = db_cursor.fetchone()
entry = Entry(data['id'], data['concept'], data['entry'], data['date'],
data['mood_id'])
return json.dumps(entry.__dict__)
def storeInDatabase():
# Get json data from the endpoint
try:
response = requests.get("https://api.publicapis.org/entries").json()
except Exception as e:
error = f"Error! {e}"
return render_template('index.html', err=error)
else:
limit = 0
for entry in response["entries"]:
entry = Entry(entry["API"], entry["Description"], entry["Auth"],
entry["HTTPS"], entry["Cors"], entry["Link"],
entry["Category"])
try:
db.session.add(entry)
except Exception as e:
error = f"Error! {e}"
return render_template('index.html', err=error)
# Limit to insert only 50 records
limit += 1
if limit == 50:
break
try:
# Commit the new records to DB
db.session.commit()
message = "Data Successfully Stored in database"
except SQLAlchemyError as e:
error = e.__dict__['orig']
return render_template('index.html', err=error)
else:
# Get all entries from database
entries = Entry.query.all()
return render_template('index.html',
success=message,
entries=entries)
def get_entries_with_value(value):
with sqlite3.connect("./dailyjournal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
queryString = f'%{value}%'
# Use a ? parameter to inject a variable's value
# into the SQL statement.
entries = []
db_cursor.execute(
"""
SELECT
a.id,
a.concept,
a.entry,
a.date,
a.moods_id
FROM JournalEntries a
WHERE a.entry LIKE ?
""", (queryString, ))
data = db_cursor.fetchall()
for row in data:
# Create an entry instance from the current row.
# Note that the database fields are specified in
# exact order of the parameters defined in the
# entry class above.
entry = Entry(row['id'], row['concept'], row['entry'], row['date'],
row['moods_id'])
entries.append(entry.__dict__)
return json.dumps(entries)
sheema, errors = user_schema.load({
'username': '******',
'email': 'sheema@email',
'password': '******',
'password_confirmation': 'pass',
# 'entries': [breakfast],
# 'tasks': [project],
# 'events': [friday_drinks]
})
if errors:
raise Exception(errors)
entry_one = Entry(
date='2019-07-29',
what=
'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.',
image='image-url',
user=sheema)
entry_two = Entry(
date='2019-07-30',
what=
'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.',
image='image-url',
user=sheema)
entry_three = Entry(
date='2019-07-20',
what=
'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.',
image='image-url',
def test_canonical(self, use_database=True):
new_canonicals_amount = 0
old_canonicals_amount = 0
# Use data from the database
if use_database:
old_entries = get_data(session=get_engine_session(),
limit=100,
offset=5000,
order_descending=True,
canonical_url=None)
# Or use a single entry as specified below
else:
old_entries = [
Entry(
original_url=
"www.google.com/amp/s/abc3340.com/amp/news/inside-your-world/the-federal-government-spends-billions-each-year-maintaining-empty-buildings-nationwide",
canonical_url=
"https://abc3340.com/news/inside-your-world/the-federal-government-spends-billions-each-year-maintaining-empty-buildings-nationwide"
)
]
# Loop through every old entry and try to find the canonicals, compare the results
for old_entry in old_entries:
if old_entry.canonical_url:
old_canonicals_amount += 1
urls = get_urls(old_entry.original_url)
urls_info = get_urls_info(urls)
if urls_info:
for link in urls_info:
if link.canonical:
new_canonicals_amount += 1
if link.canonical.url == old_entry.canonical_url:
log.info("Canonical URLs match")
else:
log.warning("Canonical URLs do not match!")
log.info(f"BODY : {old_entry.original_url}")
log.info(f"OLD : {old_entry.canonical_url}")
log.info(
f"NEW : {link.canonical.url if link.canonical else None}"
)
else:
log.warning(f"No canonicals found")
log.info(
f"\nCanonicals found: Old: {old_canonicals_amount}, New: {new_canonicals_amount}"
)
# If same as before, great!
if new_canonicals_amount == old_canonicals_amount:
self.assertEqual(new_canonicals_amount, old_canonicals_amount)
# If better than before, great!
if new_canonicals_amount > old_canonicals_amount:
self.assertGreater(new_canonicals_amount, old_canonicals_amount)
# If it is worse than before, not good.
if new_canonicals_amount < old_canonicals_amount:
self.assertLess(old_canonicals_amount, new_canonicals_amount)
def test_canonical(self, use_database=True):
amount_of_canonicals = 0
old_amount_of_canonicals = 0
# Use data from the database
if use_database:
old_entries = get_data(session=get_engine_session(),
limit=500,
offset=5000,
order_descending=True,
canonical_url=False)
# Or use a single entry as specified below
else:
old_entries = [
Entry(
original_url=
"https://www.google.com/amp/s/abc3340.com/amp/news/inside-your-world/the-federal-government-spends-billions-each-year-maintaining-empty-buildings-nationwide",
canonical_url=
"https://abc3340.com/news/inside-your-world/the-federal-government-spends-billions-each-year-maintaining-empty-buildings-nationwide"
)
]
# Loop through every old entry and try to find the canonicals, compare the results
for old_entry in old_entries:
if old_entry.canonical_url:
old_amount_of_canonicals += 1
urls = get_urls(old_entry.original_url)
urls_info = get_urls_info(urls)
if urls_info:
for link in urls_info:
log.info(link.canonical_alt)
if link.amp_canonical:
log.info(link.amp_canonical)
if link.canonical:
amount_of_canonicals += 1
log.info(f"BODY : {old_entry.original_url}")
log.info(f"OLD : {old_entry.canonical_url}")
log.info(f"NEW : {link.canonical}")
if link.canonical == old_entry.canonical_url:
log.info("It's the same!")
else:
log.info("It's not the same!")
"""if link.canonical:
similarity = get_article_similarity(old_entry.original_url, link.canonical, log_articles=False)
log.info(f"Article similarity= {similarity}")"""
else:
log.warning(f"No canonicals found")
log.info(
f"\nCanonicals found: Old: {old_amount_of_canonicals}, New: {amount_of_canonicals}"
)
# If same as before, great!
if amount_of_canonicals == old_amount_of_canonicals:
self.assertEqual(amount_of_canonicals, old_amount_of_canonicals)
# If it is better than before, great!
if amount_of_canonicals > old_amount_of_canonicals:
self.assertGreater(amount_of_canonicals, old_amount_of_canonicals)
# If it is worse than before, not good.
if amount_of_canonicals < old_amount_of_canonicals:
self.assertLess(old_amount_of_canonicals, amount_of_canonicals)
def get_data(session,
limit,
offset,
order_descending,
entry_id=None,
entry_type=None,
handled_utc=None,
original_url=None,
canonical_url=None,
canonical_type=None,
note=None) -> [Entry]:
log.info(f"Getting data of type {entry_type} from {Table.__tablename__},"
f" limit = {limit}, order_descending = {order_descending}")
# Store the values in a dict
filter_options = {
Table.entry_id: entry_id,
Table.entry_type: entry_type,
Table.handled_utc: handled_utc,
Table.original_url: original_url,
Table.canonical_url: canonical_url,
Table.canonical_type: canonical_type,
Table.note: note
}
# Create a new query
q = session.query(Table)
for attr, value in filter_options.items():
log.debug(f"attr= {attr}, value={value}")
# If true, select not NULL
if value is True:
q = q.filter(attr.isnot(None))
# If false, select NULL
elif value is False:
q = q.filter(attr == None)
# If anything else, select X
elif value is not None:
q = q.filter(attr == value)
# Sort descending (returns most recent rows)
if order_descending:
q = q.order_by(Table.entry_id.desc())
if offset:
q = q.offset(offset)
# Set a limit
q = q.limit(limit)
log.info(q)
log.info(f"Received data, rows: {q.count()}")
# Generate entry instance for each returned row, add these to a list
entries = []
for entry in q:
entries.append(
Entry(entry_id=entry.entry_id,
entry_type=entry.entry_type,
handled_utc=entry.handled_utc,
original_url=entry.original_url,
canonical_url=entry.canonical_url,
canonical_type=entry.canonical_type,
note=entry.note))
log.info("Entry instance(s) generated")
return entries
def response(context, flow):
"""
Called when a server response has been received. At the time of this
message both a request and a response are present and completely done.
"""
# Values are converted from float seconds to int milliseconds later.
ssl_time = -.001
connect_time = -.001
if flow.server_conn not in context.seen_server:
# Calculate the connect_time for this server_conn. Afterwards add it to
# seen list, in order to avoid the connect_time being present in entries
# that use an existing connection.
connect_time = flow.server_conn.timestamp_tcp_setup - \
flow.server_conn.timestamp_start
context.seen_server.add(flow.server_conn)
if flow.server_conn.timestamp_ssl_setup is not None:
# Get the ssl_time for this server_conn as the difference between
# the start of the successful tcp setup and the successful ssl
# setup. If no ssl setup has been made it is left as -1 since it
# doesn't apply to this connection.
ssl_time = flow.server_conn.timestamp_ssl_setup - \
flow.server_conn.timestamp_tcp_setup
# Calculate the raw timings from the different timestamps present in the
# request and response object. For lack of a way to measure it dns timings
# can not be calculated. The same goes for HAR blocked: MITMProxy will open
# a server connection as soon as it receives the host and port from the
# client connection. So the time spent waiting is actually spent waiting
# between request.timestamp_end and response.timestamp_start thus it
# correlates to HAR wait instead.
timings_raw = {
'send': flow.request.timestamp_end - flow.request.timestamp_start,
'wait': flow.response.timestamp_start - flow.request.timestamp_end,
'receive': flow.response.timestamp_end - flow.response.timestamp_start,
'connect': connect_time,
'ssl': ssl_time
}
# HAR timings are integers in ms, so we have to re-encode the raw timings to
# that format.
timings = dict([(key, int(1000 * value))
for key, value in timings_raw.iteritems()])
# The full_time is the sum of all timings. Timings set to -1 will be ignored
# as per spec.
full_time = 0
for item in timings.values():
if item > -1:
full_time += item
started_date_time = datetime.fromtimestamp(
flow.request.timestamp_start, tz=pytz.timezone('UTC')).isoformat()
request_query_string = [{
"name": k,
"value": v
} for k, v in flow.request.get_query()]
request_http_version = ".".join([str(v) for v in flow.request.httpversion])
# Cookies are shaped as tuples by MITMProxy.
request_cookies = [{
"name": k.strip(),
"value": v[0]
} for k, v in (flow.request.get_cookies() or {}).iteritems()]
request_headers = [{
"name": k,
"value": v
} for k, v in flow.request.headers]
request_headers_size = len(str(flow.request.headers))
request_body_size = len(flow.request.content)
response_http_version = ".".join(
[str(v) for v in flow.response.httpversion])
# Cookies are shaped as tuples by MITMProxy.
response_cookies = [{
"name": k.strip(),
"value": v[0]
} for k, v in (flow.response.get_cookies() or {}).iteritems()]
response_headers = [{
"name": k,
"value": v
} for k, v in flow.response.headers]
response_headers_size = len(str(flow.response.headers))
response_body_size = len(flow.response.content)
response_body_decoded_size = len(flow.response.get_decoded_content())
response_body_compression = response_body_decoded_size - response_body_size
response_mime_type = flow.response.headers.get_first('Content-Type', '')
response_redirect_url = flow.response.headers.get_first('Location', '')
container_id = os.environ['HOSTNAME']
entry = Entry()
entry.startedDateTime = started_date_time
entry.time = full_time
entry.container_id = container_id
entry.request = {
"method": flow.request.method,
"url": flow.request.url,
"httpVersion": request_http_version,
"cookies": request_cookies,
"headers": request_headers,
"queryString": request_query_string,
"headersSize": request_headers_size,
"bodySize": request_body_size,
}
entry.response = {
"url": flow.request.url,
"status": flow.response.code,
"statusText": flow.response.msg,
"httpVersion": response_http_version,
"cookies": response_cookies,
"headers": response_headers,
"content": {
"size": response_body_size,
"compression": response_body_compression,
"mimeType": response_mime_type
},
"redirectURL": response_redirect_url,
"headersSize": response_headers_size,
"bodySize": response_body_size,
}
entry.cache = {}
entry.timings = timings
#ws = create_connection("ws://localhost:9000/ws/{0}".format(entry.pageref))
ws = create_connection(
"ws://websocket.bandit.io:9000/ws/{0}".format(container_id),
sslopt={"check_hostname": False})
ws.send(
json.dumps({
"method": "Network.requestWillBeSent",
"params": {
"requestId": "7897.52",
"frameId": "7897.1",
"loaderId": "7897.3",
"documentURL": entry.request['url'],
"request": {
"url": entry.request['url'],
"method": entry.request['method'],
"headers": {
header['name']: header['value']
for header in entry.request['headers']
}
},
"timestamp": 88986.634829,
"wallTime": 1440472453.19435,
"initiator": {
"type": "other"
},
"type": "Document"
}
}))
ws.send(
json.dumps({
"method": "Network.responseReceived",
"params": {
"requestId": "7897.52",
"frameId": "7897.1",
"loaderId": "7897.3",
"timestamp": 88986.985021,
"type": "Document",
"response": {
"url": entry.response['url'],
"status": entry.response['status'],
"statusText": entry.response['statusText'],
"headers": {
header['name']: header['value']
for header in entry.response['headers']
},
"mimeType": entry.response['content']['mimeType'],
"connectionReused": False,
"fromDiskCache": False,
"fromServiceWorker": False,
"timing": {
"requestTime": 88986.636403,
"proxyStart": -1,
"proxyEnd": -1,
"dnsStart": 0,
"dnsEnd": 108.372000002419,
"connectStart": 108.372000002419,
"connectEnd": 113.420000008773,
"sslStart": -1,
"sslEnd": -1,
"serviceWorkerFetchStart": -1,
"serviceWorkerFetchReady": -1,
"serviceWorkerFetchEnd": -1,
"sendStart": 113.492999997106,
"sendEnd": 113.573000009637,
"receiveHeadersEnd": 347.90900000371
},
# "requestHeaders": {
# "If-None-Match": "\"1440455137124|#public|0|en|||0\"",
# "Accept-Encoding": "gzip, deflate, sdch",
# "Host": "www.chromium.org",
# "Accept-Language": "en-US,en;q=0.8",
# "Upgrade-Insecure-Requests": "1",
# "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36",
# "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
# "Cache-Control": "max-age=0",
# "Cookie": "_ga=GA1.2.1062414394.1440468745; _gat_SitesTracker=1; __utmt=1; __utma=221884874.1062414394.1440468745.1440468745.1440471278.2; __utmb=221884874.2.10.1440471278; __utmc=221884874; __utmz=221884874.1440468745.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); aftzc=QW1lcmljYS9Mb3NfQW5nZWxlczp3eGRhd0FxcWxWZkNYdHRkVVJ2ZStlVEpOOVE9",
# "Connection": "keep-alive",
# "If-Modified-Since": "Mon, 24 Aug 2015 22:25:37 GMT"
# },
"remoteIPAddress": "216.239.32.27",
"remotePort": 80,
"protocol":
"http/{0}".format(entry.response['httpVersion'])
}
}
}))
ws.send(
json.dumps({
"method": "Network.dataReceived",
"params": {
"requestId": "7897.52",
"timestamp": 88986.985513,
"dataLength": entry.response['content']['size'],
"encodedDataLength": entry.response['bodySize']
}
}))
ws.send(
json.dumps({
"method": "Network.loadingFinished",
"params": {
"requestId": "7897.52",
"timestamp": 88986.985401,
"encodedDataLength": entry.response['bodySize']
}
}))
#ws.send(json.dumps({"method":"Network.requestWillBeSent","params":{"requestId":"7897.52","frameId":"7897.1","loaderId":"7897.3","documentURL":"http://www.chromium.org/","request":{"url":"http://www.chromium.org/","method":"GET","headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","Upgrade-Insecure-Requests":"1","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"}},"timestamp":88986.634829,"wallTime":1440472453.19435,"initiator":{"type":"other"},"type":"Document"}}))
#ws.send(json.dumps({"method":"Network.responseReceived","params":{"requestId":"7897.52","frameId":"7897.1","loaderId":"7897.3","timestamp":88986.985021,"type":"Document","response":{"url":"http://www.chromium.org/","status":304,"statusText":"Not Modified","headers":{"Date":"Tue, 25 Aug 2015 03:14:13 GMT","Last-Modified":"Mon, 24 Aug 2015 22:25:37 GMT","Server":"GSE","X-Robots-Tag":"noarchive","ETag":"\"1440455137124|#public|0|en|||0\""},"mimeType":"text/html","connectionReused":False,"connectionId":2554,"encodedDataLength":-1,"fromDiskCache":False,"fromServiceWorker":False,"timing":{"requestTime":88986.636403,"proxyStart":-1,"proxyEnd":-1,"dnsStart":0,"dnsEnd":108.372000002419,"connectStart":108.372000002419,"connectEnd":113.420000008773,"sslStart":-1,"sslEnd":-1,"serviceWorkerFetchStart":-1,"serviceWorkerFetchReady":-1,"serviceWorkerFetchEnd":-1,"sendStart":113.492999997106,"sendEnd":113.573000009637,"receiveHeadersEnd":347.90900000371},"headersText":"HTTP/1.1 304 Not Modified\r\nX-Robots-Tag: noarchive\r\nLast-Modified: Mon, 24 Aug 2015 22:25:37 GMT\r\nETag: \"1440455137124|#public|0|en|||0\"\r\nDate: Tue, 25 Aug 2015 03:14:13 GMT\r\nServer: GSE\r\n\r\n","requestHeaders":{"If-None-Match":"\"1440455137124|#public|0|en|||0\"","Accept-Encoding":"gzip, deflate, sdch","Host":"www.chromium.org","Accept-Language":"en-US,en;q=0.8","Upgrade-Insecure-Requests":"1","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36","Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","Cache-Control":"max-age=0","Cookie":"_ga=GA1.2.1062414394.1440468745; _gat_SitesTracker=1; __utmt=1; __utma=221884874.1062414394.1440468745.1440468745.1440471278.2; __utmb=221884874.2.10.1440471278; __utmc=221884874; __utmz=221884874.1440468745.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); aftzc=QW1lcmljYS9Mb3NfQW5nZWxlczp3eGRhd0FxcWxWZkNYdHRkVVJ2ZStlVEpOOVE9","Connection":"keep-alive","If-Modified-Since":"Mon, 24 Aug 2015 22:25:37 GMT"},"requestHeadersText":"GET / HTTP/1.1\r\nHost: www.chromium.org\r\nConnection: keep-alive\r\nCache-Control: max-age=0\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36\r\nAccept-Encoding: gzip, deflate, sdch\r\nAccept-Language: en-US,en;q=0.8\r\nCookie: _ga=GA1.2.1062414394.1440468745; _gat_SitesTracker=1; __utmt=1; __utma=221884874.1062414394.1440468745.1440468745.1440471278.2; __utmb=221884874.2.10.1440471278; __utmc=221884874; __utmz=221884874.1440468745.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); aftzc=QW1lcmljYS9Mb3NfQW5nZWxlczp3eGRhd0FxcWxWZkNYdHRkVVJ2ZStlVEpOOVE9\r\nIf-None-Match: \"1440455137124|#public|0|en|||0\"\r\nIf-Modified-Since: Mon, 24 Aug 2015 22:25:37 GMT\r\n\r\n","remoteIPAddress":"216.239.32.27","remotePort":80,"protocol":"http/1.1"}}}))
#ws.send(json.dumps({"method":"Network.dataReceived","params":{"requestId":"7897.52","timestamp":88986.985513,"dataLength":23423,"encodedDataLength":190}}))
##WebSocketHandler.ws_send('test', {"method":"Page.frameNavigated","params":{"frame":{"id":"7897.1","loaderId":"7897.3","url":"http://www.chromium.org/","mimeType":"text/html","securityOrigin":"http://www.chromium.org"}}})
#ws.send(json.dumps({"method":"Network.loadingFinished","params":{"requestId":"7897.52","timestamp":88986.985401,"encodedDataLength":190}}))
ws.close()
# def done(context):
# """
# Called once on script shutdown, after any other events.
# """
# from pprint import pprint
# import json
#
# json_dump = context.HARLog.json()
# compressed_json_dump = context.HARLog.compress()
#
# if context.dump_file == '-':
# context.log(pprint.pformat(json.loads(json_dump)))
# elif context.dump_file.endswith('.zhar'):
# file(context.dump_file, "w").write(compressed_json_dump)
# else:
# file(context.dump_file, "w").write(json_dump)
# context.log(
# "HAR log finished with %s bytes (%s bytes compressed)" % (
# len(json_dump), len(compressed_json_dump)
# )
# )
# context.log(
# "Compression rate is %s%%" % str(
# 100. * len(compressed_json_dump) / len(json_dump)
# )
# )
#
#
# def print_attributes(obj, filter_string=None, hide_privates=False):
# """
# Useful helper method to quickly get all attributes of an object and its
# values.
# """
# for attr in dir(obj):
# if hide_privates and "__" in attr:
# continue
# if filter_string is not None and filter_string not in attr:
# continue
# value = getattr(obj, attr)
# print("%s.%s" % ('obj', attr), value, type(value))
from models.entry import Entry
entry = Entry()
entry.test = 'hello'
print entry.test
#
# headers = [
# {
# "name": "Host",
# "value": "www.cnn.com"
# },
# {
# "name": "User-Agent",
# "value": "curl/7.43.0"
# },
# {
# "name": "Accept",
# "value": "*/*"
# },
# {
# "name": "Proxy-Connection",
# "value": "Keep-Alive"
# },
# {
# "name": "accept-encoding",
# "value": "identity"
# }
# ]
#
#
