def create(self, patch: Patch, spoiler: Spoiler, settings: Settings) -> str: if not self.enabled: raise EnvironmentError("Database not enabled") if not spoiler is None: entry = Entry(settings.seed, patch.version, patch.patch, patch.patchName, spoiler.spoiler, spoiler.spoilerName, settings) else: entry = Entry(settings.seed, patch.version, patch.patch, patch.patchName, None, None, settings) key = self.collection.insert_one(entry.__dict__) return str(key.inserted_id)
def get_all_entries(): with sqlite3.connect("./daily-journal.db") as conn: conn.row_factory = sqlite3.Row db_cursor = conn.cursor() db_cursor.execute(""" SELECT e.id, e.concept, e.entry, e.date, e.moodId, m.id, m.label FROM entries e JOIN moods m ON m.id = e.moodId """) entries = [] dataset = db_cursor.fetchall() for row in dataset: entry = Entry(row['id'], row['concept'], row['entry'], row['date'], row['moodId']) mood = Mood(row['id'], row['label']) entry.mood = mood.__dict__ entries.append(entry.__dict__) return json.dumps(entries)
def entries_query(query): my_query = '%{}%'.format(query) with sqlite3.connect('./daily_journal.db') as conn: conn.row_factory = sqlite3.Row db_cursor = conn.cursor() db_cursor.execute( """ SELECT e.id, e.concept, e.entry, e.date, e.moodId FROM entries e WHERE e.entry LIKE ? """, (my_query, )) entries = [] dataset = db_cursor.fetchall() for row in dataset: entry = Entry(row['id'], row['concept'], row['entry'], row['date'], row['moodId']) entries.append(entry.__dict__) return json.dumps(entries)
def get_single_entry(id): with sqlite3.connect("./daily-journal.db") as conn: conn.row_factory = sqlite3.Row db_cursor = conn.cursor() db_cursor.execute( """ SELECT e.id, e.concept, e.entry, e.date, e.moodId, m.id, m.label FROM entries e JOIN moods m ON m.id = e.moodId WHERE e.id = ? """, (id, )) data = db_cursor.fetchone() entry = Entry(data['id'], data['concept'], data['entry'], data['date'], data['moodId']) mood = Mood(data['id'], data['label']) entry.mood = mood.__dict__ return json.dumps(entry.__dict__)
def search_entry(entry): with sqlite3.connect('./daily-journal.db') as conn: conn.row_factory = sqlite3.Row db_cursor = conn.cursor() db_cursor.execute(f""" SELECT e.id, e.concept, e.entry, e.date, e.moodId FROM entries e WHERE e.entry LIKE '%{entry}%' """) entries = [] dataset = db_cursor.fetchall() for row in dataset: entry = Entry( row['id'], row['concept'], row['entry'], row['date'], row['moodId'], ) entries.append(entry.__dict__) return json.dumps(entries)
def get_data(session, limit, offset=None, order_descending=False, entry_id=False, entry_type=False, handled_utc=False, original_url=False, canonical_url=False, note=False): log.info(f"Getting data of type {entry_type} from {Table.__tablename__}," f" limit = {limit}, order_descending = {order_descending}") # Store the values in a dict filter_options = { Table.entry_id: entry_id, Table.entry_type: entry_type, Table.handled_utc: handled_utc, Table.original_url: original_url, Table.canonical_url: canonical_url, Table.note: note } # Create a new query q = session.query(Table) # Loop through the dict and add it to the query if a value was specified for attr, value in filter_options.items(): log.debug(f"attr= {attr}, value={value}") if value is True: q = q.filter(attr.isnot(None)) elif value is not False: q = q.filter(attr == value) # Sort descending (returns most recent rows) if order_descending: q = q.order_by(Table.entry_id.desc()) if offset: q = q.offset(offset) # Set a limit q = q.limit(limit) log.info(q) log.info(f"Received data, amount of rows returned: {q.count()}") # Generate entry instance for each returned row, add these to a list entries = [] for entry in q: entries.append( Entry(entry_id=entry.entry_id, entry_type=entry.entry_type, handled_utc=entry.handled_utc, original_url=entry.original_url, canonical_url=entry.canonical_url, note=entry.note)) log.info("Generated entry instances for each row") return entries
def get_single_entry(id): with sqlite3.connect("./dailyjournal.db") as conn: conn.row_factory = sqlite3.Row db_cursor = conn.cursor() # Use a ? parameter to inject a variable's value # into the SQL statement. db_cursor.execute( """ SELECT a.id, a.concept, a.entry, a.date, a.moods_id FROM JournalEntries a WHERE a.id = ? """, (id, )) # Load the single result into memory data = db_cursor.fetchone() # Create an entry instance from the current row entry = Entry(data['id'], data['concept'], data['entry'], data['date'], data['moods_id']) return json.dumps(entry.__dict__)
def get_all_entries(): with sqlite3.connect("./daily_journal.db") as conn: conn.row_factory = sqlite3.Row db_cursor = conn.cursor() db_cursor.execute(""" SELECT e.id, e.concept, e.entry, e.date, e.mood_id FROM entries e """) entries = [] dataset = db_cursor.fetchall() for row in dataset: entry = Entry(row['id'], row['concept'], row['entry'], row['date'], row['mood_id']) entries.append(entry._dict_) return json.dumps(entries)
def search_for_entry(search_term): with sqlite3.connect("./dailyjournal.db") as conn: conn.row_factory = sqlite3.Row db_cursor = conn.cursor() db_cursor.execute( """ SELECT a.id, a.concept, a.entry, a.date, a.moodId FROM entries a WHERE a.entry LIKE ? """, ('%' + search_term + '%', )) entries = [] dataset = db_cursor.fetchall() for row in dataset: entry = Entry(row['id'], row['concept'], row['entry'], row['date'], row['moodId']) entries.append(entry.__dict__) return json.dumps(entries)
def get_entries(self): """Gets film entries as list comprehension Returns: list comprehension of film entries """ return [Entry(item) for item in self.data["entries"]]
def parse_entry(self, entry): return Entry( id=entry.find('id').text, title=entry.find('title').text, link=entry.find('link')['href'], updated=entry.find('updated').text, summary=entry.find('summary').text, content=entry.find('content').text.replace('\r', '').replace('\n', ''), author=Author(name=entry.find('author').find('name').text, uri=entry.find('author').find('uri').text, email=entry.find('author').find('email').text))
def __init__(self, name: str, body: dict): if body['type'] != TYPE_DIRECTORY: raise TypeError('cant parse non-directory object') self.name = name self.created_at = body['created_at'] self.updated_at = body['updated_at'] self._content = {} for k in body['content'].keys(): k_object = body['content'][k] if k_object['type'] == TYPE_DIRECTORY: self._content[k] = Directory(k, k_object) elif k_object['type'] == TYPE_ENTRY: self._content[k] = Entry(k, k_object)
def parse_entries(self, soup): results = [] entries = soup.find_all( id=re.compile('MAQA_Search_gvResults_ctl00__\d+')) for entry in entries: heading = entry.find('a') body = entry.find(id=re.compile( 'MAQA_Search_gvResults_ctl00_ctl\d+_pnl(Motion|Question)(?<!Header)$' )) if body == None: continue # body = body.text link = self.entry_link.format( re.search('\w{3}-\w{5}', heading.text).group()) entry_obj = Entry(heading, body, link) results.append(entry_obj) return results
def insert_entries(base_url, db_path, event_ids_dict): current_file = os.path.basename(__file__) current_file_name = os.path.splitext(current_file)[0] for key in event_ids_dict: for event_id in event_ids_dict[key]: url = base_url + "/" + current_file_name + "/" + str( event_id) + "/" try: print(url) response = requests.get(url) except requests.exceptions.RequestException as e: print(e) sys.exit(1) if response.status_code == 200: doc = pq(response.text) connection = sqlite3.connect(db_path) try: # Entries startlist = doc("table.results") startlist('td.entry-sct > span.text-danger').parents( 'tr').remove() # Remove course cars for tr in startlist('tr').items(): entry = Entry(event_id, tr) if entry.driver_id: connection.execute( '''INSERT INTO entries (event_id,car_number,driver_id,codriver_id,team,car,plate,tyres,category,startlist_m,championship,created_at,updated_at,deleted_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', entry.get_tuple()) connection.commit() except Exception as e: connection.rollback() raise e finally: connection.close()
def database_entry(): path = request.args['path'] entry_name = request.args['entry_name'] if request.method == 'POST': try: e = Entry( entry_name, dict(type=TYPE_ENTRY, created_at=get_current_date(), updated_at=get_current_date(), content=request.json)) return created(session_global.database.decrypted.new_entry( path, e)) except KeyError as k: return internal(k) elif request.method == 'GET': try: return retrieved( session_global.database.decrypted.get_entry(path, entry_name)) except KeyError as k: return not_found(k) except TypeError as te: return internal(te) elif request.method == 'PUT': try: return edited( session_global.database.decrypted.update_entry( path, entry_name, request.json)) except KeyError as k: return not_found(k) elif request.method == 'DELETE': try: return deleted( session_global.database.decrypted.delete_entry( path, entry_name)) except KeyError as k: return not_found(k) except TypeError as te: return internal(te)
def test_get_canonical_from_database_by_url(self, use_database=False): amount_of_correct_retrievals = 0 session = get_engine_session() # Use data from the database if use_database: old_entries = get_data(session=session, limit=100, offset=5000, order_descending=True, canonical_url=True) # Or use a single entry as specified below else: old_entries = [ Entry( original_url= "https://www.mynbc5.com/amp/article/emily-ferlazzo-joseph-bolton-vermont-missing-update/38004866", canonical_url= "https://abc3340.com/news/inside-your-world/the-federal-government-spends-billions-each-year-maintaining-empty-buildings-nationwide" ) ] for old_entry in old_entries: log.info("OLD") log.info(old_entry.entry_id) log.info(old_entry.canonical_url) found_entry = get_entry_by_original_url(old_entry.original_url, session) if found_entry: log.info("NEW") log.info(found_entry.entry_id) log.info(old_entry.canonical_url) if old_entry.entry_id == found_entry.entry_id: amount_of_correct_retrievals += 1 else: log.warning("No entry found!") self.assertEqual(amount_of_correct_retrievals, old_entries.len)
def get_all_entries(): # Open a connection to the database with sqlite3.connect("./dailyjournal.db") as conn: # Just use these. It's a Black Box. conn.row_factory = sqlite3.Row db_cursor = conn.cursor() # Write the SQL query to get the information you want db_cursor.execute(""" SELECT a.id, a.concept, a.entry, a.date, a.moods_id FROM JournalEntries a """) # Initialize an empty list to hold all entry representations entries = [] # Convert rows of data into a Python list dataset = db_cursor.fetchall() # Iterate list of data returned from database for row in dataset: # Create an entry instance from the current row. # Note that the database fields are specified in # exact order of the parameters defined in the # entry class above. entry = Entry(row['id'], row['concept'], row['entry'], row['date'], row['moods_id']) entries.append(entry.__dict__) # Use `json` package to properly serialize list as JSON return json.dumps(entries)
def get_single_entry(id): with sqlite3.connect("./daily_journal.db") as conn: conn.row_factory = sqlite3.Row db_cursor = conn.cursor() db_cursor.execute( """ SELECT e.id, e.concept, e.entry, e.date, e.mood_id FROM entries e WHERE e.id = ? """, (id, )) data = db_cursor.fetchone() entry = Entry(data['id'], data['concept'], data['entry'], data['date'], data['mood_id']) return json.dumps(entry.__dict__)
def storeInDatabase(): # Get json data from the endpoint try: response = requests.get("https://api.publicapis.org/entries").json() except Exception as e: error = f"Error! {e}" return render_template('index.html', err=error) else: limit = 0 for entry in response["entries"]: entry = Entry(entry["API"], entry["Description"], entry["Auth"], entry["HTTPS"], entry["Cors"], entry["Link"], entry["Category"]) try: db.session.add(entry) except Exception as e: error = f"Error! {e}" return render_template('index.html', err=error) # Limit to insert only 50 records limit += 1 if limit == 50: break try: # Commit the new records to DB db.session.commit() message = "Data Successfully Stored in database" except SQLAlchemyError as e: error = e.__dict__['orig'] return render_template('index.html', err=error) else: # Get all entries from database entries = Entry.query.all() return render_template('index.html', success=message, entries=entries)
def get_entries_with_value(value): with sqlite3.connect("./dailyjournal.db") as conn: conn.row_factory = sqlite3.Row db_cursor = conn.cursor() queryString = f'%{value}%' # Use a ? parameter to inject a variable's value # into the SQL statement. entries = [] db_cursor.execute( """ SELECT a.id, a.concept, a.entry, a.date, a.moods_id FROM JournalEntries a WHERE a.entry LIKE ? """, (queryString, )) data = db_cursor.fetchall() for row in data: # Create an entry instance from the current row. # Note that the database fields are specified in # exact order of the parameters defined in the # entry class above. entry = Entry(row['id'], row['concept'], row['entry'], row['date'], row['moods_id']) entries.append(entry.__dict__) return json.dumps(entries)
sheema, errors = user_schema.load({ 'username': '******', 'email': 'sheema@email', 'password': '******', 'password_confirmation': 'pass', # 'entries': [breakfast], # 'tasks': [project], # 'events': [friday_drinks] }) if errors: raise Exception(errors) entry_one = Entry( date='2019-07-29', what= 'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.', image='image-url', user=sheema) entry_two = Entry( date='2019-07-30', what= 'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.', image='image-url', user=sheema) entry_three = Entry( date='2019-07-20', what= 'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.', image='image-url',
def test_canonical(self, use_database=True): new_canonicals_amount = 0 old_canonicals_amount = 0 # Use data from the database if use_database: old_entries = get_data(session=get_engine_session(), limit=100, offset=5000, order_descending=True, canonical_url=None) # Or use a single entry as specified below else: old_entries = [ Entry( original_url= "www.google.com/amp/s/abc3340.com/amp/news/inside-your-world/the-federal-government-spends-billions-each-year-maintaining-empty-buildings-nationwide", canonical_url= "https://abc3340.com/news/inside-your-world/the-federal-government-spends-billions-each-year-maintaining-empty-buildings-nationwide" ) ] # Loop through every old entry and try to find the canonicals, compare the results for old_entry in old_entries: if old_entry.canonical_url: old_canonicals_amount += 1 urls = get_urls(old_entry.original_url) urls_info = get_urls_info(urls) if urls_info: for link in urls_info: if link.canonical: new_canonicals_amount += 1 if link.canonical.url == old_entry.canonical_url: log.info("Canonical URLs match") else: log.warning("Canonical URLs do not match!") log.info(f"BODY : {old_entry.original_url}") log.info(f"OLD : {old_entry.canonical_url}") log.info( f"NEW : {link.canonical.url if link.canonical else None}" ) else: log.warning(f"No canonicals found") log.info( f"\nCanonicals found: Old: {old_canonicals_amount}, New: {new_canonicals_amount}" ) # If same as before, great! if new_canonicals_amount == old_canonicals_amount: self.assertEqual(new_canonicals_amount, old_canonicals_amount) # If better than before, great! if new_canonicals_amount > old_canonicals_amount: self.assertGreater(new_canonicals_amount, old_canonicals_amount) # If it is worse than before, not good. if new_canonicals_amount < old_canonicals_amount: self.assertLess(old_canonicals_amount, new_canonicals_amount)
def test_canonical(self, use_database=True): amount_of_canonicals = 0 old_amount_of_canonicals = 0 # Use data from the database if use_database: old_entries = get_data(session=get_engine_session(), limit=500, offset=5000, order_descending=True, canonical_url=False) # Or use a single entry as specified below else: old_entries = [ Entry( original_url= "https://www.google.com/amp/s/abc3340.com/amp/news/inside-your-world/the-federal-government-spends-billions-each-year-maintaining-empty-buildings-nationwide", canonical_url= "https://abc3340.com/news/inside-your-world/the-federal-government-spends-billions-each-year-maintaining-empty-buildings-nationwide" ) ] # Loop through every old entry and try to find the canonicals, compare the results for old_entry in old_entries: if old_entry.canonical_url: old_amount_of_canonicals += 1 urls = get_urls(old_entry.original_url) urls_info = get_urls_info(urls) if urls_info: for link in urls_info: log.info(link.canonical_alt) if link.amp_canonical: log.info(link.amp_canonical) if link.canonical: amount_of_canonicals += 1 log.info(f"BODY : {old_entry.original_url}") log.info(f"OLD : {old_entry.canonical_url}") log.info(f"NEW : {link.canonical}") if link.canonical == old_entry.canonical_url: log.info("It's the same!") else: log.info("It's not the same!") """if link.canonical: similarity = get_article_similarity(old_entry.original_url, link.canonical, log_articles=False) log.info(f"Article similarity= {similarity}")""" else: log.warning(f"No canonicals found") log.info( f"\nCanonicals found: Old: {old_amount_of_canonicals}, New: {amount_of_canonicals}" ) # If same as before, great! if amount_of_canonicals == old_amount_of_canonicals: self.assertEqual(amount_of_canonicals, old_amount_of_canonicals) # If it is better than before, great! if amount_of_canonicals > old_amount_of_canonicals: self.assertGreater(amount_of_canonicals, old_amount_of_canonicals) # If it is worse than before, not good. if amount_of_canonicals < old_amount_of_canonicals: self.assertLess(old_amount_of_canonicals, amount_of_canonicals)
def get_data(session, limit, offset, order_descending, entry_id=None, entry_type=None, handled_utc=None, original_url=None, canonical_url=None, canonical_type=None, note=None) -> [Entry]: log.info(f"Getting data of type {entry_type} from {Table.__tablename__}," f" limit = {limit}, order_descending = {order_descending}") # Store the values in a dict filter_options = { Table.entry_id: entry_id, Table.entry_type: entry_type, Table.handled_utc: handled_utc, Table.original_url: original_url, Table.canonical_url: canonical_url, Table.canonical_type: canonical_type, Table.note: note } # Create a new query q = session.query(Table) for attr, value in filter_options.items(): log.debug(f"attr= {attr}, value={value}") # If true, select not NULL if value is True: q = q.filter(attr.isnot(None)) # If false, select NULL elif value is False: q = q.filter(attr == None) # If anything else, select X elif value is not None: q = q.filter(attr == value) # Sort descending (returns most recent rows) if order_descending: q = q.order_by(Table.entry_id.desc()) if offset: q = q.offset(offset) # Set a limit q = q.limit(limit) log.info(q) log.info(f"Received data, rows: {q.count()}") # Generate entry instance for each returned row, add these to a list entries = [] for entry in q: entries.append( Entry(entry_id=entry.entry_id, entry_type=entry.entry_type, handled_utc=entry.handled_utc, original_url=entry.original_url, canonical_url=entry.canonical_url, canonical_type=entry.canonical_type, note=entry.note)) log.info("Entry instance(s) generated") return entries
def response(context, flow): """ Called when a server response has been received. At the time of this message both a request and a response are present and completely done. """ # Values are converted from float seconds to int milliseconds later. ssl_time = -.001 connect_time = -.001 if flow.server_conn not in context.seen_server: # Calculate the connect_time for this server_conn. Afterwards add it to # seen list, in order to avoid the connect_time being present in entries # that use an existing connection. connect_time = flow.server_conn.timestamp_tcp_setup - \ flow.server_conn.timestamp_start context.seen_server.add(flow.server_conn) if flow.server_conn.timestamp_ssl_setup is not None: # Get the ssl_time for this server_conn as the difference between # the start of the successful tcp setup and the successful ssl # setup. If no ssl setup has been made it is left as -1 since it # doesn't apply to this connection. ssl_time = flow.server_conn.timestamp_ssl_setup - \ flow.server_conn.timestamp_tcp_setup # Calculate the raw timings from the different timestamps present in the # request and response object. For lack of a way to measure it dns timings # can not be calculated. The same goes for HAR blocked: MITMProxy will open # a server connection as soon as it receives the host and port from the # client connection. So the time spent waiting is actually spent waiting # between request.timestamp_end and response.timestamp_start thus it # correlates to HAR wait instead. timings_raw = { 'send': flow.request.timestamp_end - flow.request.timestamp_start, 'wait': flow.response.timestamp_start - flow.request.timestamp_end, 'receive': flow.response.timestamp_end - flow.response.timestamp_start, 'connect': connect_time, 'ssl': ssl_time } # HAR timings are integers in ms, so we have to re-encode the raw timings to # that format. timings = dict([(key, int(1000 * value)) for key, value in timings_raw.iteritems()]) # The full_time is the sum of all timings. Timings set to -1 will be ignored # as per spec. full_time = 0 for item in timings.values(): if item > -1: full_time += item started_date_time = datetime.fromtimestamp( flow.request.timestamp_start, tz=pytz.timezone('UTC')).isoformat() request_query_string = [{ "name": k, "value": v } for k, v in flow.request.get_query()] request_http_version = ".".join([str(v) for v in flow.request.httpversion]) # Cookies are shaped as tuples by MITMProxy. request_cookies = [{ "name": k.strip(), "value": v[0] } for k, v in (flow.request.get_cookies() or {}).iteritems()] request_headers = [{ "name": k, "value": v } for k, v in flow.request.headers] request_headers_size = len(str(flow.request.headers)) request_body_size = len(flow.request.content) response_http_version = ".".join( [str(v) for v in flow.response.httpversion]) # Cookies are shaped as tuples by MITMProxy. response_cookies = [{ "name": k.strip(), "value": v[0] } for k, v in (flow.response.get_cookies() or {}).iteritems()] response_headers = [{ "name": k, "value": v } for k, v in flow.response.headers] response_headers_size = len(str(flow.response.headers)) response_body_size = len(flow.response.content) response_body_decoded_size = len(flow.response.get_decoded_content()) response_body_compression = response_body_decoded_size - response_body_size response_mime_type = flow.response.headers.get_first('Content-Type', '') response_redirect_url = flow.response.headers.get_first('Location', '') container_id = os.environ['HOSTNAME'] entry = Entry() entry.startedDateTime = started_date_time entry.time = full_time entry.container_id = container_id entry.request = { "method": flow.request.method, "url": flow.request.url, "httpVersion": request_http_version, "cookies": request_cookies, "headers": request_headers, "queryString": request_query_string, "headersSize": request_headers_size, "bodySize": request_body_size, } entry.response = { "url": flow.request.url, "status": flow.response.code, "statusText": flow.response.msg, "httpVersion": response_http_version, "cookies": response_cookies, "headers": response_headers, "content": { "size": response_body_size, "compression": response_body_compression, "mimeType": response_mime_type }, "redirectURL": response_redirect_url, "headersSize": response_headers_size, "bodySize": response_body_size, } entry.cache = {} entry.timings = timings #ws = create_connection("ws://localhost:9000/ws/{0}".format(entry.pageref)) ws = create_connection( "ws://websocket.bandit.io:9000/ws/{0}".format(container_id), sslopt={"check_hostname": False}) ws.send( json.dumps({ "method": "Network.requestWillBeSent", "params": { "requestId": "7897.52", "frameId": "7897.1", "loaderId": "7897.3", "documentURL": entry.request['url'], "request": { "url": entry.request['url'], "method": entry.request['method'], "headers": { header['name']: header['value'] for header in entry.request['headers'] } }, "timestamp": 88986.634829, "wallTime": 1440472453.19435, "initiator": { "type": "other" }, "type": "Document" } })) ws.send( json.dumps({ "method": "Network.responseReceived", "params": { "requestId": "7897.52", "frameId": "7897.1", "loaderId": "7897.3", "timestamp": 88986.985021, "type": "Document", "response": { "url": entry.response['url'], "status": entry.response['status'], "statusText": entry.response['statusText'], "headers": { header['name']: header['value'] for header in entry.response['headers'] }, "mimeType": entry.response['content']['mimeType'], "connectionReused": False, "fromDiskCache": False, "fromServiceWorker": False, "timing": { "requestTime": 88986.636403, "proxyStart": -1, "proxyEnd": -1, "dnsStart": 0, "dnsEnd": 108.372000002419, "connectStart": 108.372000002419, "connectEnd": 113.420000008773, "sslStart": -1, "sslEnd": -1, "serviceWorkerFetchStart": -1, "serviceWorkerFetchReady": -1, "serviceWorkerFetchEnd": -1, "sendStart": 113.492999997106, "sendEnd": 113.573000009637, "receiveHeadersEnd": 347.90900000371 }, # "requestHeaders": { # "If-None-Match": "\"1440455137124|#public|0|en|||0\"", # "Accept-Encoding": "gzip, deflate, sdch", # "Host": "www.chromium.org", # "Accept-Language": "en-US,en;q=0.8", # "Upgrade-Insecure-Requests": "1", # "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36", # "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", # "Cache-Control": "max-age=0", # "Cookie": "_ga=GA1.2.1062414394.1440468745; _gat_SitesTracker=1; __utmt=1; __utma=221884874.1062414394.1440468745.1440468745.1440471278.2; __utmb=221884874.2.10.1440471278; __utmc=221884874; __utmz=221884874.1440468745.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); aftzc=QW1lcmljYS9Mb3NfQW5nZWxlczp3eGRhd0FxcWxWZkNYdHRkVVJ2ZStlVEpOOVE9", # "Connection": "keep-alive", # "If-Modified-Since": "Mon, 24 Aug 2015 22:25:37 GMT" # }, "remoteIPAddress": "216.239.32.27", "remotePort": 80, "protocol": "http/{0}".format(entry.response['httpVersion']) } } })) ws.send( json.dumps({ "method": "Network.dataReceived", "params": { "requestId": "7897.52", "timestamp": 88986.985513, "dataLength": entry.response['content']['size'], "encodedDataLength": entry.response['bodySize'] } })) ws.send( json.dumps({ "method": "Network.loadingFinished", "params": { "requestId": "7897.52", "timestamp": 88986.985401, "encodedDataLength": entry.response['bodySize'] } })) #ws.send(json.dumps({"method":"Network.requestWillBeSent","params":{"requestId":"7897.52","frameId":"7897.1","loaderId":"7897.3","documentURL":"http://www.chromium.org/","request":{"url":"http://www.chromium.org/","method":"GET","headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","Upgrade-Insecure-Requests":"1","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"}},"timestamp":88986.634829,"wallTime":1440472453.19435,"initiator":{"type":"other"},"type":"Document"}})) #ws.send(json.dumps({"method":"Network.responseReceived","params":{"requestId":"7897.52","frameId":"7897.1","loaderId":"7897.3","timestamp":88986.985021,"type":"Document","response":{"url":"http://www.chromium.org/","status":304,"statusText":"Not Modified","headers":{"Date":"Tue, 25 Aug 2015 03:14:13 GMT","Last-Modified":"Mon, 24 Aug 2015 22:25:37 GMT","Server":"GSE","X-Robots-Tag":"noarchive","ETag":"\"1440455137124|#public|0|en|||0\""},"mimeType":"text/html","connectionReused":False,"connectionId":2554,"encodedDataLength":-1,"fromDiskCache":False,"fromServiceWorker":False,"timing":{"requestTime":88986.636403,"proxyStart":-1,"proxyEnd":-1,"dnsStart":0,"dnsEnd":108.372000002419,"connectStart":108.372000002419,"connectEnd":113.420000008773,"sslStart":-1,"sslEnd":-1,"serviceWorkerFetchStart":-1,"serviceWorkerFetchReady":-1,"serviceWorkerFetchEnd":-1,"sendStart":113.492999997106,"sendEnd":113.573000009637,"receiveHeadersEnd":347.90900000371},"headersText":"HTTP/1.1 304 Not Modified\r\nX-Robots-Tag: noarchive\r\nLast-Modified: Mon, 24 Aug 2015 22:25:37 GMT\r\nETag: \"1440455137124|#public|0|en|||0\"\r\nDate: Tue, 25 Aug 2015 03:14:13 GMT\r\nServer: GSE\r\n\r\n","requestHeaders":{"If-None-Match":"\"1440455137124|#public|0|en|||0\"","Accept-Encoding":"gzip, deflate, sdch","Host":"www.chromium.org","Accept-Language":"en-US,en;q=0.8","Upgrade-Insecure-Requests":"1","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36","Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","Cache-Control":"max-age=0","Cookie":"_ga=GA1.2.1062414394.1440468745; _gat_SitesTracker=1; __utmt=1; __utma=221884874.1062414394.1440468745.1440468745.1440471278.2; __utmb=221884874.2.10.1440471278; __utmc=221884874; __utmz=221884874.1440468745.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); aftzc=QW1lcmljYS9Mb3NfQW5nZWxlczp3eGRhd0FxcWxWZkNYdHRkVVJ2ZStlVEpOOVE9","Connection":"keep-alive","If-Modified-Since":"Mon, 24 Aug 2015 22:25:37 GMT"},"requestHeadersText":"GET / HTTP/1.1\r\nHost: www.chromium.org\r\nConnection: keep-alive\r\nCache-Control: max-age=0\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36\r\nAccept-Encoding: gzip, deflate, sdch\r\nAccept-Language: en-US,en;q=0.8\r\nCookie: _ga=GA1.2.1062414394.1440468745; _gat_SitesTracker=1; __utmt=1; __utma=221884874.1062414394.1440468745.1440468745.1440471278.2; __utmb=221884874.2.10.1440471278; __utmc=221884874; __utmz=221884874.1440468745.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); aftzc=QW1lcmljYS9Mb3NfQW5nZWxlczp3eGRhd0FxcWxWZkNYdHRkVVJ2ZStlVEpOOVE9\r\nIf-None-Match: \"1440455137124|#public|0|en|||0\"\r\nIf-Modified-Since: Mon, 24 Aug 2015 22:25:37 GMT\r\n\r\n","remoteIPAddress":"216.239.32.27","remotePort":80,"protocol":"http/1.1"}}})) #ws.send(json.dumps({"method":"Network.dataReceived","params":{"requestId":"7897.52","timestamp":88986.985513,"dataLength":23423,"encodedDataLength":190}})) ##WebSocketHandler.ws_send('test', {"method":"Page.frameNavigated","params":{"frame":{"id":"7897.1","loaderId":"7897.3","url":"http://www.chromium.org/","mimeType":"text/html","securityOrigin":"http://www.chromium.org"}}}) #ws.send(json.dumps({"method":"Network.loadingFinished","params":{"requestId":"7897.52","timestamp":88986.985401,"encodedDataLength":190}})) ws.close() # def done(context): # """ # Called once on script shutdown, after any other events. # """ # from pprint import pprint # import json # # json_dump = context.HARLog.json() # compressed_json_dump = context.HARLog.compress() # # if context.dump_file == '-': # context.log(pprint.pformat(json.loads(json_dump))) # elif context.dump_file.endswith('.zhar'): # file(context.dump_file, "w").write(compressed_json_dump) # else: # file(context.dump_file, "w").write(json_dump) # context.log( # "HAR log finished with %s bytes (%s bytes compressed)" % ( # len(json_dump), len(compressed_json_dump) # ) # ) # context.log( # "Compression rate is %s%%" % str( # 100. * len(compressed_json_dump) / len(json_dump) # ) # ) # # # def print_attributes(obj, filter_string=None, hide_privates=False): # """ # Useful helper method to quickly get all attributes of an object and its # values. # """ # for attr in dir(obj): # if hide_privates and "__" in attr: # continue # if filter_string is not None and filter_string not in attr: # continue # value = getattr(obj, attr) # print("%s.%s" % ('obj', attr), value, type(value))
from models.entry import Entry entry = Entry() entry.test = 'hello' print entry.test # # headers = [ # { # "name": "Host", # "value": "www.cnn.com" # }, # { # "name": "User-Agent", # "value": "curl/7.43.0" # }, # { # "name": "Accept", # "value": "*/*" # }, # { # "name": "Proxy-Connection", # "value": "Keep-Alive" # }, # { # "name": "accept-encoding", # "value": "identity" # } # ] # #