def get_all_entries():
with sqlite3.connect("./daily-journal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute("""
SELECT
e.id,
e.concept,
e.entry,
e.date,
e.moodId,
m.id,
m.label
FROM entries e
JOIN moods m
ON m.id = e.moodId
""")
entries = []
dataset = db_cursor.fetchall()
for row in dataset:
entry = Entry(row['id'], row['concept'], row['entry'], row['date'],
row['moodId'])
mood = Mood(row['id'], row['label'])
entry.mood = mood.__dict__
entries.append(entry.__dict__)
return json.dumps(entries)
def get_single_entry(id):
with sqlite3.connect("./daily-journal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute(
"""
SELECT
e.id,
e.concept,
e.entry,
e.date,
e.moodId,
m.id,
m.label
FROM entries e
JOIN moods m
ON m.id = e.moodId
WHERE e.id = ?
""", (id, ))
data = db_cursor.fetchone()
entry = Entry(data['id'], data['concept'], data['entry'], data['date'],
data['moodId'])
mood = Mood(data['id'], data['label'])
entry.mood = mood.__dict__
return json.dumps(entry.__dict__)
def post(self):
url = self.request.get("url", "")
if url:
slug = nice_guid()
entry = Entry(time=datetime.datetime.now(),
url=url, slug=slug, packaged=False)
entry.put()
self.render_template("code.html", slug=slug)
else:
return self.get()
def post(self):
upload_files = self.get_uploads('package')
blob_info = upload_files[0]
slug = nice_guid()
entry = Entry(time=datetime.datetime.now(),
url=str(blob_info.key()), slug=slug, packaged=True)
entry.put()
self.render_template("code.html", slug=slug)
def test_insert_entry__invalid_entry_type__expect_raise_error(self):
connection = Connection('postgres', 'root', '127.0.0.1:5432', 'planting_manager_teste')
session = connection.session()
meeiro_id = create_meeiro(name='tadeu', cpf='55584447213', rg='50658045x')
session.expunge_all()
session.close()
with self.assertRaises(RowNotFound):
Entry.insert(meeiro_id=meeiro_id, entry_date=datetime(2018, 10, 1),
entry_type_id=23, entry_value=100.0,
description='veneno', db_session=connection.session())
def test_insert_entry__invalid_meeiro__expect_raise_error(self):
connection = Connection('postgres', 'root', '127.0.0.1:5432', 'planting_manager_teste')
session = connection.session()
entry_type_id = create_entry_type('despesas', connection)
session.expunge_all()
session.close()
with self.assertRaises(RowNotFound):
Entry.insert(meeiro_id=21, entry_date=datetime(2018, 10, 1),
entry_type_id=entry_type_id, entry_value=100.0,
description='veneno', db_session=connection.session())
def create(self, patch: Patch, spoiler: Spoiler, settings: Settings) -> str:
if not self.enabled:
raise EnvironmentError("Database not enabled")
if not spoiler is None:
entry = Entry(settings.seed, patch.version, patch.patch, patch.patchName, spoiler.spoiler, spoiler.spoilerName, settings)
else:
entry = Entry(settings.seed, patch.version, patch.patch, patch.patchName, None, None, settings)
key = self.collection.insert_one(entry.__dict__)
return str(key.inserted_id)
def post(self):
entry = Entry(
author = users.get_current_user().email(),
title = self.request.get('title'),
slug = self.request.get("slug"),
# tags = self.request.get("tags"),
template = self.request.get("template"),
body = self.request.get("body"),
draft = bool(self.request.get("draft"))
)
entry.put()
entry.index()
self.redirect("/" + entry.category.slug + "/" + entry.slug)
def search_entry(entry):
with sqlite3.connect('./daily-journal.db') as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute(f"""
SELECT
e.id,
e.concept,
e.entry,
e.date,
e.moodId
FROM entries e
WHERE e.entry LIKE '%{entry}%'
""")
entries = []
dataset = db_cursor.fetchall()
for row in dataset:
entry = Entry(
row['id'],
row['concept'],
row['entry'],
row['date'],
row['moodId'],
)
entries.append(entry.__dict__)
return json.dumps(entries)
def wp_editPage(
blogid,
pageid,
struct,
publish,
):
entry = Entry.get_by_id(int(pageid))
if struct.has_key('wp_slug'):
entry.slug = struct['wp_slug']
if struct.has_key('wp_page_order'):
entry.menu_order = int(struct['wp_page_order'])
entry.title = struct['title']
entry.content = struct['description']
if struct.has_key('mt_text_more'):
entry.content = entry.content + '<!--more-->' + struct['mt_text_more']
if publish:
entry.publish(True)
else:
entry.save()
return True
def entries_query(query):
my_query = '%{}%'.format(query)
with sqlite3.connect('./daily_journal.db') as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute(
"""
SELECT
e.id,
e.concept,
e.entry,
e.date,
e.moodId
FROM entries e
WHERE e.entry LIKE ?
""", (my_query, ))
entries = []
dataset = db_cursor.fetchall()
for row in dataset:
entry = Entry(row['id'], row['concept'], row['entry'], row['date'],
row['moodId'])
entries.append(entry.__dict__)
return json.dumps(entries)
def action_updatelink(self):
link_format = self.param('linkfmt')
if link_format:
link_format = link_format.strip()
g_blog.link_format = link_format
g_blog.save()
for entry in Entry.all():
vals = {
'year': entry.date.year,
'month': str(entry.date.month).zfill(2),
'day': entry.date.day,
'postname': entry.slug,
'post_id': entry.post_id,
}
if entry.slug:
newlink = link_format % vals
else:
newlink = '?p=%(post_id)s' % vals
if entry.link != newlink:
entry.link = newlink
entry.put()
self.write('"ok"')
else:
self.write('"Please input url format."')
def get_data(session,
limit,
offset=None,
order_descending=False,
entry_id=False,
entry_type=False,
handled_utc=False,
original_url=False,
canonical_url=False,
note=False):
log.info(f"Getting data of type {entry_type} from {Table.__tablename__},"
f" limit = {limit}, order_descending = {order_descending}")
# Store the values in a dict
filter_options = {
Table.entry_id: entry_id,
Table.entry_type: entry_type,
Table.handled_utc: handled_utc,
Table.original_url: original_url,
Table.canonical_url: canonical_url,
Table.note: note
}
# Create a new query
q = session.query(Table)
# Loop through the dict and add it to the query if a value was specified
for attr, value in filter_options.items():
log.debug(f"attr= {attr}, value={value}")
if value is True:
q = q.filter(attr.isnot(None))
elif value is not False:
q = q.filter(attr == value)
# Sort descending (returns most recent rows)
if order_descending:
q = q.order_by(Table.entry_id.desc())
if offset:
q = q.offset(offset)
# Set a limit
q = q.limit(limit)
log.info(q)
log.info(f"Received data, amount of rows returned: {q.count()}")
# Generate entry instance for each returned row, add these to a list
entries = []
for entry in q:
entries.append(
Entry(entry_id=entry.entry_id,
entry_type=entry.entry_type,
handled_utc=entry.handled_utc,
original_url=entry.original_url,
canonical_url=entry.canonical_url,
note=entry.note))
log.info("Generated entry instances for each row")
return entries
def action_updatecomments(self):
for entry in Entry.all():
cnt = entry.comments().count()
if cnt != entry.commentcount:
entry.commentcount = cnt
entry.put()
self.write('"ok"')
def get_single_entry(id):
with sqlite3.connect("./dailyjournal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
# Use a ? parameter to inject a variable's value
# into the SQL statement.
db_cursor.execute(
"""
SELECT
a.id,
a.concept,
a.entry,
a.date,
a.moods_id
FROM JournalEntries a
WHERE a.id = ?
""", (id, ))
# Load the single result into memory
data = db_cursor.fetchone()
# Create an entry instance from the current row
entry = Entry(data['id'], data['concept'], data['entry'], data['date'],
data['moods_id'])
return json.dumps(entry.__dict__)
def get(self):
urls = []
def addurl(loc, lastmod=None, changefreq=None, priority=None):
url_info = {
'location': loc,
'lastmod': lastmod,
'changefreq': changefreq,
'priority': priority,
}
urls.append(url_info)
addurl(g_blog.baseurl, changefreq='daily', priority=1)
entries = Entry.all().filter('published =', True).order('-date').fetch(g_blog.sitemap_entries)
for item in entries:
loc = '%s/%s' % (g_blog.baseurl, item.link)
addurl(loc, item.date, 'daily', 0.9)
if g_blog.sitemap_include_category:
cats = Category.all()
for cat in cats:
loc = '%s/category/%s' % (g_blog.baseurl, cat.slug)
addurl(loc, None, 'weekly', 0.8)
if g_blog.sitemap_include_tag:
tags = Tag.all()
for tag in tags:
loc = '%s/tag/%s' % (g_blog.baseurl, urlencode(tag.tag))
addurl(loc, None, 'weekly', 0.8)
self.response.headers['Content-Type'] = 'text/xml; charset=utf-8'
self.render2('views/sitemap.xml', {'urlset': urls})
def doget(self, page):
try:
page = int(page)
max_page = (self.blog.entrycount - 1) / self.blog.posts_per_page
if max_page < 0:
max_page = 0
if page < 0 or page > max_page:
return self.error(404)
query = Entry.gql("WHERE entrytype = 'post' AND published = TRUE ORDER BY date DESC")
entries = query.fetch(self.blog.posts_per_page, page * self.blog.posts_per_page)
show_prev = entries and not page == 0
show_next = entries and not page == max_page
logging.info("generated main page")
return self.render('index', {
'entries': entries,
'show_prev': show_prev,
'show_next': show_next,
'pageindex': page,
'ishome': True,
})
except:
logging.error(traceback.format_exc())
return self.error(404)
def get(self, tags=None):
entries = Entry.all().order('-date')
cates = Category.all()
tags = Tag.all()
self.response.headers['Content-Type'] = 'binary/octet-stream' # 'application/atom+xml'
self.render2('views/wordpress.xml', {'entries': entries, 'cates': cates, 'tags': tags})
def get_all_entries():
with sqlite3.connect("./daily_journal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute("""
SELECT
e.id,
e.concept,
e.entry,
e.date,
e.mood_id
FROM entries e
""")
entries = []
dataset = db_cursor.fetchall()
for row in dataset:
entry = Entry(row['id'], row['concept'], row['entry'], row['date'],
row['mood_id'])
entries.append(entry._dict_)
return json.dumps(entries)
def get(self, slug):
app = Entry.all().filter('slug =', slug).get()
if app.packaged:
self.render_template("install.html", url='/minifest/%s' % slug,
packaged=True)
else:
self.render_template("install.html", url=app.url, packaged=False)
def search_for_entry(search_term):
with sqlite3.connect("./dailyjournal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute(
"""
SELECT
a.id,
a.concept,
a.entry,
a.date,
a.moodId
FROM entries a
WHERE a.entry LIKE ?
""", ('%' + search_term + '%', ))
entries = []
dataset = db_cursor.fetchall()
for row in dataset:
entry = Entry(row['id'], row['concept'], row['entry'], row['date'],
row['moodId'])
entries.append(entry.__dict__)
return json.dumps(entries)
def update_site_entry(self):
for rss_entry in self.entries:
entry = Entry.find_by_link(rss_entry.link)
if entry.empty:
Entry.create(rss_entry, self.site_url)
elif entry.entry_md5 != rss_entry.entry_md5:
# print "rss_link : %s" %(rss_entry.link)
# print "link : %s" %(entry['link'])
# print "entry_md5 : %s" %(entry.entry_md5())
# print "rss_entry_md5 : %s" %(rss_entry.entry_md5())
entry['description'] = rss_entry.description
entry['title'] = rss_entry.title
# print "After entry_md5 : %s" %(entry.entry_md5())
entry.save()
def get_entries(self):
"""Gets film entries as list comprehension
Returns:
list comprehension of film entries
"""
return [Entry(item) for item in self.data["entries"]]
def _test_url(self, url):
self.response.write('%s<br>' % url)
resp = urlfetch.fetch('%s/media/revision.txt' % url)
assert resp.status_code == 200
rev = resp.content.strip()
resp = urlfetch.fetch('%s?mobile=true' % url)
if resp.status_code != 200:
return
size = len(resp.content)
asset_size = 0
css_size = 0
js_size = 0
for asset in (m.group(2) for m in
asset_url_pattern.finditer(resp.content)):
# Handle relative URLs
if '://' not in asset:
asset = url + asset
if ('.js?' in asset or '.css' in asset or
asset.endswith('.js')):
self.response.write('%s<br>' % asset)
try:
data = urlfetch.fetch(asset).content
except Exception:
continue
if data:
data_len = len(data)
if '.js?' in asset:
js_size += data_len
elif '.css' in asset:
css_size += data_len
asset_size += data_len
entry = Entry(time=datetime.datetime.now(), size=size,
domain=url, with_assets=size + asset_size,
commit=rev, size_css=css_size, size_js=js_size)
entry.put()
self.response.write('Size: %d<br>' % size)
self.response.write('Assets Size: %d ' % asset_size)
self.response.write('(CSS: %d, JS: %d)<br>' % (css_size, js_size))
def get(self, tags=None):
entries = Entry.all().filter('entrytype =', 'post').filter('published =', True).order('-date').fetch(20)
if entries and entries[0]:
last_updated = entries[0].date
last_updated = last_updated.strftime('%Y-%m-%dT%H:%M:%SZ')
for e in entries:
e.formatted_date = e.date.strftime('%Y-%m-%dT%H:%M:%SZ')
self.response.headers['Content-Type'] = 'application/atom+xml'
self.render2('views/atom.xml', {'entries': entries, 'last_updated': last_updated})
def wp_newPage(blogid, struct, publish):
entry = Entry(title=struct['title'], content=struct['description'])
if struct.has_key('mt_text_more'):
entry.content = entry.content + '<!--more-->' + struct['mt_text_more']
if struct.has_key('wp_slug'):
entry.slug = struct['wp_slug']
if struct.has_key('wp_page_order'):
entry.menu_order = int(struct['wp_page_order'])
entry.entrytype = 'page'
if publish:
entry.publish(True)
else:
entry.save()
postid = entry.key().id()
return str(postid)
def insert_entries(base_url, db_path, event_ids_dict):
current_file = os.path.basename(__file__)
current_file_name = os.path.splitext(current_file)[0]
for key in event_ids_dict:
for event_id in event_ids_dict[key]:
url = base_url + "/" + current_file_name + "/" + str(
event_id) + "/"
try:
print(url)
response = requests.get(url)
except requests.exceptions.RequestException as e:
print(e)
sys.exit(1)
if response.status_code == 200:
doc = pq(response.text)
connection = sqlite3.connect(db_path)
try:
# Entries
startlist = doc("table.results")
startlist('td.entry-sct > span.text-danger').parents(
'tr').remove() # Remove course cars
for tr in startlist('tr').items():
entry = Entry(event_id, tr)
if entry.driver_id:
connection.execute(
'''INSERT INTO entries
(event_id,car_number,driver_id,codriver_id,team,car,plate,tyres,category,startlist_m,championship,created_at,updated_at,deleted_at)
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)''', entry.get_tuple())
connection.commit()
except Exception as e:
connection.rollback()
raise e
finally:
connection.close()
def get(self, slug=None, postid=None):
if postid:
entries = Entry.all().filter("published =", True).filter("post_id =", postid).fetch(1)
else:
slug = urldecode(slug)
entries = Entry.all().filter("published =", True).filter("link =", slug).fetch(1)
if not entries or len(entries) == 0:
return self.error(404)
entry = entries[0]
comments = Comment.all().filter("entry =", entry)
commentuser = ["", "", ""]
if entry.entrytype == "post":
self.render(
"single",
{
"entry": entry,
"relateposts": entry.relateposts,
"comments": comments,
"user_name": commentuser[0],
"user_email": commentuser[1],
"user_url": commentuser[2],
"checknum1": random.randint(1, 10),
"checknum2": random.randint(1, 10),
},
)
else:
self.render(
"page",
{
"entry": entry,
"relateposts": entry.relateposts,
"comments": comments,
"user_name": commentuser[0],
"user_email": commentuser[1],
"user_url": commentuser[2],
"checknum1": random.randint(1, 10),
"checknum2": random.randint(1, 10),
},
)
def test_insert_entry__expect_correct_insert(self):
connection = Connection('postgres', 'root', '127.0.0.1:5432', 'planting_manager_teste')
session = connection.session()
meeiro_id = create_meeiro(name='tadeu', cpf='55584447213', rg='50658045x')
entry_type_id = create_entry_type('despesas', connection)
session.expunge_all()
session.close()
Entry.insert(meeiro_id=meeiro_id, entry_date=datetime(2018, 10, 1),
entry_type_id=entry_type_id, entry_value=100.0,
description='veneno', db_session=connection.session())
america_timezone = pytz.timezone('America/Sao_Paulo')
entry = session.query(EntryMapping).one()
self.assertEqual(entry.entry_type, entry_type_id)
self.assertEqual(str(entry.entry_date), str(america_timezone.localize(datetime(2018, 10, 1))))
self.assertEqual(entry.meeiro_id, meeiro_id)
self.assertEqual(entry.description, 'veneno')
self.assertEqual(entry.entry_value, 100.0)
def action_init_blog(self, slug=None):
for com in Comment.all():
com.delete()
for entry in Entry.all():
entry.delete()
g_blog.entrycount = 0
self.write('"Init has succeed."')
def initialize(self, request, response):
try:
BaseRequestHandler.initialize(self, request, response)
m_pages = (
Entry.all()
.filter("entrytype =", "page")
.filter("published =", True)
.filter("entry_parent =", 0)
.order("menu_order")
)
blogroll = Link.all().filter("linktype =", "blogroll")
query = Entry.gql("WHERE entrytype = 'post' AND published = TRUE ORDER BY date")
entries = query.fetch(1)
start_date = end_date = None
if entries:
start_date = entries[0].date
query = Entry.gql("WHERE entrytype = 'post' AND published = TRUE ORDER BY date DESC")
entries = query.fetch(1)
if entries:
end_date = entries[0].date
end_year = 0
if end_date:
end_year = end_date.year
self.template_vals.update(
{
"dates": self.build_dates(start_date, end_date),
"end_year": end_year,
"menu_pages": m_pages,
"tags": Tag.all().order("tag"),
"blogroll": blogroll,
"recent_comments": Comment.all().order("-date").fetch(5),
}
)
logging.info("base public page initialized")
except:
logging.error(traceback.format_exc())
return self.error(404)
def parse_entry(self, entry):
return Entry(
id=entry.find('id').text,
title=entry.find('title').text,
link=entry.find('link')['href'],
updated=entry.find('updated').text,
summary=entry.find('summary').text,
content=entry.find('content').text.replace('\r',
'').replace('\n', ''),
author=Author(name=entry.find('author').find('name').text,
uri=entry.find('author').find('uri').text,
email=entry.find('author').find('email').text))
def get_recent_data(domain):
key = '%s:recent' % domain
data = memcache.get(key)
if data is not None:
return pickle.loads(data)
else:
# ~Two weeks of data.
data = list(Entry.all().filter('domain =', urls[domain])
.order('-time')
.run(limit=336))
memcache.add(key, pickle.dumps(data), time=3600)
return data
def test_get_entries__filter_by_range_date__expected_correct_result(self):
america_timezone = pytz.timezone('America/Sao_Paulo')
connection = Connection('postgres', 'root', '127.0.0.1:5432', 'planting_manager_teste')
session = connection.session()
meeiro_id = create_meeiro(name='tadeu', cpf='55584447213', rg='50658045x')
entry_type_id = create_entry_type('despesas', connection)
session.expunge_all()
session.close()
execute_session = connection.session()
Entry.insert(meeiro_id=meeiro_id, entry_date=america_timezone.localize(datetime(2018, 10, 2, 1, 1)),
entry_type_id=entry_type_id, entry_value=100.0,
description='veneno', db_session=execute_session)
Entry.insert(meeiro_id=meeiro_id, entry_date=america_timezone.localize(datetime(2018, 10, 1, 23, 58)),
entry_type_id=entry_type_id, entry_value=1520.0,
description='combustivel', db_session=execute_session)
filters_ = Entry.get_filters_to_query_entry(db_session=execute_session,
date_filter=
DateRange(min_date=america_timezone.localize(datetime(2018, 10, 1, 23, 59)),
max_date=america_timezone.localize(datetime(2018, 10, 2, 1, 10)),
use_equal=True),
)
entries = Entry.list(execute_session, filters_)
self.assertEqual(len(entries), 1)
expected_entry = entries[0]
self.assertEqual(expected_entry._entry_type_id, entry_type_id)
self.assertEqual(str(expected_entry._entry_date), str(america_timezone.localize(datetime(2018, 10, 2, 1, 1))))
self.assertEqual(expected_entry._meeiro_id, meeiro_id)
self.assertEqual(expected_entry._description, 'veneno')
self.assertEqual(expected_entry._entry_value, 100.0)
def action_update_tags(self, slug=None):
for tag in Tag.all():
tag.delete()
for entry in Entry.all().filter('entrytype =', 'post'):
if entry.tags:
for t in entry.tags:
try:
logging.info('sss:' + t)
Tag.add(t)
except:
traceback.print_exc()
self.write('"All tags for entry have been updated."')
def get(self, slug):
self.response.headers.add_header(
"Content-type", "application/x-web-app-manifest+json")
try:
app = Entry.all().filter('slug =', slug).get()
except Exception:
self.response.write('{"error":"Not found."}')
return
try:
blob_reader = blobstore.BlobReader(app.url)
except Exception:
self.response.write('{"error":"Could not retrieve package."}')
return
try:
package = ZipFile(StringIO(blob_reader.read()))
except Exception:
self.response.write('{"error":"Could not retrieve package."}')
return
try:
manifest = package.read("manifest.webapp")
except Exception:
self.response.write('{"error":"Could not open manifest."}')
return
try:
unjsoned = json.loads(manifest)
except Exception:
self.response.write('{"error":"Could not parse manifest."}')
return
try:
name = unjsoned["name"]
except Exception:
self.response.write('{"error":"Could not read app name."}')
return
try:
version = unjsoned["version"]
except Exception:
self.response.write('{"error":"Could not read app version."}')
return
self.response.write(json.dumps({
"name": name,
"package_path": "/serve/%s" % app.url,
"version": version,
}))
def get(self, slug=None):
logging.info("browsing by tag %s" % slug)
if not slug:
self.error(404)
return
try:
page_index = int(self.param('page'))
except:
page_index = 1
import urllib
slug = urldecode(slug)
entries = Entry.all().filter('tags =', slug).order('-date')
(entries, links) = Pager(query=entries).fetch(page_index)
self.render('tag', {'entries': entries, 'tag': slug, 'pager': links})
def post(self, slug):
key = self.request.get("key", None)
entry = Entry.get(key)
entry.title = self.request.get('title')
entry.slug = self.request.get("slug")
# entry.published = datetime.datetime.strptime( self.request.get("published"), "%Y-%m-%d %H:%M:%S" )
entry.excerpt = self.request.get("excerpt")
entry.body = self.request.get("body")
entry.draft = bool(self.request.get("draft"))
entry.put()
entry.index()
entry.indexed_title_changed()
self.redirect("/entry/" + entry.slug)
def get(self, number):
offset = 10*(int(number)-1)
entries_count = Entry.all(keys_only=True).count()
if entries_count / 10 >= number:
number = number+1
else:
number = False
entries = db.Query(Entry).filter('draft = ', False).order('-published').fetch(limit=10, offset=offset)
for i in range(len(entries)):
body = markdown.markdown(entries[i].body)
entries[i].body = body
path = os.path.join(template_dir, 'blog.html')
data = {'entries': entries, 'sitetitle': 'The Staydecent™ Web Design & Development Blog', 'number':number}
self.response.out.write(template.render(path, data))
def get(self, slug=None):
if not slug:
self.error(404)
return
try:
page_index = int(self.param('page'))
except:
page_index = 1
slug = urllib.unquote(slug).decode('utf8')
cats = Category.all().filter('slug =', slug).fetch(1)
if cats:
entries = Entry.all().filter('categorie_keys =', cats[0].key()).order('-date')
(entries, links) = Pager(query=entries).fetch(page_index)
self.render('category', {'entries': entries, 'category': cats[0], 'pager': links})
else:
self.error(414, slug)
def __init__(self, name: str, body: dict):
if body['type'] != TYPE_DIRECTORY:
raise TypeError('cant parse non-directory object')
self.name = name
self.created_at = body['created_at']
self.updated_at = body['updated_at']
self._content = {}
for k in body['content'].keys():
k_object = body['content'][k]
if k_object['type'] == TYPE_DIRECTORY:
self._content[k] = Directory(k, k_object)
elif k_object['type'] == TYPE_ENTRY:
self._content[k] = Entry(k, k_object)
def parse_entries(self, soup):
results = []
entries = soup.find_all(
id=re.compile('MAQA_Search_gvResults_ctl00__\d+'))
for entry in entries:
heading = entry.find('a')
body = entry.find(id=re.compile(
'MAQA_Search_gvResults_ctl00_ctl\d+_pnl(Motion|Question)(?<!Header)$'
))
if body == None:
continue
# body = body.text
link = self.entry_link.format(
re.search('\w{3}-\w{5}', heading.text).group())
entry_obj = Entry(heading, body, link)
results.append(entry_obj)
return results
def get(self, year, month):
gql = "WHERE date > DATETIME(%s, %s, 1, 0, 0, 0) AND date < DATETIME(%s, %s, %d, 23, 59, 59) ORDER BY date DESC" % (year, month, year, month, self.last_day_of_month(year, month))
query = Entry.gql(gql)
entries = query.fetch(1000)
show_prev = False
show_next = False
return self.render('index', {
'entries': entries,
'show_prev': show_prev,
'show_next': show_next,
'pageindex': 0,
'ishome': True,
'end_year': int(year),
})
def database_entry():
path = request.args['path']
entry_name = request.args['entry_name']
if request.method == 'POST':
try:
e = Entry(
entry_name,
dict(type=TYPE_ENTRY,
created_at=get_current_date(),
updated_at=get_current_date(),
content=request.json))
return created(session_global.database.decrypted.new_entry(
path, e))
except KeyError as k:
return internal(k)
elif request.method == 'GET':
try:
return retrieved(
session_global.database.decrypted.get_entry(path, entry_name))
except KeyError as k:
return not_found(k)
except TypeError as te:
return internal(te)
elif request.method == 'PUT':
try:
return edited(
session_global.database.decrypted.update_entry(
path, entry_name, request.json))
except KeyError as k:
return not_found(k)
elif request.method == 'DELETE':
try:
return deleted(
session_global.database.decrypted.delete_entry(
path, entry_name))
except KeyError as k:
return not_found(k)
except TypeError as te:
return internal(te)
def test_get_canonical_from_database_by_url(self, use_database=False):
amount_of_correct_retrievals = 0
session = get_engine_session()
# Use data from the database
if use_database:
old_entries = get_data(session=session,
limit=100,
offset=5000,
order_descending=True,
canonical_url=True)
# Or use a single entry as specified below
else:
old_entries = [
Entry(
original_url=
"https://www.mynbc5.com/amp/article/emily-ferlazzo-joseph-bolton-vermont-missing-update/38004866",
canonical_url=
"https://abc3340.com/news/inside-your-world/the-federal-government-spends-billions-each-year-maintaining-empty-buildings-nationwide"
)
]
for old_entry in old_entries:
log.info("OLD")
log.info(old_entry.entry_id)
log.info(old_entry.canonical_url)
found_entry = get_entry_by_original_url(old_entry.original_url,
session)
if found_entry:
log.info("NEW")
log.info(found_entry.entry_id)
log.info(old_entry.canonical_url)
if old_entry.entry_id == found_entry.entry_id:
amount_of_correct_retrievals += 1
else:
log.warning("No entry found!")
self.assertEqual(amount_of_correct_retrievals, old_entries.len)
def update_user_entry(self, user, callback=None):
logging.info('user:'******'id'])
for feed in feeds:
logging.info(feed)
entries = Entry.get_new_entries(feed['site_url'], feed['updated_at'])
user_entries = []
for entry in entries:
user_entries.append((entry['id'], user['id'], entry['site_url'] ))
if len(user_entries) > 0:
UserEntry.insert_entries(user_entries)
user.update_feed_update_at(feed['site_url'])
groups = user.get_groups()
callback(groups)
def metaWeblog_editPost(postid, struct, publish):
if struct.has_key('categories'):
cates = struct['categories']
else:
cates = []
newcates = []
for cate in cates:
c = Category.all().filter('name =', cate).fetch(1)
if c:
newcates.append(c[0].key())
entry = Entry.get_by_id(int(postid))
if struct.has_key('mt_keywords'):
entry.settags(struct['mt_keywords'])
if struct.has_key('wp_slug'):
entry.slug = struct['wp_slug']
if struct.has_key('mt_excerpt'):
entry.excerpt = struct['mt_excerpt']
entry.title = struct['title']
entry.content = struct['description']
if struct.has_key('mt_text_more'):
content = struct['mt_text_more']
if content:
entry.content = entry.content + '<!--more-->' + struct['mt_text_more']
entry.categorie_keys = newcates
if struct.has_key('mt_tags'):
tags = struct['mt_tags']
else:
tags = []
entry.settags(tags)
if publish:
entry.publish(True)
else:
entry.save()
return True
def get_all_entries():
# Open a connection to the database
with sqlite3.connect("./dailyjournal.db") as conn:
# Just use these. It's a Black Box.
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
# Write the SQL query to get the information you want
db_cursor.execute("""
SELECT
a.id,
a.concept,
a.entry,
a.date,
a.moods_id
FROM JournalEntries a
""")
# Initialize an empty list to hold all entry representations
entries = []
# Convert rows of data into a Python list
dataset = db_cursor.fetchall()
# Iterate list of data returned from database
for row in dataset:
# Create an entry instance from the current row.
# Note that the database fields are specified in
# exact order of the parameters defined in the
# entry class above.
entry = Entry(row['id'], row['concept'], row['entry'], row['date'],
row['moods_id'])
entries.append(entry.__dict__)
# Use `json` package to properly serialize list as JSON
return json.dumps(entries)
def action_getcomments(self):
key = self.param('key')
entry = Entry.get(key)
comments = Comment.all().filter('entry =', key)
commentuser = self.request.cookies.get('comment_user', '')
if commentuser:
commentuser = commentuser.split('#@#')
else:
commentuser = ['', '', '']
vals = {
'entry': entry,
'comments': comments,
'user_name': commentuser[0],
'user_email': commentuser[1],
'user_url': commentuser[2],
'checknum1': random.randint(1, 10),
'checknum2': random.randint(1, 10),
}
html = self.get_render('comments', vals)
self.write(simplejson.dumps(html.decode('utf8')))
def get_single_entry(id):
with sqlite3.connect("./daily_journal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
db_cursor.execute(
"""
SELECT
e.id,
e.concept,
e.entry,
e.date,
e.mood_id
FROM entries e
WHERE e.id = ?
""", (id, ))
data = db_cursor.fetchone()
entry = Entry(data['id'], data['concept'], data['entry'], data['date'],
data['mood_id'])
return json.dumps(entry.__dict__)
def get_entries_with_value(value):
with sqlite3.connect("./dailyjournal.db") as conn:
conn.row_factory = sqlite3.Row
db_cursor = conn.cursor()
queryString = f'%{value}%'
# Use a ? parameter to inject a variable's value
# into the SQL statement.
entries = []
db_cursor.execute(
"""
SELECT
a.id,
a.concept,
a.entry,
a.date,
a.moods_id
FROM JournalEntries a
WHERE a.entry LIKE ?
""", (queryString, ))
data = db_cursor.fetchall()
for row in data:
# Create an entry instance from the current row.
# Note that the database fields are specified in
# exact order of the parameters defined in the
# entry class above.
entry = Entry(row['id'], row['concept'], row['entry'], row['date'],
row['moods_id'])
entries.append(entry.__dict__)
return json.dumps(entries)
def storeInDatabase():
# Get json data from the endpoint
try:
response = requests.get("https://api.publicapis.org/entries").json()
except Exception as e:
error = f"Error! {e}"
return render_template('index.html', err=error)
else:
limit = 0
for entry in response["entries"]:
entry = Entry(entry["API"], entry["Description"], entry["Auth"],
entry["HTTPS"], entry["Cors"], entry["Link"],
entry["Category"])
try:
db.session.add(entry)
except Exception as e:
error = f"Error! {e}"
return render_template('index.html', err=error)
# Limit to insert only 50 records
limit += 1
if limit == 50:
break
try:
# Commit the new records to DB
db.session.commit()
message = "Data Successfully Stored in database"
except SQLAlchemyError as e:
error = e.__dict__['orig']
return render_template('index.html', err=error)
else:
# Get all entries from database
entries = Entry.query.all()
return render_template('index.html',
success=message,
entries=entries)
from models.entry import Entry
entry = Entry()
entry.test = 'hello'
print entry.test
#
# headers = [
# {
# "name": "Host",
# "value": "www.cnn.com"
# },
# {
# "name": "User-Agent",
# "value": "curl/7.43.0"
# },
# {
# "name": "Accept",
# "value": "*/*"
# },
# {
# "name": "Proxy-Connection",
# "value": "Keep-Alive"
# },
# {
# "name": "accept-encoding",
# "value": "identity"
# }
# ]
#
#
def response(context, flow):
"""
Called when a server response has been received. At the time of this
message both a request and a response are present and completely done.
"""
# Values are converted from float seconds to int milliseconds later.
ssl_time = -.001
connect_time = -.001
if flow.server_conn not in context.seen_server:
# Calculate the connect_time for this server_conn. Afterwards add it to
# seen list, in order to avoid the connect_time being present in entries
# that use an existing connection.
connect_time = flow.server_conn.timestamp_tcp_setup - \
flow.server_conn.timestamp_start
context.seen_server.add(flow.server_conn)
if flow.server_conn.timestamp_ssl_setup is not None:
# Get the ssl_time for this server_conn as the difference between
# the start of the successful tcp setup and the successful ssl
# setup. If no ssl setup has been made it is left as -1 since it
# doesn't apply to this connection.
ssl_time = flow.server_conn.timestamp_ssl_setup - \
flow.server_conn.timestamp_tcp_setup
# Calculate the raw timings from the different timestamps present in the
# request and response object. For lack of a way to measure it dns timings
# can not be calculated. The same goes for HAR blocked: MITMProxy will open
# a server connection as soon as it receives the host and port from the
# client connection. So the time spent waiting is actually spent waiting
# between request.timestamp_end and response.timestamp_start thus it
# correlates to HAR wait instead.
timings_raw = {
'send': flow.request.timestamp_end - flow.request.timestamp_start,
'wait': flow.response.timestamp_start - flow.request.timestamp_end,
'receive': flow.response.timestamp_end - flow.response.timestamp_start,
'connect': connect_time,
'ssl': ssl_time
}
# HAR timings are integers in ms, so we have to re-encode the raw timings to
# that format.
timings = dict([(key, int(1000 * value))
for key, value in timings_raw.iteritems()])
# The full_time is the sum of all timings. Timings set to -1 will be ignored
# as per spec.
full_time = 0
for item in timings.values():
if item > -1:
full_time += item
started_date_time = datetime.fromtimestamp(
flow.request.timestamp_start, tz=pytz.timezone('UTC')).isoformat()
request_query_string = [{
"name": k,
"value": v
} for k, v in flow.request.get_query()]
request_http_version = ".".join([str(v) for v in flow.request.httpversion])
# Cookies are shaped as tuples by MITMProxy.
request_cookies = [{
"name": k.strip(),
"value": v[0]
} for k, v in (flow.request.get_cookies() or {}).iteritems()]
request_headers = [{
"name": k,
"value": v
} for k, v in flow.request.headers]
request_headers_size = len(str(flow.request.headers))
request_body_size = len(flow.request.content)
response_http_version = ".".join(
[str(v) for v in flow.response.httpversion])
# Cookies are shaped as tuples by MITMProxy.
response_cookies = [{
"name": k.strip(),
"value": v[0]
} for k, v in (flow.response.get_cookies() or {}).iteritems()]
response_headers = [{
"name": k,
"value": v
} for k, v in flow.response.headers]
response_headers_size = len(str(flow.response.headers))
response_body_size = len(flow.response.content)
response_body_decoded_size = len(flow.response.get_decoded_content())
response_body_compression = response_body_decoded_size - response_body_size
response_mime_type = flow.response.headers.get_first('Content-Type', '')
response_redirect_url = flow.response.headers.get_first('Location', '')
container_id = os.environ['HOSTNAME']
entry = Entry()
entry.startedDateTime = started_date_time
entry.time = full_time
entry.container_id = container_id
entry.request = {
"method": flow.request.method,
"url": flow.request.url,
"httpVersion": request_http_version,
"cookies": request_cookies,
"headers": request_headers,
"queryString": request_query_string,
"headersSize": request_headers_size,
"bodySize": request_body_size,
}
entry.response = {
"url": flow.request.url,
"status": flow.response.code,
"statusText": flow.response.msg,
"httpVersion": response_http_version,
"cookies": response_cookies,
"headers": response_headers,
"content": {
"size": response_body_size,
"compression": response_body_compression,
"mimeType": response_mime_type
},
"redirectURL": response_redirect_url,
"headersSize": response_headers_size,
"bodySize": response_body_size,
}
entry.cache = {}
entry.timings = timings
#ws = create_connection("ws://localhost:9000/ws/{0}".format(entry.pageref))
ws = create_connection(
"ws://websocket.bandit.io:9000/ws/{0}".format(container_id),
sslopt={"check_hostname": False})
ws.send(
json.dumps({
"method": "Network.requestWillBeSent",
"params": {
"requestId": "7897.52",
"frameId": "7897.1",
"loaderId": "7897.3",
"documentURL": entry.request['url'],
"request": {
"url": entry.request['url'],
"method": entry.request['method'],
"headers": {
header['name']: header['value']
for header in entry.request['headers']
}
},
"timestamp": 88986.634829,
"wallTime": 1440472453.19435,
"initiator": {
"type": "other"
},
"type": "Document"
}
}))
ws.send(
json.dumps({
"method": "Network.responseReceived",
"params": {
"requestId": "7897.52",
"frameId": "7897.1",
"loaderId": "7897.3",
"timestamp": 88986.985021,
"type": "Document",
"response": {
"url": entry.response['url'],
"status": entry.response['status'],
"statusText": entry.response['statusText'],
"headers": {
header['name']: header['value']
for header in entry.response['headers']
},
"mimeType": entry.response['content']['mimeType'],
"connectionReused": False,
"fromDiskCache": False,
"fromServiceWorker": False,
"timing": {
"requestTime": 88986.636403,
"proxyStart": -1,
"proxyEnd": -1,
"dnsStart": 0,
"dnsEnd": 108.372000002419,
"connectStart": 108.372000002419,
"connectEnd": 113.420000008773,
"sslStart": -1,
"sslEnd": -1,
"serviceWorkerFetchStart": -1,
"serviceWorkerFetchReady": -1,
"serviceWorkerFetchEnd": -1,
"sendStart": 113.492999997106,
"sendEnd": 113.573000009637,
"receiveHeadersEnd": 347.90900000371
},
# "requestHeaders": {
# "If-None-Match": "\"1440455137124|#public|0|en|||0\"",
# "Accept-Encoding": "gzip, deflate, sdch",
# "Host": "www.chromium.org",
# "Accept-Language": "en-US,en;q=0.8",
# "Upgrade-Insecure-Requests": "1",
# "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36",
# "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
# "Cache-Control": "max-age=0",
# "Cookie": "_ga=GA1.2.1062414394.1440468745; _gat_SitesTracker=1; __utmt=1; __utma=221884874.1062414394.1440468745.1440468745.1440471278.2; __utmb=221884874.2.10.1440471278; __utmc=221884874; __utmz=221884874.1440468745.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); aftzc=QW1lcmljYS9Mb3NfQW5nZWxlczp3eGRhd0FxcWxWZkNYdHRkVVJ2ZStlVEpOOVE9",
# "Connection": "keep-alive",
# "If-Modified-Since": "Mon, 24 Aug 2015 22:25:37 GMT"
# },
"remoteIPAddress": "216.239.32.27",
"remotePort": 80,
"protocol":
"http/{0}".format(entry.response['httpVersion'])
}
}
}))
ws.send(
json.dumps({
"method": "Network.dataReceived",
"params": {
"requestId": "7897.52",
"timestamp": 88986.985513,
"dataLength": entry.response['content']['size'],
"encodedDataLength": entry.response['bodySize']
}
}))
ws.send(
json.dumps({
"method": "Network.loadingFinished",
"params": {
"requestId": "7897.52",
"timestamp": 88986.985401,
"encodedDataLength": entry.response['bodySize']
}
}))
#ws.send(json.dumps({"method":"Network.requestWillBeSent","params":{"requestId":"7897.52","frameId":"7897.1","loaderId":"7897.3","documentURL":"http://www.chromium.org/","request":{"url":"http://www.chromium.org/","method":"GET","headers":{"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","Upgrade-Insecure-Requests":"1","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"}},"timestamp":88986.634829,"wallTime":1440472453.19435,"initiator":{"type":"other"},"type":"Document"}}))
#ws.send(json.dumps({"method":"Network.responseReceived","params":{"requestId":"7897.52","frameId":"7897.1","loaderId":"7897.3","timestamp":88986.985021,"type":"Document","response":{"url":"http://www.chromium.org/","status":304,"statusText":"Not Modified","headers":{"Date":"Tue, 25 Aug 2015 03:14:13 GMT","Last-Modified":"Mon, 24 Aug 2015 22:25:37 GMT","Server":"GSE","X-Robots-Tag":"noarchive","ETag":"\"1440455137124|#public|0|en|||0\""},"mimeType":"text/html","connectionReused":False,"connectionId":2554,"encodedDataLength":-1,"fromDiskCache":False,"fromServiceWorker":False,"timing":{"requestTime":88986.636403,"proxyStart":-1,"proxyEnd":-1,"dnsStart":0,"dnsEnd":108.372000002419,"connectStart":108.372000002419,"connectEnd":113.420000008773,"sslStart":-1,"sslEnd":-1,"serviceWorkerFetchStart":-1,"serviceWorkerFetchReady":-1,"serviceWorkerFetchEnd":-1,"sendStart":113.492999997106,"sendEnd":113.573000009637,"receiveHeadersEnd":347.90900000371},"headersText":"HTTP/1.1 304 Not Modified\r\nX-Robots-Tag: noarchive\r\nLast-Modified: Mon, 24 Aug 2015 22:25:37 GMT\r\nETag: \"1440455137124|#public|0|en|||0\"\r\nDate: Tue, 25 Aug 2015 03:14:13 GMT\r\nServer: GSE\r\n\r\n","requestHeaders":{"If-None-Match":"\"1440455137124|#public|0|en|||0\"","Accept-Encoding":"gzip, deflate, sdch","Host":"www.chromium.org","Accept-Language":"en-US,en;q=0.8","Upgrade-Insecure-Requests":"1","User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36","Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8","Cache-Control":"max-age=0","Cookie":"_ga=GA1.2.1062414394.1440468745; _gat_SitesTracker=1; __utmt=1; __utma=221884874.1062414394.1440468745.1440468745.1440471278.2; __utmb=221884874.2.10.1440471278; __utmc=221884874; __utmz=221884874.1440468745.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); aftzc=QW1lcmljYS9Mb3NfQW5nZWxlczp3eGRhd0FxcWxWZkNYdHRkVVJ2ZStlVEpOOVE9","Connection":"keep-alive","If-Modified-Since":"Mon, 24 Aug 2015 22:25:37 GMT"},"requestHeadersText":"GET / HTTP/1.1\r\nHost: www.chromium.org\r\nConnection: keep-alive\r\nCache-Control: max-age=0\r\nAccept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8\r\nUpgrade-Insecure-Requests: 1\r\nUser-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36\r\nAccept-Encoding: gzip, deflate, sdch\r\nAccept-Language: en-US,en;q=0.8\r\nCookie: _ga=GA1.2.1062414394.1440468745; _gat_SitesTracker=1; __utmt=1; __utma=221884874.1062414394.1440468745.1440468745.1440471278.2; __utmb=221884874.2.10.1440471278; __utmc=221884874; __utmz=221884874.1440468745.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); aftzc=QW1lcmljYS9Mb3NfQW5nZWxlczp3eGRhd0FxcWxWZkNYdHRkVVJ2ZStlVEpOOVE9\r\nIf-None-Match: \"1440455137124|#public|0|en|||0\"\r\nIf-Modified-Since: Mon, 24 Aug 2015 22:25:37 GMT\r\n\r\n","remoteIPAddress":"216.239.32.27","remotePort":80,"protocol":"http/1.1"}}}))
#ws.send(json.dumps({"method":"Network.dataReceived","params":{"requestId":"7897.52","timestamp":88986.985513,"dataLength":23423,"encodedDataLength":190}}))
##WebSocketHandler.ws_send('test', {"method":"Page.frameNavigated","params":{"frame":{"id":"7897.1","loaderId":"7897.3","url":"http://www.chromium.org/","mimeType":"text/html","securityOrigin":"http://www.chromium.org"}}})
#ws.send(json.dumps({"method":"Network.loadingFinished","params":{"requestId":"7897.52","timestamp":88986.985401,"encodedDataLength":190}}))
ws.close()
# def done(context):
# """
# Called once on script shutdown, after any other events.
# """
# from pprint import pprint
# import json
#
# json_dump = context.HARLog.json()
# compressed_json_dump = context.HARLog.compress()
#
# if context.dump_file == '-':
# context.log(pprint.pformat(json.loads(json_dump)))
# elif context.dump_file.endswith('.zhar'):
# file(context.dump_file, "w").write(compressed_json_dump)
# else:
# file(context.dump_file, "w").write(json_dump)
# context.log(
# "HAR log finished with %s bytes (%s bytes compressed)" % (
# len(json_dump), len(compressed_json_dump)
# )
# )
# context.log(
# "Compression rate is %s%%" % str(
# 100. * len(compressed_json_dump) / len(json_dump)
# )
# )
#
#
# def print_attributes(obj, filter_string=None, hide_privates=False):
# """
# Useful helper method to quickly get all attributes of an object and its
# values.
# """
# for attr in dir(obj):
# if hide_privates and "__" in attr:
# continue
# if filter_string is not None and filter_string not in attr:
# continue
# value = getattr(obj, attr)
# print("%s.%s" % ('obj', attr), value, type(value))
def new_entry(self, path: str, entry: Entry):
d = self.access_path(path)
d.new_entry(entry)
return entry.to_dict()
def receive(self, message):
# Combine message bodies and decode html
bodies = message.bodies("text/plain")
for content_type, body in bodies:
decoded_body = body.decode()
# Split body from any optional fields
body_list = decoded_body.split("---\n\n", 1)
body_len = len(body_list)
fields = body_list[0]
body = body_list[1]
# grab the first paragraph as
body_parts = body.split("\n\n", 1)
excerpt = body_parts[0]
body = body_parts[1]
# Grab any data from fields
slug = message.subject.replace(" ", "-").lower()
cat_name = "Bit"
draft = False
published = None
if fields:
logging.info("We have YAML front matter.")
for config in yaml.load_all(fields):
logging.info("Checking each YFM.")
logging.debug(type(config))
# hard-coded data extraction
# Probably should do this a better way
if "slug" in config:
slug = config["slug"]
if "published" in config:
published = config["published"]
if "category" in config:
cat_name = config["category"]
if "draft" in config:
draft = True
else:
logging.error("Malformed YFM: --------- " + fields)
if published is None:
date = message.date.split(" -")
published = datetime.datetime.strptime(date[0], "%a, %d %b %Y %H:%M:%S")
else:
published = datetime.datetime.strptime(published, "%a, %d %b %Y %H:%M:%S")
# get the category object
category = db.Query(Category).filter("name =", cat_name).get()
logging.info("Entry in cat: " + category.name + " from: " + cat_name)
# Process attachments!!!!!!
images = []
thumbnail = None
if cat_name == "Work":
if message.attachments:
# Check file extension of attachments
re_filetype = re.compile("(.gif|.jpg|.jpeg|.png|.GIF|.JPG|.JPEG|.PNG)")
for att in message.attachments:
# att[0] = name, att[1] = payload
is_image = re_filetype.search(att[0])
filename, encoded_data = att
data = encoded_data.payload
if encoded_data.encoding:
data = data.decode(encoded_data.encoding)
if is_image:
# upload image
if thumbnail is None:
thumb = image.resize(data, 250)
thumbnail = s3.upload_image(filename, thumb, message.subject)
new_image = s3.upload_image(filename, data, message.subject)
images.append(new_image)
else:
logging.info(att)
logging.info("Attachment was not an image.")
# save
# Okay, just hardcode email
# author property is now string
# check user.email() to author when in admin
if message.sender == "":
entry = Entry(
author=message.sender,
title=message.subject,
slug=slug,
category=category.key(),
body=body,
excerpt=excerpt,
published=published,
draft=draft,
images=images,
thumbnail=thumbnail,
)
entry.put()
entry.index()
sheema, errors = user_schema.load({
'username': '******',
'email': 'sheema@email',
'password': '******',
'password_confirmation': 'pass',
# 'entries': [breakfast],
# 'tasks': [project],
# 'events': [friday_drinks]
})
if errors:
raise Exception(errors)
entry_one = Entry(
date='2019-07-29',
what=
'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.',
image='image-url',
user=sheema)
entry_two = Entry(
date='2019-07-30',
what=
'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.',
image='image-url',
user=sheema)
entry_three = Entry(
date='2019-07-20',
what=
'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.',
image='image-url',
def get_data(session,
limit,
offset,
order_descending,
entry_id=None,
entry_type=None,
handled_utc=None,
original_url=None,
canonical_url=None,
canonical_type=None,
note=None) -> [Entry]:
log.info(f"Getting data of type {entry_type} from {Table.__tablename__},"
f" limit = {limit}, order_descending = {order_descending}")
# Store the values in a dict
filter_options = {
Table.entry_id: entry_id,
Table.entry_type: entry_type,
Table.handled_utc: handled_utc,
Table.original_url: original_url,
Table.canonical_url: canonical_url,
Table.canonical_type: canonical_type,
Table.note: note
}
# Create a new query
q = session.query(Table)
for attr, value in filter_options.items():
log.debug(f"attr= {attr}, value={value}")
# If true, select not NULL
if value is True:
q = q.filter(attr.isnot(None))
# If false, select NULL
elif value is False:
q = q.filter(attr == None)
# If anything else, select X
elif value is not None:
q = q.filter(attr == value)
# Sort descending (returns most recent rows)
if order_descending:
q = q.order_by(Table.entry_id.desc())
if offset:
q = q.offset(offset)
# Set a limit
q = q.limit(limit)
log.info(q)
log.info(f"Received data, rows: {q.count()}")
# Generate entry instance for each returned row, add these to a list
entries = []
for entry in q:
entries.append(
Entry(entry_id=entry.entry_id,
entry_type=entry.entry_type,
handled_utc=entry.handled_utc,
original_url=entry.original_url,
canonical_url=entry.canonical_url,
canonical_type=entry.canonical_type,
note=entry.note))
log.info("Entry instance(s) generated")
return entries
def test_canonical(self, use_database=True):
amount_of_canonicals = 0
old_amount_of_canonicals = 0
# Use data from the database
if use_database:
old_entries = get_data(session=get_engine_session(),
limit=500,
offset=5000,
order_descending=True,
canonical_url=False)
# Or use a single entry as specified below
else:
old_entries = [
Entry(
original_url=
"https://www.google.com/amp/s/abc3340.com/amp/news/inside-your-world/the-federal-government-spends-billions-each-year-maintaining-empty-buildings-nationwide",
canonical_url=
"https://abc3340.com/news/inside-your-world/the-federal-government-spends-billions-each-year-maintaining-empty-buildings-nationwide"
)
]
# Loop through every old entry and try to find the canonicals, compare the results
for old_entry in old_entries:
if old_entry.canonical_url:
old_amount_of_canonicals += 1
urls = get_urls(old_entry.original_url)
urls_info = get_urls_info(urls)
if urls_info:
for link in urls_info:
log.info(link.canonical_alt)
if link.amp_canonical:
log.info(link.amp_canonical)
if link.canonical:
amount_of_canonicals += 1
log.info(f"BODY : {old_entry.original_url}")
log.info(f"OLD : {old_entry.canonical_url}")
log.info(f"NEW : {link.canonical}")
if link.canonical == old_entry.canonical_url:
log.info("It's the same!")
else:
log.info("It's not the same!")
"""if link.canonical:
similarity = get_article_similarity(old_entry.original_url, link.canonical, log_articles=False)
log.info(f"Article similarity= {similarity}")"""
else:
log.warning(f"No canonicals found")
log.info(
f"\nCanonicals found: Old: {old_amount_of_canonicals}, New: {amount_of_canonicals}"
)
# If same as before, great!
if amount_of_canonicals == old_amount_of_canonicals:
self.assertEqual(amount_of_canonicals, old_amount_of_canonicals)
# If it is better than before, great!
if amount_of_canonicals > old_amount_of_canonicals:
self.assertGreater(amount_of_canonicals, old_amount_of_canonicals)
# If it is worse than before, not good.
if amount_of_canonicals < old_amount_of_canonicals:
self.assertLess(old_amount_of_canonicals, amount_of_canonicals)
def posts(self):
return Entry.all().filter('entrytype =', 'post').filter('categorie_keys =', self)
