def trainwreck(n=100):
"""
Like sequential, but with updates, because you're a glutton for pain.
"""
node_index = 0
def get_node():
"""
Returns the node corresponding to `node_index`.
Wraps around if `node_index` > `len(nodes)`.
"""
return nodes[node_index % len(nodes)]
results = []
for i in range(n):
while True:
node_index += 1
url = '/'.join([make_url(get_node()), 'test'])
doc = {
'value': i
}
try:
r = requests.post(url,
data=json.dumps(doc),
headers={
'content-type': 'application/json'
})
except requests.exceptions.ConnectionError:
pass
else:
results.append(r.json())
break
# now for the updates
results2 = []
for i, result in enumerate(results):
if 'ok' in result:
while True:
node_index += 1
url = '/'.join([make_url(get_node()), 'test', result['id']])
doc = {
'value': i,
'changed': True,
'_id': result['id'],
'_rev': result['rev']
}
try:
r = requests.put(url,
data=json.dumps(doc),
headers={
'content-type': 'application/json'
})
except requests.exceptions.ConnectionError:
pass
else:
results2.append(r.json())
break
else:
results2.append(result)
return results2
def GET(self, action, badge_id):
cart_id = view.getCookie()
if not cart_id:
raise web.seeother(config.make_url("/"))
elif "status" in db.getCart(cart_id):
view.destroyCookie()
raise web.seeother(config.make_url("/"))
return render.base(view.cartListing(cart_id, (action, badge_id)))
def GET(self, action, badge_id):
cart_id = view.getCookie()
if not cart_id:
raise web.seeother(config.make_url("/"))
elif "status" in db.getCart(cart_id):
view.destroyCookie()
raise web.seeother(config.make_url("/"))
return render.base(view.cartListing(cart_id, (action, badge_id)))
def random(n=100):
"""
Assigns work amongst the nodes randomly
"""
def get_node():
"""
Returns a random node; pretends to be a Heroku router. (zing!)
"""
return rand.choice(nodes)
results = []
for i in range(n):
doc = {
'value': i
}
while True:
url = '/'.join([make_url(get_node()), 'test'])
try:
r = requests.post(url,
data=json.dumps(doc),
headers={
'content-type': 'application/json'
})
except requests.exceptions.ConnectionError:
pass
else:
results.append(r.json())
break
return results
def sequential(n=100):
"""
Each write goes to a different node, in sequence
"""
node_index = 0
def get_node():
"""
Returns the node corresponding to `node_index`.
Wraps around if `node_index` > `len(nodes)`.
"""
return nodes[node_index % len(nodes)]
results = []
for i in range(n):
while True:
node_index += 1
url = '/'.join([make_url(get_node()), 'test'])
doc = {
'value': i
}
try:
r = requests.post(url,
data=json.dumps(doc),
headers={
'content-type': 'application/json'
})
except requests.exceptions.ConnectionError:
pass
else:
results.append(r.json())
break
return results
def direct(n=100):
"""
Writes to a single master. Picks the next node on failure.
"""
node_index = 0
def get_node():
"""
Returns the node corresponding to `node_index`.
Wraps around if `node_index` > `len(nodes)`.
"""
return nodes[node_index % len(nodes)]
results = []
for i in range(n):
while True:
url = '/'.join([make_url(get_node()), 'test'])
doc = {
'value': i
}
try:
r = requests.post(url,
data=json.dumps(doc),
headers={
'content-type': 'application/json'
})
except requests.exceptions.ConnectionError:
node_index += 1
else:
results.append(r.json())
break
return results
def GET(self):
cart_id = web.cookies().get(config.cookieName)
if cart_id:
f = badgeForm()
return render.base(render.badge(f), "Nouvel insigne ~ New badge", True)
else:
raise web.seeother(config.make_url('/error'))
def trainwreck_get(nodes):
results = []
for node in nodes:
try:
r = requests.get('/'.join([make_url(node), 'test', '_all_docs']),
params={
'include_docs': True
})
except requests.exceptions.ConnectionError:
pass
else:
results.extend(r.json()['rows'])
ids = {}
for row in [row['doc'] for row in results]:
if row['_id'] in ids and row['_rev'] != ids[row['_id']]['rev']:
print "Document conflict with %s" % row['_id']
ids[row['_id']] = {
'rev': row['_rev'],
'value': row['value']
}
for id in ids.keys()[:1]:
doc = get_one_by_quorum(nodes, id)
if doc:
print "Got %s by quorum!" % id
else:
print "Failed to get %s by quorum" % id
return ids
def GET(self):
cart_id = web.cookies().get(config.cookieName)
if cart_id:
f = badgeForm()
return render.base(render.badge(f), "Nouvel insigne ~ New badge",
True)
else:
raise web.seeother(config.make_url('/error'))
def get_author_recent_published_detail(
session_obj: requests.Session,
author_id: int,
exclude_book_name: str,
) -> Dict:
# https://ridibooks.com/author/2317?id=2317&order=recent#books
author_detail_page = call_with_response_check(
f"작가 [{author_id}] 페이지 읽기",
200,
session_obj.get,
config.make_author_url(author_id),
params={
"id": str(author_id),
"order": "recent"
},
)
author_detail_soup = BeautifulSoup(author_detail_page.text, 'html.parser')
author_detail_find_args = config.ridi.SOUP_FIND_ARGS["AUTHOR_DETAIL_PAGE"]
get_book_detail_find_args = author_detail_find_args["GET_BOOK_DETAIL"]
author_books = author_detail_soup.find_all(name="div",
class_="book_macro_landscape")
last_published_book_detail = {}
for author_book in author_books:
title_html = author_book.find(
**get_book_detail_find_args["TITLE_LINK"])
title = title_html.text.strip()
if exclude_book_name in title or title in exclude_book_name:
continue
title_link = title_html['href']
star_rate_score_html = author_book.find(
**get_book_detail_find_args["STAR_RATE_SCORE"])
star_rate_participant_count_html = author_book.find(
**get_book_detail_find_args["STAR_RATE_PARTICIPANT_COUNT"])
last_published_book_detail = {
"b_id": title_link.split('/')[-1],
"link": config.make_url(title_link),
"title": title_html.text.strip(),
"rating": {
"buyer_rating_score":
star_rate_score_html.text.split("점")[0]
if star_rate_score_html else "0",
"buyer_rating_count":
star_rate_participant_count_html.text.split("명")[0]
if star_rate_participant_count_html else "0",
},
}
break
return last_published_book_detail
def POST(self):
cart_id = web.cookies().get(config.cookieName)
if cart_id:
# We have a cart, proceed
f = badgeForm()
if not f.validates():
return render.base(render.badge(f),
"Nouvel insigne ~ New badge", True)
else:
# Everything is valid, let's build the query
badge = {
"prenom": f.d.prenom,
"nom": f.d.nom,
"texte_insigne": f.d.texte_insigne,
"courriel": f.d.courriel,
"adresse": {
"ligne_1": f.d.adresse_1,
"ligne_2": f.d.adresse_2,
"ville": f.d.ville,
"province": f.d.province,
"code_postal": f.d.code_postal
},
"type": f.d.type,
"telephone_urgence": f.d.telephone_urgence,
"extra": {
"noiz": f.d.noiz,
"tshirt": f.d.tshirt,
"dvd": f.d.dvd
},
"instructions_speciales": f.d.instructions_speciales,
"status": "IN_CART",
"date": datetime.datetime.utcnow()
}
# Adding to DB
badge_id = db.addBadge(badge, cart_id)
raise web.seeother(config.make_url("/cart/add/%s" % badge_id))
else:
raise web.seeother(config.make_url('/error'))
def get_all(nodes):
results = {}
for node in nodes:
try:
r = requests.get('/'.join([make_url(node), 'test', '_all_docs']),
params={
'include_docs': True
})
except requests.exceptions.ConnectionError:
print "%s down :(" % node
else:
results[node] = r
return results
def POST(self):
cart_id = web.cookies().get(config.cookieName)
if cart_id:
# We have a cart, proceed
f = badgeForm()
if not f.validates():
return render.base(render.badge(f), "Nouvel insigne ~ New badge", True)
else:
# Everything is valid, let's build the query
badge = {
"prenom": f.d.prenom,
"nom": f.d.nom,
"texte_insigne": f.d.texte_insigne,
"courriel": f.d.courriel,
"adresse": {
"ligne_1": f.d.adresse_1,
"ligne_2": f.d.adresse_2,
"ville": f.d.ville,
"province": f.d.province,
"code_postal": f.d.code_postal
},
"type": f.d.type,
"telephone_urgence": f.d.telephone_urgence,
"extra": {
"noiz": f.d.noiz,
"tshirt": f.d.tshirt,
"dvd": f.d.dvd
},
"instructions_speciales": f.d.instructions_speciales,
"status": "IN_CART",
"date": datetime.datetime.utcnow()}
# Adding to DB
badge_id = db.addBadge(badge, cart_id)
raise web.seeother(config.make_url("/cart/add/%s" % badge_id))
else:
raise web.seeother(config.make_url('/error'))
def sync(nodes=nodes):
for i, node in enumerate(nodes):
# create test db in source
r = requests.put('/'.join([make_url(node), 'test']))
if r.status_code not in [201, 412]:
print r.status_code
raise Exception(r.json())
# get list of nodes to create replications with
other_nodes = nodes[::]
other_nodes.pop(i)
# push replication docs
for other_node in other_nodes:
doc = sync_doc % make_url(other_node)
# print doc
headers = {
'Content-Type': 'application/json'
}
r = requests.post('/'.join([make_url(node), '_replicator']),
data=doc,
headers=headers)
if 'error' in r.json():
raise Exception(r.json())
print "Synced."
def reset():
results = []
for i, node in enumerate(nodes):
try:
r = requests.get('/'.join([make_url(node), '_replicator', '_all_docs']))
rows = [row for row in r.json()['rows'] if '_design' not in row['id']]
for row in rows:
doc = {
'id': row['key'],
'rev': row['value']['rev']
}
r = requests.delete('/'.join([make_url(node), '_replicator', doc['id']]),
params={'rev': doc['rev']})
results.append(r)
r = requests.delete('/'.join([make_url(node), 'test']))
results.append(r)
except requests.exceptions.ConnectionError:
pass
errors = [result for result in results if 'error' in result.json()]
if not len(errors) == len(results):
reset()
else:
print "Reset."
def GET(self, cart_id, hash):
"""Validate an email and forward to the pickup page"""
try:
bson.objectid.ObjectId(cart_id)
except bson.errors.InvalidId:
return render.base(render.validate(None, ("bad", "Invalid ObjectId")), "Confirmation", True)
cart = db.getCart(cart_id)
if not cart:
return render.base("""<h2>Ce courriel est invalide</h2>""", "Confirmation", True)
vEmail = hashlib.sha1()
vEmail.update(cart["email"])
vEmail.update(config.salt)
if vEmail.hexdigest() == hash:
db.cartValidEmail(cart_id)
raise web.seeother(config.make_url("/pickup/%s" % (cart_id, )))
else:
return render.base("""<h2>Ce courriel est invalide</h2>""", "Confirmation", True)
def GET(self, badge_id):
cart_id = web.cookies().get(config.cookieName)
if cart_id:
f = badgeForm()
if not db.validObjectId(badge_id):
raise web.seeother('/error')
badge = db.getBadge(badge_id)
badge["adresse_1"] = badge["adresse"]["ligne_1"]
badge["adresse_2"] = badge["adresse"]["ligne_2"]
badge["ville"] = badge["adresse"]["ville"]
badge["province"] = badge["adresse"]["province"]
badge["code_postal"] = badge["adresse"]["code_postal"]
badge["noiz"] = badge["extra"]["noiz"]
badge["tshirt"] = badge["extra"]["tshirt"]
badge["dvd"] = badge["extra"]["dvd"]
f.fill(badge)
return render.base(render.badge(f), "Editer une insigne ~ Edit a badge", True)
else:
raise web.seeother(config.make_url('/error'))
def get_one_by_quorum(nodes, id):
n = 3
docs = {}
votes = Counter()
for node in nodes:
try:
r = requests.get('/'.join([make_url(node), 'test', id]))
except requests.exceptions.ConnectionError:
pass
else:
doc = r.json()
if '_rev' in doc:
docs[doc['_rev']] = doc
votes[doc['_rev']] += 1
if votes.most_common(1)[0][1] >= n:
break
else:
print votes
return None
return docs[votes.most_common(1)[0][0]]
def GET(self, badge_id):
cart_id = web.cookies().get(config.cookieName)
if cart_id:
f = badgeForm()
if not db.validObjectId(badge_id):
raise web.seeother('/error')
badge = db.getBadge(badge_id)
badge["adresse_1"] = badge["adresse"]["ligne_1"]
badge["adresse_2"] = badge["adresse"]["ligne_2"]
badge["ville"] = badge["adresse"]["ville"]
badge["province"] = badge["adresse"]["province"]
badge["code_postal"] = badge["adresse"]["code_postal"]
badge["noiz"] = badge["extra"]["noiz"]
badge["tshirt"] = badge["extra"]["tshirt"]
badge["dvd"] = badge["extra"]["dvd"]
f.fill(badge)
return render.base(render.badge(f),
"Editer une insigne ~ Edit a badge", True)
else:
raise web.seeother(config.make_url('/error'))
def get_by_quorum(nodes):
results = []
for node in nodes:
try:
r = requests.get('/'.join([make_url(node), 'test', '_all_docs']),
params={
'include_docs': True
})
except requests.exceptions.ConnectionError:
pass
else:
results.extend(r.json()['rows'])
ids = {}
for row in results:
ids[row['id']] = row['doc']['value']
for id in ids.keys()[:1]:
doc = get_one_by_quorum(nodes, id)
if doc:
print "Got %s by quorum!" % id
else:
print "Failed to get %s by quorum" % id
return ids
def scrape_event(
session_obj: requests.Session,
event_info: Dict,
) -> List:
event_url = config.make_url(event_info['url'])
event_page = call_with_response_check(f"이벤트[{event_info['title']}] 페이지 읽기",
200, session_obj.get, event_url)
event_detail_soup = BeautifulSoup(event_page.text, 'html.parser')
event_link_find_args = config.ridi.SOUP_FIND_ARGS["EVENT_PAGE"]
get_book_detail_find_args = event_link_find_args["GET_BOOK_DETAIL"]
event_book_items = []
for i, event_book in enumerate(
event_detail_soup.find_all(
**event_link_find_args["GET_BOOK_LIST"])):
title_html = event_book.find(**get_book_detail_find_args["TITLE_LINK"])
title_link = title_html['href']
star_rate_score_html = event_book.find(
**get_book_detail_find_args["STAR_RATE_SCORE"])
star_rate_participant_count_html = event_book.find(
**get_book_detail_find_args["STAR_RATE_PARTICIPANT_COUNT"])
event_book_items.append({
"b_id": title_link.split('/')[-1],
"title": title_html.text.strip(),
"rating": {
"buyer_rating_score":
star_rate_score_html.text.split("점")[0]
if star_rate_score_html else "0",
"buyer_rating_count":
star_rate_participant_count_html.text.split("명")[0].replace(
",", "") if star_rate_participant_count_html else "0",
},
})
return event_book_items
def GET(self, action, badge_id):
cart_id = view.getCookie()
if not cart_id:
raise web.seeother(config.make_url("/"))
return render.base(view.cartListing(cart_id, (action, badge_id)))
def GET(self, action, badge_id):
cart_id = view.getCookie()
if not cart_id:
raise web.seeother(config.make_url("/"))
return render.base(view.cartListing(cart_id, (action, badge_id)))