def exctract_metadata(text, isbn, container=dict()):
soup = BeautifulSoup(response_txt, "html.parser")
Option(soup.find("span", {"id": "productTitle"}))\
.map(lambda item: item.text)\
.map(lambda item: setitem(container, "title", item))\
.get_or(None)
Option(soup.find("span", {"class": "author"}))\
.map(lambda item: item.find("a"))\
.map(lambda item: item.text)\
.map(lambda item: setitem(container, "authors", item))\
.get_or(None)
Option(soup.find("div", {"id": "bookDescription_feature_div"}))\
.map(lambda item: item.find("noscript"))\
.map(lambda item: item.find("div"))\
.map(lambda item: item.text)\
.map(lambda item: setitem(container, "description", item))\
.get_or(None)
Option(soup.find("div", {"id": "img-canvas"}))\
.map(lambda item: item.find("img"))\
.map(lambda item: item.get("data-a-dynamic-image", None))\
.map(lambda item: item.split("\""))\
.map(lambda item: item[1])\
.map(lambda item: setitem(container, "imageLinks", item))\
.get_or(None)
container["industryIdentifiers"] = isbn
container["source"] = "AM"
return container
def get_data():
return (
("facti", Option(map(lambda item: evaluate_func(facti, item), DEF_VALUES)).map(lambda item: list(item)).map(lambda item: [statistics.mean(element) for element in item]).get_or([])),
("fact", Option(map(lambda item: evaluate_func(fact, item), DEF_VALUES)).map(lambda item: list(item)).map(lambda item: [statistics.mean(element) for element in item]).get_or([])),
("fact_nt", Option(map(lambda item: evaluate_func(fact_nt, item), DEF_VALUES)).map(lambda item: list(item)).map(lambda item: [statistics.mean(element) for element in item]).get_or([])),
("fact_m", Option(map(lambda item: evaluate_func(fact_m, item), DEF_VALUES)).map(lambda item: list(item)).get_or([])),
)
def create_strand(name, target, args=None, kwargs=None, use_process=False):
"""
Use this to create a multiprocessing.Process.
Handles log configuration.
Must call setup_log() from the main process before calling this.
:param target: the function to call. Must be a top-level function so that it can be pickled.
:type target: function
:param args: all data you want to pass to target
:type args: iterable
:param kwargs: all data you want to pass to target
:type kwargs: dict
:param use_process: True iff you want to launch a process instead of a thread. Not that even if
you specify True, may not launch a process if memdam.config.debug_processes is set
:type use_process: bool
"""
if memdam.is_threaded_logging_setup():
queue = memdam._logger.handlers[0].queue
else:
if memdam.config.debug_logging:
queue = None
else:
raise Exception("setup_log() must be called before calling create_strand()")
kwargs = Option.from_value(kwargs).get_or({})
args = [target, queue] + list(Option.from_value(args).get_or(()))
if not use_process or memdam.config.debug_processes:
return ProcessLikeThread(name=name, target=_setup_logging_and_call, args=args, kwargs=kwargs)
else:
return multiprocessing.Process(name=name, target=_setup_logging_and_call, args=args, kwargs=kwargs)
def secondary_type_option(name):
"""
:param name: The name of the field to parse
:type name: string
:returns: an Option of the secondary type for a field with the given name (empty if there is
no secondary type)
:rtype: Option(secondaryType)
"""
data = name.upper().split('__')
if len(data) > 2:
return Option.from_value(data.pop())
return Option.from_value(None)
def oauth_uri(cls, callback, **kwargs):
cls.last_callback = kwargs.get('raw_callback', callback)
return 'https://login.microsoftonline.com/common/oauth2/v2.0/authorize?client_id={}&nonce={}&scope=openid+profile+User.Read&response_type=code&redirect_uri={}{}'.format(
MICROSOFT_CLIENT_ID, time.time(),
quote(kwargs.get('raw_callback', callback)),
Option.from_call(kwargs.get, 'state').map(json.dumps).map(
'&state={}'.format).get_or(''))
def get_embedding_user(cls, adapter, embedding):
return Option(Oauth.query \
.filter(Oauth.oauth_adapter == adapter) \
.filter(Oauth.embedding_user == embedding) \
.first()) \
.map(lambda x: x.user.username) \
.get_or(None)
def upsert_record(application, key, content, all, username, role, raw_payload):
raw_content = raw_payload.get('content')
content = raw_content
if raw_content is not None:
content = Option.from_call(json.dumps, raw_content).get_or(raw_content)
result = PluginStorageManager \
.upsert_record(application, key, content, None if bool(all) else username,
raw_payload.get('index'))
return {'success': not not result}
def add_image(self, file: FileStorage, file_name: str,
convert: str) -> Option:
if file and self.is_supported_filename(file_name):
filename = secure_filename(file.filename)
if not os.path.exists(self.upload_dir):
os.mkdir(self.upload_dir)
filename = datetime.datetime.now().strftime(
"Upload_%Y%m%d%H%M%S_") + filename
filename = ImageConverter(file, filename,
convert).save(self.upload_dir)
return Option((self.get_url(filename), filename))
return Empty()
def exctract_metadata(text, isbn, container=dict()):
soup = BeautifulSoup(response_txt, "html.parser")
Option(soup.find("span", {"itemprop": "name"}))\
.map(lambda item: item.text)\
.map(lambda item: setitem(container, "title", item))\
.get_or(None)
Option(soup.find("div", {"class": "head-intro"}))\
.map(lambda item: item.find("h2"))\
.map(lambda item: item.find("a"))\
.map(lambda item: item.text)\
.map(lambda item: setitem(container, "authors", item))\
.get_or(None)
Option(soup.find("div", {"id": "detail-content"}))\
.map(lambda item: item.find("div", {"class": "block-content"}))\
.map(lambda item: item.find("p"))\
.map(lambda item: item.text)\
.map(lambda item: setitem(container, "description", item))\
.get_or(None)
Option(soup.find("img", {"itemprop": "image"}))\
.map(lambda item: "http:" + item["src"])\
.map(lambda item: setitem(container, "imageLinks", item))\
.get_or(None)
Option(soup.find("div", {"id": "block-more-info"}))\
.map(lambda item: item.find("ul"))\
.map(lambda item: item.find("li"))\
.map(lambda item: item.find("div"))\
.map(lambda item: item.find("a"))\
.map(lambda item: setitem(container, "tags", item.text))\
.get_or(None)
container["industryIdentifiers"] = isbn
container["source"] = "LAF"
print(container)
return container
def get_comments(p):
limit = request.args.get('limit', None, int)
page = request.args.get('page', 1, int)
offset = (page - 1) * limit if limit and page else None
p_int = Option.from_call(int, p)
result = p_int \
.map(lambda pid:
CommentManager.get_comments(p, limit, offset))
total_count = p_int \
.map(lambda pid: CommentManager.get_comment_count(pid))
total_pages = total_count \
.map(lambda count: count // limit + (count % limit > 0))
return {
'success': not result.empty,
'result': result.get_or(None),
'pages': total_pages.get_or(None),
'total': total_count.get_or(None)
}
def remove_comment(comment, username, role, raw_payload):
remove_token = raw_payload.get('remove_token', None)
if username:
stat = CommentManager.delete_comment(comment, username)
elif remove_token:
cid = dct_processor.check_remove_token(remove_token)
if cid and Option.from_call(int, comment).filter(lambda x: x == cid).get_or(None):
stat = CommentManager.force_delete_comment(cid)
else:
stat = False
else:
stat = False
if stat:
emit('comment_removed', {
'id': int(comment)
})
return {
'success': stat
}
def add_comment(username, role, content, to_post, raw_payload):
nickname = raw_payload.get('nickname')
name = username or nickname
if not name:
return {
'success': False,
'msg': 'Missing nickname or token expired.'
}
to_post = Option.from_call(int, to_post).get_or(None)
if not to_post:
return {
'success': False,
'msg': 'to_post should be a number'
}
state, msg = CommentManager.add_comment(to_post, content, raw_payload.get('to_comment'), username, nickname)
if state:
token = dct_processor.iss_remove_token(msg)
emit('comment_added', {
'post_id': to_post,
'comment_id': msg
})
raw_post = PostManager().get_db_post(to_post)
author = raw_post.author.username
if not nickname:
name = UserManager.find_user(username).nickname
emit('post_commented', {
'post_id': raw_post.post_id,
'title': raw_post.title,
'by_name': name
}, room=author)
else:
token = ''
return {
'success': state,
'msg': 'Success' if state else msg,
'comment_id': msg,
'remove_token': token
}
def index(request):
amazon_url = "http://www.amazon.it/gp/search/search-alias=stripbooks&field-isbn="
laf_url = "http://www.lafeltrinelli.it/libri/rfrrrfrre/rdferd/"
flattened_qs = Option(request.query_string)\
.map(lambda item: item.split("&"))\
.map(lambda exqs: [(item.split("=")[0], item.split("=")[1]) for item in exqs])\
.map(lambda item: dict(item)).get_or(dict())
data = yield from asyncio.gather(*[
asyncio.Task(
handle_amazon_scraping(url="%s%s" %
(amazon_url, flattened_qs["isbn"]),
isbn=flattened_qs["isbn"])),
asyncio.Task(
handle_laf_scraping(url="%s%s" % (laf_url, flattened_qs["isbn"]),
isbn=flattened_qs["isbn"]))
])
return aiohttp.web.Response(text=json.dumps(data),
content_type="application/json")
def addphoto(request):
def save_content(content):
path = os.path.join("uploaded", str(random.random()) + ".jpeg")
with open(path, "wb") as output:
output.write(content)
return path
data = yield from request.post()
Option(data.get("file", None))\
.map(lambda item: getattr(item, "file"))\
.map(lambda item: save_content(item.read()))\
.map(lambda item: resize_images(item))\
.map(lambda item: get_all_images())\
.get_or([])
r = web.Response(text=json.dumps(
[dict(src="../uploaded/" + value) for value in get_all_images()]),
content_type="application/json")
r.headers['Access-Control-Allow-Origin'] = '*'
return r
def new_record(cls,
application: str,
key: str,
value: str,
username=None,
index_key=None):
user = Option(username).map(UserManager.find_user).get_or(None)
if username:
record = cls.raw_query(application, key, username)
else:
record = cls.raw_query(application, key)
if record.count():
return False
record = PluginStorage()
if user:
record.user = user.user_id
record.application = application
record.key = key
record.index_key = index_key
record.content = value
db.session.add(record)
db.session.commit()
return True
def GetEncoding(root) -> Option(str):
for meta in root.find(HEAD_TAG).iter(META_TAG):
if meta.get('charset') is not None:
return meta.get('charset')
return None
def username(self):
return Option(self.token) \
.map(token_processor.get_username) \
.filter(itemgetter(0)) \
.map(itemgetter(1)).map(itemgetter('username')).get_or(None)
def find_user_option(cls, by):
return Option(
User.query.get(by) if isinstance(by, int)
else User.query.filter(sql_funcs.func.lower(User.username) == sql_funcs.func.lower(by)).first()
)
async def get_optional_product_with_id(product_id):
return Option.from_value(head(filter(_['id'] == product_id, products)))
def get_user_id(cls, username):
return Option(username) \
.map(UserManager.find_user) \
.map(lambda x: x.user_id) \
.get_or(None)
def new_record(application, key, content, all, username, role, raw_payload):
result = PluginStorageManager.new_record(
application, key,
Option.from_call(json.dumps, content).get_or(content),
None if bool(all) else username, raw_payload.get('index'))
return {'success': not not result}
def get_db_post(self, pid) -> Post:
return Option(
Post.query.filter(
sql_funcs.func.lower(Post.display_id) ==
sql_funcs.func.lower(pid)).first()
if isinstance(pid, str) else Post.query.get(pid)).get_or(None)
def GetMeta(root, name) -> Option(str):
for meta in root.find(HEAD_TAG).iter(META_TAG):
if meta.get('name') == name:
return meta.get('content')
return None
