def process_post(cls, data):
with app.app_context():
url = data["url"]
source_id = data["source"]
coordinates = data["latlong"]
source = SourceModel.find_by_uuid(source_id)
request = requests.get(url)
html = BeautifulSoup(request.text, "html.parser")
content = ""
tagsText = ""
tagsArray = []
title = html.select_one(source.title_selector)
if source.subtitle_selector:
subtitle = html.select_one(source.subtitle_selector)
if source.content_selector:
elements_body = html.select(source.content_selector)
for el in elements_body:
content += el.text + "\n\n"
if source.tags_selector:
elements_tags = html.select(source.tags_selector)
for el in elements_tags:
tagsArray.append(el.text.replace("\n", ""))
tagsText = ",".join(tagsArray)
if source.author_selector:
author = html.select(source.author_selector)
body = {
"source_id": source_id,
"state_id": source_id,
"municipality_id": source_id,
"title": title.text.replace("\n", ""),
"subtitle": subtitle.text.replace("\n", ""),
"location": {
"type": "Point",
"coordinates": coordinates
},
"content": content,
"tags": tagsText.upper(),
"author": author.text.replace("\n", ""),
"link": url,
"timestamp": datetime.now(),
}
es.index(index="contents", body=body)
def edit_country(cls, id, name):
res = es.index(index=os.environ.get("INDEX"),
doc_type='country',
id=id,
body={"name": name})
if "result" in res and res["result"] == "updated":
return True
return False
def create(cls, name):
id = int(datetime.timestamp(datetime.now()) * 1000)
res = es.index(index=os.environ.get("INDEX"),
doc_type='country',
id=id,
body={"name": name})
if "created" in res and res["created"]:
return True
return False
def create(cls, name, state):
state_rec = State.get(state)
if state_rec:
id = int(datetime.timestamp(datetime.now()) * 1000)
body = {"name": name, "state": state_rec["name"]}
res = es.index(index=os.environ.get("INDEX"),
doc_type='city',
id=id,
parent=state_rec["id"],
body=body)
if "created" in res and res["created"]:
return True
return False
def create(cls, name, country):
country_rec = Country.get(country)
if country_rec:
id = int(datetime.timestamp(datetime.now()) * 1000)
body = {"name": name, "country": country_rec["name"]}
res = es.index(index=os.environ.get("INDEX"),
doc_type='state',
id=id,
parent=country_rec["id"],
body=body)
if "created" in res and res["created"]:
return True
return False
def edit(cls, id, name, state):
state_rec = State.get(state)
if state_rec:
res = es.index(index=os.environ.get("INDEX"),
doc_type='city',
id=id,
parent=state_rec["id"],
body={
"name": name,
"state": state_rec["name"]
})
if "result" in res and res["result"] == "updated":
return True
return False
def edit(cls, id, name, country):
country_rec = Country.get(country)
if country_rec:
res = es.index(index=os.environ.get("INDEX"),
doc_type='state',
id=id,
parent=country_rec["id"],
body={
"name": name,
"country": country_rec["name"]
})
if "result" in res and res["result"] == "updated":
return True
return False