def db_put(url, results=None, errors=None):
db_errs= []
# save the results
try:
from models import Result
if errors:
result = Result(
url=url,
datetime=f"{datetime.utcnow()} (UTC)",
errors=errors
)
else:
result = Result(
url=url,
datetime=f"{datetime.utcnow()} (UTC)",
results=results
)
db.session.add(result)
db.session.commit()
return result.id
except Exception as err:
db_errs.append(
f"Unable to add item to database:{str(err)}"
)
return {"errors": db_errs}
def setUp(self):
self.app = tested_app.test_client()
self.main = tested_app.app_context()
with tested_app.app_context():
db.init_app(tested_app) # removes cyclic dependency??
db.create_all()
db.session.add(Result(year=1994, type="birth", event="Alice"))
db.session.add(Result(year=1995, type="death", event="Bob"))
db.session.add(
Result(year=1996, month=1, type="birth", event="John"))
db.session.add(
Result(year=1997, month=2, day=1, type="birth", event="Ann"))
db.session.commit()
def scraper():
print('About to start the scraper.')
base_rows = get_rows(base_scraper_url)
base_for_today = base_rows[0]
base_datacells = base_for_today.find_all('td')
today_url_cell = base_datacells[0].find('a')
base_dict['url'] = unescape(base_url + today_url_cell.attrs['href'])
base_dict['date'] = parse(today_url_cell.text)
todays_rows = get_rows(base_dict.get('url'))
for row in todays_rows:
datacells = row.find_all('td')
doc = {
'url': base_dict.get('url'),
'date': base_dict.get('date'),
'source': datacells[1].text,
'agency': datacells[2].text,
'fsg': datacells[3].text,
'title': datacells[4].text,
'keywords': datacells[5].text,
'url_2':
unescape(base_url + datacells[4].find('a').attrs.get('href')),
}
article = get_article(doc.get('url_2'))
if article:
try:
doc['description'] = get_body(article)
result = Result(**doc)
db.session.add(result)
db.session.commit()
except Exception as e:
result = Result(**doc)
db.session.add(result)
db.session.commit()
print('Scraper completed execution')
return article
def index():
errors = []
results = {}
r = None # prevents uninitialization error, which happens on Heroku but not my laptop
if request.method == 'POST':
# get the URL entered
try:
url = request.form['url']
r = requests.get(url)
except:
errors.append("Unable to get URL - try again")
if r is not None:
(raw_counts, stop_removed_count) = count_words_from_html(r)
# package results for web display
results = sorted(stop_removed_count.items(), key=operator.itemgetter(1), reverse=True)[:10]
# store results in the database
try:
db_result = Result(
url=url,
result_all=raw_counts,
result_no_stop_words=stop_removed_count
)
db.session.add(db_result)
db.session.commit()
except Exception as e:
err = "Unable to add results to the database: %s" % e
errors.append(err)
return render_template('index.html', errors=errors, results=results)
def index():
form = LivechatForm()
if request.method == 'POST' and form.validate():
message = form.Message.data
gender = form.Gender.data
Text_analyser = TextAnalyser(message)
cl, pos, neg = Text_analyser.opinion()
entity = Text_analyser.entity('templates/entity.html')
result = Result(message=message,
gender=gender,
cl=cl,
pos=pos,
neg=neg)
db.session.add(result)
db.session.commit()
else:
message = None
gender = None
cl = None
pos = None
neg = None
return render_template('index.html',
form=form,
message=message,
gender=gender,
cl=cl,
pos=pos,
neg=neg)
async def post_result(request):
logging.info("Create Result for Task")
data = await request.json()
tid = request.match_info['id']
try:
task = Task.objects.get(id=tid)
except DoesNotExist:
return web.json_response({}, headers=response_headers)
# Get latest result
latest_result = get_latest_result_for_task(tid)
result_data = data.get('data', "")
new_result = Result(data=result_data)
new_result.task = task
new_result.save()
# Update Task with Result
task.results.append(new_result)
task.save()
diff_results(new_result.data, latest_result.data, task.notification_type,
task.notification_args)
return web.json_response(new_result.to_json(),
dumps=str,
headers=response_headers)
def cnnFetch():
script = Scraper().scrape('cnn')
result = Result(source="cnn",
date=datetime.date.today(),
titles=" ".join(script))
db.session.add(result)
db.session.commit()
def index():
errors = []
results = {}
if request.method == 'POST':
try:
url = request.form['url']
r = requests.get(url)
except:
errors.append('Unable to get URL')
return render_template('index.html', errors=errors)
if r:
raw = BeautifulSoup(r.text, 'html.parser').get_text()
nltk.data.path.append('./nltk/data')
tokens = nltk.word_tokenize(raw)
text = nltk.Text(tokens)
nonPunct = re.compile('.*[A-za-z]*.')
raw_words = [w for w in text if nonPunct.match(w)]
raw_words_count = Counter(raw_words)
no_stop_words = [w for w in raw_words if w.lower() not in stops]
no_stop_words_count = Counter(no_stop_words)
results = sorted(no_stop_words_count.items(),
key=operator.itemgetter(1),
reverse=True)
try:
result = Result(url=url,
result_all=raw_words_count,
results_no_stop_words=no_stop_words_count)
db.session.add(result)
db.session.commit()
except:
errors.append('Unable to add item to database')
return render_template('index.html', errors=errors, results=results)
def generate_feed_for_query(query):
global db
logger.debug("generate_feed_for_query()")
results = db.get_top_results_for_query(query.id)
fg = FeedGenerator()
fg.id(f"{settings.BASE_URL}/results/{query.id}")
fg.title(f"Results for {query.search}")
fg.author({"name": "Reef", "email": "*****@*****.**"})
fg.description("A list of latest results for a search")
fg.link(href=settings.BASE_URL)
fg.language("en")
for result_raw in results:
result = Result(json=result_raw)
logger.debug(
f"adding entry for {result.id}: {result.title}: {result.content}")
fe = fg.add_entry()
fe.id(f"{settings.BASE_URL}/results/{query.id}/{result.id}")
fe.title(result.title)
fe.link(href=result.content)
if settings.FEED_FORMAT == "rss":
return fg.rss_str(pretty=True)
# else...
return fg.atom_str(pretty=True)
def get_results(job_key):
errors = []
job = Job.fetch(job_key, connection=conn)
if job.is_finished:
# save the results
url, raw_word_count, no_stop_words_count = job.result
try:
result_record = Result(
url=url,
result_all=raw_word_count,
result_no_stop_words=no_stop_words_count
)
db.session.add(result_record)
db.session.commit()
# return result.id
except:
errors.append("Unable to add item to database.")
return {"error": errors}
result = Result.query.filter_by(id=result_record.id).first()
results = sorted(
result.result_no_stop_words.items(),
key=operator.itemgetter(1),
reverse=True
)[:10]
return jsonify(results)
else:
return "Nay!", 202
def add_result():
event = request.data['event']
first_name = request.data['first_name']
first_institution = request.data['first_institution']
second_name = request.data['second_name']
second_institution = request.data['second_institution']
third_name = request.data['third_name']
third_institution = request.data['third_institution']
print('event:' + event + 'firstname:' + first_name + 'first_institution:' +
first_institution +
'-----------------------------------------------')
if not session.query(Event).filter_by(id=event).first():
session.close()
return {'status': 'BAD REQUEST', 'message': 'EVENT DOES NOT EXIST'}
info = Result(event_id=event, first_name=first_name, first_institution=first_institution, \
second_name=second_name, second_institution=second_institution, third_name=third_name, \
third_institution=third_institution)
session.add(info)
try:
session.commit()
except:
session.rollback()
flash(config.UNEXPECTED_ERROR)
finally:
session.close()
return redirect('/dashboard')
def index():
errors = []
results = {}
if request.method == "POST":
# get url that the user has entered
try:
h = request.form['height']
w = request.form['weight']
height = float(h)
weight = float(w)
print(str(height) + " " + str(weight))
except:
errors.append(
"Unable to get Weight & Height. Please make sure it's valid and try again."
)
return render_template('index.html', errors=errors)
if height and weight:
bmi = round((weight / (height * height)) * 703)
print "BMI: " + str(bmi)
# save the results
results = bmi
try:
from models import Result
result = Result(height=height, weight=weight, bmi=bmi)
db.session.add(result)
db.session.commit()
except:
errors.append("Unable to add item to database.")
return render_template('index.html', errors=errors, results=results)
def send_request(r_type, task_id, method, url, body=None, headers=None):
print("Send request")
if not headers:
headers = {'Content-Type': 'application/json'}
if method == "GET":
response = requests.get(url, headers=headers)
elif method == "POST":
response = requests.post(url, body, headers=headers)
if r_type == "sync":
data = response.json()
# Get latest result
latest_result = get_latest_result_for_task(task_id)
# Create new Result
new_result = Result(data=json.dumps(data))
task = Task.objects.get(id=task_id)
new_result.task = task
new_result.save()
# Update Task with Result
task.results.append(new_result)
task.save()
diff_results(new_result.data, latest_result.data,
task.notification_type, task.notification_args)
elif r_type == "poll":
pass
return
def loader(start, stop):
"""ডাটা লোড করার মেইন ফাংশন।
start থেকে stop প্যারামিটারের সকল রোল সেভ করবে (start এবং stop রোল সহ।) এবং লগ ইন্টারেক্টিভ শেলে প্রিন্ট করবে, কোন কারনে ছয়বার এরর হলে বা ডাটা লোড করতে না পারলে
লুপ থেমে যাবে।
ইনপুটঃ শুরু এবং শেষের রোল নং ইন্টেজার।
আউটপুটঃ নান"""
error_count = 0
for roll in range(start, stop+1):
if error_count > 5: break
try:
payload['Roll'] = roll
r = requests.post(result_url,payload,headers=headers)
r.encoding = 'utf-8'
result = beautify(r)
data_row = Result(roll, result)
session.add(data_row)
session.commit()
print('roll {} added to database'.format(roll))
except:
error_count +=1
print('one error occured')
continue
if error_count > 4:
print( "data loading abort due to 5 error!!")
else:
print("Data loaded succesfully")
def __get_row_data(element):
columns = element.find_elements_by_xpath("td")
return Result(name=columns[0].text,
price=int(columns[1].text.replace(",", "")),
quantity=columns[2].text,
s_name=columns[3].text,
where=columns[4].text)
def process_result(self,
message: Message,
user: User,
bot: TeleBot,
competitor: Competitor,
final=False):
opponent, opponent_user = get_opponent_and_opponent_user(competitor)
if not opponent or not opponent_user:
return teardown_challenge(
competitor, message, user, bot,
'challenge_confirm_cannot_find_opponent_msg' if not opponent
else 'challenge_confirm_cannot_fin_opponents_user_msg')
res = user.check_result()
if not res:
res = Result(player_a=competitor,
player_b=opponent,
confirmged=False)
res.save()
user.current_result = res
user.save()
text = render_result(res, final)
bot.send_message(
message.chat.id,
text,
reply_markup=self.__base_keyboard(confirmation_stage=final),
parse_mode='html')
return RET.OK, None, None, None
def WordCount2():
from pycompss.api.api import compss_wait_on
'''
WordCount Test
- Wordcount task receives a PSCO and returns a dictionary.
- Reduce task receives a INOUT PSCO (result) where accumulates the partial results.
'''
words = [
Words('This is a test'),
Words('This is a test'),
Words('This is a test'),
Words('This is a test')
]
for w in words:
w.makePersistent()
result = Result()
result.makePersistent()
localResults = []
for w in words:
partialResults = wordcountTask(w)
reduceTaskPSCOs(result, partialResults)
final = compss_wait_on(result)
print(final.myd)
result = final.get()
if result['This'] == 4 and result['is'] == 4 and result[
'a'] == 4 and result['test'] == 4:
print("- Python Wordcount 2 with PSCOs: OK")
return True
else:
print("- Python Wordcount 2 with PSCOs: ERROR")
return False
def process(self):
raw = BeautifulSoup(self.r.text, 'html.parser').get_text()
nltk.data.path.append('./nltk_data/') # set the path
tokens = nltk.word_tokenize(raw)
text = nltk.Text(tokens)
# remove punctuation, count raw words
nonPunct = re.compile('.*[A-Za-z].*')
raw_words = [w for w in text if nonPunct.match(w)]
raw_word_count = Counter(raw_words)
#stop words
non_stop_words = [w for w in raw_words if w.lower() not in stops]
non_stop_words_count = Counter(non_stop_words)
self.results = sorted(non_stop_words_count.items(),
key=operator.itemgetter(1),
reverse=True) # add [:10] to limit to first 10
try:
result = Result(url=self.url,
result_all=raw_word_count,
result_no_stop_words=non_stop_words_count)
db.session.add(result)
db.session.commit()
self.job_id = result.id
except Exception as error:
print("Unable to add item to database.", error)
self.errors.append("Unable to add item to database.")
return {"errors": self.errors, "job_id": self.job_id}
def save(request):
results = simplejson.loads(request.POST['results'])
session_key = request.session.session_key #request.COOKIES[settings.SESSION_COOKIE_NAME]
session = Session(key=session_key, global_time=time())
session.save()
# Convert user results to Result table row
db_results = []
for result in results:
db_result = Result(
session=session,
time=result['time'],
selection=result['selection'],
)
db_result.image_id = result['id']
db_results.append(db_result)
try:
# This could be done better with a transaction
for db_result in db_results:
db_result.save()
except Exception as e:
print e
pass
return HttpResponseRedirect('/static/thankyou.html')
def processEnterMarksForm(request, enterMarksForm, gamename):
groupid = extractGroupId(gamename)
groupExamIds = examIDsinGroup(gamename)
userResults = Result.objects.filter(user=request.user.id,
exam_id__in=groupExamIds)
examname = enterMarksForm.data['exam']
enteredMark = enterMarksForm.data['mark']
if str(examname) == "":
return "The Exam field in Enter Marks Form cannot be left empty!"
elif str(enteredMark) == "":
return "The Mark field in Enter Marks Form cannot be left empty!"
try:
int(enteredMark)
except:
return "Your mark must be a whole number from 0 to 100."
examid = extractExamIDgivenGroup(examname, gamename)
alreadyEnteredExams = []
for result in userResults.values():
alreadyEnteredExams.append(result['exam_id'])
if examid in alreadyEnteredExams:
return "You have already entered your mark for this exam!"
exam = Exam.objects.get(pk=examid)
if not (int(enteredMark) >= 0 and int(enteredMark) <= 100):
return "The entered mark is invalid! It must be a whole number between 0 and 100."
else:
newResult = Result(exam=exam, user=request.user, mark=enteredMark)
newResult.save()
calculateWinner(request.user.id, examid, enteredMark, gamename)
return False
def count_and_save_words(url):
errors = []
try:
r = requests.get(url)
except:
errors.append(
"Unable to get URL. Please make sure it's valid and try again.")
raw = BeautifulSoup(r.text, 'html.parser').get_text()
nltk.data.path.append('./nltk_data/')
tokens = nltk.word_tokenize(raw)
text = nltk.Text(tokens)
non_punct = re.compile('.*[A-Za-z].*')
raw_words = [w for w in text if non_punct.match(w)]
raw_words_count = Counter(raw_words)
no_stop_words = [w for w in raw_words if w.lower() not in stops]
no_stop_words_count = Counter(no_stop_words)
try:
result = Result(url=url,
result_all=raw_words_count,
result_no_stop_words=no_stop_words_count)
db.session.add(result)
db.session.commit()
return result.id
except:
errors.append('Unable to add item to database.')
return {"error": errors}
def count_and_save_words(url):
"""Count the times a word shows up and save them."""
errors = []
try:
r = requests.get(url)
except Exception:
errors.append(
'Unable to get URL. Please make sure it is valid and try again.')
return {"error": errors}
raw = BeautifulSoup(r.text).get_text()
nltk.data.path.append('./nltk_data/') # set the path
tokens = nltk.word_tokenize(raw)
text = nltk.Text(tokens)
# remove punctuation, count raw words
non_punct = re.compile('.*[A-Za-z].*')
raw_words = [w for w in text if non_punct.match(w)]
raw_word_count = Counter(raw_words)
# stop words
no_stop_words = [w for w in raw_words if w.lower() not in stops]
no_stop_words_count = Counter(no_stop_words)
# save the results
try:
result = Result(url=url,
result_all=raw_word_count,
result_no_stop_words=no_stop_words_count)
db.session.add(result)
db.session.commit()
return result.id
except Exception:
errors.append('Unable to add item to database.')
return {"error": errors}
def index():
errors = []
results = {}
if request.method == "POST":
try:
url = request.form['url']
r = requests.get(url)
print(r.text)
except:
errors.append("유효하지 않는 url. 다시 확인부탁")
render_template('index.html', errors=errors)
if r:
print(r)
raw = BeautifulSoup(r.text, 'html.parser').get_text()
nltk.data.path.append('./nltk_data/')
tokens = nltk.word_tokenize(raw)
text = nltk.Text(tokens)
nonPunct = re.compile('.*[A-Za-z].*')
raw_words = [w for w in text if nonPunct.match(w)]
raw_word_count = Counter(raw_words)
no_stop_words = [w for w in raw_words if w.lower() not in stops]
no_stop_words_count = Counter(no_stop_words)
results = sorted(no_stop_words_count.items(),
key=operator.itemgetter(1),
reverse=True)
try:
result = Result(url=url,
result_all=raw_word_count,
result_no_stop_words=no_stop_words_count)
db.session.add(result)
db.session.commit()
except:
errors.append("db에 데이터를 추가할 수 없음")
return render_template('index.html', errors=errors, results=results)
def post_result(db: SessionClass, room_id, user_id, user_hand):
with session_manager(db) as session:
if session.query(Room).filter(Room.id == room_id).count() == 0:
raise HTTPException(status_code=404)
room = session.query(Room).filter(Room.id == room_id).first()
if user_id == room.host_user_id:
result = Result(room_id=room.id,
user_id=user_id,
stage=room.latest_stage,
hand=user_hand)
session.add(result)
client_users_result = session.query(Result).filter(
Result.room_id == room_id,
Result.stage == room.latest_stage).all()
for client_user_result in client_users_result:
if (client_user_result.hand - user_hand) % 3 == 1:
client_user_result.is_win = True
else:
client_user_result.is_win = False
session.commit()
else:
host_results = session.query(Result).filter(
Result.user_id == room.host_user_id, Result.room_id == room.id,
Result.stage == room.latest_stage)
if host_results.count() == 0:
result = Result(room_id=room.id,
user_id=user_id,
stage=room.latest_stage,
hand=user_hand)
session.add(result)
else:
host_result = host_results.first()
if (user_hand - host_result.hand) % 3 == 1:
result = Result(room_id=room.id,
user_id=user_id,
is_win=True,
stage=room.latest_stage,
hand=user_hand)
session.add(result)
else:
result = Result(room_id=room.id,
user_id=user_id,
is_win=False,
stage=room.latest_stage,
hand=user_hand)
session.add(result)
return result
async def parse(url, session):
async with SEMA:
task = [asyncio.ensure_future(fetch(url, session))]
response = await asyncio.gather(*task)
result = Result(data=response)
db_session.add(result)
db_session.commit()
db_session.refresh(result)
def aljFetch():
script = Scraper().scrape('alj')
result = Result(source="alj",
date=datetime.date.today(),
titles=" ".join(script))
db.session.add(result)
db.session.commit()
return "202"
def nytFetch():
script = Scraper().scrape('nyt')
result = Result(source="nyt",
date=datetime.date.today(),
titles=" ".join(script))
print(script)
db.session.add(result)
db.session.commit()
def post(self, poll_id):
data = request.get_json(force=True)
data = json.dumps(data)
polls[poll_id] = data
print(polls)
hasil_lengkap = Result(polls)
db.session.add(hasil_lengkap)
db.session.commit()
return {poll_id: data}
def get_or_create_car(vin, year, car):
'''checks to see if the car exists in the db, if not, add it'''
result = Result(vin, year, car["color"], car["build_date"], car["stripe"],
car["electronics"], car["convenience"])
try:
db.session.add(result)
db.session.commit()
except:
print("Unable to add item to database.")
def seed():
print("Add seed data to results table.")
NUM_VAL_SAMPLES = 20
db.session.add(Result(
[("/validation/unet_lr0.0001_seed23_losstype0_augTrue_ver1/orig/%d.jpg" % i) for i in range(NUM_VAL_SAMPLES)],
[("/validation/unet_lr0.0001_seed23_losstype0_augTrue_ver1/pred/%d.png" % i) for i in range(NUM_VAL_SAMPLES)],
[("/validation/unet_lr0.0001_seed23_losstype0_augTrue_ver1/ref/%d.png" % i) for i in range(NUM_VAL_SAMPLES)],
))
db.session.commit()
