def scrape_job_data(user):
try:
if request.method == 'POST':
print(request)
site = request.json["site"]
job_type = request.json["type"]
city = request.json["city"]
country = request.json["country"]
province = request.json["province"]
technologies = request.json["technologies"]
print(job_type, technologies)
output = scrape(job_type, technologies , site) # Passing to the Scrape function
if len(output["jobs"]) == 0:
return jsonify({"type": "Error", "message": "No results found..."}),400
query = {}
if user:
query = Query(site=site, job_type=job_type,city=city, country=country, province=province, user_id=user.id)
db.session.add(query)
db.session.commit()
save_to_db(query.id, output["jobs"])
output["query"] = query
print("NUM JOBS ", len(output["jobs"]))
return jsonify(output)
except Exception as e:
print("Error", e)
return jsonify({"type": "Error", "message": "Error fetching job results, please try again"}),400
def register_request(query_dic, flag):
image_local_path, hash = None, None
if query_dic.get("NumMedia") == '1':
image_local_path, hash = save_media(query_dic.get("MediaUrl0"),
query_dic.get("MediaContentType0"))
if query_dic.get("localPath") and query_dic.get("img"):
image_local_path, hash = query_dic.get("localPath"), hash_local_file(
query_dic.get("localPath"))
query = Query(
media_type=query_dic.get("MediaContentType0")) # e.g. image/jpeg
query.sms_message_id = query_dic.get("SmsMessageSid")
query.num_media = query_dic.get("NumMedia")
query.sms_id = query_dic.get("SmsSid")
query.sms_status = query_dic.get("SmsStatus")
query.body = query_dic.get("Body")
query.to_num = query_dic.get("To")
query.num_sug = query_dic.get("NumSegments")
query.message_id = query_dic.get("MessageSid")
query.account_id = query_dic.get("AccountSid")
query.from_num = query_dic.get("From")
query.ip_address = query_dic.get("IP")
query.image_url = query_dic.get("MediaUrl0")
query.image_local_path = image_local_path
query.image_hash = hash # hash this
query.video_url = "Not Supported" #get binary in a different field later
query.ip_address = query_dic.get("IP")
query.query_type = flag
query.save()
return query
def process_response(self, request, response):
path = request.get_full_path()
user = None
user_id = request.session.get('user_id', None)
start_time = request.session.get('time')
if start_time:
run_time = time.time() - start_time
else:
run_time = -1
if user_id:
del request.session['user_id']
user = Users.objects.get(pk=user_id)
if not self.is_filter_path(path):
data = {
'remote_address': request.META['REMOTE_ADDR'],
'request_method': request.method,
'request_path': path,
'request_body': request.body,
'response_status': response.status_code,
'user': user,
'run_time': run_time
}
Query(**data).save()
return response
def create_query(count, approximate, selected, excluded_pks, image_data_url):
query = Query()
query.count = count
if excluded_pks:
query.excluded_index_entries_pk = [int(k) for k in excluded_pks]
query.selected_indexers = selected
query.approximate = approximate
image_data = base64.decodestring(image_data_url[22:])
if settings.HEROKU_DEPLOY:
query.image_data = image_data
query.save()
dv = Video()
dv.name = 'query_{}'.format(query.pk)
dv.dataset = True
dv.query = True
dv.parent_query = query
dv.save()
create_video_folders(dv)
query_path = "{}/queries/{}.png".format(settings.MEDIA_ROOT, query.pk)
query_frame_path = "{}/{}/frames/0.png".format(settings.MEDIA_ROOT, dv.pk)
if settings.HEROKU_DEPLOY:
s3 = boto3.resource('s3')
s3.Bucket(settings.MEDIA_BUCKET).put_object(Key=query_path,
Body=image_data)
s3.Bucket(settings.MEDIA_BUCKET).put_object(Key=query_frame_path,
Body=image_data)
else:
with open(query_path, 'w') as fh:
fh.write(image_data)
with open(query_frame_path, 'w') as fh:
fh.write(image_data)
return query, dv
def get_search_results(query_id, query_postings):
# Check previously calculated queries for changes in the corpus
query = Query.objects(id=query_id).only("results",
"total_frequency").first()
total_frequency = Entry.objects(id__in=query_postings.iterkeys()).only(
"total_frequency").sum("total_frequency")
if not query or total_frequency != query.total_frequency:
results = []
avg_length = Meme.objects.only("length").aggregate_average("length")
idf, relevant_docs = get_idf_relevant_docs(query_postings)
for meme in relevant_docs: # Iterate through relevant documents to calculate its score
bm25 = calculate_bm25(avg_length, idf, meme, query_postings)
result = SearchResult(id=MemeId(source=meme.id.source,
meme_id=meme.id.meme_id),
name=meme.name,
title=meme.title,
caption=meme.caption,
score=meme.score,
url=meme.url,
image=meme.image,
bm25=bm25)
results.append(result)
results = sorted(results, key=lambda result: result.bm25,
reverse=True)[:200]
query = Query(id=query_id,
results=results,
total_frequency=total_frequency)
query.save()
return query.results
def process_query(self, entity, attribute, verbose=False):
'''
Process the user query and return the aggregated sentiment and related entries.
:param entity: str
:param attribute: str
:param verbose: bool
:rtype: (float, List[AttributeEntry])
'''
query = Query(entity, attribute)
if verbose:
print("Query parsed.")
relevant_entries = self.data_source.lookup(query)
count = len(relevant_entries)
if verbose:
print("{} relevant entr{} found.".format(
count, "y" if count == 1 else "ies"))
if count == 0:
return None, []
aggregateData = [{
'sentiment': expr.sentiment,
'is_header': expr.is_header
} for entry in relevant_entries for expr in entry.expressions]
score = self.aggregator_service.aggregate_sentiment(aggregateData)
if verbose:
print("Sentiment scores aggregated.")
return score, relevant_entries
def get_all_queries(self) -> list:
data = self.__query_sheets('Queries!A:ZZ')
result = list()
for entry in data:
try:
result.append(Query(keyword=entry[0], maximum_price=int(entry[1]), ignore=entry[2:]))
except:
logging.error("Failed to process row {}".format(entry))
return result
def read_config():
global db
if settings.CONFIG_FILE:
logger.info("reading config file: " + settings.CONFIG_FILE)
config = json.loads(get_file_or_s3(settings.CONFIG_FILE))
for query_config in config["queries"]:
query = Query(json=query_config)
db.save_query(query)
logger.debug("saved query " + query.id + ": " + query.search)
def sendMessage():
form = FeedbackForm()
if form.validate_on_submit():
new = Query(content=form.content.data,
name=current_user.username,
patient_id=current_user.id,
doctor_id=current_user.mydoctor)
db.session.add(new)
db.session.commit()
flash('Message sent', 'message')
return redirect(url_for('home'))
return render_template('sendMessage.html', form=form)
def query(request):
ConditionSet = formset_factory(ConditionForm, extra=5)
if request.method == 'POST':
selectionset = ConditionSet(request.POST,request.FILES)
if selectionset.is_valid():
query=Query(qid=makeQID(),query=constructQuery(selectionset.cleaned_data))
query.save()
return HttpResponseRedirect('/portal/results/%s/'%query.qid)
else:
selectionset = ConditionSet(initial=[
{'lower': u'5000',
'upper': u'5050',
'parameter':'RadTransWavelengthExperimentalValue',
'connection':True,
},
])
return render_to_response('portal/query.html', {'selectionset': selectionset})
def load_model(city_code):
# check database, if not found make api call
print("loding model ***********************************")
model = Query.query.filter_by(city_code=city_code).first()
if not model:
print("api call ***********************************")
dataset = Dataset(city_code)
model = PricePredictionModel(dataset)
model = Query(city_code=model.city_code, model=model)
db.session.add(model)
db.session.commit()
print("model loaded ***********************************")
return model
def search(input: str, size: int = 100) -> Query:
"""Adds search query into a queue for importing and processing
"""
# TODO - Take into account the rate limit, the importer will fail and need adding back to the queue if hit
# TODO - multi depends_on maybe coming soon. It would allow a end of processing notification to exist
query = Query()
query.save(input=input, size=size, stream='twitter')
source_queue = Queue(name='source')
process_queue = Queue(name='process')
# Import tweets
import_job = source_queue.enqueue(import_raw, query=query)
# Process tweets
process_queue.enqueue(aggregate_pixels, query=query, depends_on=import_job)
process_queue.enqueue(aggregate_adjectives,
query=query,
depends_on=import_job)
return query
def receive_message():
user_text_message = request.values.get('Text')
phone_number = '+' + request.values.get('From')
wifi_request = 'Wifi' in request.values
user = get_user(phone_number)
user_query = Query()
user_query.save()
tokenizer = Tokenizer(user_text_message)
if tokenizer.partial_text:
return ""
arguments = tokenizer.arguments_dict
if 'c' in arguments:
country_tag = arguments["c"]
else:
country_tag = 'US'
if 'v' in arguments:
version_number = int(arguments["v"])
else:
version_number = 10000
should_rotate = False
if tokenizer.api != "onboard" and version_number > 6:
should_rotate = True
results = process_message(user, user_text_message, tokenizer)
key = results.get("key", "")
# if user.is_over_limit():
# user_text_message = "limit default: key: %s" % key[1:]
# results = process_message(user, user_text_message, user_query)
# key = results.get("key" "")
messages_list = results.get("messages")
user_query.response = messages_list[0] if len(
messages_list) else "Feedback"
user_query.save()
user.queries.append(user_query)
user.save()
if not wifi_request:
distribute(str(phone_number), messages_list, key, country_tag,
should_rotate)
return ""
return jsonify(results=prepend_key(messages_list, key))
def single_review(request, review_name_slug):
context = {}
try:
review = Review.objects.get(user=request.user, slug=review_name_slug)
context['review_title'] = review.title
queries = Query.objects.filter(review=review)
paper = Paper.objects.filter(review=review)
context['queries'] = queries
context['review'] = review
context['paper'] = paper
if request.method == "POST":
if request.POST.get('delete_query', "") != "":
query_to_delete = Query.objects.get(
name=request.POST.get('delete_query'))
if query_to_delete != None:
query_to_delete.delete()
context[
'alert_message'] = "Query deleted: " + query_to_delete.name
else:
#making sure an empty query will not be saved, removing spaces on both sides of the name
new_query = request.POST.get('queryField', default="")
new_query = new_query.strip()
if new_query != "" and not queries.filter(
name=new_query).exists():
query = Query(review=review, name=new_query)
query.save()
context[
'alert_message'] = "Query saved: " + request.POST.get(
'queryField')
else:
context[
'alert_message'] = "You are either saving an empty query or a query with this name alreday exists."
queries = Query.objects.filter(review=review)
context['queries'] = queries
context['review'] = review
except Review.DoesNotExist:
pass
return render(request, 'ultimatereview/querybuilder.html', context)
def lifecycle():
global db
queries = db.get_all_queries()
for query_raw in queries:
query = Query(json=query_raw)
logger.debug(f'checking query {query.id}')
logger.debug("performing search...")
results = search(query.search, num_results=10)
logger.info(f"query {query.id}: hits: {len(results)}")
new_items = False
for result_raw in results:
#logger.debug(f'result_raw: {result_raw[0]}\t {result_raw[1]}')
result = Result(id=str(uuid.uuid4()),
query=query.id,
timestamp=str(get_now_as_timestamp()),
title=result_raw[0],
content=result_raw[1])
logger.debug(
"result: " +
json.dumps(result.as_dict(), indent=4, sort_keys=True))
if not db.result_exists(result):
logger.debug("result has not previously been recorded")
new_items = True
db.save_result(result)
if new_items:
feed_string = generate_feed_for_query(query)
write_to_file_or_s3(uri=settings.RESULTS_FOLDER,
query=query,
body=feed_string)
def spell_check():
if not 'nonce' in session or not 'token' in session:
return redirect(url_for('login'))
ses = session['token'] + session['nonce']
l = Login.query.filter_by(session_key=ses).filter_by(active=True).first()
if not l:
return redirect(url_for('login'))
csrf_token = hashlib.sha256(session['nonce'].encode('ascii')).hexdigest()
if request.method == "GET":
return render_template("spell_check.html", csrf_token=csrf_token)
if request.form['csrf'] != csrf_token:
return redirect(url_for('login'))
tmpfname = ''.join(random.choices(string.ascii_letters, k=40))
inputtext = request.form['inputtext']
with open(tmpfname.encode('ascii'), 'w') as tmp:
tmp.write(inputtext)
cmd = "./a.out {} wordlist.txt".format(tmpfname)
res = subprocess.check_output(shlex.split(cmd)).decode('ascii').split('\n')
res = ','.join(res)
os.remove(tmpfname)
q = Query(l.user, inputtext, res)
db.session.add(q)
db.session.commit()
db.session.refresh(q)
return render_template('spell_check.html',
inwords=request.form['inputtext'],
misspelled=res,
csrf_token=csrf_token)
def create_query(count, approximate, selected, excluded_pks, image_data_url):
query = Query()
query.count = count
if excluded_pks:
query.excluded_index_entries_pk = [int(k) for k in excluded_pks]
query.selected_indexers = selected
query.approximate = approximate
query.save()
dv = Video()
dv.name = 'query_{}'.format(query.pk)
dv.dataset = True
dv.query = True
dv.parent_query = query
dv.save()
create_video_folders(dv)
image_data = base64.decodestring(image_data_url[22:])
query_path = "{}/queries/{}.png".format(settings.MEDIA_ROOT, query.pk)
query_frame_path = "{}/{}/frames/0.png".format(settings.MEDIA_ROOT, dv.pk)
with open(query_path, 'w') as fh:
fh.write(image_data)
with open(query_frame_path, 'w') as fh:
fh.write(image_data)
return query, dv
def _save_query(chat_id, query):
query_doc = Query(query=query)
query_doc.save()
user_doc = User.objects.get(chat_id=chat_id)
user_doc.queries.append(query_doc)
user_doc.save()
def get_all(cls, *object_ids):
return Query(cls.table()).get_all(*object_ids)
def filter(cls, filter_hash):
return Query(cls.table()).filter(filter_hash)
def find_first(cls, find_hash):
object_data = list(Query(cls.table()).filter(find_hash).limit(1).run())
return cls(object_data[0]) if object_data else None
def insert(cls, insert_hash, conflict="error"):
return Query(cls.table()).insert(insert_hash, conflict=conflict)
def index():
query = Query()
company = query.get_company()
query.close()
return render_template('home.html', data=enumerate(company))
def query(cls):
return Query(cls.table())
def uuid(cls, string):
return Query(r).uuid(string)
def update(self, update_hash):
return Query(self.table()).get(self.get_id()).update(update_hash)
def delete(self):
return Query(self.table()).get(self.get_id()).delete()
from csv_parser import parse_csv_data
from models import Query
users = parse_csv_data('data.csv')
over_september = Query().retention_query(users,
start_date="2014-09-01",
end_date="2014-09-30")
september8_10_android = Query().retention_query(users,
start_date="2014-09-08",
end_date="2014-09-10",
filter=True,
os_name="android")
# TODO: filter over_september query after first making...
over_september_ios_175 = Query().retention_query(users,
start_date="2014-09-01",
end_date="2014-09-30",
filter=True,
os_name="IOS",
sdk_version="1.7.5")
print "Day7 UI Retention over the month of September: %s" % (
over_september.retention_rate)
print "Day7 UI Retention from September 8 through September 10 for the Android SDK: %s" % (
september8_10_android.retention_rate)
print "Day7 UI Retention over the month of September for version 1.7.5 of the iOS SDK: %s" % (
over_september_ios_175.retention_rate)
from time import time
from man import *
from models import User, Params, Query
from loading import loading
start = time()
loading = loading()
len_name = set()
requests_count = 0
reset_time = datetime.now()
user = User()
if user.authorized:
params = Params()
params.validation()
query = Query(params)
print(
f"Start from: {params.start_date}\nTo: {params.end_date}\nbranch: {params.branch}\n"
)
else:
print(man_no_auth)
exit()
# convert string created time to <datetime> whithout 'Z'
def date_format(j):
return datetime.fromisoformat(j['created_at'][:-1:])
async def fetch(session, url, params={}):
global requests_count
def run_query_through_api(request, project_id):
user = request.user
# TODO fix this
user = User.objects.get(id=1)
project = Project.objects.get(id=project_id)
query_name = request.POST.get("query_name", "")
from_date = request.POST.get("datepicker_from", "")
to_date = request.POST.get("datepicker_to", "")
language = request.POST.get("lan", "")
query = Query(name=query_name,
venn=request.POST.get("query_logic", ""),
from_date=datetime.strptime(from_date, "%d/%m/%Y"),
to_date=datetime.strptime(to_date, "%d/%m/%Y"),
created=timezone.now(),
created_by=user,
owned_by=project)
query.save()
keywords = request.POST.get("keywords", "")
category = Category.objects.get(name="Keywords")
query_property = Query_properties(query=query,
category=category,
properties=keywords)
query_property.save()
twitter = request.POST.get("twitter", "")
category = Category.objects.get(name="Twitter")
query_property = Query_properties(query=query,
category=category,
properties=twitter)
query_property.save()
facebook = request.POST.get("facebook", "")
category = Category.objects.get(name="Facebook")
query_property = Query_properties(query=query,
category=category,
properties=facebook)
query_property.save()
brands = request.POST.get("brands", "")
try:
category = Category.objects.filter(name="brands")
except ValueError:
print ValueError.message
if category.__len__(): #exists already the category
category = category[0]
## otherwise create the category
else:
#print "is empty"
category = Category(name="brands")
category.save()
query_property = Query_properties(query=query,
category=category,
properties=brands)
query_property.save()
query_lan = Query_languages(query=query, language=language)
query_lan.save()
##handle dynamic properties
i = 0
prop_value = "prop-value-%s" % i
prop_name = "prop-name-%s" % i
while request.POST.get(prop_value, ""):
property_name = request.POST.get(prop_name, "")
property_value = request.POST.get(prop_value, "")
try:
## try to find if the category already exists - in lowercase
category = Category.objects.filter(
name=(str(property_name).lower()))
except ValueError:
#print ValueError.message
continue
if category.__len__(): #exists already the category
category = category[0]
## otherwise create the category
else:
category = Category(name=str(property_name).lower())
category.save()
## end store the properties in the category
query_property = Query_properties(query=query,
category=category,
properties=property_value)
query_property.save()
i += 1
prop_value = "prop-value-%s" % i
prop_name = "prop-name-%s" % i
return query.id
