def scrape_job_data(user): try: if request.method == 'POST': print(request) site = request.json["site"] job_type = request.json["type"] city = request.json["city"] country = request.json["country"] province = request.json["province"] technologies = request.json["technologies"] print(job_type, technologies) output = scrape(job_type, technologies , site) # Passing to the Scrape function if len(output["jobs"]) == 0: return jsonify({"type": "Error", "message": "No results found..."}),400 query = {} if user: query = Query(site=site, job_type=job_type,city=city, country=country, province=province, user_id=user.id) db.session.add(query) db.session.commit() save_to_db(query.id, output["jobs"]) output["query"] = query print("NUM JOBS ", len(output["jobs"])) return jsonify(output) except Exception as e: print("Error", e) return jsonify({"type": "Error", "message": "Error fetching job results, please try again"}),400
def register_request(query_dic, flag): image_local_path, hash = None, None if query_dic.get("NumMedia") == '1': image_local_path, hash = save_media(query_dic.get("MediaUrl0"), query_dic.get("MediaContentType0")) if query_dic.get("localPath") and query_dic.get("img"): image_local_path, hash = query_dic.get("localPath"), hash_local_file( query_dic.get("localPath")) query = Query( media_type=query_dic.get("MediaContentType0")) # e.g. image/jpeg query.sms_message_id = query_dic.get("SmsMessageSid") query.num_media = query_dic.get("NumMedia") query.sms_id = query_dic.get("SmsSid") query.sms_status = query_dic.get("SmsStatus") query.body = query_dic.get("Body") query.to_num = query_dic.get("To") query.num_sug = query_dic.get("NumSegments") query.message_id = query_dic.get("MessageSid") query.account_id = query_dic.get("AccountSid") query.from_num = query_dic.get("From") query.ip_address = query_dic.get("IP") query.image_url = query_dic.get("MediaUrl0") query.image_local_path = image_local_path query.image_hash = hash # hash this query.video_url = "Not Supported" #get binary in a different field later query.ip_address = query_dic.get("IP") query.query_type = flag query.save() return query
def process_response(self, request, response): path = request.get_full_path() user = None user_id = request.session.get('user_id', None) start_time = request.session.get('time') if start_time: run_time = time.time() - start_time else: run_time = -1 if user_id: del request.session['user_id'] user = Users.objects.get(pk=user_id) if not self.is_filter_path(path): data = { 'remote_address': request.META['REMOTE_ADDR'], 'request_method': request.method, 'request_path': path, 'request_body': request.body, 'response_status': response.status_code, 'user': user, 'run_time': run_time } Query(**data).save() return response
def create_query(count, approximate, selected, excluded_pks, image_data_url): query = Query() query.count = count if excluded_pks: query.excluded_index_entries_pk = [int(k) for k in excluded_pks] query.selected_indexers = selected query.approximate = approximate image_data = base64.decodestring(image_data_url[22:]) if settings.HEROKU_DEPLOY: query.image_data = image_data query.save() dv = Video() dv.name = 'query_{}'.format(query.pk) dv.dataset = True dv.query = True dv.parent_query = query dv.save() create_video_folders(dv) query_path = "{}/queries/{}.png".format(settings.MEDIA_ROOT, query.pk) query_frame_path = "{}/{}/frames/0.png".format(settings.MEDIA_ROOT, dv.pk) if settings.HEROKU_DEPLOY: s3 = boto3.resource('s3') s3.Bucket(settings.MEDIA_BUCKET).put_object(Key=query_path, Body=image_data) s3.Bucket(settings.MEDIA_BUCKET).put_object(Key=query_frame_path, Body=image_data) else: with open(query_path, 'w') as fh: fh.write(image_data) with open(query_frame_path, 'w') as fh: fh.write(image_data) return query, dv
def get_search_results(query_id, query_postings): # Check previously calculated queries for changes in the corpus query = Query.objects(id=query_id).only("results", "total_frequency").first() total_frequency = Entry.objects(id__in=query_postings.iterkeys()).only( "total_frequency").sum("total_frequency") if not query or total_frequency != query.total_frequency: results = [] avg_length = Meme.objects.only("length").aggregate_average("length") idf, relevant_docs = get_idf_relevant_docs(query_postings) for meme in relevant_docs: # Iterate through relevant documents to calculate its score bm25 = calculate_bm25(avg_length, idf, meme, query_postings) result = SearchResult(id=MemeId(source=meme.id.source, meme_id=meme.id.meme_id), name=meme.name, title=meme.title, caption=meme.caption, score=meme.score, url=meme.url, image=meme.image, bm25=bm25) results.append(result) results = sorted(results, key=lambda result: result.bm25, reverse=True)[:200] query = Query(id=query_id, results=results, total_frequency=total_frequency) query.save() return query.results
def process_query(self, entity, attribute, verbose=False): ''' Process the user query and return the aggregated sentiment and related entries. :param entity: str :param attribute: str :param verbose: bool :rtype: (float, List[AttributeEntry]) ''' query = Query(entity, attribute) if verbose: print("Query parsed.") relevant_entries = self.data_source.lookup(query) count = len(relevant_entries) if verbose: print("{} relevant entr{} found.".format( count, "y" if count == 1 else "ies")) if count == 0: return None, [] aggregateData = [{ 'sentiment': expr.sentiment, 'is_header': expr.is_header } for entry in relevant_entries for expr in entry.expressions] score = self.aggregator_service.aggregate_sentiment(aggregateData) if verbose: print("Sentiment scores aggregated.") return score, relevant_entries
def get_all_queries(self) -> list: data = self.__query_sheets('Queries!A:ZZ') result = list() for entry in data: try: result.append(Query(keyword=entry[0], maximum_price=int(entry[1]), ignore=entry[2:])) except: logging.error("Failed to process row {}".format(entry)) return result
def read_config(): global db if settings.CONFIG_FILE: logger.info("reading config file: " + settings.CONFIG_FILE) config = json.loads(get_file_or_s3(settings.CONFIG_FILE)) for query_config in config["queries"]: query = Query(json=query_config) db.save_query(query) logger.debug("saved query " + query.id + ": " + query.search)
def sendMessage(): form = FeedbackForm() if form.validate_on_submit(): new = Query(content=form.content.data, name=current_user.username, patient_id=current_user.id, doctor_id=current_user.mydoctor) db.session.add(new) db.session.commit() flash('Message sent', 'message') return redirect(url_for('home')) return render_template('sendMessage.html', form=form)
def query(request): ConditionSet = formset_factory(ConditionForm, extra=5) if request.method == 'POST': selectionset = ConditionSet(request.POST,request.FILES) if selectionset.is_valid(): query=Query(qid=makeQID(),query=constructQuery(selectionset.cleaned_data)) query.save() return HttpResponseRedirect('/portal/results/%s/'%query.qid) else: selectionset = ConditionSet(initial=[ {'lower': u'5000', 'upper': u'5050', 'parameter':'RadTransWavelengthExperimentalValue', 'connection':True, }, ]) return render_to_response('portal/query.html', {'selectionset': selectionset})
def load_model(city_code): # check database, if not found make api call print("loding model ***********************************") model = Query.query.filter_by(city_code=city_code).first() if not model: print("api call ***********************************") dataset = Dataset(city_code) model = PricePredictionModel(dataset) model = Query(city_code=model.city_code, model=model) db.session.add(model) db.session.commit() print("model loaded ***********************************") return model
def search(input: str, size: int = 100) -> Query: """Adds search query into a queue for importing and processing """ # TODO - Take into account the rate limit, the importer will fail and need adding back to the queue if hit # TODO - multi depends_on maybe coming soon. It would allow a end of processing notification to exist query = Query() query.save(input=input, size=size, stream='twitter') source_queue = Queue(name='source') process_queue = Queue(name='process') # Import tweets import_job = source_queue.enqueue(import_raw, query=query) # Process tweets process_queue.enqueue(aggregate_pixels, query=query, depends_on=import_job) process_queue.enqueue(aggregate_adjectives, query=query, depends_on=import_job) return query
def receive_message(): user_text_message = request.values.get('Text') phone_number = '+' + request.values.get('From') wifi_request = 'Wifi' in request.values user = get_user(phone_number) user_query = Query() user_query.save() tokenizer = Tokenizer(user_text_message) if tokenizer.partial_text: return "" arguments = tokenizer.arguments_dict if 'c' in arguments: country_tag = arguments["c"] else: country_tag = 'US' if 'v' in arguments: version_number = int(arguments["v"]) else: version_number = 10000 should_rotate = False if tokenizer.api != "onboard" and version_number > 6: should_rotate = True results = process_message(user, user_text_message, tokenizer) key = results.get("key", "") # if user.is_over_limit(): # user_text_message = "limit default: key: %s" % key[1:] # results = process_message(user, user_text_message, user_query) # key = results.get("key" "") messages_list = results.get("messages") user_query.response = messages_list[0] if len( messages_list) else "Feedback" user_query.save() user.queries.append(user_query) user.save() if not wifi_request: distribute(str(phone_number), messages_list, key, country_tag, should_rotate) return "" return jsonify(results=prepend_key(messages_list, key))
def single_review(request, review_name_slug): context = {} try: review = Review.objects.get(user=request.user, slug=review_name_slug) context['review_title'] = review.title queries = Query.objects.filter(review=review) paper = Paper.objects.filter(review=review) context['queries'] = queries context['review'] = review context['paper'] = paper if request.method == "POST": if request.POST.get('delete_query', "") != "": query_to_delete = Query.objects.get( name=request.POST.get('delete_query')) if query_to_delete != None: query_to_delete.delete() context[ 'alert_message'] = "Query deleted: " + query_to_delete.name else: #making sure an empty query will not be saved, removing spaces on both sides of the name new_query = request.POST.get('queryField', default="") new_query = new_query.strip() if new_query != "" and not queries.filter( name=new_query).exists(): query = Query(review=review, name=new_query) query.save() context[ 'alert_message'] = "Query saved: " + request.POST.get( 'queryField') else: context[ 'alert_message'] = "You are either saving an empty query or a query with this name alreday exists." queries = Query.objects.filter(review=review) context['queries'] = queries context['review'] = review except Review.DoesNotExist: pass return render(request, 'ultimatereview/querybuilder.html', context)
def lifecycle(): global db queries = db.get_all_queries() for query_raw in queries: query = Query(json=query_raw) logger.debug(f'checking query {query.id}') logger.debug("performing search...") results = search(query.search, num_results=10) logger.info(f"query {query.id}: hits: {len(results)}") new_items = False for result_raw in results: #logger.debug(f'result_raw: {result_raw[0]}\t {result_raw[1]}') result = Result(id=str(uuid.uuid4()), query=query.id, timestamp=str(get_now_as_timestamp()), title=result_raw[0], content=result_raw[1]) logger.debug( "result: " + json.dumps(result.as_dict(), indent=4, sort_keys=True)) if not db.result_exists(result): logger.debug("result has not previously been recorded") new_items = True db.save_result(result) if new_items: feed_string = generate_feed_for_query(query) write_to_file_or_s3(uri=settings.RESULTS_FOLDER, query=query, body=feed_string)
def spell_check(): if not 'nonce' in session or not 'token' in session: return redirect(url_for('login')) ses = session['token'] + session['nonce'] l = Login.query.filter_by(session_key=ses).filter_by(active=True).first() if not l: return redirect(url_for('login')) csrf_token = hashlib.sha256(session['nonce'].encode('ascii')).hexdigest() if request.method == "GET": return render_template("spell_check.html", csrf_token=csrf_token) if request.form['csrf'] != csrf_token: return redirect(url_for('login')) tmpfname = ''.join(random.choices(string.ascii_letters, k=40)) inputtext = request.form['inputtext'] with open(tmpfname.encode('ascii'), 'w') as tmp: tmp.write(inputtext) cmd = "./a.out {} wordlist.txt".format(tmpfname) res = subprocess.check_output(shlex.split(cmd)).decode('ascii').split('\n') res = ','.join(res) os.remove(tmpfname) q = Query(l.user, inputtext, res) db.session.add(q) db.session.commit() db.session.refresh(q) return render_template('spell_check.html', inwords=request.form['inputtext'], misspelled=res, csrf_token=csrf_token)
def create_query(count, approximate, selected, excluded_pks, image_data_url): query = Query() query.count = count if excluded_pks: query.excluded_index_entries_pk = [int(k) for k in excluded_pks] query.selected_indexers = selected query.approximate = approximate query.save() dv = Video() dv.name = 'query_{}'.format(query.pk) dv.dataset = True dv.query = True dv.parent_query = query dv.save() create_video_folders(dv) image_data = base64.decodestring(image_data_url[22:]) query_path = "{}/queries/{}.png".format(settings.MEDIA_ROOT, query.pk) query_frame_path = "{}/{}/frames/0.png".format(settings.MEDIA_ROOT, dv.pk) with open(query_path, 'w') as fh: fh.write(image_data) with open(query_frame_path, 'w') as fh: fh.write(image_data) return query, dv
def _save_query(chat_id, query): query_doc = Query(query=query) query_doc.save() user_doc = User.objects.get(chat_id=chat_id) user_doc.queries.append(query_doc) user_doc.save()
def get_all(cls, *object_ids): return Query(cls.table()).get_all(*object_ids)
def filter(cls, filter_hash): return Query(cls.table()).filter(filter_hash)
def find_first(cls, find_hash): object_data = list(Query(cls.table()).filter(find_hash).limit(1).run()) return cls(object_data[0]) if object_data else None
def insert(cls, insert_hash, conflict="error"): return Query(cls.table()).insert(insert_hash, conflict=conflict)
def index(): query = Query() company = query.get_company() query.close() return render_template('home.html', data=enumerate(company))
def query(cls): return Query(cls.table())
def uuid(cls, string): return Query(r).uuid(string)
def update(self, update_hash): return Query(self.table()).get(self.get_id()).update(update_hash)
def delete(self): return Query(self.table()).get(self.get_id()).delete()
from csv_parser import parse_csv_data from models import Query users = parse_csv_data('data.csv') over_september = Query().retention_query(users, start_date="2014-09-01", end_date="2014-09-30") september8_10_android = Query().retention_query(users, start_date="2014-09-08", end_date="2014-09-10", filter=True, os_name="android") # TODO: filter over_september query after first making... over_september_ios_175 = Query().retention_query(users, start_date="2014-09-01", end_date="2014-09-30", filter=True, os_name="IOS", sdk_version="1.7.5") print "Day7 UI Retention over the month of September: %s" % ( over_september.retention_rate) print "Day7 UI Retention from September 8 through September 10 for the Android SDK: %s" % ( september8_10_android.retention_rate) print "Day7 UI Retention over the month of September for version 1.7.5 of the iOS SDK: %s" % ( over_september_ios_175.retention_rate)
from time import time from man import * from models import User, Params, Query from loading import loading start = time() loading = loading() len_name = set() requests_count = 0 reset_time = datetime.now() user = User() if user.authorized: params = Params() params.validation() query = Query(params) print( f"Start from: {params.start_date}\nTo: {params.end_date}\nbranch: {params.branch}\n" ) else: print(man_no_auth) exit() # convert string created time to <datetime> whithout 'Z' def date_format(j): return datetime.fromisoformat(j['created_at'][:-1:]) async def fetch(session, url, params={}): global requests_count
def run_query_through_api(request, project_id): user = request.user # TODO fix this user = User.objects.get(id=1) project = Project.objects.get(id=project_id) query_name = request.POST.get("query_name", "") from_date = request.POST.get("datepicker_from", "") to_date = request.POST.get("datepicker_to", "") language = request.POST.get("lan", "") query = Query(name=query_name, venn=request.POST.get("query_logic", ""), from_date=datetime.strptime(from_date, "%d/%m/%Y"), to_date=datetime.strptime(to_date, "%d/%m/%Y"), created=timezone.now(), created_by=user, owned_by=project) query.save() keywords = request.POST.get("keywords", "") category = Category.objects.get(name="Keywords") query_property = Query_properties(query=query, category=category, properties=keywords) query_property.save() twitter = request.POST.get("twitter", "") category = Category.objects.get(name="Twitter") query_property = Query_properties(query=query, category=category, properties=twitter) query_property.save() facebook = request.POST.get("facebook", "") category = Category.objects.get(name="Facebook") query_property = Query_properties(query=query, category=category, properties=facebook) query_property.save() brands = request.POST.get("brands", "") try: category = Category.objects.filter(name="brands") except ValueError: print ValueError.message if category.__len__(): #exists already the category category = category[0] ## otherwise create the category else: #print "is empty" category = Category(name="brands") category.save() query_property = Query_properties(query=query, category=category, properties=brands) query_property.save() query_lan = Query_languages(query=query, language=language) query_lan.save() ##handle dynamic properties i = 0 prop_value = "prop-value-%s" % i prop_name = "prop-name-%s" % i while request.POST.get(prop_value, ""): property_name = request.POST.get(prop_name, "") property_value = request.POST.get(prop_value, "") try: ## try to find if the category already exists - in lowercase category = Category.objects.filter( name=(str(property_name).lower())) except ValueError: #print ValueError.message continue if category.__len__(): #exists already the category category = category[0] ## otherwise create the category else: category = Category(name=str(property_name).lower()) category.save() ## end store the properties in the category query_property = Query_properties(query=query, category=category, properties=property_value) query_property.save() i += 1 prop_value = "prop-value-%s" % i prop_name = "prop-name-%s" % i return query.id