def parse_news(self, response): try: body_news = response.css(response.meta.get('body')) text = w3lib.html.replace_entities( w3lib.html.replace_escape_chars( w3lib.html.remove_tags(' '.join(body_news.extract())))) text = re.compile(r'\W+', re.UNICODE).split(text.lower()) text = ' '.join(text) news = response.meta.get('news') news.content = text #db.session.add(news) words = frequency.get_frequency([text, response.meta.get('title')]) tags = words.pop(0).to_dict() for stock in words: for key, value in stock.to_dict().items(): if key in tags: tags[key] += value else: tags[key] = value for tag, value in tags.items(): try: tag = models.Tag.query.filter_by(name=tag).one() except Exception as e: tag = models.Tag(name=tag) #db.session.add(tag) if value > 0: db.session.add( models.NewsTag(news=news, tag=tag, rank=value)) db.session.commit() yield { 'title': response.meta.get('title'), 'href': response.meta.get('href'), } except Exception as e: print(e)
def _row_to_tag(row: Optional[Any]) -> Optional[models.Tag]: if row is None: return row return models.Tag(db_id=row[0], id=row[1], name=row[2], created_at=datetime.fromtimestamp(row[3]))
def post(self): ''' method to add a tag ''' args = self.reqparse.parse_args() tag = models.Tag(args["name"]) db.session.add(tag) db.session.commit() return {"tag": marshal(tag, TAG_FIELDS)}, 201
def new_tag(): form = TagForm() if form.validate_on_submit(): flash('Tag with name %r created successfully!' % (form.name.data)) new_tag = models.Tag(name=form.name.data) db.session.add(new_tag) db.session.commit() return redirect('/admin/new_tag') return render_template('new_tag.html', title="New Tag - Admin", form=form, user=current_user)
def api_upload_file(token): try: user = m.User.query.filter_by(token=token).one() except NoResultFound: return jsonify(exceptions=["API Token is missing or invalid"], status=404) if request.method == "POST": traceFile = request.files["file"] filename = traceFile.filename filetype = os.path.splitext(filename)[1].strip(".") uuid_filename = ".".join([str(uuid.uuid4()), filetype]) traceFile.save(os.path.join(pcap.upload_folder, uuid_filename)) else: filename = request.args.get("filename") filetype = os.path.splitext(filename)[1].strip(".") uuid_filename = f"{uuid.uuid4()}.{filetype}" with open(os.path.join(pcap.upload_folder, uuid_filename), "w") as f: f.write(request.stream.read()) if not allowed_file(filename): os.remove(os.path.join(pcap.upload_folder, uuid_filename)) return jsonify( exceptions=["Not a valid file type. (pcap, pcapng, cap)"], status=406) new_file = m.TraceFile( id=str(uuid.uuid4())[:8], name=secure_filename(os.path.splitext(filename)[0]), user_id=user.id, filename=uuid_filename, filetype=filetype, filesize=os.path.getsize( os.path.join(pcap.upload_folder, uuid_filename)), packet_count=pcap.get_capture_count(uuid_filename), date_added=datetime.datetime.now(), ) db.session.add(new_file) db.session.commit() db.session.refresh(new_file) # add tags if request.form.getlist("additional_tags"): for tag in request.form.getlist("additional_tags")[0].split(","): if tag.strip(",") != "": new_tag = m.Tag(name=tag.strip(","), file_id=new_file.id) db.session.add(new_tag) db.session.commit() log("info", f"File uploaded by {user.username!r}: {filename!r}.") return jsonify(filename=filename, id=new_file.id, status=202)
def fill_tags(self, cnt): objs = list() for i in range(cnt): objs.append(models.Tag( title=f.sentence()[:128], count=f.random_int(min=0, max=10) )) if i % 1e7 == 0 and i != 0: models.Tag.objects.bulk_create(objs=objs) objs.clear() models.Tag.objects.bulk_create(objs=objs)
def add_tag(file_id: str): tag_name = request.data.decode("utf-8") trace_file = m.TraceFile.query.get_or_404(file_id) if tag_name in {t.name for t in trace_file.tags}: return "Tag exists" new_tag = m.Tag(name=tag_name.lower()) trace_file.tags.append(new_tag) db.session.commit() return f"Tag {tag_name!r} has been added."
def create_tag(user_id: int, tag: schemas.TagCreate, db: Session) -> schemas.Tag: db_tag = (db.query( models.Tag).filter(models.Tag.user_id == user_id).filter( models.Tag.name == tag.name).first()) if db_tag: raise HTTPException(status_code=400, detail=f"Tag with name {tag.name} already exists") tag_dict = tag.dict() tag_dict["user_id"] = user_id db_tag = models.Tag(**tag_dict) db.add(db_tag) db.commit() db.refresh(db_tag) return db_tag
def tag_add(): form = TagForm() if form.validate_on_submit(): data = form.data tag = app_models.Tag.query.filter_by(name=data['name']).count() if tag == 1: flash('名称已经存在', 'err') return redirect(url_for('admin.tag_add')) tag = app_models.Tag(name=data['name']) db.session.add(tag) flash('添加标签成功!', 'ok') oplog = app_models.Oplog(admin_id=session['admin_id'], ip=request.remote_addr, reason='添加标签 - [%s]' % data['name']) db.session.add(oplog) db.session.commit() redirect(url_for('admin.tag_add')) return render_template('admin/tag_add.html', form=form)
def make_mock_picture(user=None, tags=None, despriction=None, address=None): _picture = models.Picture(userId=str(user.id), despriction=despriction or 'testdes', address=address or 'testaddress') _picture.tags = [models.Tag(tag=tags or 'testtags')] db.session.add(_picture) db.session.commit() # _tags = models.Tags( # picId=_picture.id, # tag='testtag' # ) # db.session.add(_tags) # db.session.commit() return _picture
for title in forms: form = models.FeedbackForm(title=title, is_active=True, is_published=True) form.save() print('%s created' % title) for i in range(0, len(questions), 3): question = questions[i].strip() question_type = models.QuestionType.objects.get( title=questions[i + 1].strip()) tag = questions[i + 2].strip() cur_tag = None try: cur_tag = models.Tag.objects.get(tag_title=tag) except: if cur_tag == None and tag != "None": cur_tag = models.Tag(tag_title=tag) cur_tag.save() models.Question(text=question, type=question_type, tag=cur_tag, feedback_form=form).save() print('question created:', question, cur_tag) all_questions = models.Question.objects.filter(feedback_form=form) question_dict = {} for a_q in all_questions: if not a_q.type.title in question_dict: question_dict[a_q.type.title] = [] question_dict[a_q.type.title].append(a_q) for student in models.Student.objects.all():
def gen_tags(): print("generating tags") for tag in TAGS: models.Tag(tag_title=tag).save()
datetime.datetime.strptime("20151003050936", "%Y%m%d%H%M%S"), user_1.id), models.Post('Ninth Post', POSTS_CONTENT[0], datetime.datetime.strptime("20151002050936", "%Y%m%d%H%M%S"), user_1.id), models.Post('Tenth Post', POSTS_CONTENT[0], datetime.datetime.strptime("20151001050936", "%Y%m%d%H%M%S"), user_1.id), ] for post in posts: db.session.add(post) db.session.commit() tags = [ models.Tag('tag_a'), models.Tag('tag_b'), models.Tag('tag_c'), models.Tag('d'), models.Tag('e'), models.Tag('f'), models.Tag('g'), models.Tag('h'), models.Tag('i'), models.Tag('j'), models.Tag('k'), models.Tag('l'), models.Tag('m'), models.Tag('n'), models.Tag('o'), models.Tag('p'),
def _create_tag(self, name='Default'): tag = models.Tag(name) db.session.add(tag) db.session.commit() return models.Tag.query.filter(models.Tag.name == name).first()
# # obj.tags.append(tag_in_db) # # db.session.add(obj) # prod_data = pandas.read_csv("data/objects_elo7.txt") for idx, row in prod_data.iterrows(): obj = models.Object(row["oid"], row["title"], row["description"], row["img"], 0, "product") for string in row["tags"].split(","): if string in tag_map: tag_in_db = tag_map[string] else: tag_in_db = models.Tag(string) tag_map[string] = tag_in_db obj.tags.append(tag_in_db) db.session.add(obj) db.session.commit() objs = models.Object.query.all() for obj in objs: print(obj), for tag in obj.tags: print(tag), print
def get_json(): # payload = "token=nB2bAolyYp4uDT0hoqUyfVyv&team_id=T02V23TD2&team_domain=optimizely&channel_id=CLF247QJK&channel_name=hackweek2019-seeker&user_id=UGBCTP0AZ&user_name=trent.robbins&command=%2Fseeker-payload&text=&response_url=https%3A%2F%2Fhooks.slack.com%2Fcommands%2FT02V23TD2%2F690346166849%2FqeJF824OcWVD8tuLhmvwlYvt&trigger_id=696685010704.2988129444.7f616d9c52435a42645e726f26a42269" # resp wrapped in dict because it's sometimes an array # we should catch if there is no text in the command e.g. "/command <text>" where <text> is empty # because that is the case where it could just be a dict in a dict #response = make_response(jsonify(resp),200) response_payload = {} parsed_payload = text_parser.parse(request.form['text']) # repeat for each command, we can fix the structure later # testing shim # parsed_payload = {'command': 'help'} # /seeker, /seeker help if parsed_payload['command'] == 'help' or parsed_payload['command'] == None: help_json_template = json_templates.seeker_help() response_payload = jsonify(help_json_template) # /seeker tags elif parsed_payload['command'] == 'tags': tag_list = models.Tag.query.distinct(models.Tag.name).all() arr = [] for tag in tag_list: arr.append('`' + tag.name + '`') tag_json_template = json_templates.seeker_tags(arr) response_payload = jsonify(tag_json_template) # tag an existing message elif parsed_payload['command'] == 'tag': response_payload = 'undefined error in tag' tokens = parsed_payload['payload'] if len(tokens) != 2: response_payload = 'Syntax for tagging and untagging is /seeker tag <tag> <message_id>' else: try: message_id = int(tokens[0]) app.logger.info( "user provided message_id: {}".format(message_id)) except ValueError: response_payload = 'The message id was not an integer.' return jsonify({'message': response_payload}) # get the tag, create tag if it doesn't exist try: # return first or None object if none tag = db.session.query( models.Tag).filter(models.Tag.name == tokens[1]).first() if not tag: tag = models.Tag() tag.name = tokens[1] db.session.add(tag) db.session.commit() except Exception as e: response_payload = 'New tag creation failed {}'.format(e) try: # TODO: this is somehow getting an empty message or something message = db.session.query(models.SlackMessage).get(message_id) if not message: raise except Exception as e: response_payload = 'Message id was not found in the seeker database, try a seeker save on the message URL first. {}'.format( e) try: for mtag in message.tags: if mtag.name == tag.name: response_payload = 'Tag already found on message.' else: message.tags.append(tag) db.session.commit() response_payload = 'Tag {} was added to message {}.'.format( tag.name, message_id) except Exception as e: response_payload = 'Some unidentified issue in tagging {}'.format( e) # package it into a response response_payload = jsonify({'message': response_payload}) # untag an existing message elif parsed_payload['command'] == 'untag': response_payload = 'undefined error in untag' tokens = parsed_payload['payload'] flag_tag_found = False # set to true if the tag is found and excluded (aka removed) if len(tokens) != 2: response_payload = 'Syntax for tagging and untagging is /seeker tag <tag> <message_id>' else: try: message_id = int(tokens[0]) app.logger.info( "user provided message_id: {}".format(message_id)) except ValueError: response_payload = 'The message id was not an integer.' return jsonify({'message': response_payload}) try: message = db.session.query(models.SlackMessage).get(message_id) if not message: raise except Exception as e: response_payload = 'Message id was not found in the seeker database, try a seeker save on the message URL first. {}'.format( e) try: tag = db.session.query( models.Tag).filter(models.Tag.name == tokens[1]).first() new_message_tags = list() for tag in message.tags: if tag.name != tokens[1]: new_message_tags.append(tag) else: flag_tag_found = True message.tags = new_message_tags db.session.commit() if flag_tag_found: response_payload = 'Message tag was removed from the message successfully.' else: response_payload = 'Message tag was not found on that message.' except Exception as e: response_payload = 'Message id was not found in the seeker database, try a seeker save on the message URL first. {}'.format( e) # package it into a response response_payload = jsonify({'message': response_payload}) elif parsed_payload['command'] == 'show': tokens = parsed_payload['payload'] if len(tokens) == 0: messages = helper.get_all_messages() json_template = json_templates.seeker_show_no_tag(messages) response_payload = jsonify(json_template) return response_payload tag = tokens[0] message_urls = helper.get_all_messages_by_tag(tag) app.logger.info(message_urls) if len(message_urls) == 0: return jsonify( {"message": "No message urls found with the given tag"}) show_json_template = json_templates.seeker_show(tag, message_urls) response_payload = jsonify(show_json_template) # response_payload = jsonify({"list of message urls":message_urls}) elif parsed_payload['command'] in [ 'share', 'save', 'add', 'new', 'create' ]: tokens = parsed_payload['payload'] app.logger.info(tokens) message_Url = tokens["message_URL"] message_arr = message_Url.split('/') channel_id = message_arr[4] message = message_arr[5] post_message_id = message[-6:] post_message_id_start_index = message.find(post_message_id) pre_message_id = message[1:post_message_id_start_index] params = { 'token': os.getenv('SLACK_TOKEN'), 'channel': channel_id, 'oldest': pre_message_id + '.' + post_message_id, 'limit': 1, 'inclusive': True, } slack_message_resp = requests.get( "https://slack.com/api/conversations.history", params).json() tags = tokens["tags"] description = tokens["description"] message_text = slack_message_resp['messages'][0]['text'] annotator = request.form['user_name'] # author = 'test' db_message = models.SlackMessage.query.filter_by( url=message_Url).first() app.logger.info(db_message) save_json_template = {} if not db_message is None: save_json_template = json_templates.seeker_already_save(db_message) else: save_json_template = json_templates.seeker_save( message_Url, tags, description) helper.saveMessage(url=message_Url, description=description, message_text=message_text, tags=tags, annotator=annotator) response_payload = jsonify(save_json_template) elif parsed_payload['command'] == 'search': terms = parsed_payload['payload'] terms[0] = terms[0].strip("\"").strip("\u201c").strip("\u201d") terms[-1] = terms[-1].strip("\"").strip("\u201c").strip("\u201d") message_q = Q.PriorityQueue() if "|" in terms: tags = [] tokens = [] start_index = -1 for i in range(len(terms)): if terms[i] == "|" and start_index == -1: start_index = i if start_index == -1: tokens.append(terms[i]) elif i > start_index: tags.append(terms[i]) message_q = helper.searchMessageByTags(tokens, tags) else: message_q = helper.searchMessage(terms) search_json_template = json_templates.seeker_search(terms, message_q) response_payload = jsonify(search_json_template) # if command not recognized else: unrecognized_json_template = json_templates.seeker_unrecognized() response_payload = jsonify(unrecognized_json_template) response_payload.headers['Access-Control-Allow-Origin'] = '*' return response_payload
pool -= set2 set3 = pool # 3rd random subset for u in set1: u.team = 0 db.session.add(u) db.session.commit() for u in set2: u.team = 1 db.session.add(u) db.session.commit() for u in set3: u.team = 2 db.session.add(u) db.session.commit() # Clear all tags tags = models.Tag.query.all() for t in tags: db.session.delete(t) db.session.commit() # Make new tags for u in users: u2 = [x for x in users if x.id != u.id and x.team != u.team ] # List of users excluding the tagging user and same team targets = random.sample(u2, 3) for target in targets: t = models.Tag(source=u.id, destination=target.id) db.session.add(t) db.session.commit()