def parse_news(self, response):
try:
body_news = response.css(response.meta.get('body'))
text = w3lib.html.replace_entities(
w3lib.html.replace_escape_chars(
w3lib.html.remove_tags(' '.join(body_news.extract()))))
text = re.compile(r'\W+', re.UNICODE).split(text.lower())
text = ' '.join(text)
news = response.meta.get('news')
news.content = text
#db.session.add(news)
words = frequency.get_frequency([text, response.meta.get('title')])
tags = words.pop(0).to_dict()
for stock in words:
for key, value in stock.to_dict().items():
if key in tags:
tags[key] += value
else:
tags[key] = value
for tag, value in tags.items():
try:
tag = models.Tag.query.filter_by(name=tag).one()
except Exception as e:
tag = models.Tag(name=tag)
#db.session.add(tag)
if value > 0:
db.session.add(
models.NewsTag(news=news, tag=tag, rank=value))
db.session.commit()
yield {
'title': response.meta.get('title'),
'href': response.meta.get('href'),
}
except Exception as e:
print(e)
def _row_to_tag(row: Optional[Any]) -> Optional[models.Tag]:
if row is None:
return row
return models.Tag(db_id=row[0],
id=row[1],
name=row[2],
created_at=datetime.fromtimestamp(row[3]))
def post(self):
'''
method to add a tag
'''
args = self.reqparse.parse_args()
tag = models.Tag(args["name"])
db.session.add(tag)
db.session.commit()
return {"tag": marshal(tag, TAG_FIELDS)}, 201
def new_tag():
form = TagForm()
if form.validate_on_submit():
flash('Tag with name %r created successfully!' % (form.name.data))
new_tag = models.Tag(name=form.name.data)
db.session.add(new_tag)
db.session.commit()
return redirect('/admin/new_tag')
return render_template('new_tag.html', title="New Tag - Admin", form=form, user=current_user)
def api_upload_file(token):
try:
user = m.User.query.filter_by(token=token).one()
except NoResultFound:
return jsonify(exceptions=["API Token is missing or invalid"],
status=404)
if request.method == "POST":
traceFile = request.files["file"]
filename = traceFile.filename
filetype = os.path.splitext(filename)[1].strip(".")
uuid_filename = ".".join([str(uuid.uuid4()), filetype])
traceFile.save(os.path.join(pcap.upload_folder, uuid_filename))
else:
filename = request.args.get("filename")
filetype = os.path.splitext(filename)[1].strip(".")
uuid_filename = f"{uuid.uuid4()}.{filetype}"
with open(os.path.join(pcap.upload_folder, uuid_filename), "w") as f:
f.write(request.stream.read())
if not allowed_file(filename):
os.remove(os.path.join(pcap.upload_folder, uuid_filename))
return jsonify(
exceptions=["Not a valid file type. (pcap, pcapng, cap)"],
status=406)
new_file = m.TraceFile(
id=str(uuid.uuid4())[:8],
name=secure_filename(os.path.splitext(filename)[0]),
user_id=user.id,
filename=uuid_filename,
filetype=filetype,
filesize=os.path.getsize(
os.path.join(pcap.upload_folder, uuid_filename)),
packet_count=pcap.get_capture_count(uuid_filename),
date_added=datetime.datetime.now(),
)
db.session.add(new_file)
db.session.commit()
db.session.refresh(new_file)
# add tags
if request.form.getlist("additional_tags"):
for tag in request.form.getlist("additional_tags")[0].split(","):
if tag.strip(",") != "":
new_tag = m.Tag(name=tag.strip(","), file_id=new_file.id)
db.session.add(new_tag)
db.session.commit()
log("info", f"File uploaded by {user.username!r}: {filename!r}.")
return jsonify(filename=filename, id=new_file.id, status=202)
def fill_tags(self, cnt):
objs = list()
for i in range(cnt):
objs.append(models.Tag(
title=f.sentence()[:128],
count=f.random_int(min=0, max=10)
))
if i % 1e7 == 0 and i != 0:
models.Tag.objects.bulk_create(objs=objs)
objs.clear()
models.Tag.objects.bulk_create(objs=objs)
def add_tag(file_id: str):
tag_name = request.data.decode("utf-8")
trace_file = m.TraceFile.query.get_or_404(file_id)
if tag_name in {t.name for t in trace_file.tags}:
return "Tag exists"
new_tag = m.Tag(name=tag_name.lower())
trace_file.tags.append(new_tag)
db.session.commit()
return f"Tag {tag_name!r} has been added."
def create_tag(user_id: int, tag: schemas.TagCreate,
db: Session) -> schemas.Tag:
db_tag = (db.query(
models.Tag).filter(models.Tag.user_id == user_id).filter(
models.Tag.name == tag.name).first())
if db_tag:
raise HTTPException(status_code=400,
detail=f"Tag with name {tag.name} already exists")
tag_dict = tag.dict()
tag_dict["user_id"] = user_id
db_tag = models.Tag(**tag_dict)
db.add(db_tag)
db.commit()
db.refresh(db_tag)
return db_tag
def tag_add():
form = TagForm()
if form.validate_on_submit():
data = form.data
tag = app_models.Tag.query.filter_by(name=data['name']).count()
if tag == 1:
flash('名称已经存在', 'err')
return redirect(url_for('admin.tag_add'))
tag = app_models.Tag(name=data['name'])
db.session.add(tag)
flash('添加标签成功！', 'ok')
oplog = app_models.Oplog(admin_id=session['admin_id'],
ip=request.remote_addr,
reason='添加标签 - [%s]' % data['name'])
db.session.add(oplog)
db.session.commit()
redirect(url_for('admin.tag_add'))
return render_template('admin/tag_add.html', form=form)
def make_mock_picture(user=None,
tags=None,
despriction=None,
address=None):
_picture = models.Picture(userId=str(user.id),
despriction=despriction or 'testdes',
address=address or 'testaddress')
_picture.tags = [models.Tag(tag=tags or 'testtags')]
db.session.add(_picture)
db.session.commit()
# _tags = models.Tags(
# picId=_picture.id,
# tag='testtag'
# )
# db.session.add(_tags)
# db.session.commit()
return _picture
for title in forms:
form = models.FeedbackForm(title=title, is_active=True, is_published=True)
form.save()
print('%s created' % title)
for i in range(0, len(questions), 3):
question = questions[i].strip()
question_type = models.QuestionType.objects.get(
title=questions[i + 1].strip())
tag = questions[i + 2].strip()
cur_tag = None
try:
cur_tag = models.Tag.objects.get(tag_title=tag)
except:
if cur_tag == None and tag != "None":
cur_tag = models.Tag(tag_title=tag)
cur_tag.save()
models.Question(text=question,
type=question_type,
tag=cur_tag,
feedback_form=form).save()
print('question created:', question, cur_tag)
all_questions = models.Question.objects.filter(feedback_form=form)
question_dict = {}
for a_q in all_questions:
if not a_q.type.title in question_dict:
question_dict[a_q.type.title] = []
question_dict[a_q.type.title].append(a_q)
for student in models.Student.objects.all():
def gen_tags():
print("generating tags")
for tag in TAGS:
models.Tag(tag_title=tag).save()
datetime.datetime.strptime("20151003050936", "%Y%m%d%H%M%S"),
user_1.id),
models.Post('Ninth Post', POSTS_CONTENT[0],
datetime.datetime.strptime("20151002050936", "%Y%m%d%H%M%S"),
user_1.id),
models.Post('Tenth Post', POSTS_CONTENT[0],
datetime.datetime.strptime("20151001050936", "%Y%m%d%H%M%S"),
user_1.id),
]
for post in posts:
db.session.add(post)
db.session.commit()
tags = [
models.Tag('tag_a'),
models.Tag('tag_b'),
models.Tag('tag_c'),
models.Tag('d'),
models.Tag('e'),
models.Tag('f'),
models.Tag('g'),
models.Tag('h'),
models.Tag('i'),
models.Tag('j'),
models.Tag('k'),
models.Tag('l'),
models.Tag('m'),
models.Tag('n'),
models.Tag('o'),
models.Tag('p'),
def _create_tag(self, name='Default'):
tag = models.Tag(name)
db.session.add(tag)
db.session.commit()
return models.Tag.query.filter(models.Tag.name == name).first()
#
# obj.tags.append(tag_in_db)
#
# db.session.add(obj)
#
prod_data = pandas.read_csv("data/objects_elo7.txt")
for idx, row in prod_data.iterrows():
obj = models.Object(row["oid"], row["title"], row["description"],
row["img"], 0, "product")
for string in row["tags"].split(","):
if string in tag_map:
tag_in_db = tag_map[string]
else:
tag_in_db = models.Tag(string)
tag_map[string] = tag_in_db
obj.tags.append(tag_in_db)
db.session.add(obj)
db.session.commit()
objs = models.Object.query.all()
for obj in objs:
print(obj),
for tag in obj.tags:
print(tag),
print
def get_json():
# payload = "token=nB2bAolyYp4uDT0hoqUyfVyv&team_id=T02V23TD2&team_domain=optimizely&channel_id=CLF247QJK&channel_name=hackweek2019-seeker&user_id=UGBCTP0AZ&user_name=trent.robbins&command=%2Fseeker-payload&text=&response_url=https%3A%2F%2Fhooks.slack.com%2Fcommands%2FT02V23TD2%2F690346166849%2FqeJF824OcWVD8tuLhmvwlYvt&trigger_id=696685010704.2988129444.7f616d9c52435a42645e726f26a42269"
# resp wrapped in dict because it's sometimes an array
# we should catch if there is no text in the command e.g. "/command <text>" where <text> is empty
# because that is the case where it could just be a dict in a dict
#response = make_response(jsonify(resp),200)
response_payload = {}
parsed_payload = text_parser.parse(request.form['text'])
# repeat for each command, we can fix the structure later
# testing shim
# parsed_payload = {'command': 'help'}
# /seeker, /seeker help
if parsed_payload['command'] == 'help' or parsed_payload['command'] == None:
help_json_template = json_templates.seeker_help()
response_payload = jsonify(help_json_template)
# /seeker tags
elif parsed_payload['command'] == 'tags':
tag_list = models.Tag.query.distinct(models.Tag.name).all()
arr = []
for tag in tag_list:
arr.append('`' + tag.name + '`')
tag_json_template = json_templates.seeker_tags(arr)
response_payload = jsonify(tag_json_template)
# tag an existing message
elif parsed_payload['command'] == 'tag':
response_payload = 'undefined error in tag'
tokens = parsed_payload['payload']
if len(tokens) != 2:
response_payload = 'Syntax for tagging and untagging is /seeker tag <tag> <message_id>'
else:
try:
message_id = int(tokens[0])
app.logger.info(
"user provided message_id: {}".format(message_id))
except ValueError:
response_payload = 'The message id was not an integer.'
return jsonify({'message': response_payload})
# get the tag, create tag if it doesn't exist
try:
# return first or None object if none
tag = db.session.query(
models.Tag).filter(models.Tag.name == tokens[1]).first()
if not tag:
tag = models.Tag()
tag.name = tokens[1]
db.session.add(tag)
db.session.commit()
except Exception as e:
response_payload = 'New tag creation failed {}'.format(e)
try:
# TODO: this is somehow getting an empty message or something
message = db.session.query(models.SlackMessage).get(message_id)
if not message:
raise
except Exception as e:
response_payload = 'Message id was not found in the seeker database, try a seeker save on the message URL first. {}'.format(
e)
try:
for mtag in message.tags:
if mtag.name == tag.name:
response_payload = 'Tag already found on message.'
else:
message.tags.append(tag)
db.session.commit()
response_payload = 'Tag {} was added to message {}.'.format(
tag.name, message_id)
except Exception as e:
response_payload = 'Some unidentified issue in tagging {}'.format(
e)
# package it into a response
response_payload = jsonify({'message': response_payload})
# untag an existing message
elif parsed_payload['command'] == 'untag':
response_payload = 'undefined error in untag'
tokens = parsed_payload['payload']
flag_tag_found = False # set to true if the tag is found and excluded (aka removed)
if len(tokens) != 2:
response_payload = 'Syntax for tagging and untagging is /seeker tag <tag> <message_id>'
else:
try:
message_id = int(tokens[0])
app.logger.info(
"user provided message_id: {}".format(message_id))
except ValueError:
response_payload = 'The message id was not an integer.'
return jsonify({'message': response_payload})
try:
message = db.session.query(models.SlackMessage).get(message_id)
if not message:
raise
except Exception as e:
response_payload = 'Message id was not found in the seeker database, try a seeker save on the message URL first. {}'.format(
e)
try:
tag = db.session.query(
models.Tag).filter(models.Tag.name == tokens[1]).first()
new_message_tags = list()
for tag in message.tags:
if tag.name != tokens[1]:
new_message_tags.append(tag)
else:
flag_tag_found = True
message.tags = new_message_tags
db.session.commit()
if flag_tag_found:
response_payload = 'Message tag was removed from the message successfully.'
else:
response_payload = 'Message tag was not found on that message.'
except Exception as e:
response_payload = 'Message id was not found in the seeker database, try a seeker save on the message URL first. {}'.format(
e)
# package it into a response
response_payload = jsonify({'message': response_payload})
elif parsed_payload['command'] == 'show':
tokens = parsed_payload['payload']
if len(tokens) == 0:
messages = helper.get_all_messages()
json_template = json_templates.seeker_show_no_tag(messages)
response_payload = jsonify(json_template)
return response_payload
tag = tokens[0]
message_urls = helper.get_all_messages_by_tag(tag)
app.logger.info(message_urls)
if len(message_urls) == 0:
return jsonify(
{"message": "No message urls found with the given tag"})
show_json_template = json_templates.seeker_show(tag, message_urls)
response_payload = jsonify(show_json_template)
# response_payload = jsonify({"list of message urls":message_urls})
elif parsed_payload['command'] in [
'share', 'save', 'add', 'new', 'create'
]:
tokens = parsed_payload['payload']
app.logger.info(tokens)
message_Url = tokens["message_URL"]
message_arr = message_Url.split('/')
channel_id = message_arr[4]
message = message_arr[5]
post_message_id = message[-6:]
post_message_id_start_index = message.find(post_message_id)
pre_message_id = message[1:post_message_id_start_index]
params = {
'token': os.getenv('SLACK_TOKEN'),
'channel': channel_id,
'oldest': pre_message_id + '.' + post_message_id,
'limit': 1,
'inclusive': True,
}
slack_message_resp = requests.get(
"https://slack.com/api/conversations.history", params).json()
tags = tokens["tags"]
description = tokens["description"]
message_text = slack_message_resp['messages'][0]['text']
annotator = request.form['user_name']
# author = 'test'
db_message = models.SlackMessage.query.filter_by(
url=message_Url).first()
app.logger.info(db_message)
save_json_template = {}
if not db_message is None:
save_json_template = json_templates.seeker_already_save(db_message)
else:
save_json_template = json_templates.seeker_save(
message_Url, tags, description)
helper.saveMessage(url=message_Url,
description=description,
message_text=message_text,
tags=tags,
annotator=annotator)
response_payload = jsonify(save_json_template)
elif parsed_payload['command'] == 'search':
terms = parsed_payload['payload']
terms[0] = terms[0].strip("\"").strip("\u201c").strip("\u201d")
terms[-1] = terms[-1].strip("\"").strip("\u201c").strip("\u201d")
message_q = Q.PriorityQueue()
if "|" in terms:
tags = []
tokens = []
start_index = -1
for i in range(len(terms)):
if terms[i] == "|" and start_index == -1:
start_index = i
if start_index == -1:
tokens.append(terms[i])
elif i > start_index:
tags.append(terms[i])
message_q = helper.searchMessageByTags(tokens, tags)
else:
message_q = helper.searchMessage(terms)
search_json_template = json_templates.seeker_search(terms, message_q)
response_payload = jsonify(search_json_template)
# if command not recognized
else:
unrecognized_json_template = json_templates.seeker_unrecognized()
response_payload = jsonify(unrecognized_json_template)
response_payload.headers['Access-Control-Allow-Origin'] = '*'
return response_payload
pool -= set2
set3 = pool # 3rd random subset
for u in set1:
u.team = 0
db.session.add(u)
db.session.commit()
for u in set2:
u.team = 1
db.session.add(u)
db.session.commit()
for u in set3:
u.team = 2
db.session.add(u)
db.session.commit()
# Clear all tags
tags = models.Tag.query.all()
for t in tags:
db.session.delete(t)
db.session.commit()
# Make new tags
for u in users:
u2 = [x for x in users if x.id != u.id and x.team != u.team
] # List of users excluding the tagging user and same team
targets = random.sample(u2, 3)
for target in targets:
t = models.Tag(source=u.id, destination=target.id)
db.session.add(t)
db.session.commit()
