def create_tokens(collection_id, files, is_g2p):
tokens = []
num_errors = 0
error_line, error_file = None, None
for file in files:
if is_g2p:
i_f = file.stream.read().decode("utf-8").split('\n')
for idx, line in enumerate(i_f):
try:
text, src, scr, *pron = line.split('\t')[0:]
pron = '\t'.join(p for p in pron)
token = Token(text,
file.filename,
collection_id,
score=scr,
pron=pron,
source=src)
tokens.append(token)
db.session.add(token)
except ValueError as error:
num_errors += 1
error_line = idx + 1
error_file = file.filename
continue
else:
content = file.stream.read().decode("utf-8").strip().split('\n')
for c in content:
# we split each file by new lines
if c[-1] == ',':
# this is a hack for the SQL stuff.
c = c[:-1]
token = Token(c, file.filename, collection_id)
tokens.append(token)
# add token to database
db.session.add(token)
if num_errors > 0:
flash(
f'{num_errors} villur komu upp, fyrsta villan í {error_file} í línu {error_line}',
category='danger')
db.session.commit()
# save token text to file
for token in tokens:
token.save_to_disk()
db.session.commit()
# reset the number of tokens in the collection
collection = Collection.query.get(collection_id)
collection.update_numbers()
db.session.commit()
return tokens
def consumer_create(request):
if request.method == "POST":
form = ConsumerCreateForm(request.POST)
if form.is_valid():
consumer = Consumer()
consumer.name = form.cleaned_data['name']
consumer.description = form.cleaned_data['description']
consumer.user_id = request.user.username
consumer.refresh_key_secret()
consumer.save()
token = Token()
token.key = Token.generate_token()
token.secret = Token.generate_token()
token.consumer = consumer
token.user = request.user.username
token.type = 'A'
token.save()
return HttpResponseRedirect('/accounts/profile')
else:
form = ConsumerCreateForm()
params = {'form': form}
return render_to_response('oauth/consumer_form.tpl', params,
context_instance=RequestContext(request))
def save_token(token, request, *args, **kwargs):
client_id = request.client.client_id
user_id = request.user.id
# Make sure there is only one grant token for every (client, user)
mongo.db.tokens.remove({'client_id': client_id, 'user_id': user_id})
expires_in = token.pop('expires_in')
expires = datetime.utcnow() + timedelta(seconds=expires_in)
token = Token(
client_id=request.client.client_id,
user_id=user_id,
token_type=token['token_type'],
access_token=token['access_token'],
refresh_token=token['refresh_token'],
expires=expires,
)
# Add the access token to the Redis cache and set it to
# expire at the appropriate time.
redis.setex(token.access_token, expires_in, user_id)
spec = {'user_id': user_id, 'client_id': client_id}
# Replace token if it exists already, insert otherwise.
mongo.db.tokens.update(spec, _to_json(token), upsert=True)
def create_token(request, url_id=None, **kwargs):
kwargs['extra_context'] = {}
if request.method == 'POST':
form = TokenAddForm(request.POST)
if form.is_valid():
email = form.cleaned_data['email']
token = Token(
url=form.cleaned_data['url'],
valid_until=form.cleaned_data['valid_until'],
forward_count=form.cleaned_data['forward_count'],
email=email,
name=form.cleaned_data['name'],
)
token.save()
messages.add_message(
request, messages.SUCCESS,
'Token successfully created for %s.' % token.email)
return HttpResponseRedirect(reverse('token_list'))
else:
initial_data = None
if not url_id is None:
url = ProtectedURL.objects.get(id=url_id)
initial_data = {
'url': url.url,
}
form = TokenAddForm(initial=initial_data)
kwargs['extra_context']['form'] = form
return direct_to_template(request,
template='token_auth/create_token.html',
**kwargs)
def _tokensetter(self, token, req, *args, **kwargs):
from models import Token
from flask import current_app as app
session = app.db.session
if req.user:
toks = req.user and Token.query.filter_by(
client_id=req.client.client_id,
user_id=req.user.id) or Token.query.filter_by(
client_id=req.client.client_id)
# make sure that every client has only one token connected to a user
for t in toks:
session.delete(t)
expires_in = token.get('expires_in')
expires = datetime.utcnow() + timedelta(seconds=expires_in)
tok = Token(
access_token=token['access_token'],
refresh_token=token['refresh_token'],
token_type=token['token_type'],
_scopes=token['scope'],
expires=expires,
client_id=req.client.client_id,
user_id=req.user and req.user.id or None,
)
session.add(tok)
session.commit()
return tok
def generate_token():
token = ''.join(random.choice(string.ascii_uppercase + string.digits)
for x in range(32))
insertToken = Token(
name='first_token',
token=token
)
session.add(insertToken)
session.commit()
try:
newToken = session.query(Token).filter_by(token=token).one()
return make_response(
jsonify(
status=200,
result = {
'token': newToken.token
}
), 200
)
except Exception:
return make_response(
jsonify(
status=500,
result = {
'message': 'Something went wrong'
}
), 500
)
async def get_token(form_data: OAuth2PasswordRequestForm = Depends()):
"""
Used for getting the jwt token,
takes a form data with {"username": ... , "password": ...}
but username is email in our case
"""
credentials_exception = HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Incorrect username or password",
headers={"WWW-Authenticate": "Bearer"},
)
# checks if the username is in the db
user = await User.get(email=form_data.username
).values_list("email", "password_hash")
if not user:
raise credentials_exception
# list of tuples
user_email = user[0][0]
user_hash = user[0][1]
# checks if password matches
if not verify_password(user_hash, form_data.password):
raise credentials_exception
access_token = create_access_token(data={"sub": user_email})
return Token(access_token=access_token, token_type="bearer")
def set_token(token, request, *args, **kwargs):
logger.debug('set token - user: {}\n'.format(request.user.userId))
tok = Token(**token)
tok.userId = request.user.userId
tok.client_id = request.client.client_id
tok.save()
return tok
def lexer(input: str, skip_whitespace: bool = True) -> [Token]:
tokens = []
pos = 0
while pos <= len(input):
if skip_whitespace:
match = re_ws_skip.search(input, pos)
if match:
pos = match.start()
else:
return tokens
matches = [re.match(input, pos) for re, _ in rules]
try:
match = [(match, rules[idx][1])
for idx, match in enumerate(matches) if match][0]
except IndexError as e:
print(input[pos])
raise LexerError(pos)
if match[1] is not 'COMMENT':
tokens.append(Token(match[1], match[0].group(0)))
pos = match[0].end()
return tokens
def save_token(token, request, *args, **kwargs):
print("token setter")
toks = Token.query.filter_by(
client_id=request.client.client_id,
user_id=request.user.id
)
print(toks)
# make sure that every client has only one token connected to a user
for t in toks:
db.session.delete(t)
expires_in = token.pop('expires_in')
expires = datetime.utcnow() + timedelta(seconds=expires_in)
tok = Token(
access_token=token['access_token'],
refresh_token=token['refresh_token'],
token_type=token['token_type'],
_scopes=token['scope'],
expires=expires,
client_id=request.client.client_id,
user_id=request.user.id,
)
print(tok)
db.session.add(tok)
db.session.commit()
return tok
def save_token(token, request, *args, **kwargs):
''' Saves a token.
This will delete any other tokens associated with the client once run.
This means that any existing refresh tokens will be lost and the new
one is the only refresh that can be used from that point onward. Which
is a reasonable way to handle things, although technically the length
a refresh token is valid for is not specified in the spec, so if you
want longer lived refresh tokens this will have to be modified.
'''
toks = Token.query.filter_by(client_id=request.client.client_id,
user_id=request.user.id)
# make sure that every client has only one token connected to a user
for t in toks:
db.session.delete(t)
expires_in = token.pop('expires_in')
expires = datetime.utcnow() + timedelta(seconds=expires_in)
tok = Token(
access_token=token['access_token'],
refresh_token=token['refresh_token'],
token_type=token['token_type'],
_scopes=token['scope'],
expires=expires,
client_id=request.client.client_id,
user_id=request.user.id,
)
db.session.add(tok)
db.session.commit()
return tok
def make_refresh_token(refresh_token: str) -> dict:
""" формирование нового access_token по refresh_token """
token = Token.query.filter(
and_(Token.refresh_token == refresh_token, Token.used == False))
if token.count() > 0:
token = token.first()
new_token = Token(token.user_client_id)
db.session.add(new_token)
db.session.commit()
token.used = True
db.session.add(token)
db.session.commit()
result = {
"access_token": new_token.access_token,
"token_type": "bearer",
"expires_in": 86400,
"refresh_token": new_token.refresh_token,
}
else:
result = {
"description": "Refresh token not found or it was used",
"errors": [{
"type": "bad request"
}],
}
return result
def forgot_password():
content = request.get_json(silent=True)
with db.connect() as session:
user = (session.query(User).filter_by(
username=content["username"]).first())
if user and content["email"] == user.email:
data = user.get_data()
data["uuid"] = str(uuid4())
# Store token in database
expiry_date = datetime.utcnow() + timedelta(seconds=600)
token = Token(token=data["uuid"],
token_expire=expiry_date,
token_type="PW_RESET",
blacklisted=False)
session.add(token)
# Create and send email
token = encode_email_token(data)
url = url_for("reset_password", token=token, _external=True)
send_forgot_email(url, content["email"])
return {"succes": True, "message": "Reset request sent."}
return {"success": False, "message": "Nope. Didn't work! :("}
def mockdata():
user = User(
username="******",
email="*****@*****.**",
is_admin=False,
)
user.set_password("pass")
question_cool = Question(body="Is it cool?")
question_neat = Question(body="Is this neat?")
token_cool = Token(name="cool")
token_neat = Token(name="neat")
mod_cool = Modifier(yes_modifier=1,
no_modifier=-1,
token=token_cool,
question=question_cool)
mod_neat = Modifier(yes_modifier=1,
no_modifier=-1,
token=token_neat,
question=question_neat)
resp_yes_neat = Response(user=user, answer="yes", question=question_neat)
resp_no_cool = Response(user=user, answer="no", question=question_cool)
data = [
user, question_cool, question_neat, token_cool, token_neat, mod_cool,
mod_neat, resp_no_cool, resp_yes_neat
]
app.logger.info("Inserting mock data...")
for datum in data:
db.session.add(datum)
try:
db.session.commit()
except Exception as e:
db.session.rollback()
app.logger.error("Error inserting mock data: {}".format(e))
finally:
db.session.close()
app.logger.info("Mock data inserted")
def sign_stuff():
# note: sign_hash() blocks gevent event loop for about a second.
ksisig = ksi.sign_hash(HashVal(hash_in, parsed_args['algorithm']))
token = Token(hash=binascii.unhexlify(hash_in),
sig=ksisig.serialize(),
by='myself')
session.add(token)
session.commit()
def create_token(uid):
'''
生成token
'''
shal = '%s=%s=%s' % ('Token', uid, time.time())
shal_uid = hashlib.sha1(shal.encode('utf-8')).hexdigest()
token = Token(id=next_id(), uid=uid, key=shal_uid)
yield from token.save()
return token['key']
def generate_token():
global token
auth = ''.join(
random.choice(string.ascii_uppercase + string.digits +
string.ascii_lowercase) for _ in range(15))
token = Token(auth)
with open('token.json', 'w') as f:
f.flush()
f.write(token.toJSON())
def create_access_token(
data: dict,
expires_delta: timedelta = timedelta(days=ACCESS_TOKEN_EXPIRE_DAYS)
) -> Token:
"""Creates a JWT with default time delta of 30 minutes"""
expire = datetime.utcnow() + expires_delta
to_encode = data.copy()
to_encode.update({"exp": expire})
access_token = jwt.encode(to_encode, key=SECRET, algorithm=ALGORITHM)
return Token(access_token=access_token, token_type=TOKEN_TYPE)
def regist_token(self, request):
if Token.query(Token.token == request.token).fetch():
return RegistTokenResult(result="DUP")
token = Token(token=request.token, os='ios')
token.put()
push_to(token, u"환영합니다! 벙커원 교회 주보 알림목록에 등록되었습니다!", 0)
return RegistTokenResult(result="OK")
def create_user(name, email, token):
user = User(name=name, email=email)
session.add(user)
session.commit()
# TODO: generate unique tokens for each user
# temporary implementation
token = Token(user_id=user.id, key=token)
session.add(token)
session.commit()
return token.key
def load_token():
global token
print("[DEBUG] Loading token")
try:
with open('token.json') as f:
data = json.load(f)
token = Token(data["auth"])
return True
except:
return False
def releaseTokens(user_id):
ret = {}
ret['refresh_token'] = uuid.uuid4().hex
ret['token'] = jwt.encode({'id': user_id},
config.secret,
algorithm='HS256')
ret['token'] = ret['token'].decode('utf-8')
print('[+] New token is ', ret['refresh_token'])
new_token = Token(user_id=user_id, token=ret['refresh_token'])
db.session.add(new_token)
db.session.commit()
return ret
def notification():
if request.method == "POST":
token = request.get_json()
db.create_all()
print(token['token'])
obj = Token(tokenID=token['token'])
db.session.add(obj)
db.session.commit()
return jsonify(sucess=True)
return render_template("notification.html")
def auth(mail, pwhash):
user = User.query.filter_by(mail=mail).first()
if (user != None):
if (user.pwhash == pwhash):
oldt = user.token[0] if len(user.token) == 1 else None
if (oldt != None):
oldt.token = generate_token()
oldt.update_expiration()
else:
db.session.add(Token(uid=user.uid, token=generate_token()))
db.session.commit()
return jsonify(user.json())
return "denied"
def save_token_to_database(token):
if not Token.query.filter_by(user_id=current_user.id).first():
token = Token(
user_id=current_user.id,
token=token,
)
db.session.add(token)
else:
token_instance = Token.query.filter_by(user_id=current_user.id).first()
token_instance.token = token
db.session.commit()
return token
def tokenize(text, content):
if content == '':
return []
if isinstance(content, str):
content = unicode(content, 'utf8')
tokens = []
for seq, word in enumerate(vc.tokenize(content)):
token = Token(word=word,
corpus=text.corpus,
text=text,
seq=seq)
tokens.append(token)
return tokens
def add_user_token():
# Creates a fresh UUID and stores it in the database
# If a token for a user_id already exists, return the existing token
req_data = request.get_json()
user_id = User.query.filter_by(username=req_data['username']).first().id
user_token = Token.query.filter_by(user_id=user_id).first()
if user_token:
return user_token.token
else:
new_uuid = str(uuid4())
token = Token(token=new_uuid, user_id=user_id)
db.session.add(token)
db.session.commit()
return new_uuid
def token(): # 生成token，生成后存储到数据库。
appid_in = Appid.query.get(1)
appid = appid_in.appid
secret = appid_in.secret
url = "https://api.weixin.qq.com/cgi-bin/token?grant_type=client_credential&appid={}&secret={}".format(
appid, secret)
ret = requests.get(url=url)
content = ret.content.decode("utf-8")
js = json.loads(content)
access_token = js['access_token']
token = Token(token=access_token)
db.session.add(token)
db.session.commit()
return access_token
def regist_token(self, request):
if Token.query(Token.token == request.token).fetch():
return RegistTokenResult(result="DUP")
token = urllib.unquote(request.token)
token = Token(token=request.token,
os=request.os,
width=request.width,
height=request.height)
token.put()
push_to(token, u"환영합니다! 벙커원 교회 주보 알림목록에 등록되었습니다!", 0)
return RegistTokenResult(result="OK")
def monitor_token():
"""
监听 kafka 日志数据，出现新数据时，过滤出 transfer 函数对应的日志，从 web3 获取 token
:return:
"""
if not config.get('kafka', 'log_topic', fallback=None):
logger_err.error('config.ini 中没有 log_topic 参数，退出 monitor_token 任务')
return
elif not config.get('kafka', 'token_topic', fallback=None):
logger_err.error('config.ini 中没有 token_topic 参数，退出 monitor_token 任务')
return
consumer = kafka_consumer(config.get('kafka', 'log_topic'), group_id='monitor_token')
last_block_height = None
tx_cnt = 0 # 已处理的交易数
for msg in consumer:
logs = msg.value
block_number = logs[0]['block_number']
if last_block_height != block_number:
logger.info(f'区块 {last_block_height} 共处理交易 {tx_cnt} 笔')
logger.info(f'开始处理区块高度 {block_number} 下各交易的 receipt')
last_block_height = block_number
tx_cnt = 1
else:
tx_cnt += 1
# 筛选出 token 地址
addresses = set()
for log in logs:
if log['topics'].startswith(TOPIC_TRANSFER):
addresses.add(log['address'])
# 获取 token
for address in addresses:
# 如果是已经处理过的地址，则直接处理
if address in valid_token_cache:
valid_token_cache[address].save()
continue
elif address in invalid_token_cache:
invalid_token_cache[address] = 1
continue
try:
contract = w3.eth.contract(address, abi=ERC20_ABI)
token = Token(data=address, contract=contract, block_number=block_number)
token.save()
valid_token_cache[address] = token
except InvalidAddress:
invalid_token_cache[address] = 1
logger_err.debug(f'无法处理 token 合约地址 {address}')
