def test_no_uid_with_admin(self):
token = create_token("barfoo", None, {"admin": True})
self.assertIsInstance(token, six.string_types)
token = create_token("barfoo", {}, {"admin": True})
self.assertIsInstance(token, six.string_types)
token = create_token("barfoo", {"foo": "bar"}, {"admin": True})
self.assertIsInstance(token, six.string_types)
def test_invalid_uid_with_admin(self):
with self.assertRaises(ValueError):
token = create_token("barfoo", {"uid": 1}, {"admin": True})
with self.assertRaises(ValueError):
token = create_token("barfoo", {"uid": None}, {"admin": True})
with self.assertRaises(ValueError):
token = create_token("barfoo", "foo", {"admin": True})
def login(self):
data.validate(required_fields=['email', 'password'], error_message='Invalid User / Password')
user = get_verified_user_by_email(data.email.lower())
password = web_auth.signed_password(data.password)
if not user:
return error('Invalid User / Password')
elif password != user.password:
return error('Invalid User / Password')
org = user.org[0].get()
org_id = org.key.id()
if org.secure_signup_step == 'Deactivated':
return error('This account has been deactivated')
web_auth.set_cookie_for_user(user.email, response)
response.set_cookie("org_id", org_id, max_age=7200)
response.set_cookie("is_demo", 'False')
auth_payload = {'uid': user.email, 'org': org_id}
token = create_token(FIREBASE_SECRET, auth_payload)
return {
'message': 'Logged In',
'user': user.email,
'org': org_id,
'fbaseToken': token,
'isTemporaryPassword': user.is_temporary_password
}
def get_auth_token(self):
auth_payload = {'uid': str(self.user_id)}
expire = datetime.datetime.now() + timedelta(days=2)
token = create_token(self.secret,
auth_payload,
options={'expires': expire.strftime('%s')})
return token
def get_flame_token(uid):
"""
Creates a public key using the user id and the secret key.
FLAME_FIREBASE_SECRET_KEY = 'z22bA3KOg54gfwr9lmoWyi1sg8dL8uwOg8txRyfl2'
"""
token = create_token(settings.FLAME_FIREBASE_SECRET_KEY, {"uid": str(uid)})
return token
def retrieve(self, request, *args, **kwargs):
"""gets basic information about the user
:param request: a WSGI request object
:param args: inline arguments (optional)
:param kwargs: keyword arguments (optional)
:return: `rest_framework.response.Response`
"""
data = UserSerializer().to_representation(request.user)
# add superuser flag only if user is a superuser, putting it here so users can only
# tell if they are themselves superusers
if request.user.is_superuser:
data['is_superuser'] = True
# attempt to add a firebase token if we have a firebase secret
secret = getattr(settings, 'FIREBASE_SECRET', None)
if secret:
# use firebase auth to provide auth variables to firebase security api
firebase_auth_payload = {
'id': request.user.pk,
'username': request.user.username,
'email': request.user.email,
'is_staff': request.user.is_staff
}
data['firebase_token'] = create_token(secret,
firebase_auth_payload)
return Response(data)
def __init__(self, fire_base_url, fire_base_secret):
if not fire_base_url.endswith('/'):
url = ''.join([fire_base_url, '/'])
else:
url = fire_base_url
# find db name between http:// and .firebaseio.com
db_name = re.search('https://(.*).firebaseio.com', fire_base_url)
if db_name:
name = db_name.group(1)
else:
db_name = re.search('(.*).firebaseio.com', fire_base_url)
name = db_name.group(1)
# default to admin
auth_payload = {"uid": "1"}
options = {"admin": True}
self.token = create_token(fire_base_secret, auth_payload, options)
self.requests = requests.Session()
adapter = requests.adapters.HTTPAdapter(max_retries=3)
for scheme in ('http://', 'https://'):
self.requests.mount(scheme, adapter)
self.fire_base_url = url
self.fire_base_name = name
self.secret = fire_base_secret
self.path = ""
self.build_query = {}
self.last_push_time = 0
self.last_rand_chars = []
def __init__(self, fire_base_url, fire_base_secret):
if not fire_base_url.endswith('/'):
url = ''.join([fire_base_url, '/'])
else:
url = fire_base_url
# find db name between http:// and .firebaseio.com
db_name = re.search('https://(.*).firebaseio.com', fire_base_url)
if db_name:
name = db_name.group(1)
else:
db_name = re.search('(.*).firebaseio.com', fire_base_url)
name = db_name.group(1)
# default to admin
auth_payload = {"uid": "1"}
options = {"admin": True}
self.token = create_token(fire_base_secret, auth_payload, options)
self.requests = requests.Session()
self.fire_base_url = url
self.fire_base_name = name
self.secret = fire_base_secret
self.path = ""
self.build_query = {}
self.last_push_time = 0
self.last_rand_chars = []
def _create_auth_token(request, organization_ids):
roles = []
user = request.user
if user.is_authenticated():
roles.append(ROLE_PARTICIPANT)
# TODO: is_active=True
user_id = request.user.username
name = '@' + user_id
if IsModerator(request=request).is_authorized():
roles.append(ROLE_MODERATOR)
organization_ids = list(organization_ids)
if len(organization_ids) > 0:
roles.append(ROLE_ORGANIZATION_ADMIN)
else:
roles.append(ROLE_ANONYMOUS)
if 'firebase' not in request.session:
if 'redis' in settings.CACHES:
cache = caches['redis']
weekly_key = 'firebase_anon_id_%d' % datetime.date.today(
).isocalendar()[1]
uid = str(cache.incr(weekly_key))
else:
uid = str(uuid4().int)[:6]
request.session['firebase'] = {'id': uid, 'name': '+anon_' + uid}
user_id = request.session['firebase']['id']
name = request.session['firebase']['name']
role = max(roles)
return create_token(
settings.FIREBASE['secret'], {
'uid': '-'.join([ROLE_PREFIXES[role], user_id]),
'name': name,
'role': role,
'organizations': organization_ids
})
def retrieve_token(userid, secret):
'''
Return a token for the backend of annotations.
It uses the course id to retrieve a variable that contains the secret
token found in inheritance.py. It also contains information of when
the token was issued. This will be stored with the user along with
the id for identification purposes in the backend.
'''
# the following five lines of code allows you to include the default timezone in the iso format
# for more information: http://stackoverflow.com/questions/3401428/how-to-get-an-isoformat-datetime-string-including-the-default-timezone
dtnow = datetime.datetime.now()
dtutcnow = datetime.datetime.utcnow()
delta = dtnow - dtutcnow
newhour, newmin = divmod(
(delta.days * 24 * 60 * 60 + delta.seconds + 30) // 60, 60)
newtime = "%s%+02d:%02d" % (dtnow.isoformat(), newhour, newmin)
# uses the issued time (UTC plus timezone), the consumer key and the user's email to maintain a
# federated system in the annotation backend server
custom_data = {
"issuedAt": newtime,
"consumerKey": secret,
"userId": userid,
"ttl": 86400
}
newtoken = create_token(secret, custom_data)
return newtoken
def _create_auth_token(request, organization_ids):
roles = []
user = request.user
if user.is_authenticated():
roles.append(ROLE_PARTICIPANT)
# TODO: is_active=True
user_id = request.user.username
name = '@' + user_id
if IsModerator(request=request).is_authorized():
roles.append(ROLE_MODERATOR)
organization_ids = list(organization_ids)
if len(organization_ids) > 0:
roles.append(ROLE_ORGANIZATION_ADMIN)
else:
roles.append(ROLE_ANONYMOUS)
if 'firebase' not in request.session:
if 'redis' in settings.CACHES:
cache = caches['redis']
weekly_key = 'firebase_anon_id_%d' % datetime.date.today().isocalendar()[1]
uid = str(cache.incr(weekly_key))
else:
uid = str(uuid4().int)[:6]
request.session['firebase'] = {
'id': uid,
'name': '+anon_' + uid
}
user_id = request.session['firebase']['id']
name = request.session['firebase']['name']
role = max(roles)
return create_token(settings.FIREBASE['secret'], {
'uid': '-'.join([ROLE_PREFIXES[role], user_id]),
'name': name,
'role': role,
'organizations': organization_ids
})
def retrieve_token(userid, apikey, secret):
'''
Return a token for the backend of annotations.
It uses the course id to retrieve a variable that contains the secret
token found in inheritance.py. It also contains information of when
the token was issued. This will be stored with the user along with
the id for identification purposes in the backend.
'''
apikey = apikey
secret = secret
# the following five lines of code allows you to include the defaulttimezone in the iso format
# for more information: http://stackoverflow.com/questions/3401428/how-to-get-an-isoformat-datetime-string-including-the-default-timezone
dtnow = datetime.datetime.now()
dtutcnow = datetime.datetime.utcnow()
delta = dtnow - dtutcnow
newhour, newmin = divmod((delta.days * 24 * 60 * 60 + delta.seconds + 30) // 60, 60)
newtime = "%s%+02d:%02d" % (dtnow.isoformat(), newhour, newmin)
# uses the issued time (UTC plus timezone), the consumer key and the user's email to maintain a
# federated system in the annotation backend server
custom_data = {"issuedAt": newtime, "consumerKey": apikey, "uid": userid, "ttl": 172800}
newtoken = create_token(secret, custom_data)
return newtoken
def auth_token(self):
custom_data = self.auth_data
admin = False
options = {'admin': admin}
token = create_token(self.data.firebase_secret, custom_data, options)
print "auth_token", token
return token
def retrieve(self, request, *args, **kwargs):
"""gets basic information about the user
:param request: a WSGI request object
:param args: inline arguments (optional)
:param kwargs: keyword arguments (optional)
:return: `rest_framework.response.Response`
"""
data = UserSerializer().to_representation(request.user)
# add superuser flag only if user is a superuser, putting it here so users can only
# tell if they are themselves superusers
if request.user.is_superuser:
data['is_superuser'] = True
# attempt to add a firebase token if we have a firebase secret
secret = getattr(settings, 'FIREBASE_SECRET', None)
if secret:
# use firebase auth to provide auth variables to firebase security api
firebase_auth_payload = {
'id': request.user.pk,
'username': request.user.username,
'email': request.user.email,
'is_staff': request.user.is_staff
}
data['firebase_token'] = create_token(secret, firebase_auth_payload)
return Response(data)
def get_auth_token(context, request):
plone_userid = get_allowed_userid(context, request)
if plone_userid is False:
# If the user is not allowed, return a void token.
return ''
if plone_userid is None:
# If the user is anonymous (not logged in), we do not
# allow it either. Return a void token.
return ''
# Admin is always false for now.
admin = False
custom_data = {
'ploneUserid': plone_userid,
'userPrefix': '/users/%s' % (plone_userid, ),
}
options = {
'admin': admin,
}
config = get_config()
token = create_token(config['firebase_secret'], custom_data, options)
return token
def getFirebaseToken(secret, uid):
return create_token(secret, {
"uid": uid
},
{
"admin": True
})
def get_flame_token(uid):
"""
Creates a public key using the user id and the secret key.
FLAME_FIREBASE_SECRET_KEY = 'z22bA3KOg54gfwr9lmoWyi1sg8dL8uwOg8txRyfl2'
"""
token = create_token(settings.FLAME_FIREBASE_SECRET_KEY, {"uid": str(uid)})
return token
def test_uid_max_length(self):
#length: 10 20 30 40 50 60 70 80 90 100 110 120 130 140 150 160 170 180 190 200 210 220 230 240 250 256
token = create_token(
"barfoo", {
"uid":
"1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456"
})
self.assertIsInstance(token, basestring)
def test_uid_too_long(self):
with self.assertRaises(ValueError):
#length: 10 20 30 40 50 60 70 80 90 100 110 120 130 140 150 160 170 180 190 200 210 220 230 240 250 257
token = create_token(
"barfoo", {
"uid":
"12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567"
})
def createUsersDatabase():
auth_payload = {"uid": "bhptnfQoq1eEytRBbjjGDrv40oC2"}
token = create_token("neftLmN0eBpzsRLAasLcer70wt6KqM6OZmoHKgFd",
auth_payload)
fire = Firebase("https://roofpik-948d0.firebaseio.com/users/data/",
auth_token=token)
users_all = fire.get()
for i in users_all:
createUserJSON(i)
def generate_firebase_token(request):
if request.user.is_authenticated():
auth_payload = {
"uid": str(request.user.id),
"fullname": request.user.first_name,
"email": request.user.email
}
token = create_token(settings.FIRECHAT_SECURITY_TOKEN, auth_payload)
return HttpResponse(token)
else:
return HttpResponse("You need to login first.")
def createauthtoken(request):
response = None
rdio_user_key = request.GET.get('userKey')
if rdio_user_key:
custom_data = {'rdio_user_key': rdio_user_key}
options = {'debug': settings.DEBUG}
firebase_token = create_token(settings.FIREBASE_TOKEN, custom_data, options)
response = { "token": firebase_token }
else:
response = {"error": "userKey is a required GET param"}
return HttpResponse(json.dumps(response), content_type = "application/json")
def get_token():
print 'This is the request header right now\n' + str(request.headers)
if g.current_user.is_anonymous() or g.token_used:
return jsonify(make_response(unauthorized('Invalid credentials')))
##return token.fb token and user object
payload = {"id": g.current_user.userid}
token = create_token("FgUjXoxvFKgsUDdgfONGnOqP3dOi9ZZe3Kkb5bXK",payload)
return jsonify({'token': unicode(g.current_user.generate_auth_token(
expiration=3600)), 'expiration': 3600, 'user':g.current_user.to_json(),'fbToken':token})
def createauthtoken(request):
response = None
rdio_user_key = request.GET.get('userKey')
if rdio_user_key:
custom_data = {'rdio_user_key': rdio_user_key}
options = {'debug': settings.DEBUG}
firebase_token = create_token(settings.FIREBASE_TOKEN, custom_data,
options)
response = {"token": firebase_token}
else:
response = {"error": "userKey is a required GET param"}
return HttpResponse(json.dumps(response), content_type="application/json")
def get_auth_token_for_admin():
"""Create an auth token with full admin rights.
Used from privilegized console scripts.
This needs the url, and the secret from env variables.
"""
config = get_env_config()
custom_data = {
'ploneUserid': 'admin',
}
options = {
'admin': True,
}
token = create_token(config['firebase_secret'], custom_data, options)
return token
def get_auth(self):
if self.user:
org = self.user.org[0].id()
auth_payload = {'uid': self.user.email, 'org': org}
token = create_token(FIREBASE_SECRET, auth_payload)
return {
'message': 'Logged In',
'user': self.user.email,
'org': org,
'fbaseToken': token,
'isTemporaryPassword': self.user.is_temporary_password
}
return error('User not logged in.')
def retrieve_token(userid, secret):
'''
Return a token for the backend of annotations.
It uses the course id to retrieve a variable that contains the secret
token found in inheritance.py. It also contains information of when
the token was issued. This will be stored with the user along with
the id for identification purposes in the backend.
'''
dtnow = datetime.datetime.now()
dtutcnow = datetime.datetime.utcnow()
delta = dtnow - dtutcnow
newhour, newmin = divmod((delta.days * 24 * 60 * 60 + delta.seconds + 30) // 60, 60)
newtime = "%s%+02d:%02d" % (dtnow.isoformat(), newhour, newmin)
custom_data = {"issuedAt": newtime, "consumerKey": secret, "userId": userid, "ttl": 86400}
newtoken = create_token(secret, custom_data)
return newtoken
def get_point_firebase():
points = {}
# initiate the connexion to Firebase
token = create_token(FIREBASE_SECRET , AUTH_PAYLOAD)
firebase_project = FIREBASE_PROJECT + '/' + ROOT_OBJECT + '.json'
firebase = Firebase(firebase_project, token)
results = firebase.get()
if results is not None:
for result in results:
points[results[result]['id']] = {
'key': result,
'data': results[result]
}
return points
def get_point_firebase():
points = {}
# initiate the connexion to Firebase
token = create_token(FIREBASE_SECRET, AUTH_PAYLOAD)
firebase_project = FIREBASE_PROJECT + '/' + ROOT_OBJECT + '.json'
firebase = Firebase(firebase_project, token)
results = firebase.get()
if results is not None:
for result in results:
points[results[result]['id']] = {
'key': result,
'data': results[result]
}
return points
def get_user(self,
expires=None,
not_before=None,
admin=False,
debug=False,
simulate=False):
"""
Method that gets the authenticated user. The returning user has
the token, email and the provider data.
"""
options = {'admin': admin, 'debug': debug, 'simulate': simulate}
if expires is not None:
options['expires'] = expires
if not_before is not None:
options['notBefore'] = not_before
token = create_token(self.secret, self.auth_payload, options)
user_id = self.auth_payload.get('uid')
return FirebaseUser(self.email, token, self.provider, user_id)
def __init__(self, rdio_oauth2):
rdio_user_key = rdio_oauth2.extra_data['rdio_id']
custom_data = {
'rdio_user_key': rdio_user_key,
}
options = {
'debug': settings.DEBUG,
}
firebase_token = create_token(settings.FIREBASE_TOKEN, custom_data, options)
extra_data = rdio_oauth2.extra_data
self.username = extra_data['rdio_username']
self.stream_region = extra_data['rdio_stream_region']
self.id = extra_data['rdio_id']
self.icon_url = extra_data['rdio_icon_url']
self.profile_url = 'http://rdio.com%s' % extra_data['rdio_profile_url']
self.access_token = extra_data['access_token']
self.refresh_token = extra_data['refresh_token']
self.client_id = settings.RDIO_OAUTH2_KEY
self.firebase_token = firebase_token
def __init__(self, rdio_oauth2):
rdio_user_key = rdio_oauth2.extra_data['rdio_id']
custom_data = {
'rdio_user_key': rdio_user_key,
}
options = {
'debug': settings.DEBUG,
}
firebase_token = create_token(settings.FIREBASE_TOKEN, custom_data,
options)
extra_data = rdio_oauth2.extra_data
self.username = extra_data['rdio_username']
self.stream_region = extra_data['rdio_stream_region']
self.id = extra_data['rdio_id']
self.icon_url = extra_data['rdio_icon_url']
self.profile_url = 'http://rdio.com%s' % extra_data['rdio_profile_url']
self.access_token = extra_data['access_token']
self.refresh_token = extra_data['refresh_token']
self.client_id = settings.RDIO_OAUTH2_KEY
self.firebase_token = firebase_token
def __init__(self):
try:
with open(secrets_filename, "r") as f:
data = f.read()
except:
err = "Error processing {} - make sure to fill in {}!".format(secrets_filename, secrets_filename)
with open(secrets_filename, "w") as f:
f.write(document)
raise Exception(err)
try:
conf = yaml.load(data)
self.firebase_url = conf["firebase_url"]
self.secret = conf["secret"]
self.data_bucket = conf["data_bucket"]
except:
raise Exception("Error parsing {}!".format(secrets_filename))
if self.firebase_url is None or self.secret is None:
raise Exception("Error parsing {} - not all fields have been filled in!".format(secrets_filename))
auth_payload = {"uid": "1", "auth_data": "foo", "other_auth_data": "bar"}
token = create_token(self.secret, auth_payload)
self._client = Firebase(self.firebase_url + self.data_bucket, auth_token=token)
def smsReciever():
"""
Handle the SMS when is coming and save it in Firebase
"""
r = twiml.Response()
# Limit the request to my phone number
if not request.form.has_key('From') \
and request.form['Body'] != current_app.config['MY_PHONE_NUMBER']:
r.message('Not allowed to send a message !')
return Response(response=str(r), status=401, mimetype="text/plain")
# initiate the connexion to Firebase
token = create_token(FIREBASE_SECRET, AUTH_PAYLOAD)
firebase_project = FIREBASE_PROJECT + '/' + ROOT_OBJECT + '.json'
firebase = Firebase(firebase_project, token)
# Get the message
if request.form.has_key('Body'):
body = request.form['Body']
# Get the next firebase Id and parse the content to firebase
next_id = get_next_id(firebase)
content = get_content(body, next_id)
# saved the content in firebase
if content is not None:
try:
firebase.post(content)
r.message("Record saved ! id = {}".format(next_id))
except Exception:
r.message("Can't save the record in firebase")
else:
r.message("Bad body format")
else:
r.message("Can't get the body message")
return str(r)
def create_user(num, username, password):
'''create user in heroku database
if user already exists(and password correct) return token without saving
if password incorrect return None
else save and return firebase token'''
uid = num+password
uid = base64.b64encode(uid.encode())
uid = uid.decode() #byte back to string
# auth_payload = {"uid": uid, "auth_data": "foo", "other_auth_data": "bar"}
auth_payload = {"uid": uid}
token = create_token(FIREBASE_SECRET, auth_payload)
user = None
try:
user = User.objects.get(num=num)
except:
user = User(num=num, password=password, username=username)
user.save()
return token
if user.password != password:
return None
return token
def test_uid_max_length(self):
#length: 10 20 30 40 50 60 70 80 90 100 110 120 130 140 150 160 170 180 190 200 210 220 230 240 250 256
token = create_token("barfoo", {"uid": "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456"})
self.assertIsInstance(token, six.string_types)
def test_invalid_uid(self):
with self.assertRaises(ValueError):
token = create_token("barfoo", {"uid": 5, "blah": 5})
def token(number, uid):
from firebase_token_generator import create_token
auth_payload = {"uid": uid, "auth_data": number}
options = {"debug": False}
return create_token(db_admin_secret, auth_payload, options)
def test_uid_min_length(self):
token = create_token("barfoo", {"uid": ""})
self.assertIsInstance(token, basestring)
def trainModel():
print 'Setting up firebase'
if not localTraining:
ref = firebase.FirebaseApplication(url, None)
users = firebase.FirebaseApplication(url + '/users', None)
auth_payload = {"uid": uid}
token = create_token(secret, auth_payload)
user = '******' + uid + '/'
print 'Pre-loading test data'
Xtest, ytest = loadData(dataDir, testFeatures, testLabels, Ntest, Dtest,
Ktest)
while True:
paramIter = -1
weightIter = -1
if not localTraining: # Read all params from server
print ' '
print 'Downloading parameters from server'
paramIter = np.int(
ref.get('/parameters/paramIter', None, params={"auth": token}))
params['D'] = np.int(
ref.get('/parameters/D', None, params={"auth": token}))
params['K'] = np.int(
ref.get('/parameters/K', None, params={"auth": token}))
params['L'] = np.double(
ref.get('/parameters/L', None, params={"auth": token}))
params['N'] = np.int(
ref.get('/parameters/N', None, params={"auth": token}))
params['clientBatchSize'] = np.int(
ref.get('/parameters/clientBatchSize',
None,
params={"auth": token}))
params['featureSource'] = ref.get('/parameters/featureSource',
None,
params={"auth": token})
params['labelSource'] = ref.get('/parameters/labelSource',
None,
params={"auth": token})
params['lossFunction'] = ref.get('/parameters/lossFunction',
None,
params={"auth": token})
params['noiseDistribution'] = ref.get(
'/parameters/noiseDistribution', None, params={"auth": token})
params['noiseScale'] = np.double(
ref.get('/parameters/noiseScale', None, params={"auth":
token}))
print params
print 'Loading training data'
X, y = loadData(dataDir, params['featureSource'],
params['labelSource'], params['N'], params['D'],
params['K'])
# Re-init w
if (params['lossFunction'] == 'Hinge'):
w = loss_hinge.init(params['D'])
elif (params['lossFunction'] == 'LogReg'):
w = loss_logreg.init(params['D'])
elif (params['lossFunction'] == 'Softmax'):
w = loss_softmax.init(params['D'], params['K'])
elif (params['lossFunction'] == 'NNdemo1'):
w = loss_nndemo1.init(params['D'], params['K'])
else:
print 'Unknown loss type'
exit()
print 'Begin iteration'
for gradIter in range(1, maxiter + 1):
print ' '
print 'paramIter = ', str(paramIter)
print 'weightIter = ', str(weightIter)
print 'gradIter = ', str(gradIter), '/', str(maxiter)
# Ready to send weights?
reset = False
print 'Checking server status'
while not localTraining:
if (gradIter == 1): # beginning
break
print '.',
time.sleep(1.) # sleep for 1 sec
paramIter_server = np.int(
ref.get('parameters/paramIter',
None,
params={"auth": token}))
if (paramIter_server >
paramIter): # parameter has changed. Reset
reset = True
break
gradientProcessed = ref.get(user + 'gradientProcessed',
None,
params={"auth": token})
gradIter_server = np.int(
ref.get(user + 'gradIter', None, params={"auth": token}))
#print 'gradientProcessed:',str(gradientProcessed),', gradIter_server:',str(gradIter_server)
if (gradientProcessed and gradIter_server == gradIter - 1):
break
print ' '
if reset:
print 'Parameter changed !!!'
break
# Randomly choose (clientBatchSize) samples
ind = np.random.choice(range(params['N']),
size=(params['clientBatchSize'], ),
replace=False)
tX = X[ind, :]
ty = y[ind]
# Fetch iteration number and weight
if localTraining:
weightIter = gradIter
else:
#print 'Fetching weights'
weightIter = np.int(
ref.get('/trainingWeights/iteration',
None,
params={"auth": token}))
#print 'weightIter= ', weightIter
w = np.array(ref.get('/trainingWeights/weights',
None,
params={"auth": token}),
dtype=np.double)
# Use one of loss functions.
# The output is the averaged gradient
if (params['lossFunction'] == 'Hinge'):
g, l = loss_hinge.getAvgGradient(w, tX, ty, params['L'])
elif (params['lossFunction'] == 'LogReg'):
g, l = loss_logreg.getAvgGradient(w, tX, ty, params['L'])
elif (params['lossFunction'] == 'Softmax'):
g, l = loss_softmax.getAvgGradient(w, tX, ty, params['L'],
params['K'])
elif (params['lossFunction'] == 'NNdemo1'):
g, l = loss_nndemo1.getAvgGradient(w, tX, ty, params['L'],
params['K'])
else:
print 'Unknown loss type'
exit()
if (params['noiseDistribution'] == 'NoNoise'):
noise = np.zeros(w.shape)
elif (params['noiseDistribution'] == 'Gauss'):
noise = GenerateGaussianNoise(params['noiseScale'], w.shape)
elif (params['noiseDistribution'] == 'Laplace'):
noise = GenerateLaplaceNoise(params['noiseScale'], w.shape)
else:
print 'Unknown noise type'
exit()
g += noise
if np.isnan(g).any():
print 'Nan in gradient'
exit()
#print str((g**2).sum())
print 'loss = ', str(l)
if localTraining:
# Simple learning rate
w -= naught / gradIter * g
else:
print 'Uploading gradients'
gradJson = g.tolist()
ref.put(user, 'paramIter', paramIter, params={"auth": token})
ref.put(user, 'weightIter', weightIter, params={"auth": token})
ref.put(user, 'gradIter', gradIter, params={"auth": token})
ref.put(user, 'gradients', gradJson, params={"auth": token})
ref.put(user,
'gradientProcessed',
False,
params={"auth": token})
## Iteration ended
if (gradIter == maxiter):
testModel(w, Xtest, ytest, params['K'], params['lossFunction'])
if localTraining:
break
def trainModel():
print 'Setting up firebase'
if not standalone:
ref = firebase.FirebaseApplication(url, None)
users = firebase.FirebaseApplication(url+'/users', None)
auth_payload = {"uid": uid}
token = create_token(secret, auth_payload)
user = '******'+uid+'/'
print 'Pre-loading test data'
Xtest,ytest = loadData(dataDir,testFeatures,testLabels,Ntest,Dtest,Ktest)
while True:
paramIter = -1
weightIter = -1
if not standalone: # Read all params from server
print ' '
print 'Downloading parameters from server'
paramIter = np.int(ref.get('/parameters/paramIter', None, params = {"auth":token}))
params['D'] = np.int(ref.get('/parameters/D', None, params = {"auth":token}))
params['K'] = np.int(ref.get('/parameters/K', None, params = {"auth":token}))
params['L'] = np.double(ref.get('/parameters/L', None, params = {"auth":token}))
params['N'] = np.int(ref.get('/parameters/N', None, params = {"auth":token}))
params['naughtRate'] = np.int(ref.get('/parameters/naughtRate', None, params = {"auth":token}))
params['clientBatchSize'] = np.int(ref.get('/parameters/clientBatchSize', None, params = {"auth":token}))
params['featureSource'] = ref.get('/parameters/featureSource', None, params = {"auth":token})
params['labelSource'] = ref.get('/parameters/labelSource', None, params = {"auth":token})
params['lossFunction'] = ref.get('/parameters/lossFunction', None, params = {"auth":token})
params['noiseDistribution'] = ref.get('/parameters/noiseDistribution', None, params = {"auth":token})
params['noiseScale'] = np.double(ref.get('/parameters/noiseScale', None, params = {"auth":token}))
params['localUpdateNum'] = np.int(ref.get('/parameters/localUpdateNum', None, params = {"autho":token}))
print params
print 'Loading training data'
X,y = loadData(dataDir,params['featureSource'],params['labelSource'],params['N'],params['D'],params['K'])
# Re-init w
if (params['lossFunction']=='Hinge'):
w = loss_hinge.init(params['D'])
elif (params['lossFunction']=='LogReg'):
w = loss_logreg.init(params['D'])
elif (params['lossFunction']=='Softmax'):
w = loss_softmax.init(params['D'],params['K'])
elif (params['lossFunction']=='NNdemo1'):
w = loss_nndemo1.init(params['D'],params['K'])
else:
print 'Unknown loss type'
exit()
print 'Begin iteration'
for gradIter in range(1,maxiter+1):
print ' '
print 'paramIter = ', str(paramIter)
print 'weightIter = ', str(weightIter)
print 'gradIter = ', str(gradIter),'/',str(maxiter)
# Ready to send weights?
reset = False
print 'Checking server status'
while not standalone:
if (gradIter==1): # beginning
break;
print '.',
time.sleep(1.) # sleep for 1 sec
paramIter_server = np.int(ref.get('parameters/paramIter', None, params = {"auth":token}))
if (paramIter_server > paramIter): # parameter has changed. Reset
reset = True
break
gradientProcessed = ref.get(user+'gradientProcessed', None, params = {"auth":token})
gradIter_server = np.int(ref.get(user+'gradIter', None, params = {"auth":token}))
#print 'gradientProcessed:',str(gradientProcessed),', gradIter_server:',str(gradIter_server)
if (gradientProcessed and gradIter_server == gradIter-1):
break
print ' '
if reset:
print 'Parameter changed !!!'
break;
# Fetch iteration number and weight
if standalone:
weightIter = gradIter
else:
#print 'Fetching weights'
weightIter = np.int(ref.get('/trainingWeights/iteration', None, params = {"auth":token}))
#print 'weightIter= ', weightIter
w = np.array(ref.get('/trainingWeights/weights', None, params = {"auth":token}),dtype=np.double)
if params['localUpdateNum']<=0 :
# SGD mode: compute and send the gradient
tX,ty = sampleData(X,y,params)
g, l = computeLossGradient(w,tX,ty,params)
xi = sampleNoise(w,params)
g += xi
else: # Parameter averaging mode: compute and send the parameters
for s in range(params['localUpdateNum']):
tX,ty = sampleData(X,y,params)
g,l = computeLossGradient(w,tX,ty,params)
# Simple learning rate
#w -= naught/gradIter*g
w -= params['naughtRate']/np.sqrt(params['localUpdateNum']*gradIter+s)*g
xi = sampleNoise(w,params)
w += xi
print 'loss = ',str(l)
if standalone:
if params['localUpdateNum']<=0:
# Simple learning rate
#w -= naughtRate/gradIter*g
w -= params['naughtRate']/np.sqrt(gradIter)*g
else:
pass # Do nothing
else:
print 'Uploading gradients'
if params['localUpdateNum']<=0:
gradJson = g.tolist()
else:
gradJson = w.tolist()
ref.put(user, 'paramIter', paramIter, params = {"auth":token})
ref.put(user, 'weightIter', weightIter, params = {"auth":token})
ref.put(user, 'gradIter', gradIter, params = {"auth":token})
ref.put(user, 'gradients', gradJson, params = {"auth":token})
ref.put(user ,'gradientProcessed', False, params = {"auth":token})
## Iteration ended
if (gradIter==maxiter):
testModel(w,Xtest,ytest,params['K'],params['lossFunction'])
if standalone:
break
def test_token_too_long(self):
with self.assertRaises(RuntimeError):
create_token("barfoo", {"uid": "blah", "long_var": "123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345612345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234561234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456"})
def token():
auth_payload = {"uid": "1", "auth_data": "foo", "other_auth_data": "bar"}
options = {"admin": True}
token = create_token("<AIzaSyARXSs-bBjRg4FGya6yV7opSEaHibywSj0>", auth_payload, options)
return token
from firebase_token_generator import create_token
import secrets
auth_payload = {"uid": "admin"}
token = create_token(secrets.firebase_secret, auth_payload)
print 'token:', token
import json
import urllib
from google.appengine.api import urlfetch
import firebase_token_generator as fb_token_generator
secrets = {}
with open('./secrets.json') as secrets_data:
secrets = json.load(secrets_data)
auth_payload = {"uid": "gae-instance", "provider": "gae"}
gae_auth_token = fb_token_generator.create_token(
secrets["firebaseSecret"], auth_payload)
FIREBASE_URL = "https://fb-channel.firebaseio.com"
def create_token(auth_payload, options=None):
return fb_token_generator.create_token(
secrets["firebaseSecret"], auth_payload, options)
# GET - Read Firebase data
def fb_get(path, params=None):
_params = {"auth": gae_auth_token}
if params is not None:
_params.update(params)
url = FIREBASE_URL + path + ".json"
_params = urllib.urlencode(_params)
response = urlfetch.fetch(url + "?" + _params, method=urlfetch.GET)
if response.content != "null":
from firebase import firebase
from firebase_token_generator import create_token
from pprint import pprint
import time
import sys
import fontv
""" Creates the Token necesarry for Authorization """
# "private_key" in the credentials JSON
private_key = '-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCPxdH3NfDAY/aV\n14gZIBLv1fZfz/WfFnRUSxCSgVR6lp0t1+oOk70GDn0JoHZIp15a6Ht5JBSG8LRa\nQNvBeC7CbBFyeGGLBqW1oUqcf9wLmh7pWFNvzNGjwIQfp71QKm5lzVV1UWBZBUKL\n/josBB1yydAMSF9vBN7abOiFvm6S3uVx4rKy4Dkw2p6+YnXclkB7a3GjSLGJiEZV\nCjSlQdO5iCmkRZV1Reztv1aSJFRneL2Ufdy61RgWm7fiHij2ySdOs91C2S0Dwarg\nxWf/oM8dG3atc/t+erURVvOVNRu4QH2JFA7cy4BBJKoINxmH5AANVZNFDXQN5YSE\nudiUDanhAgMBAAECggEAWB1fj+lpQiCB8lgEO5HiyUcTFqm32ebDKR5Qa9oD0pYY\ngw4/juuQ//NFQu8rT1/0fjbZ5ebtBb7kaiQyCyMDVXkyQDvzXGeYi/bsaXobfKja\npRzTFkQrf/bvgw8lAcrfmlj8NUjIigalQHNxZ6Sl/8IcVkrM4pfTlX8GC2IpFq3F\n9nOkg2N3UV/Da75CppLjcYpz2DrhFrJ9KHLmLc9VYnE4PncrqxT425RUwjLQq4UE\nVNLdOcOeNEoODEJJnZV4hwpDjbfCIfhWpOX0bTQ7JF/66nLnMkrmeSdV10AfK1bY\nI+juh7VTD5xSBBZWXTFQAYM1OnqpXf08539R8GQVmQKBgQDJUT9S/uVltHplB2ly\ne+OSDZzJpteUZMB/1tIR8pO+XAAQK8MWpTjl4KF9ApRR5TbyK0evp65nR+1XHKVL\nHc6h2RKo1JPp3f19Cv1x4vb7xYWV/N8kMU8tVDXcXeGsUrv+g4zloCWKTRo7EWsr\nsII7TZ3IbP0JTKQ/gIlHZENt9wKBgQC20ynwVnrdwTDJ1iq5cHGOaxE5uCapkYQN\nCtHFt1K0YwjXQGaXUuIOtW85pSIjE1VwVX86uRMjBZ6YTvkT5WZswJpk4fEu8fJE\nxaBv2iUTNsevZN1jFBnExYSgXwVHrhYy52MSPK/84bokUH7lPXMy7fgsWTsvPAs9\n35DcwNoQ5wKBgQC75xvF6ahPv56mAb8IXl5QimgXR6TcGxYZvHaizdncyZ49ebtD\nvb9XwMy6E9hAgENC0ea9KzN7rwT2VARsEkw1gJkMAC53EP5y4fkhfYiuAvTmw/m6\nnFOcqYcEQpRK9YYGzt6JzWp8pe6fx2hShRNpKs7kreWX8nMz2lNH0WumjQKBgGAF\ntnBBq5SO5EYOEzCEa1AG0exD2SKUiAullBnJOEn//eO4MmnmIU7iYkGfhrdvbx9p\n+EHqdK+fQvXx/IZDpTEXU8AKn3ctojYWqjY1F+XojqaDTne8VfBwYUEtxwMn3wbr\nWrB4aWJjWDW8hXl3derY69C3KQuc+LaNlnMrYx0fAoGAVaFu4ODKIjGUOyJq3Cby\nIGht8suKIPw7UtO1vBG/UkSMAyCDIBNoGtJPp9oeu6g9DwWGbDA+rHh5o58l1Zod\nUSBR2Mqu68+Ihuj31KGzdDGTtTwFef3yOlBe7l7uKC4J2omTET/DSyJzRkVhfdHO\nnHWe69Hdh9YwV4sgcvlQ41g=\n-----END PRIVATE KEY-----\n'
# "client_email" in the credentials JSON
service_account_email = "*****@*****.**"
auth_payload = {"auth_data": "foo", "other_auth_data": "bar"}
token = create_token(service_account_email, private_key, "user1", auth_payload)
"""Access the firebase database"""
# FIREBASE -> PYTHON
fbs = firebase.FirebaseApplication('https://project-4674828252354873374.firebaseio.com/', None)
authentication = firebase.FirebaseAuthentication('nU9t12pCYNaGUIP7EKbEy21b0xDkKmYoB8VE1vs5', '*****@*****.**', extra={'id': 123})
fbs.authentication = authentication
print authentication.extra
#AUTHENTICATIONS
user = authentication.get_user()
user.firebase_auth_token = token
print user.firebase_auth_token
import json
import urllib
from google.appengine.api import urlfetch
import firebase_token_generator as fb_token_generator
secrets = {}
with open('./secrets.json') as secrets_data:
secrets = json.load(secrets_data)
auth_payload = {"uid": "gae-instance", "provider": "gae"}
gae_auth_token = fb_token_generator.create_token(secrets["firebaseSecret"],
auth_payload)
FIREBASE_URL = "https://fb-channel.firebaseio.com"
def create_token(auth_payload, options=None):
return fb_token_generator.create_token(secrets["firebaseSecret"],
auth_payload, options)
# GET - Read Firebase data
def fb_get(path, params=None):
_params = {"auth": gae_auth_token}
if params is not None:
_params.update(params)
url = FIREBASE_URL + path + ".json"
_params = urllib.urlencode(_params)
def test_uid_too_long(self):
with self.assertRaises(ValueError):
#length: 10 20 30 40 50 60 70 80 90 100 110 120 130 140 150 160 170 180 190 200 210 220 230 240 250 257
token = create_token("barfoo", {"uid": "12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567"})
def put_point_firebase(key, point):
# initiate the connexion to Firebase
token = create_token(FIREBASE_SECRET, AUTH_PAYLOAD)
firebase_project = FIREBASE_PROJECT + '/' + ROOT_OBJECT + '/' + key + '/.json'
firebase = Firebase(firebase_project, token)
firebase.put(point)
def test_uid_min_length(self):
token = create_token("barfoo", {"uid": ""})
self.assertIsInstance(token, six.string_types)
from firebase import Firebase
from firebase_token_generator import create_token
import time
import random
auth_payload = {"uid":"k6wJ1M33MnaTiuRGU5IrFTwM5H53"}
token = create_token("1ufUtMReQxZlUe8kHdCs2WDfcmWdDKNeIziMg49o", auth_payload)
for i in range (1,100):
spotId = random.randrange(1,127)
f = Firebase("https://parking-monitor.firebaseio.com/lots/S17", auth_token=token)
distance = random.randrange(0,400)
randTime = random.randrange(1435255107,1465265107)
rp = f.post({"date":randTime, "serial":"HC-SR04-1", "distance":distance, "spot":spotId})
print("POST: " + str(rp))
time.sleep(1)
def getFirebaseToken(secret, uid):
return create_token(secret, {"uid": uid}, {"admin": True})
def get_auth_token(self):
auth_payload = {'uid': str(self.user_id)}
expire = datetime.datetime.now() + timedelta(days=2)
token = create_token(self.secret, auth_payload, options={'expires': expire.strftime('%s')})
return token
def test_smoke_test(self):
token = create_token("barfoo", {"uid": "foo"})
self.assertIsInstance(token, six.string_types)
def test_malformed_key(self):
with self.assertRaises(ValueError):
token = create_token(1234567890, {"uid": "foo"})
def test_smoke_test(self):
token = create_token("barfoo", {"uid": "foo"})
self.assertIsInstance(token, basestring)
def test_no_uid(self):
with self.assertRaises(ValueError):
token = create_token("barfoo", {"blah": 5})
def create_token(auth_payload, options=None):
return fb_token_generator.create_token(
secrets["firebaseSecret"], auth_payload, options)
def create_token(auth_payload, options=None):
return fb_token_generator.create_token(secrets["firebaseSecret"],
auth_payload, options)
