def delete_contact():
client = ndb.Client()
with client.context():
contact = Contact.get_by_id(int(request.form.get('uid')))
contact.key.delete()
return redirect('/contacts/{}'.format(contact.key.id()))
def get_client(app=None):
"""Singleton for the Datastore client."""
global client
if not client:
client = ndb.Client(**ndb_kwargs(app=app))
return client
def get_pending_jobs():
parent_id = request.args['parent_id']
child_name = request.args['child_name']
client = ndb.Client()
with client.context():
child_dict = get_child(parent_id, child_name).to_dict()
return '{ "array":' + json.dumps(child_dict['pending_jobs']) + '}'
def client_context(namespace):
client = ndb.Client()
context_manager = client.context(
cache_policy=False, legacy_data=False, namespace=namespace,
)
with context_manager as context:
yield context
def test_sync_projects_update(self):
"""Testing sync_projects() updating a schedule."""
cloud_scheduler_client = CloudSchedulerClient()
with ndb.Client().context():
Project(name='test1',
schedule='0 8 * * *',
project_yaml_contents='',
dockerfile_contents='').put()
Project(name='test2',
schedule='0 9 * * *',
project_yaml_contents='',
dockerfile_contents='').put()
projects = {
'test1': ProjectMetadata('0 8 * * *', '', ''),
'test2': ProjectMetadata('0 7 * * *', '', '')
}
sync_projects(cloud_scheduler_client, projects)
projects_query = Project.query()
self.assertEqual({
'test1': '0 8 * * *',
'test2': '0 7 * * *'
}, {project.name: project.schedule
for project in projects_query})
def test_can_add_user():
client = ndb.Client()
with client.context():
ndb_user_repository = NDBUserRepository()
number_users_before_add = len(ndb_user_repository.get_all())
ndb_user_repository.add(user)
assert len(ndb_user_repository.get_all()) == number_users_before_add + 1
def collab(environ, start_response):
forms = cgi.FieldStorage(fp=environ['wsgi.input'], environ=environ)
try:
n = int(forms["n"].value)
except:
n = -1
try:
s = forms["s"].value
except:
s = None
isUpdate = (0 <= n <= 99) and (s in ("0", "1"))
client = ndb.Client()
with client.context():
result = Grid.get_by_id("STARS")
if result:
data = result.data
else:
data = "0" * 100
if isUpdate:
# Change the requested value.
data = data[:n] + s + data[n + 1:]
# Store the new grid.
with client.context():
row = Grid(id="STARS", data=data)
row.put()
headers = [("Content-Type", "text/plain")]
start_response("200 OK", headers)
return [data.encode("utf-8")]
def add_policy():
"""
Add policy.
Returns:
"""
logging.debug(json.dumps(request.json))
name = request.json["name"]
tags = request.json["tags"]
projects = request.json["projects"]
schedule_name = request.json["schedulename"]
client = ndb.Client()
with client.context():
res = SchedulesModel.query(SchedulesModel.Name == schedule_name).get()
if not res:
return "not found", 404
policy_model = PolicyModel()
policy_model.Name = name
policy_model.Tags = tags
policy_model.Projects = projects
policy_model.Schedule = schedule_name
policy_model.key = ndb.Key("PolicyModel", name)
policy_model.put()
return "ok", 200
def test_get_coverage_build_steps(self, mocked_url, mocked_corpora_steps,
mocked_time):
"""Test for get_build_steps."""
del mocked_url, mocked_corpora_steps, mocked_time
datetime.datetime = test_utils.SpoofedDatetime
project_yaml_contents = ('language: c++\n'
'sanitizers:\n'
' - address\n'
'architectures:\n'
' - x86_64\n')
dockerfile_contents = 'test line'
image_project = 'oss-fuzz'
base_images_project = 'oss-fuzz-base'
expected_build_steps_file_path = test_utils.get_test_data_file_path(
'expected_coverage_build_steps.json')
with open(expected_build_steps_file_path) as expected_build_steps_file:
expected_coverage_build_steps = json.load(expected_build_steps_file)
with ndb.Client().context():
Project(name='test-project',
project_yaml_contents=project_yaml_contents,
dockerfile_contents=dockerfile_contents).put()
dockerfile_lines = dockerfile_contents.split('\n')
build_steps = get_build_steps('test-project', project_yaml_contents,
dockerfile_lines, image_project,
base_images_project)
self.assertEqual(build_steps, expected_coverage_build_steps)
async def send_query_results(request: VTAPI):
client = ndb.Client()
try:
decoded = jwt.decode(request.jwt_token,
get_secret('jwt_secret'),
algorithms=['HS256'])
except jwt.exceptions.InvalidSignatureError:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail="Access forbidden")
if (datetime.now() - datetime.fromisoformat(decoded['issued'])).seconds > 60:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail="Token expired")
email = None
with client.context():
email_row = ndb.Key("UserEmail", request.api_key).get()
if email_row is None:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN, detail="API Key is not valid")
else:
email = email_row.email
log.info("Send an email to {}.\nBody:\n{}".format(
email, json.dumps(jsonable_encoder(request.data))))
def team_list() -> str:
page = 1
client = ndb.Client()
with client.context():
page_labels = ['1-999']
cur_page_label = page_labels[0]
teams_1 = Team.query(Team.team_number >= 0,
Team.team_number <= 500).fetch_async()
teams_2 = Team.query(Team.team_number >= 501,
Team.team_number <= 1000).fetch_async()
teams = teams_1.get_result() + teams_2.get_result()
num_teams = len(teams)
middle_value = num_teams // 2
if num_teams % 2 != 0:
middle_value += 1
teams_a, teams_b = teams[:middle_value], teams[middle_value:]
template_values = {
"teams_a": teams_a,
"teams_b": teams_b,
"num_teams": num_teams,
"page_labels": page_labels,
"cur_page_label": cur_page_label,
"current_page": page
}
return render_template("team_list.html", **template_values)
def test_parallel_threads(dispose_of, namespace):
client = ndb.Client(namespace=namespace)
class SomeKind(ndb.Model):
foo = ndb.IntegerProperty()
bar = ndb.StringProperty()
def insert(foo):
with client.context(cache_policy=False):
entity = SomeKind(foo=foo, bar="none")
key = entity.put()
dispose_of(key._key)
retrieved = key.get()
assert retrieved.foo == foo
assert retrieved.bar == "none"
thread1 = threading.Thread(target=insert, args=[42], name="one")
thread2 = threading.Thread(target=insert, args=[144], name="two")
thread1.start()
thread2.start()
thread1.join()
thread2.join()
def ndb_wsgi_middleware(wsgi_app):
"""
This is helpful for Flask and NDB to play nice together.
https://cloud.google.com/appengine/docs/standard/python3/migrating-to-cloud-ndb
We need to be able to access NDB in the application context.
If we're running a local datastore, make up a dummy project name.
"""
project = emulator and 'glowscript-dev' or None
client = ndb.Client(
project=project) # for user data, folders, and programs
def middleware(environ, start_response):
if False and environ.get('REQUEST_METHOD') == 'PUT':
#
# this can be useful for debugging late exceptions in PUT operations
# just remove 'False' above.
#
import pdb
pdb.set_trace()
with client.context():
return wsgi_app(environ, start_response)
return middleware
def app(environ, start_response):
forms = cgi.FieldStorage(fp=environ['wsgi.input'], environ=environ)
out = '<html><body><center>BIPES Blocks shared by the Community :-)</center>'
client = ndb.Client()
with client.context():
res = Xml.query().fetch()
if not res:
out += "<br><center>No shared blocks :-(<br>"
else:
out += "<br><br><center>Shared blocks (click to access the program):<br>"
out += "<table border=1><tr><td>Link to access</td><td><center>Author</center></td><td><center>Date</center></td><td><center>Description</center></td></tr>"
for k in res:
linkOld = str(k.xml_hash)
link = xmlToKey(
k.xml_content
) # I dont like this, but cant access id() from ndb
#link = str(k.id())
try:
out += "<tr><td><a href=http://bipes.net.br/beta2/ui/#" + link + " target=_blank>" + link + "</a></td><td>" + k.xml_author + "</td><td>" + k.xml_date + "</td><td>" + k.xml_desc + "</td></tr>"
except:
out += "<tr><td><a href=http://bipes.net.br/beta2/ui/#" + link + " target=_blank>" + link + "</a></td><td>.</td><td>.</td><td>-</td></tr>"
#out+="Valor: " + str(k.xml_hash) + " "
out += "</table>"
out += '<center><A HREF="javascript:history.go(0)">Refresh list</A></center>'
out += "</body></html>"
headers = [("Content-Type", "text/html")]
start_response("200 OK", headers)
return [out.encode("utf-8")]
def get_or_add(cls, user_id, email):
u = cls.get(user_id)
if not u:
with ndb.Client().context():
um = UserModel(id=user_id, email=email)
um.put()
u = cls(user_id, email)
return u
def ndb_django_middleware(get_response):
client = ndb.Client()
def middleware(request):
with client.context():
return get_response(request)
return middleware
def get_env_variable(key):
if os.environ.get('FLASK_ENV') == 'production':
client = ndb.Client()
with client.context():
return env_variables.get(key)
else:
# If in development environment, get user specified environment variables
return os.environ.get(key)
def ndb_wsgi_middleware(wsgi_app):
client = ndb.Client()
def middleware(environ, start_response):
with client.context():
return wsgi_app(environ, start_response)
return middleware
def get_redeemed_bonds():
parent_id = request.args['parent_id']
child_name = request.args['child_name']
client = ndb.Client()
with client.context():
child_dict = get_child(parent_id, child_name).to_dict()
return '{ "array":' + json.dumps(
[bond for bond in child_dict['bonds'] if bond['redeemed']]) + '}'
def get_config_from_datastore() -> AppConfig:
# This happens at application startup, so we use a new NDB context.
client = ndb.Client()
with client.context():
obj = AppConfig.singleton()
return obj
def test_can_add_topic():
client = ndb.Client()
with client.context():
ndb_topic_repository = NDBTopicRepository()
number_users_before_add = len(ndb_topic_repository.get_all())
ndb_topic_repository.add(topic)
assert len(
ndb_topic_repository.get_all()) == number_users_before_add + 1
def test_can_get_all_users():
client = ndb.Client()
with client.context():
all_users = NDBUserRepository().get_all()
assert type(all_users) == list
assert len(all_users) > 0
for user in all_users:
assert type(user) == User
def test_build_history_no_existing_project(self):
"""Testing build history when build history object is missing."""
with ndb.Client().context():
update_build_history('test-project', '1', 'fuzzing')
expected_build_ids = ['1']
self.assertEqual(BuildsHistory.query().get().build_ids,
expected_build_ids)
def test_can_get_all_topics():
client = ndb.Client()
with client.context():
topic_repo = NDBTopicRepository()
all_topics = topic_repo.get_all()
assert type(all_topics) == list
assert len(all_topics) > 0
for topic in all_topics:
assert type(topic) == Topic
def ndb_client(datastore):
client = ndb.Client()
yield client
# Now delete all entities.
with client.context():
for key in ndb.Query().iter(keys_only=True):
key.delete_async()
async def run_queries(x_appengine_cron: Optional[str] = Header(None)):
client = ndb.Client()
if x_appengine_cron != 'true':
raise HTTPException(403, 'Access forbidden')
with client.context():
for user in models.Userdata.query():
async with aiohttp.ClientSession() as httpSession:
# Run VT query.
async with httpSession.get(
f'https://www.virustotal.com/api/v3/intelligence/search?query={user.vt_query}',
headers={'x-apikey': user.apikey},
ssl=ssl_context) as vt_resp:
payload = {'api_key': user.apikey}
payload.update(await vt_resp.json())
if vt_resp.status != 200:
logging.error(
f'VT query failed with {vt_resp.status}: {vt_resp.text()}'
)
raise HTTPException(400, 'Bad request')
# Auth to webhook, get JWT token.
async with httpSession.post(
'https://webhook-dot-virustotal-step-2020.ew.r.appspot.com/auth/',
json={
'access_key': get_secret('access_key'),
'vt_key': user.apikey
},
ssl=ssl_context) as auth_resp:
jwt_token = await auth_resp.text()
payload['jwt_token'] = jwt_token.strip(
'"') # Webhook sends token with quotes.
if auth_resp.status != 200:
logging.error(
f'Authentication on webhook failed with {auth_resp.status}: {auth_resp.text()}'
)
raise HTTPException(403, 'Authentication failed')
# Send data with JWT to webhook.
async with httpSession.post(
user.webhook, json=payload,
ssl=ssl_context) as webhook_resp:
result = await webhook_resp.text()
if webhook_resp.status != 200:
logging.error(
f'Post to webhook failed with {webhook_resp.status}: {result}'
)
raise HTTPException(400, 'Bad request')
return 'Success'
def contact_details(uid):
client = ndb.Client()
with client.context():
contact = Contact.get_by_id(int(uid))
if not contact:
return redirect('/', code=301)
return render_template('contact.html', contact=contact)
def test_can_get_topic():
client = ndb.Client()
with client.context():
topic_repo = NDBTopicRepository()
topic_from_ndb = topic_repo.get(topic.id)
assert topic_from_ndb.author_uuid == topic.author_uuid
assert topic_from_ndb.id == topic.id
assert topic_from_ndb.topic_name == topic.topic_name
def get_sibling_names():
parent_id = request.args['parent_id']
child_name = request.args['child_name']
client = ndb.Client()
with client.context():
parent = get_parent(parent_id)
child_list = [child.get().name for child in parent.children]
return '{ "array":' + json.dumps(
[cn for cn in child_list if not cn == child_name]) + '}'
def sync(event, context):
"""Sync projects with cloud datastore."""
del event, context #unused
with ndb.Client().context():
github_client = Github(get_access_token())
repo = github_client.get_repo('google/oss-fuzz')
projects = get_projects(repo)
cloud_scheduler_client = scheduler_v1.CloudSchedulerClient()
sync_projects(cloud_scheduler_client, projects)
