def test_items_to_tasks(self):
"""
Verifies that multiple datastore page fetches result in properly emitted items.
"""
Org(id='test1').put()
Org(id='test2').put()
Org(id='test3').put()
emitted_orgs = []
for org in datastore_utils.emit_items(Org.query()):
emitted_orgs.append(org)
orgs = Org.query().fetch()
self.assertListEqual(orgs, emitted_orgs)
def reset_endpoints():
"""
Handler which creates reset endpoint tasks for selected endpoints/orgs.
"""
endpoint_indexes = request.form.getlist('endpoint_index')
org_uid = request.form.get('org_uid')
if not endpoint_indexes:
flash("At least one endpoint is required")
return render_template('select_endpoints.html',
endpoints=ENDPOINTS,
org_uid=org_uid), 200
if org_uid:
taskqueue.add(target='admin',
url='/admin/reset_endpoints_task/{}'.format(org_uid),
params={'endpoint_index': endpoint_indexes})
flash("Kicked off reset of {} endpoints for {}".format(
len(endpoint_indexes), org_uid))
return redirect(prefix('/'))
else:
count = query_to_tasks(
query=Org.query(Org.status == CONNECTED),
queue=Queue('admin'),
task_generator=lambda key: Task(
url='/admin/reset_endpoints_task/{}'.format(key.string_id()),
params={'endpoint_index': endpoint_indexes}))
flash("Kicked off reset of {} endpoints for {} orgs".format(
len(endpoint_indexes), count))
return redirect(prefix('/commands'))
def test_query_to_tasks(self):
"""
Verifies that multiple pages of tasks get queued up properly.
"""
Org(id='test1', status=CONNECTED).put()
Org(id='test2', status=CONNECTED).put()
Org(id='test3', status=DISCONNECTED).put()
count = task_utils.query_to_tasks(
query=Org.query(Org.status == CONNECTED),
queue=Queue('adapter-update'),
task_generator=lambda key: Task(url='/something/{}'.format(
key.string_id())))
self.assertEqual(count, 2)
task_count = len(self.taskqueue.get_filtered_tasks())
self.assertEqual(task_count, 2)
def init_all_updates():
"""
Initialises update cycle for each connected org by putting a task onto the update queue (which ends up calling
init_new_changeset(org_uid)).
"""
count = query_to_tasks(
query=Org.query(
Org.status == CONNECTED, Org.last_update_cycle_completed_at <
datetime.utcnow() - SYNC_INTERVAL,
Org.provider.IN(API_PROVIDERS)).order(
-Org.last_update_cycle_completed_at,
Org.key), # Queries involving IN need to be ordered by key
queue=Queue('adapter-update'),
task_generator=lambda key: Task(url='/adapter/{}/init_update'.format(
key.string_id())))
logging.info("queued {} tasks for a sync update".format(count))
def replay_item_types():
"""
Kicks off dataflow replay job for the selected item types and one or all orgs.
"""
item_types = request.form.getlist('item_type')
org_uid = request.form.get('org_uid')
template_name = request.form.get('action')
if not item_types:
flash("At least one item type is required")
return render_template('select_item_types.html',
item_types=ITEM_TYPES,
org_uid=org_uid), 200
job_params = {}
job_params['datatypes'] = ','.join(item_types)
if org_uid:
job_params['orgs'] = org_uid
else:
job_params['orgs'] = ','.join(
[key.string_id() for key in Org.query().fetch(keys_only=True)])
logging.info("starting dataflow template '{}' with params: {}".format(
template_name, job_params))
try:
flash(
escape(
str(
start_template(template_name,
'{} all that is good'.format(template_name),
job_params))))
except Exception as e:
logging.exception("failed to start dataflow template")
flash("Failed to start dataflow template with error: {}".format(
escape(str(e))))
if org_uid:
return redirect(prefix('/'))
else:
return redirect(prefix('/commands'))
def org_list():
"""
Renders the org listing page.
Returns:
(str, int): org listing page
"""
cursor = Cursor(urlsafe=request.args.get('cursor'))
orgs, next_cursor, more = Org.query().order(-Org.created_at).fetch_page(
20, start_cursor=cursor)
connect_org_uid = request.args.get('connect_org_uid')
message = None
if connect_org_uid:
error_code = request.args.get('error_code')
if error_code == 'cancelled':
message = "Failed to connect {} - OAuth flow cancelled".format(
connect_org_uid)
if error_code == 'source_mismatch':
message = "Failed to connect {} - attempt to re-connect to different file".format(
connect_org_uid)
if error_code == 'invalid_credentials':
message = 'Failed to connect {} - Invalid username or password'.format(
connect_org_uid)
else:
message = "{} has been connected".format(connect_org_uid)
show_connect = os.environ.get('SHOW_CONNECT_BUTTON', False) == "1"
return render_template('org_list.html',
orgs=orgs,
next_cursor=next_cursor,
more=more,
changesets=_get_changesets(orgs),
message=message,
show_connect=show_connect), 200
