def test_items_to_tasks(self): """ Verifies that multiple datastore page fetches result in properly emitted items. """ Org(id='test1').put() Org(id='test2').put() Org(id='test3').put() emitted_orgs = [] for org in datastore_utils.emit_items(Org.query()): emitted_orgs.append(org) orgs = Org.query().fetch() self.assertListEqual(orgs, emitted_orgs)
def reset_endpoints(): """ Handler which creates reset endpoint tasks for selected endpoints/orgs. """ endpoint_indexes = request.form.getlist('endpoint_index') org_uid = request.form.get('org_uid') if not endpoint_indexes: flash("At least one endpoint is required") return render_template('select_endpoints.html', endpoints=ENDPOINTS, org_uid=org_uid), 200 if org_uid: taskqueue.add(target='admin', url='/admin/reset_endpoints_task/{}'.format(org_uid), params={'endpoint_index': endpoint_indexes}) flash("Kicked off reset of {} endpoints for {}".format( len(endpoint_indexes), org_uid)) return redirect(prefix('/')) else: count = query_to_tasks( query=Org.query(Org.status == CONNECTED), queue=Queue('admin'), task_generator=lambda key: Task( url='/admin/reset_endpoints_task/{}'.format(key.string_id()), params={'endpoint_index': endpoint_indexes})) flash("Kicked off reset of {} endpoints for {} orgs".format( len(endpoint_indexes), count)) return redirect(prefix('/commands'))
def test_query_to_tasks(self): """ Verifies that multiple pages of tasks get queued up properly. """ Org(id='test1', status=CONNECTED).put() Org(id='test2', status=CONNECTED).put() Org(id='test3', status=DISCONNECTED).put() count = task_utils.query_to_tasks( query=Org.query(Org.status == CONNECTED), queue=Queue('adapter-update'), task_generator=lambda key: Task(url='/something/{}'.format( key.string_id()))) self.assertEqual(count, 2) task_count = len(self.taskqueue.get_filtered_tasks()) self.assertEqual(task_count, 2)
def init_all_updates(): """ Initialises update cycle for each connected org by putting a task onto the update queue (which ends up calling init_new_changeset(org_uid)). """ count = query_to_tasks( query=Org.query( Org.status == CONNECTED, Org.last_update_cycle_completed_at < datetime.utcnow() - SYNC_INTERVAL, Org.provider.IN(API_PROVIDERS)).order( -Org.last_update_cycle_completed_at, Org.key), # Queries involving IN need to be ordered by key queue=Queue('adapter-update'), task_generator=lambda key: Task(url='/adapter/{}/init_update'.format( key.string_id()))) logging.info("queued {} tasks for a sync update".format(count))
def replay_item_types(): """ Kicks off dataflow replay job for the selected item types and one or all orgs. """ item_types = request.form.getlist('item_type') org_uid = request.form.get('org_uid') template_name = request.form.get('action') if not item_types: flash("At least one item type is required") return render_template('select_item_types.html', item_types=ITEM_TYPES, org_uid=org_uid), 200 job_params = {} job_params['datatypes'] = ','.join(item_types) if org_uid: job_params['orgs'] = org_uid else: job_params['orgs'] = ','.join( [key.string_id() for key in Org.query().fetch(keys_only=True)]) logging.info("starting dataflow template '{}' with params: {}".format( template_name, job_params)) try: flash( escape( str( start_template(template_name, '{} all that is good'.format(template_name), job_params)))) except Exception as e: logging.exception("failed to start dataflow template") flash("Failed to start dataflow template with error: {}".format( escape(str(e)))) if org_uid: return redirect(prefix('/')) else: return redirect(prefix('/commands'))
def org_list(): """ Renders the org listing page. Returns: (str, int): org listing page """ cursor = Cursor(urlsafe=request.args.get('cursor')) orgs, next_cursor, more = Org.query().order(-Org.created_at).fetch_page( 20, start_cursor=cursor) connect_org_uid = request.args.get('connect_org_uid') message = None if connect_org_uid: error_code = request.args.get('error_code') if error_code == 'cancelled': message = "Failed to connect {} - OAuth flow cancelled".format( connect_org_uid) if error_code == 'source_mismatch': message = "Failed to connect {} - attempt to re-connect to different file".format( connect_org_uid) if error_code == 'invalid_credentials': message = 'Failed to connect {} - Invalid username or password'.format( connect_org_uid) else: message = "{} has been connected".format(connect_org_uid) show_connect = os.environ.get('SHOW_CONNECT_BUTTON', False) == "1" return render_template('org_list.html', orgs=orgs, next_cursor=next_cursor, more=more, changesets=_get_changesets(orgs), message=message, show_connect=show_connect), 200