def test_dict_keys_substr_invalid(self): with self.assertRaises(orjson.JSONEncodeError): orjson.dumps({SubStr("\ud800"): True}, option=orjson.OPT_NON_STR_KEYS)
def do_test_realm_update_api(self, name: str) -> None: """Test updating realm properties. If new realm properties have been added to the Realm model but the test_values dict below has not been updated, this will raise an assertion error. """ bool_tests: List[bool] = [False, True] test_values: Dict[str, Any] = dict( default_language=["de", "en"], default_code_block_language=["javascript", ""], description=["Realm description", "New description"], digest_weekday=[0, 1, 2], message_retention_days=[10, 20], name=["Zulip", "New Name"], waiting_period_threshold=[10, 20], create_stream_policy=[ Realm.POLICY_ADMINS_ONLY, Realm.POLICY_MEMBERS_ONLY, Realm.POLICY_FULL_MEMBERS_ONLY, Realm.POLICY_MODERATORS_ONLY, ], user_group_edit_policy=[ Realm.USER_GROUP_EDIT_POLICY_ADMINS, Realm.USER_GROUP_EDIT_POLICY_MEMBERS, ], private_message_policy=[ Realm.PRIVATE_MESSAGE_POLICY_UNLIMITED, Realm.PRIVATE_MESSAGE_POLICY_DISABLED, ], invite_to_stream_policy=[ Realm.POLICY_ADMINS_ONLY, Realm.POLICY_MEMBERS_ONLY, Realm.POLICY_FULL_MEMBERS_ONLY, Realm.POLICY_MODERATORS_ONLY, ], wildcard_mention_policy=[ Realm.WILDCARD_MENTION_POLICY_EVERYONE, Realm.WILDCARD_MENTION_POLICY_MEMBERS, Realm.WILDCARD_MENTION_POLICY_FULL_MEMBERS, Realm.WILDCARD_MENTION_POLICY_STREAM_ADMINS, Realm.WILDCARD_MENTION_POLICY_ADMINS, Realm.WILDCARD_MENTION_POLICY_NOBODY, Realm.WILDCARD_MENTION_POLICY_MODERATORS, ], bot_creation_policy=[1, 2], email_address_visibility=[ Realm.EMAIL_ADDRESS_VISIBILITY_EVERYONE, Realm.EMAIL_ADDRESS_VISIBILITY_ADMINS, Realm.EMAIL_ADDRESS_VISIBILITY_NOBODY, ], video_chat_provider=[ dict( video_chat_provider=orjson.dumps( Realm.VIDEO_CHAT_PROVIDERS["jitsi_meet"]["id"] ).decode(), ), ], giphy_rating=[ Realm.GIPHY_RATING_OPTIONS["y"]["id"], Realm.GIPHY_RATING_OPTIONS["r"]["id"], ], message_content_delete_limit_seconds=[1000, 1100, 1200], invite_to_realm_policy=[ Realm.POLICY_ADMINS_ONLY, Realm.POLICY_MEMBERS_ONLY, Realm.POLICY_FULL_MEMBERS_ONLY, Realm.POLICY_MODERATORS_ONLY, ], move_messages_between_streams_policy=[ Realm.POLICY_ADMINS_ONLY, Realm.POLICY_MEMBERS_ONLY, Realm.POLICY_FULL_MEMBERS_ONLY, Realm.POLICY_MODERATORS_ONLY, ], ) vals = test_values.get(name) if Realm.property_types[name] is bool: vals = bool_tests if vals is None: raise AssertionError(f"No test created for {name}") if name == "video_chat_provider": self.set_up_db(name, vals[0][name]) realm = self.update_with_api_multiple_value(vals[0]) self.assertEqual(getattr(realm, name), orjson.loads(vals[0][name])) else: self.set_up_db(name, vals[0]) realm = self.update_with_api(name, vals[1]) self.assertEqual(getattr(realm, name), vals[1]) realm = self.update_with_api(name, vals[0]) self.assertEqual(getattr(realm, name), vals[0])
def nest_get_spikes_by_spikerecorder(spikerecorder_id, from_time=None, to_time=None, node_ids=None, skip=None, top=None): # noqa: E501 """Retrieves the spikes for the given time range (optional) and node IDs (optional) from one spike detector. If no time range or node list is specified, it will return the spikes for whole time or all nodes respectively. # noqa: E501 :param spikedetector_id: The ID of the spike detector to query. :type spikedetector_id: int :param from_time: The start time in milliseconds (including) to be queried. :type from_time: float :param to_time: The end time in milliseconds (excluding) to be queried. :type to_time: float :param node_ids: A list of node IDs queried for spike data. :type node_ids: List[int] :param skip: The offset into the result. :type skip: int :param top: The maximum number of entries to be returned. :type top: int :rtype: Spikes """ spikes = Spikes([], []) simulation_times = [] node_id_list = [] lastFrame = False for node in simulation_nodes.nest_simulation_nodes: if node_ids is not None: node_id_param = ",".join(map(str, node_ids)) else: node_id_param = None response = requests.get(node + "/spikes", params={ "fromTime": from_time, "toTime": to_time, "nodeIds": node_id_param, "spikedetectorId": spikerecorder_id }) response = orjson.loads(response.content) lastFrame = response["lastFrame"] simulation_times = simulation_times + response["simulationTimes"] node_id_list = node_id_list + response["nodeIds"] #sort sorted_lists = sort_together([simulation_times, node_id_list]) if sorted_lists != []: spikes.simulation_times = sorted_lists[0] spikes.node_ids = sorted_lists[1] # offset and limit if (skip is None): skip = 0 if (top is None or (top + skip) > len(spikes.node_ids)): top = len(spikes.node_ids) - skip spikes.node_ids = spikes.node_ids[skip:skip + top] spikes.simulation_times = spikes.simulation_times[skip:skip + top] json_string = orjson.dumps({ "nodeIds": spikes.node_ids, "simulationTimes": spikes.simulation_times, "lastFrame": lastFrame }) return ConnexionResponse(status_code=200, content_type='application/json', mimetype='text/plain', body=json_string)
def get_ui_display(self, record): return orjson.dumps(LiteratureDetailSchema().dump(record).data).decode("utf-8")
def _dump_message(data): return orjson.dumps(data) + __CRLF
def write(self, elems): for elem in elems: self.fh.write(orjson.dumps(elem) + b"\n")
class TestOutgoingWebhookMessaging(ZulipTestCase): def create_outgoing_bot(self, bot_owner: UserProfile) -> UserProfile: return self.create_test_bot( "outgoing-webhook", bot_owner, full_name="Outgoing Webhook bot", bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, service_name="foo-service", ) def test_multiple_services(self) -> None: bot_owner = self.example_user("othello") bot = do_create_user( bot_owner=bot_owner, bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, full_name="Outgoing Webhook Bot", email="whatever", realm=bot_owner.realm, password=None, acting_user=None, ) add_service( "weather", user_profile=bot, interface=Service.GENERIC, base_url="weather_url", token="weather_token", ) add_service( "qotd", user_profile=bot, interface=Service.GENERIC, base_url="qotd_url", token="qotd_token", ) sender = self.example_user("hamlet") with mock.patch("zerver.worker.queue_processors.do_rest_call") as m: self.send_personal_message( sender, bot, content="some content", ) url_token_tups = set() for item in m.call_args_list: args = item[0] base_url = args[0] request_data = orjson.loads(args[1]) tup = (base_url, request_data["token"]) url_token_tups.add(tup) message_data = request_data["message"] self.assertEqual(message_data["content"], "some content") self.assertEqual(message_data["sender_id"], sender.id) self.assertEqual( url_token_tups, { ("weather_url", "weather_token"), ("qotd_url", "qotd_token"), }, ) @mock.patch( "requests.request", return_value=ResponseMock( 200, orjson.dumps( {"response_string": "Hidley ho, I'm a webhook responding!"})), ) def test_pm_to_outgoing_webhook_bot( self, mock_requests_request: mock.Mock) -> None: bot_owner = self.example_user("othello") bot = self.create_outgoing_bot(bot_owner) sender = self.example_user("hamlet") self.send_personal_message(sender, bot, content="foo") last_message = self.get_last_message() self.assertEqual(last_message.content, "Hidley ho, I'm a webhook responding!") self.assertEqual(last_message.sender_id, bot.id) self.assertEqual( last_message.recipient.type_id, sender.id, ) self.assertEqual( last_message.recipient.type, Recipient.PERSONAL, ) @mock.patch( "requests.request", return_value=ResponseMock( 200, orjson.dumps( {"response_string": "Hidley ho, I'm a webhook responding!"})), ) def test_stream_message_to_outgoing_webhook_bot( self, mock_requests_request: mock.Mock) -> None: bot_owner = self.example_user("othello") bot = self.create_outgoing_bot(bot_owner) self.send_stream_message(bot_owner, "Denmark", content=f"@**{bot.full_name}** foo", topic_name="bar") last_message = self.get_last_message() self.assertEqual(last_message.content, "Hidley ho, I'm a webhook responding!") self.assertEqual(last_message.sender_id, bot.id) self.assertEqual(last_message.topic_name(), "bar") display_recipient = get_display_recipient(last_message.recipient) self.assertEqual(display_recipient, "Denmark")
def test_create_external_account_field(self) -> None: self.login('iago') realm = get_realm('zulip') data: Dict[str, Union[str, int, Dict[str, str]]] = {} data["name"] = "Twitter" data["field_type"] = CustomProfileField.EXTERNAL_ACCOUNT data['field_data'] = 'invalid' result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, "Bad value for 'field_data': invalid") data['field_data'] = orjson.dumps({}).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, "subtype key is missing from field_data") data["field_data"] = orjson.dumps({ 'subtype': '', }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'field_data["subtype"] cannot be blank.') data["field_data"] = orjson.dumps({ 'subtype': '123', }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'Invalid external account type') non_default_external_account = 'linkedin' data["field_data"] = orjson.dumps({ 'subtype': non_default_external_account, }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'Invalid external account type') data["field_data"] = orjson.dumps({ 'subtype': 'twitter', }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_success(result) twitter_field = CustomProfileField.objects.get(name="Twitter", realm=realm) self.assertEqual(twitter_field.field_type, CustomProfileField.EXTERNAL_ACCOUNT) self.assertEqual(twitter_field.name, "Twitter") self.assertEqual( orjson.loads(twitter_field.field_data)['subtype'], 'twitter') data['name'] = 'Reddit' data["field_data"] = orjson.dumps({ 'subtype': 'custom', }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error( result, 'Custom external account must define url pattern') data["field_data"] = orjson.dumps({ 'subtype': 'custom', 'url_pattern': 123, }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'field_data["url_pattern"] is not a string') data["field_data"] = orjson.dumps({ 'subtype': 'custom', 'url_pattern': 'invalid', }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'Malformed URL pattern.') data["field_data"] = orjson.dumps({ 'subtype': 'custom', 'url_pattern': 'https://www.reddit.com/%(username)s/user/%(username)s', }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'Malformed URL pattern.') data["field_data"] = orjson.dumps({ 'subtype': 'custom', 'url_pattern': 'reddit.com/%(username)s', }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'field_data["url_pattern"] is not a URL') data["field_data"] = orjson.dumps({ 'subtype': 'custom', 'url_pattern': 'https://www.reddit.com/user/%(username)s', }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_success(result) custom_field = CustomProfileField.objects.get(name="Reddit", realm=realm) self.assertEqual(custom_field.field_type, CustomProfileField.EXTERNAL_ACCOUNT) self.assertEqual(custom_field.name, "Reddit") field_data = orjson.loads(custom_field.field_data) self.assertEqual(field_data['subtype'], 'custom') self.assertEqual(field_data['url_pattern'], 'https://www.reddit.com/user/%(username)s') result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, "A field with that label already exists.")
def test_update(self) -> None: self.login('iago') realm = get_realm('zulip') result = self.client_patch( "/json/realm/profile_fields/100", info={ 'name': 'Phone Number', 'field_type': CustomProfileField.SHORT_TEXT }, ) self.assert_json_error(result, 'Field id 100 not found.') field = CustomProfileField.objects.get(name="Phone number", realm=realm) result = self.client_patch( f"/json/realm/profile_fields/{field.id}", info={ 'name': '', 'field_type': CustomProfileField.SHORT_TEXT }, ) self.assert_json_error(result, 'Label cannot be blank.') self.assertEqual(CustomProfileField.objects.count(), self.original_count) result = self.client_patch(f"/json/realm/profile_fields/{field.id}", info={ 'name': 'New phone number', 'field_type': CustomProfileField.SHORT_TEXT }) self.assert_json_success(result) field = CustomProfileField.objects.get(id=field.id, realm=realm) self.assertEqual(CustomProfileField.objects.count(), self.original_count) self.assertEqual(field.name, 'New phone number') self.assertIs(field.hint, '') self.assertEqual(field.field_type, CustomProfileField.SHORT_TEXT) result = self.client_patch(f"/json/realm/profile_fields/{field.id}", info={ 'name': '*' * 41, 'field_type': CustomProfileField.SHORT_TEXT }) msg = "name is too long (limit: 40 characters)" self.assert_json_error(result, msg) result = self.client_patch(f"/json/realm/profile_fields/{field.id}", info={ 'name': 'New phone number', 'hint': '*' * 81, 'field_type': CustomProfileField.SHORT_TEXT }) msg = "hint is too long (limit: 80 characters)" self.assert_json_error(result, msg) result = self.client_patch(f"/json/realm/profile_fields/{field.id}", info={ 'name': 'New phone number', 'hint': 'New contact number', 'field_type': CustomProfileField.SHORT_TEXT }) self.assert_json_success(result) field = CustomProfileField.objects.get(id=field.id, realm=realm) self.assertEqual(CustomProfileField.objects.count(), self.original_count) self.assertEqual(field.name, 'New phone number') self.assertEqual(field.hint, 'New contact number') self.assertEqual(field.field_type, CustomProfileField.SHORT_TEXT) result = self.client_patch( f"/json/realm/profile_fields/{field.id}", info={ 'name': 'Name ', 'field_type': CustomProfileField.SHORT_TEXT }, ) self.assert_json_success(result) field.refresh_from_db() self.assertEqual(field.name, 'Name') field = CustomProfileField.objects.get(name="Favorite editor", realm=realm) result = self.client_patch(f"/json/realm/profile_fields/{field.id}", info={ 'name': 'Favorite editor', 'field_data': 'invalid' }) self.assert_json_error(result, "Bad value for 'field_data': invalid") field_data = orjson.dumps({ 'vim': 'Vim', 'emacs': { 'order': '2', 'text': 'Emacs' }, }).decode() result = self.client_patch(f"/json/realm/profile_fields/{field.id}", info={ 'name': 'Favorite editor', 'field_data': field_data }) self.assert_json_error(result, "field_data is not a dict") field_data = orjson.dumps({ 'vim': { 'order': '1', 'text': 'Vim' }, 'emacs': { 'order': '2', 'text': 'Emacs' }, 'notepad': { 'order': '3', 'text': 'Notepad' }, }).decode() result = self.client_patch(f"/json/realm/profile_fields/{field.id}", info={ 'name': 'Favorite editor', 'field_data': field_data }) self.assert_json_success(result)
def process_initial_upgrade( user: UserProfile, licenses: int, automanage_licenses: bool, billing_schedule: int, charge_automatically: bool, free_trial: bool, ) -> None: realm = user.realm customer = update_or_create_stripe_customer(user) assert customer.stripe_customer_id is not None # for mypy assert customer.realm is not None ensure_realm_does_not_have_active_plan(customer.realm) ( billing_cycle_anchor, next_invoice_date, period_end, price_per_license, ) = compute_plan_parameters( CustomerPlan.STANDARD, automanage_licenses, billing_schedule, customer.default_discount, free_trial, ) # TODO: The correctness of this relies on user creation, deactivation, etc being # in a transaction.atomic() with the relevant RealmAuditLog entries with transaction.atomic(): # billed_licenses can greater than licenses if users are added between the start of # this function (process_initial_upgrade) and now billed_licenses = max(get_latest_seat_count(realm), licenses) plan_params = { "automanage_licenses": automanage_licenses, "charge_automatically": charge_automatically, "price_per_license": price_per_license, "discount": customer.default_discount, "billing_cycle_anchor": billing_cycle_anchor, "billing_schedule": billing_schedule, "tier": CustomerPlan.STANDARD, } if free_trial: plan_params["status"] = CustomerPlan.FREE_TRIAL plan = CustomerPlan.objects.create( customer=customer, next_invoice_date=next_invoice_date, **plan_params ) ledger_entry = LicenseLedger.objects.create( plan=plan, is_renewal=True, event_time=billing_cycle_anchor, licenses=billed_licenses, licenses_at_next_renewal=billed_licenses, ) plan.invoiced_through = ledger_entry plan.save(update_fields=["invoiced_through"]) RealmAuditLog.objects.create( realm=realm, acting_user=user, event_time=billing_cycle_anchor, event_type=RealmAuditLog.CUSTOMER_PLAN_CREATED, extra_data=orjson.dumps(plan_params, default=decimal_to_float).decode(), ) if not free_trial: stripe.InvoiceItem.create( currency="usd", customer=customer.stripe_customer_id, description="Zulip Cloud Standard", discountable=False, period={ "start": datetime_to_timestamp(billing_cycle_anchor), "end": datetime_to_timestamp(period_end), }, quantity=billed_licenses, unit_amount=price_per_license, ) if charge_automatically: collection_method = "charge_automatically" days_until_due = None else: collection_method = "send_invoice" days_until_due = DEFAULT_INVOICE_DAYS_UNTIL_DUE stripe_invoice = stripe.Invoice.create( auto_advance=True, collection_method=collection_method, customer=customer.stripe_customer_id, days_until_due=days_until_due, statement_descriptor="Zulip Cloud Standard", ) stripe.Invoice.finalize_invoice(stripe_invoice) from zerver.actions.realm_settings import do_change_realm_plan_type do_change_realm_plan_type(realm, Realm.PLAN_TYPE_STANDARD, acting_user=user)
def consume(self, event: Mapping[str, Any]) -> None: # nocoverage fn = settings.ZULIP_WORKER_TEST_FILE message = orjson.dumps(event) logging.info("TestWorker should append this message to %s: %s", fn, message.decode()) with open(fn, "ab") as f: f.write(message + b"\n")
def make_end_of_cycle_updates_if_needed( plan: CustomerPlan, event_time: datetime ) -> Tuple[Optional[CustomerPlan], Optional[LicenseLedger]]: last_ledger_entry = LicenseLedger.objects.filter(plan=plan).order_by("-id").first() last_ledger_renewal = ( LicenseLedger.objects.filter(plan=plan, is_renewal=True).order_by("-id").first() ) assert last_ledger_renewal is not None last_renewal = last_ledger_renewal.event_time if plan.is_free_trial() or plan.status == CustomerPlan.SWITCH_NOW_FROM_STANDARD_TO_PLUS: assert plan.next_invoice_date is not None next_billing_cycle = plan.next_invoice_date else: next_billing_cycle = start_of_next_billing_cycle(plan, last_renewal) if next_billing_cycle <= event_time and last_ledger_entry is not None: licenses_at_next_renewal = last_ledger_entry.licenses_at_next_renewal assert licenses_at_next_renewal is not None if plan.status == CustomerPlan.ACTIVE: return None, LicenseLedger.objects.create( plan=plan, is_renewal=True, event_time=next_billing_cycle, licenses=licenses_at_next_renewal, licenses_at_next_renewal=licenses_at_next_renewal, ) if plan.is_free_trial(): plan.invoiced_through = last_ledger_entry plan.billing_cycle_anchor = next_billing_cycle.replace(microsecond=0) plan.status = CustomerPlan.ACTIVE plan.save(update_fields=["invoiced_through", "billing_cycle_anchor", "status"]) return None, LicenseLedger.objects.create( plan=plan, is_renewal=True, event_time=next_billing_cycle, licenses=licenses_at_next_renewal, licenses_at_next_renewal=licenses_at_next_renewal, ) if plan.status == CustomerPlan.SWITCH_TO_ANNUAL_AT_END_OF_CYCLE: if plan.fixed_price is not None: # nocoverage raise NotImplementedError("Can't switch fixed priced monthly plan to annual.") plan.status = CustomerPlan.ENDED plan.save(update_fields=["status"]) discount = plan.customer.default_discount or plan.discount _, _, _, price_per_license = compute_plan_parameters( tier=plan.tier, automanage_licenses=plan.automanage_licenses, billing_schedule=CustomerPlan.ANNUAL, discount=plan.discount, ) new_plan = CustomerPlan.objects.create( customer=plan.customer, billing_schedule=CustomerPlan.ANNUAL, automanage_licenses=plan.automanage_licenses, charge_automatically=plan.charge_automatically, price_per_license=price_per_license, discount=discount, billing_cycle_anchor=next_billing_cycle, tier=plan.tier, status=CustomerPlan.ACTIVE, next_invoice_date=next_billing_cycle, invoiced_through=None, invoicing_status=CustomerPlan.INITIAL_INVOICE_TO_BE_SENT, ) new_plan_ledger_entry = LicenseLedger.objects.create( plan=new_plan, is_renewal=True, event_time=next_billing_cycle, licenses=licenses_at_next_renewal, licenses_at_next_renewal=licenses_at_next_renewal, ) realm = new_plan.customer.realm assert realm is not None RealmAuditLog.objects.create( realm=realm, event_time=event_time, event_type=RealmAuditLog.CUSTOMER_SWITCHED_FROM_MONTHLY_TO_ANNUAL_PLAN, extra_data=orjson.dumps( { "monthly_plan_id": plan.id, "annual_plan_id": new_plan.id, } ).decode(), ) return new_plan, new_plan_ledger_entry if plan.status == CustomerPlan.SWITCH_NOW_FROM_STANDARD_TO_PLUS: standard_plan = plan standard_plan.end_date = next_billing_cycle standard_plan.status = CustomerPlan.ENDED standard_plan.save(update_fields=["status", "end_date"]) (_, _, _, plus_plan_price_per_license) = compute_plan_parameters( CustomerPlan.PLUS, standard_plan.automanage_licenses, standard_plan.billing_schedule, standard_plan.customer.default_discount, ) plus_plan_billing_cycle_anchor = standard_plan.end_date.replace(microsecond=0) plus_plan = CustomerPlan.objects.create( customer=standard_plan.customer, status=CustomerPlan.ACTIVE, automanage_licenses=standard_plan.automanage_licenses, charge_automatically=standard_plan.charge_automatically, price_per_license=plus_plan_price_per_license, discount=standard_plan.customer.default_discount, billing_schedule=standard_plan.billing_schedule, tier=CustomerPlan.PLUS, billing_cycle_anchor=plus_plan_billing_cycle_anchor, invoicing_status=CustomerPlan.INITIAL_INVOICE_TO_BE_SENT, next_invoice_date=plus_plan_billing_cycle_anchor, ) standard_plan_last_ledger = ( LicenseLedger.objects.filter(plan=standard_plan).order_by("id").last() ) assert standard_plan_last_ledger is not None licenses_for_plus_plan = standard_plan_last_ledger.licenses_at_next_renewal assert licenses_for_plus_plan is not None plus_plan_ledger_entry = LicenseLedger.objects.create( plan=plus_plan, is_renewal=True, event_time=plus_plan_billing_cycle_anchor, licenses=licenses_for_plus_plan, licenses_at_next_renewal=licenses_for_plus_plan, ) return plus_plan, plus_plan_ledger_entry if plan.status == CustomerPlan.DOWNGRADE_AT_END_OF_CYCLE: process_downgrade(plan) return None, None return None, last_ledger_entry
async def send(self, data: dict) -> None: """Send data to the WebSocket""" if not self._ws: log.error("Error: Cannot send to uninitialized websocket") return await self._ws.send_bytes(orjson.dumps(data))
def _to_json(obj: Any) -> str: # type: ignore return orjson.dumps(obj).decode('utf-8')
async def post(self, request: HTTPConnection): id = request.path_params["id"] writeup = await Writeup.get(id) if writeup is None: return abort(404, "Writeup not found") if not can_edit(request, writeup.author_id): return abort(400) form = await request.form() form = WriteupForm(form) is_valid = form.validate() if not slug(form.title.data): is_valid = False form.title.errors.append( "A valid url-safe name cannot be generated for this title." ) if writeup.title != form.title.data: if ( await Writeup.query.where( sa.or_( Writeup.title == form.title.data, Writeup.slug == slug(form.title.data), ) ).gino.first() is not None ): is_valid = False form.title.errors.append( f"A writeup with the title conflicting with '{form.title.data}' already exists." ) if is_valid: await writeup.update_auto( title=form.title.data, tags=form.tags.data, content=form.content.data, private=form.private.data, ).apply() url = request.url_for("writeups_view", slug=writeup.slug) await log_edit("writeup", writeup.title, request.user.username, url) return redirect_response(url=url) images = await encoded_existing_images(request) tags = orjson.dumps(await get_all_tags(True)) return templates.TemplateResponse( "writeups/edit.j2", { "request": request, "form": form, "writeup": writeup, "existing_images": images, "existing_tags": tags, }, )
def test_update_profile_data_successfully(self) -> None: self.login('iago') realm = get_realm('zulip') fields = [ ('Phone number', '*short* text data'), ('Biography', '~~short~~ **long** text data'), ('Favorite food', 'long short text data'), ('Favorite editor', 'vim'), ('Birthday', '1909-3-5'), ('Favorite website', 'https://zulip.com'), ('Mentor', [self.example_user("cordelia").id]), ('GitHub', 'zulip-mobile'), ] data = [] for i, field_value in enumerate(fields): name, value = field_value field = CustomProfileField.objects.get(name=name, realm=realm) data.append({ 'id': field.id, 'value': value, 'field': field, }) # Update value of field result = self.client_patch( "/json/users/me/profile_data", { "data": orjson.dumps([{ "id": f["id"], "value": f["value"] } for f in data]).decode() }, ) self.assert_json_success(result) iago = self.example_user('iago') expected_value = {f['id']: f['value'] for f in data} expected_rendered_value: Dict[Union[int, float, str, None], Union[str, None]] = {} for f in data: if f['field'].is_renderable(): expected_rendered_value[f['id']] = markdown_convert(f['value']) else: expected_rendered_value[f['id']] = None for field_dict in iago.profile_data: self.assertEqual(field_dict['value'], expected_value[field_dict['id']]) self.assertEqual(field_dict['rendered_value'], expected_rendered_value[field_dict['id']]) for k in ['id', 'type', 'name', 'field_data']: self.assertIn(k, field_dict) # Update value of one field. field = CustomProfileField.objects.get(name='Biography', realm=realm) data = [{ 'id': field.id, 'value': 'foobar', }] result = self.client_patch("/json/users/me/profile_data", {'data': orjson.dumps(data).decode()}) self.assert_json_success(result) for field_dict in iago.profile_data: if field_dict['id'] == field.id: self.assertEqual(field_dict['value'], 'foobar')
def write_instrumentation_reports(full_suite: bool, include_webhooks: bool) -> None: if INSTRUMENTING: calls = INSTRUMENTED_CALLS from zproject.urls import urlpatterns, v1_api_and_json_patterns # Find our untested urls. pattern_cnt: Dict[str, int] = collections.defaultdict(int) def re_strip(r: Any) -> str: return str(r).lstrip("^").rstrip("$") def find_patterns(patterns: List[Any], prefixes: List[str]) -> None: for pattern in patterns: find_pattern(pattern, prefixes) def cleanup_url(url: str) -> str: if url.startswith("/"): url = url[1:] if url.startswith("http://testserver/"): url = url[len("http://testserver/") :] if url.startswith("http://zulip.testserver/"): url = url[len("http://zulip.testserver/") :] if url.startswith("http://testserver:9080/"): url = url[len("http://testserver:9080/") :] return url def find_pattern(pattern: Any, prefixes: List[str]) -> None: if isinstance(pattern, type(URLResolver)): return # nocoverage -- shouldn't actually happen if hasattr(pattern, "url_patterns"): return canon_pattern = prefixes[0] + re_strip(pattern.pattern.regex.pattern) cnt = 0 for call in calls: if "pattern" in call: continue url = cleanup_url(call["url"]) for prefix in prefixes: if url.startswith(prefix): match_url = url[len(prefix) :] if pattern.resolve(match_url): if call["status_code"] in [200, 204, 301, 302]: cnt += 1 call["pattern"] = canon_pattern pattern_cnt[canon_pattern] += cnt find_patterns(urlpatterns, ["", "en/", "de/"]) find_patterns(v1_api_and_json_patterns, ["api/v1/", "json/"]) assert len(pattern_cnt) > 100 untested_patterns = {p.replace("\\", "") for p in pattern_cnt if pattern_cnt[p] == 0} exempt_patterns = { # We exempt some patterns that are called via Tornado. "api/v1/events", "api/v1/events/internal", "api/v1/register", # We also exempt some development environment debugging # static content URLs, since the content they point to may # or may not exist. "coverage/(?P<path>.+)", "confirmation_key/", "node-coverage/(?P<path>.+)", "docs/(?P<path>.+)", "help/delete-a-stream", "api/delete-stream", "casper/(?P<path>.+)", "static/(?P<path>.+)", "flush_caches", *(webhook.url for webhook in WEBHOOK_INTEGRATIONS if not include_webhooks), } untested_patterns -= exempt_patterns var_dir = "var" # TODO make sure path is robust here fn = os.path.join(var_dir, "url_coverage.txt") with open(fn, "wb") as f: for call in calls: f.write(orjson.dumps(call, option=orjson.OPT_APPEND_NEWLINE)) if full_suite: print(f"INFO: URL coverage report is in {fn}") print("INFO: Try running: ./tools/create-test-api-docs") if full_suite and len(untested_patterns): # nocoverage -- test suite error handling print("\nERROR: Some URLs are untested! Here's the list of untested URLs:") for untested_pattern in sorted(untested_patterns): print(f" {untested_pattern}") sys.exit(1)
def test_create_choice_field(self) -> None: self.login('iago') data: Dict[str, Union[str, int]] = {} data["name"] = "Favorite programming language" data["field_type"] = CustomProfileField.CHOICE data['field_data'] = 'invalid' result = self.client_post("/json/realm/profile_fields", info=data) error_msg = "Bad value for 'field_data': invalid" self.assert_json_error(result, error_msg) data["field_data"] = orjson.dumps({ 'python': ['1'], 'java': ['2'], }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'field_data is not a dict') data["field_data"] = orjson.dumps({ 'python': { 'text': 'Python' }, 'java': { 'text': 'Java' }, }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, "order key is missing from field_data") data["field_data"] = orjson.dumps({ 'python': { 'text': 'Python', 'order': '' }, 'java': { 'text': 'Java', 'order': '2' }, }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'field_data["order"] cannot be blank.') data["field_data"] = orjson.dumps({ '': { 'text': 'Python', 'order': '1' }, 'java': { 'text': 'Java', 'order': '2' }, }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, "'value' cannot be blank.") data["field_data"] = orjson.dumps({ 'python': { 'text': 'Python', 'order': 1 }, 'java': { 'text': 'Java', 'order': '2' }, }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'field_data["order"] is not a string') data["field_data"] = orjson.dumps({}).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_error(result, 'Field must have at least one choice.') data["field_data"] = orjson.dumps({ 'python': { 'text': 'Python', 'order': '1' }, 'java': { 'text': 'Java', 'order': '2' }, }).decode() result = self.client_post("/json/realm/profile_fields", info=data) self.assert_json_success(result)
async def store(self, oid, old_serial, writer, obj, txn): assert oid is not None pickled = writer.serialize() # This calls __getstate__ of obj if len(pickled) >= self._large_record_size: log.info(f"Large object {obj.__class__}: {len(pickled)}") if self._store_json: json_dict = await writer.get_json() json = orjson.dumps(json_dict).decode("utf-8") else: json = None part = writer.part if part is None: part = 0 update = False statement_sql = self._sql.get("NAIVE_UPSERT", self.objects_table_name) if not obj.__new_marker__ and obj.__serial__ is not None: # we should be confident this is an object update statement_sql = self._sql.get("UPDATE", self.objects_table_name) update = True conn = await txn.get_connection() async with watch_lock(txn._lock, "store_object"): try: with watch("store_object"): result = await conn.fetch( statement_sql, oid, # The OID of the object txn._tid, # Our TID len(pickled), # Len of the object part, # Partition indicator writer.resource, # Is a resource ? writer.of, # It belogs to a main old_serial, # Old serial writer.parent_id, # Parent OID writer.id, # Traversal ID writer.type, # Guillotina type json, # JSON catalog pickled, # Pickle state) ) except asyncpg.exceptions.UniqueViolationError as ex: if "Key (parent_id, id)" in ex.detail or "Key (of, id)" in ex.detail: raise ConflictIdOnContainer(ex) raise except asyncpg.exceptions.ForeignKeyViolationError: txn.deleted[obj.__uuid__] = obj raise TIDConflictError( "Bad value inserting into database that could be caused " "by a bad cache value. This should resolve on request retry.", oid, txn, old_serial, writer, ) except asyncpg.exceptions._base.InterfaceError as ex: if "another operation is in progress" in ex.args[0]: raise ConflictError( "asyncpg error, another operation in progress.", oid, txn, old_serial, writer) raise except asyncpg.exceptions.DeadlockDetectedError: raise ConflictError("Deadlock detected.", oid, txn, old_serial, writer) if len(result) != 1 or result[0]["count"] != 1: if update: # raise tid conflict error raise TIDConflictError( "Mismatch of tid of object being updated. This is likely " "caused by a cache invalidation race condition and should " "be an edge case. This should resolve on request retry.", oid, txn, old_serial, writer, ) else: log.error("Incorrect response count from database update. " "This should not happen. tid: {}".format( txn._tid)) await txn._cache.store_object(obj, pickled)
def cron_trigger(self, trigger: schemas.ScheduleCronTrigger): self.cron_trigger_str = orjson.dumps( trigger.dict(exclude_unset=True))
def consume(self, event: Dict[str, Any]) -> None: start = time.time() if event["type"] == "mark_stream_messages_as_read": user_profile = get_user_profile_by_id(event["user_profile_id"]) for recipient_id in event["stream_recipient_ids"]: count = do_mark_stream_messages_as_read( user_profile, recipient_id) logger.info( "Marked %s messages as read for user %s, stream_recipient_id %s", count, user_profile.id, recipient_id, ) elif event["type"] == "mark_stream_messages_as_read_for_everyone": # This event is generated by the stream deactivation code path. batch_size = 100 offset = 0 while True: messages = Message.objects.filter( recipient_id=event["stream_recipient_id"]).order_by( "id")[offset:offset + batch_size] UserMessage.objects.filter(message__in=messages).extra( where=[UserMessage.where_unread()]).update( flags=F("flags").bitor(UserMessage.flags.read)) offset += len(messages) if len(messages) < batch_size: break logger.info( "Marked %s messages as read for all users, stream_recipient_id %s", offset, event["stream_recipient_id"], ) elif event["type"] == "clear_push_device_tokens": try: clear_push_device_tokens(event["user_profile_id"]) except PushNotificationBouncerRetryLaterError: def failure_processor(event: Dict[str, Any]) -> None: logger.warning( "Maximum retries exceeded for trigger:%s event:clear_push_device_tokens", event["user_profile_id"], ) retry_event(self.queue_name, event, failure_processor) elif event["type"] == "realm_export": realm = Realm.objects.get(id=event["realm_id"]) output_dir = tempfile.mkdtemp(prefix="zulip-export-") export_event = RealmAuditLog.objects.get(id=event["id"]) user_profile = get_user_profile_by_id(event["user_profile_id"]) try: public_url = export_realm_wrapper( realm=realm, output_dir=output_dir, threads=6, upload=True, public_only=True, delete_after_upload=True, ) except Exception: export_event.extra_data = orjson.dumps( dict(failed_timestamp=timezone_now().timestamp(), )).decode() export_event.save(update_fields=["extra_data"]) logging.error( "Data export for %s failed after %s", user_profile.realm.string_id, time.time() - start, ) notify_realm_export(user_profile) return assert public_url is not None # Update the extra_data field now that the export is complete. export_event.extra_data = orjson.dumps( dict(export_path=urllib.parse.urlparse(public_url).path, )).decode() export_event.save(update_fields=["extra_data"]) # Send a private message notification letting the user who # triggered the export know the export finished. with override_language(user_profile.default_language): content = _( "Your data export is complete and has been uploaded here:\n\n{public_url}" ).format(public_url=public_url) internal_send_private_message( sender=get_system_bot(settings.NOTIFICATION_BOT, realm.id), recipient_user=user_profile, content=content, ) # For future frontend use, also notify administrator # clients that the export happened. notify_realm_export(user_profile) logging.info( "Completed data export for %s in %s", user_profile.realm.string_id, time.time() - start, ) end = time.time() logger.info("deferred_work processed %s event (%dms)", event["type"], (end - start) * 1000)
output_dir = pathlib.Path(sys.argv[1]) input_dir = pathlib.Path(sys.argv[2]) json_filepaths = input_dir.glob("*.ndjson") parsed_at_timestamp = datetime.datetime.utcnow().isoformat() for in_filepath in json_filepaths: filename, _ = os.path.splitext(in_filepath.name) out_filepath = output_dir / f"{filename}.normalized.ndjson" logger.info( "normalizing %s => %s", in_filepath, out_filepath, ) with in_filepath.open("rb") as fin: with out_filepath.open("wb") as fout: # Optimization: faster to batch all of the sites into a single # write than to do ~46k separate writes. # Optimization: using orjson rather than json. fout.write(b"\n".join( orjson.dumps( _get_normalized_location( orjson.loads(site_json), parsed_at_timestamp, filename, ).dict()) for site_json in fin)) fout.write(b"\n")
def render(self, content: Any) -> bytes: return orjson.dumps(content)
def test_search(self) -> None: reset_emails_in_zulip_realm() def check_hamlet_user_query_result(result: HttpResponse) -> None: self.assert_in_success_response(['<span class="label">user</span>\n', '<h3>King Hamlet</h3>', '<b>Email</b>: [email protected]', '<b>Is active</b>: True<br>', '<b>Admins</b>: [email protected], [email protected]\n', 'class="copy-button" data-copytext="[email protected], [email protected]"', ], result) def check_zulip_realm_query_result(result: HttpResponse) -> None: zulip_realm = get_realm("zulip") self.assert_in_success_response([f'<input type="hidden" name="realm_id" value="{zulip_realm.id}"', 'Zulip Dev</h3>', '<option value="1" selected>Self Hosted</option>', '<option value="2" >Limited</option>', 'input type="number" name="discount" value="None"', '<option value="active" selected>Active</option>', '<option value="deactivated" >Deactivated</option>', 'scrub-realm-button">', 'data-string-id="zulip"'], result) def check_lear_realm_query_result(result: HttpResponse) -> None: lear_realm = get_realm("lear") self.assert_in_success_response([f'<input type="hidden" name="realm_id" value="{lear_realm.id}"', 'Lear & Co.</h3>', '<option value="1" selected>Self Hosted</option>', '<option value="2" >Limited</option>', 'input type="number" name="discount" value="None"', '<option value="active" selected>Active</option>', '<option value="deactivated" >Deactivated</option>', 'scrub-realm-button">', 'data-string-id="lear"', '<b>Name</b>: Zulip Standard', '<b>Status</b>: Active', '<b>Billing schedule</b>: Annual', '<b>Licenses</b>: 2/10 (Manual)', '<b>Price per license</b>: $80.0', '<b>Payment method</b>: Send invoice', '<b>Next invoice date</b>: 02 January 2017', ], result) def check_preregistration_user_query_result(result: HttpResponse, email: str, invite: bool=False) -> None: self.assert_in_success_response(['<span class="label">preregistration user</span>\n', f'<b>Email</b>: {email}', ], result) if invite: self.assert_in_success_response(['<span class="label">invite</span>'], result) self.assert_in_success_response(['<b>Expires in</b>: 1\xa0week, 3', '<b>Status</b>: Link has never been clicked'], result) self.assert_in_success_response([], result) else: self.assert_not_in_success_response(['<span class="label">invite</span>'], result) self.assert_in_success_response(['<b>Expires in</b>: 1\xa0day', '<b>Status</b>: Link has never been clicked'], result) def check_realm_creation_query_result(result: HttpResponse, email: str) -> None: self.assert_in_success_response(['<span class="label">preregistration user</span>\n', '<span class="label">realm creation</span>\n', '<b>Link</b>: http://testserver/accounts/do_confirm/', '<b>Expires in</b>: 1\xa0day<br>\n', ], result) def check_multiuse_invite_link_query_result(result: HttpResponse) -> None: self.assert_in_success_response(['<span class="label">multiuse invite</span>\n', '<b>Link</b>: http://zulip.testserver/join/', '<b>Expires in</b>: 1\xa0week, 3', ], result) def check_realm_reactivation_link_query_result(result: HttpResponse) -> None: self.assert_in_success_response(['<span class="label">realm reactivation</span>\n', '<b>Link</b>: http://zulip.testserver/reactivate/', '<b>Expires in</b>: 1\xa0day', ], result) self.login('cordelia') result = self.client_get("/activity/support") self.assertEqual(result.status_code, 302) self.assertEqual(result["Location"], "/login/") self.login('iago') customer = Customer.objects.create(realm=get_realm("lear"), stripe_customer_id='cus_123') now = datetime(2016, 1, 2, tzinfo=timezone.utc) plan = CustomerPlan.objects.create(customer=customer, billing_cycle_anchor=now, billing_schedule=CustomerPlan.ANNUAL, tier=CustomerPlan.STANDARD, price_per_license=8000, next_invoice_date=add_months(now, 12)) LicenseLedger.objects.create(licenses=10, licenses_at_next_renewal=10, event_time=timezone_now(), is_renewal=True, plan=plan) result = self.client_get("/activity/support") self.assert_in_success_response(['<input type="text" name="q" class="input-xxlarge search-query"'], result) result = self.client_get("/activity/support", {"q": "*****@*****.**"}) check_hamlet_user_query_result(result) check_zulip_realm_query_result(result) result = self.client_get("/activity/support", {"q": "lear"}) check_lear_realm_query_result(result) result = self.client_get("/activity/support", {"q": "http://lear.testserver"}) check_lear_realm_query_result(result) with self.settings(REALM_HOSTS={'zulip': 'localhost'}): result = self.client_get("/activity/support", {"q": "http://localhost"}) check_zulip_realm_query_result(result) result = self.client_get("/activity/support", {"q": "[email protected], lear"}) check_hamlet_user_query_result(result) check_zulip_realm_query_result(result) check_lear_realm_query_result(result) result = self.client_get("/activity/support", {"q": "lear, Hamlet <*****@*****.**>"}) check_hamlet_user_query_result(result) check_zulip_realm_query_result(result) check_lear_realm_query_result(result) self.client_post('/accounts/home/', {'email': self.nonreg_email("test")}) self.login('iago') result = self.client_get("/activity/support", {"q": self.nonreg_email("test")}) check_preregistration_user_query_result(result, self.nonreg_email("test")) check_zulip_realm_query_result(result) stream_ids = [self.get_stream_id("Denmark")] invitee_emails = [self.nonreg_email("test1")] self.client_post("/json/invites", {"invitee_emails": invitee_emails, "stream_ids": orjson.dumps(stream_ids).decode(), "invite_as": PreregistrationUser.INVITE_AS['MEMBER']}) result = self.client_get("/activity/support", {"q": self.nonreg_email("test1")}) check_preregistration_user_query_result(result, self.nonreg_email("test1"), invite=True) check_zulip_realm_query_result(result) email = self.nonreg_email('alice') self.client_post('/new/', {'email': email}) result = self.client_get("/activity/support", {"q": email}) check_realm_creation_query_result(result, email) do_create_multiuse_invite_link(self.example_user("hamlet"), invited_as=1) result = self.client_get("/activity/support", {"q": "zulip"}) check_multiuse_invite_link_query_result(result) check_zulip_realm_query_result(result) MultiuseInvite.objects.all().delete() do_send_realm_reactivation_email(get_realm("zulip")) result = self.client_get("/activity/support", {"q": "zulip"}) check_realm_reactivation_link_query_result(result) check_zulip_realm_query_result(result)
def update_with_api(self, name: str, value: Union[int, str]) -> Realm: if not isinstance(value, str): value = orjson.dumps(value).decode() result = self.client_patch("/json/realm", {name: value}) self.assert_json_success(result) return get_realm("zulip") # refresh data
def write_instrumentation_reports(full_suite: bool, include_webhooks: bool) -> None: if INSTRUMENTING: calls = INSTRUMENTED_CALLS from zproject.urls import urlpatterns, v1_api_and_json_patterns # Find our untested urls. pattern_cnt: Dict[str, int] = collections.defaultdict(int) def re_strip(r: str) -> str: assert r.startswith(r"^") if r.endswith(r"$"): return r[1:-1] else: assert r.endswith(r"\Z") return r[1:-2] def find_patterns(patterns: List[Any], prefixes: List[str]) -> None: for pattern in patterns: find_pattern(pattern, prefixes) def cleanup_url(url: str) -> str: if url.startswith("/"): url = url[1:] if url.startswith("http://testserver/"): url = url[len("http://testserver/"):] if url.startswith("http://zulip.testserver/"): url = url[len("http://zulip.testserver/"):] if url.startswith("http://testserver:9080/"): url = url[len("http://testserver:9080/"):] return url def find_pattern(pattern: Any, prefixes: List[str]) -> None: if isinstance(pattern, type(URLResolver)): return # nocoverage -- shouldn't actually happen if hasattr(pattern, "url_patterns"): return canon_pattern = prefixes[0] + re_strip( pattern.pattern.regex.pattern) cnt = 0 for call in calls: if "pattern" in call: continue url = cleanup_url(call["url"]) for prefix in prefixes: if url.startswith(prefix): match_url = url[len(prefix):] if pattern.resolve(match_url): if call["status_code"] in [200, 204, 301, 302]: cnt += 1 call["pattern"] = canon_pattern pattern_cnt[canon_pattern] += cnt find_patterns(urlpatterns, ["", "en/", "de/"]) find_patterns(v1_api_and_json_patterns, ["api/v1/", "json/"]) assert len(pattern_cnt) > 100 untested_patterns = { p.replace("\\", "") for p in pattern_cnt if pattern_cnt[p] == 0 } exempt_patterns = { # We exempt some patterns that are called via Tornado. "api/v1/events", "api/v1/events/internal", "api/v1/register", # We also exempt some development environment debugging # static content URLs, since the content they point to may # or may not exist. "coverage/(?P<path>.+)", "confirmation_key/", "node-coverage/(?P<path>.+)", "docs/(?P<path>.+)", "help/add-custom-emoji", "help/configure-who-can-add-custom-emoji", "help/change-the-topic-of-a-message", "help/configure-missed-message-emails", "help/community-topic-edits", "help/about-streams-and-topics", "help/delete-a-stream", "help/add-an-alert-word", "help/change-notification-sound", "help/configure-message-notification-emails", "help/disable-new-login-emails", "help/test-mobile-notifications", "help/troubleshooting-desktop-notifications", "for/working-groups-and-communities/", "help/only-allow-admins-to-add-emoji", "help/night-mode", "api/delete-stream", "casper/(?P<path>.+)", "static/(?P<path>.+)", "flush_caches", "external_content/(?P<digest>[^/]+)/(?P<received_url>[^/]+)", # These are SCIM2 urls overridden from django-scim2 to return Not Implemented. # We actually test them, but it's not being detected as a tested pattern, # possibly due to the use of re_path. TODO: Investigate and get them # recognized as tested. "scim/v2/", "scim/v2/.search", "scim/v2/Bulk", "scim/v2/Me", "scim/v2/ResourceTypes(?:/(?P<uuid>[^/]+))?", "scim/v2/Schemas(?:/(?P<uuid>[^/]+))?", "scim/v2/ServiceProviderConfig", "scim/v2/Groups(?:/(?P<uuid>[^/]+))?", "scim/v2/Groups/.search", *(webhook.url for webhook in WEBHOOK_INTEGRATIONS if not include_webhooks), } untested_patterns -= exempt_patterns var_dir = "var" # TODO make sure path is robust here fn = os.path.join(var_dir, "url_coverage.txt") with open(fn, "wb") as f: for call in calls: f.write(orjson.dumps(call, option=orjson.OPT_APPEND_NEWLINE)) if full_suite: print(f"INFO: URL coverage report is in {fn}") print("INFO: Try running: ./tools/create-test-api-docs") if full_suite and len( untested_patterns): # nocoverage -- test suite error handling print( "\nERROR: Some URLs are untested! Here's the list of untested URLs:" ) for untested_pattern in sorted(untested_patterns): print(f" {untested_pattern}") sys.exit(1)
def nest_get_spikes_by_node_collection(node_collection_id, from_time=None, to_time=None, skip=None, top=None): # noqa: E501 """Retrieves the spikes for the given simulation steps (optional) and node collection. This request merges the spikes recorded by all spike detectors and removes duplicates. # noqa: E501 :param node_collection_id: The identifier of the node collection. :type node_collection_id: int :param from_time: The start time (including) to be queried. :type from_time: float :param to_time: The end time (excluding) to be queried. :type to_time: float :param skip: The offset into the result. :type skip: int :param top: The maximum numbers of entries to be returned. :type top: int :rtype: Spikes """ spikes = Spikes([], []) simulation_times = [] node_id_list = [] lastFrame = False for node in simulation_nodes.nest_simulation_nodes: response = requests.get(node + "/spikes", params={ "fromTime": from_time, "toTime": to_time, "nodeCollectionId": node_collection_id }) response = orjson.loads(response.content) lastFrame = response["lastFrame"] simulation_times = simulation_times + response["simulationTimes"] node_id_list = node_id_list + response["nodeIds"] # sort sorted_lists = sort_together([simulation_times, node_id_list]) if sorted_lists != []: spikes.simulation_times = sorted_lists[0] spikes.node_ids = sorted_lists[1] # offset and limit if (skip is None): skip = 0 if (top is None or (top + skip) > len(spikes.node_ids)): top = len(spikes.node_ids) - skip spikes.node_ids = spikes.node_ids[skip:skip + top] spikes.simulation_times = spikes.simulation_times[skip:skip + top] json_string = orjson.dumps({ "nodeIds": spikes.node_ids, "simulationTimes": spikes.simulation_times, "lastFrame": lastFrame }) return ConnexionResponse(status_code=200, content_type='application/json', mimetype='text/plain', body=json_string)
def _create_response(response_body: Any, status_code: int) -> Response: return Response( response=orjson.dumps(response_body), status=status_code, mimetype="application/json", )
def nest_get_multimeter_measurements(multimeter_id, attribute_name, from_time=None, to_time=None, node_ids=None, skip=0, top=0): # noqa: E501 """Retrieves the measurements for a multimeter, attribute and node IDs (optional). # noqa: E501 :param multimeter_id: The multimeter to query :type multimeter_id: int :param attribute_name: The attribute to query (e.g., 'V_m' for the membrane potential) :type attribute_name: str :param from_time: The start time (including) to be queried. :type from_time: float :param to_time: The end time (excluding) to be queried. :type to_time: float :param node_ids: A list of node IDs queried for attribute data. :type node_ids: List[int] :param skip: The offset into the result. :type skip: int :param top: The maximum number of entries to be returned. :type top: int :rtype: MultimeterMeasurement """ #TODO Cache this mult_found, mult_nodes = nest_get_nodes_by_multimeter_id(multimeter_id) node_id_params = node_ids if node_ids == None and mult_found: node_ids = mult_nodes else: for node_id in node_ids: if node_id not in mult_nodes: error = Error(code="InvalidMultimeterRequest", message="Node " + str(node_id) + " is not monitored by given Multimeter") error_response = ErrorResponse(error) return error_response, 400 init = True sim_times = [] if simulation_nodes.nest_simulation_nodes == None: return for node in simulation_nodes.nest_simulation_nodes: if node_id_params is not None: node_id_param = ",".join(map(str, node_id_params)) else: node_id_param = None response = requests.get(node + "/multimeter_measurement", params={ "multimeterId": multimeter_id, "attribute": attribute_name, "fromTime": from_time, "toTime": to_time, "nodeIds": node_id_param }) response = orjson.loads(response.content) if init: sim_times = response["simulationTimes"] multimeter_values = [ None for _ in range(0, (len(sim_times) * len(node_ids))) ] init = False for x in range(len(response['nodeIds'])): node_id = response['nodeIds'][x] index = node_ids.index(node_id) index_offset = index * len(sim_times) for y in range(len(sim_times)): multimeter_values[index_offset + y] = response['values'][x * len(sim_times) + y] # offset and limit if skip > 0 or top > 0: print("slicing") top = len(node_ids) - skip node_ids = node_ids[skip:skip + top] multimeter_values = multimeter_values[skip * len(sim_times):(skip + top) * len(sim_times)] json_string = orjson.dumps({ "simulationTimes": sim_times, "nodeIds": node_ids, "values": multimeter_values }) return ConnexionResponse(status_code=200, content_type='application/json', mimetype='text/plain', body=json_string)
def test_dict_keys_substr(self): self.assertEqual( orjson.dumps({SubStr("aaa"): True}, option=orjson.OPT_NON_STR_KEYS), b'{"aaa":true}', )