def get_logo_path(self) -> Optional[str]: logo_file_path_svg = self.DEFAULT_LOGO_STATIC_PATH_SVG.format(name=self.name) logo_file_path_png = self.DEFAULT_LOGO_STATIC_PATH_PNG.format(name=self.name) if os.path.isfile(static_path(logo_file_path_svg)): return logo_file_path_svg elif os.path.isfile(static_path(logo_file_path_png)): return logo_file_path_png return None
def generate_integration_bots_avatars(check_missing: bool = False) -> None: missing = set() for integration in INTEGRATIONS.values(): if not integration.logo_path: continue bot_avatar_path = integration.get_bot_avatar_path() if bot_avatar_path is None: continue bot_avatar_path = os.path.join(ZULIP_PATH, "static", bot_avatar_path) if check_missing: if not os.path.isfile(bot_avatar_path): missing.add(integration.name) else: create_integration_bot_avatar(static_path(integration.logo_path), bot_avatar_path) if missing: print( "ERROR: Bot avatars are missing for these webhooks: {}.\n" "ERROR: Run ./tools/setup/generate_integration_bots_avatars.py " "to generate them.\nERROR: Commit the newly generated avatars to " "the repository.".format(", ".join(missing)) ) sys.exit(1)
def generate_integration_bots_avatars(check_missing: bool = False) -> None: missing = set() for webhook in WEBHOOK_INTEGRATIONS: if not webhook.logo_path: continue bot_avatar_path = webhook.get_bot_avatar_path() if bot_avatar_path is None: continue bot_avatar_path = os.path.join(ZULIP_PATH, 'static', bot_avatar_path) if check_missing: if not os.path.isfile(bot_avatar_path): missing.add(webhook.name) else: create_integration_bot_avatar(static_path(webhook.logo_path), bot_avatar_path) if missing: print( 'ERROR: Bot avatars are missing for these webhooks: {}.\n' 'ERROR: Run ./tools/setup/generate_integration_bots_avatars.py ' 'to generate them.\nERROR: Commit the newly generated avatars to ' 'the repository.'.format(', '.join(missing))) sys.exit(1)
def get_js_source_map() -> Optional[SourceMap]: global js_source_map if not js_source_map and not (settings.DEVELOPMENT or settings.TEST_SUITE): js_source_map = SourceMap([ static_path('webpack-bundles'), ]) return js_source_map
def render_tex(tex: str, is_inline: bool = True) -> Optional[str]: r"""Render a TeX string into HTML using KaTeX Returns the HTML string, or None if there was some error in the TeX syntax Keyword arguments: tex -- Text string with the TeX to render Don't include delimiters ('$$', '\[ \]', etc.) is_inline -- Boolean setting that indicates whether the render should be inline (i.e. for embedding it in text) or not. The latter will show the content centered, and in the "expanded" form (default True) """ katex_path = (static_path("webpack-bundles/katex-cli.js") if settings.PRODUCTION else os.path.join( settings.DEPLOY_ROOT, "node_modules/katex/cli.js")) if not os.path.isfile(katex_path): logging.error("Cannot find KaTeX for latex rendering!") return None command = ['node', katex_path] if not is_inline: command.extend(['--display-mode']) katex = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout = katex.communicate(input=tex.encode())[0] if katex.returncode == 0: # stdout contains a newline at the end assert stdout is not None return stdout.decode('utf-8').strip() else: return None
def render_tex(tex: str, is_inline: bool = True) -> Optional[str]: r"""Render a TeX string into HTML using KaTeX Returns the HTML string, or None if there was some error in the TeX syntax Keyword arguments: tex -- Text string with the TeX to render Don't include delimiters ('$$', '\[ \]', etc.) is_inline -- Boolean setting that indicates whether the render should be inline (i.e. for embedding it in text) or not. The latter will show the content centered, and in the "expanded" form (default True) """ katex_path = (static_path("webpack-bundles/katex-cli.js") if settings.PRODUCTION else os.path.join( settings.DEPLOY_ROOT, "node_modules/katex/cli.js")) if not os.path.isfile(katex_path): logging.error("Cannot find KaTeX for latex rendering!") return None command = ['node', katex_path] if not is_inline: command.extend(['--display-mode']) try: stdout = subprocess.check_output(command, input=tex, stderr=subprocess.DEVNULL, universal_newlines=True) # stdout contains a newline at the end return stdout.strip() except subprocess.CalledProcessError: return None
def generate_integration_bots_avatars(check_missing: bool=False) -> None: missing = set() for webhook in WEBHOOK_INTEGRATIONS: logo_path = webhook.get_logo_path() if not logo_path: continue if check_missing: bot_avatar_path = static_path(webhook.DEFAULT_BOT_AVATAR_PATH.format(name=webhook.name)) if not os.path.isfile(bot_avatar_path): missing.add(webhook.name) else: create_integration_bot_avatar(static_path(logo_path)) if missing: print('Bot avatars are missing for these webhooks: {}.\n' 'Run ./tools/setup/generate_integration_bots_avatars.py ' 'to generate them.'.format(', '.join(missing))) sys.exit(1)
def team_view(request: HttpRequest) -> HttpResponse: with open(static_path('generated/github-contributors.json')) as f: data = ujson.load(f) return render( request, 'zerver/team.html', context=data, )
def setUp(self) -> None: """ Manual installation which did not execute `tools/provision` would not have the `static/generated/github-contributors.json` fixture file. """ # This block has unreliable test coverage due to the implicit # caching here, so we exclude it from coverage. if not os.path.exists(static_path('generated/github-contributors.json')): # Copy the fixture file in `zerver/tests/fixtures` to `static/generated` update_script = os.path.join(os.path.dirname(__file__), '../../tools/update-authors-json') # nocoverage subprocess.check_call([update_script, '--use-fixture']) # nocoverage
def get_available_notification_sounds() -> List[str]: notification_sounds_path = static_path("audio/notification_sounds") available_notification_sounds = [] for file_name in os.listdir(notification_sounds_path): root, ext = os.path.splitext(file_name) if "." in root: # nocoverage # Exclude e.g. zulip.abcd1234.ogg (generated by production hash-naming) # to avoid spurious duplicates. continue if ext == ".ogg": available_notification_sounds.append(root) return sorted(available_notification_sounds)
def team_view(request: HttpRequest) -> HttpResponse: with open(static_path('generated/github-contributors.json')) as f: data = ujson.load(f) return render( request, 'zerver/team.html', context={ 'page_params': { 'contrib': data['contrib'], }, 'date': data['date'], }, )
def generate_dev_ldap_dir(mode: str, num_users: int = 8) -> Dict[str, Dict[str, Any]]: mode = mode.lower() ldap_data = [] for i in range(1, num_users + 1): name = f"LDAP User {i}" email = f"ldapuser{i}@zulip.com" phone_number = f"999999999{i}" birthdate = f"19{i:02}-{i:02}-{i:02}" ldap_data.append((name, email, phone_number, birthdate)) profile_images = [] for path in glob.glob(os.path.join(static_path("images/team"), "*")): with open(path, "rb") as f: profile_images.append(f.read()) ldap_dir = {} for i, user_data in enumerate(ldap_data): email = user_data[1].lower() email_username = email.split("@")[0] common_data = { "cn": [user_data[0]], "userPassword": [email_username], "phoneNumber": [user_data[2]], "birthDate": [user_data[3]], } if mode == "a": ldap_dir["uid=" + email + ",ou=users,dc=zulip,dc=com"] = dict( uid=[email], thumbnailPhoto=[profile_images[i % len(profile_images)]], userAccountControl=[LDAP_USER_ACCOUNT_CONTROL_NORMAL], **common_data, ) elif mode == "b": ldap_dir["uid=" + email_username + ",ou=users,dc=zulip,dc=com"] = dict( uid=[email_username], jpegPhoto=[profile_images[i % len(profile_images)]], **common_data, ) elif mode == "c": ldap_dir["uid=" + email_username + ",ou=users,dc=zulip,dc=com"] = dict(uid=[email_username], email=[email], **common_data) return ldap_dir
def generate_dev_ldap_dir(mode: str, num_users: int = 8) -> Dict[str, Dict[str, Any]]: mode = mode.lower() ldap_data = [] for i in range(1, num_users + 1): name = f'LDAP User {i}' email = f'ldapuser{i}@zulip.com' phone_number = f'999999999{i}' birthdate = f'19{i:02}-{i:02}-{i:02}' ldap_data.append((name, email, phone_number, birthdate)) profile_images = [ open(path, "rb").read() for path in glob.glob(os.path.join(static_path("images/team"), "*")) ] ldap_dir = {} for i, user_data in enumerate(ldap_data): email = user_data[1].lower() email_username = email.split('@')[0] common_data = { 'cn': [user_data[0]], 'userPassword': [email_username], 'phoneNumber': [user_data[2]], 'birthDate': [user_data[3]], } if mode == 'a': ldap_dir['uid=' + email + ',ou=users,dc=zulip,dc=com'] = dict( uid=[email], thumbnailPhoto=[profile_images[i % len(profile_images)]], userAccountControl=[LDAP_USER_ACCOUNT_CONTROL_NORMAL], **common_data) elif mode == 'b': ldap_dir['uid=' + email_username + ',ou=users,dc=zulip,dc=com'] = dict( uid=[email_username], jpegPhoto=[profile_images[i % len(profile_images)]], **common_data) elif mode == 'c': ldap_dir['uid=' + email_username + ',ou=users,dc=zulip,dc=com'] = dict(uid=[email_username], email=[email], **common_data) return ldap_dir
import os import re import ujson from django.utils.translation import ugettext as _ from typing import Optional, Tuple from zerver.lib.request import JsonableError from zerver.lib.storage import static_path from zerver.lib.upload import upload_backend from zerver.models import Reaction, Realm, RealmEmoji, UserProfile EMOJI_PATH = static_path("generated/emoji") NAME_TO_CODEPOINT_PATH = os.path.join(EMOJI_PATH, "name_to_codepoint.json") CODEPOINT_TO_NAME_PATH = os.path.join(EMOJI_PATH, "codepoint_to_name.json") EMOTICON_CONVERSIONS_PATH = os.path.join(EMOJI_PATH, "emoticon_conversions.json") with open(NAME_TO_CODEPOINT_PATH) as fp: name_to_codepoint = ujson.load(fp) with open(CODEPOINT_TO_NAME_PATH) as fp: codepoint_to_name = ujson.load(fp) with open(EMOTICON_CONVERSIONS_PATH) as fp: EMOTICON_CONVERSIONS = ujson.load(fp) possible_emoticons = EMOTICON_CONVERSIONS.keys() possible_emoticon_regexes = (re.escape(emoticon) for emoticon in possible_emoticons) terminal_symbols = ',.;?!()\\[\\] "\'\\n\\t' # from composebox_typeahead.js
def handle(self, **options: Any) -> None: if options["percent_huddles"] + options["percent_personals"] > 100: self.stderr.write( "Error! More than 100% of messages allocated.\n") return # Get consistent data for backend tests. if options["test_suite"]: random.seed(0) if options["delete"]: # Start by clearing all the data in our database clear_database() # Create our three default realms # Could in theory be done via zerver.lib.actions.do_create_realm, but # welcome-bot (needed for do_create_realm) hasn't been created yet create_internal_realm() zulip_realm = Realm.objects.create( string_id="zulip", name="Zulip Dev", emails_restricted_to_domains=True, description= "The Zulip development environment default organization." " It's great for testing!", invite_required=False, org_type=Realm.CORPORATE) RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com") if options["test_suite"]: mit_realm = Realm.objects.create( string_id="zephyr", name="MIT", emails_restricted_to_domains=True, invite_required=False, org_type=Realm.CORPORATE) RealmDomain.objects.create(realm=mit_realm, domain="mit.edu") lear_realm = Realm.objects.create( string_id="lear", name="Lear & Co.", emails_restricted_to_domains=False, invite_required=False, org_type=Realm.CORPORATE) # Create test Users (UserProfiles are automatically created, # as are subscriptions to the ability to receive personals). names = [ ("Zoe", "*****@*****.**"), ("Othello, the Moor of Venice", "*****@*****.**"), ("Iago", "*****@*****.**"), ("Prospero from The Tempest", "*****@*****.**"), ("Cordelia Lear", "*****@*****.**"), ("King Hamlet", "*****@*****.**"), ("aaron", "*****@*****.**"), ("Polonius", "*****@*****.**"), ] # For testing really large batches: # Create extra users with semi realistic names to make search # functions somewhat realistic. We'll still create 1000 users # like Extra222 User for some predicability. num_names = options['extra_users'] num_boring_names = 1000 for i in range(min(num_names, num_boring_names)): full_name = 'Extra%03d User' % (i, ) names.append((full_name, '*****@*****.**' % (i, ))) if num_names > num_boring_names: fnames = [ 'Amber', 'Arpita', 'Bob', 'Cindy', 'Daniela', 'Dan', 'Dinesh', 'Faye', 'François', 'George', 'Hank', 'Irene', 'James', 'Janice', 'Jenny', 'Jill', 'John', 'Kate', 'Katelyn', 'Kobe', 'Lexi', 'Manish', 'Mark', 'Matt', 'Mayna', 'Michael', 'Pete', 'Peter', 'Phil', 'Phillipa', 'Preston', 'Sally', 'Scott', 'Sandra', 'Steve', 'Stephanie', 'Vera' ] mnames = ['de', 'van', 'von', 'Shaw', 'T.'] lnames = [ 'Adams', 'Agarwal', 'Beal', 'Benson', 'Bonita', 'Davis', 'George', 'Harden', 'James', 'Jones', 'Johnson', 'Jordan', 'Lee', 'Leonard', 'Singh', 'Smith', 'Patel', 'Towns', 'Wall' ] for i in range(num_boring_names, num_names): fname = random.choice(fnames) + str(i) full_name = fname if random.random() < 0.7: if random.random() < 0.5: full_name += ' ' + random.choice(mnames) full_name += ' ' + random.choice(lnames) email = fname.lower() + '@zulip.com' names.append((full_name, email)) create_users(zulip_realm, names, tos_version=settings.TOS_VERSION) iago = get_user("*****@*****.**", zulip_realm) do_change_is_admin(iago, True) iago.is_staff = True iago.save(update_fields=['is_staff']) guest_user = get_user("*****@*****.**", zulip_realm) guest_user.role = UserProfile.ROLE_GUEST guest_user.save(update_fields=['role']) # These bots are directly referenced from code and thus # are needed for the test suite. zulip_realm_bots = [ ("Zulip Error Bot", "*****@*****.**"), ("Zulip Default Bot", "*****@*****.**"), ] for i in range(options["extra_bots"]): zulip_realm_bots.append( ('Extra Bot %d' % (i, ), '*****@*****.**' % (i, ))) create_users(zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT) zoe = get_user("*****@*****.**", zulip_realm) zulip_webhook_bots = [ ("Zulip Webhook Bot", "*****@*****.**"), ] # If a stream is not supplied in the webhook URL, the webhook # will (in some cases) send the notification as a PM to the # owner of the webhook bot, so bot_owner can't be None create_users(zulip_realm, zulip_webhook_bots, bot_type=UserProfile.INCOMING_WEBHOOK_BOT, bot_owner=zoe) aaron = get_user("*****@*****.**", zulip_realm) zulip_outgoing_bots = [("Outgoing Webhook", "*****@*****.**")] create_users(zulip_realm, zulip_outgoing_bots, bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, bot_owner=aaron) outgoing_webhook = get_user("*****@*****.**", zulip_realm) add_service("outgoing-webhook", user_profile=outgoing_webhook, interface=Service.GENERIC, base_url="http://127.0.0.1:5002", token=generate_api_key()) # Add the realm internl bots to each realm. create_if_missing_realm_internal_bots() # Create public streams. stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"] stream_dict = { "Verona": { "description": "A city in Italy" }, "Denmark": { "description": "A Scandinavian country" }, "Scotland": { "description": "Located in the United Kingdom" }, "Venice": { "description": "A northeastern Italian city" }, "Rome": { "description": "Yet another Italian city", "is_web_public": True } } # type: Dict[str, Dict[str, Any]] bulk_create_streams(zulip_realm, stream_dict) recipient_streams = [ Stream.objects.get(name=name, realm=zulip_realm).id for name in stream_list ] # type: List[int] # Create subscriptions to streams. The following # algorithm will give each of the users a different but # deterministic subset of the streams (given a fixed list # of users). For the test suite, we have a fixed list of # subscriptions to make sure test data is consistent # across platforms. subscriptions_list = [ ] # type: List[Tuple[UserProfile, Recipient]] profiles = UserProfile.objects.select_related().filter( is_bot=False).order_by("email") # type: Sequence[UserProfile] if options["test_suite"]: subscriptions_map = { '*****@*****.**': ['Verona'], '*****@*****.**': ['Verona'], '*****@*****.**': ['Verona', 'Denmark'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice'], '*****@*****.**': ['Verona', 'Denmark', 'Scotland', 'Venice', 'Rome'], '*****@*****.**': ['Verona'], } for profile in profiles: if profile.email not in subscriptions_map: raise Exception( 'Subscriptions not listed for user %s' % (profile.email, )) for stream_name in subscriptions_map[profile.email]: stream = Stream.objects.get(name=stream_name) r = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id) subscriptions_list.append((profile, r)) else: num_streams = len(recipient_streams) num_users = len(profiles) for i, profile in enumerate(profiles): # Subscribe to some streams. fraction = float(i) / num_users num_recips = int(num_streams * fraction) + 1 for type_id in recipient_streams[:num_recips]: r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id) subscriptions_list.append((profile, r)) subscriptions_to_add = [] # type: List[Subscription] event_time = timezone_now() all_subscription_logs = [] # type: (List[RealmAuditLog]) i = 0 for profile, recipient in subscriptions_list: i += 1 color = STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)] s = Subscription(recipient=recipient, user_profile=profile, color=color) subscriptions_to_add.append(s) log = RealmAuditLog( realm=profile.realm, modified_user=profile, modified_stream_id=recipient.type_id, event_last_message_id=0, event_type=RealmAuditLog.SUBSCRIPTION_CREATED, event_time=event_time) all_subscription_logs.append(log) Subscription.objects.bulk_create(subscriptions_to_add) RealmAuditLog.objects.bulk_create(all_subscription_logs) # Create custom profile field data phone_number = try_add_realm_custom_profile_field( zulip_realm, "Phone number", CustomProfileField.SHORT_TEXT, hint='') biography = try_add_realm_custom_profile_field( zulip_realm, "Biography", CustomProfileField.LONG_TEXT, hint='What are you known for?') favorite_food = try_add_realm_custom_profile_field( zulip_realm, "Favorite food", CustomProfileField.SHORT_TEXT, hint="Or drink, if you'd prefer") field_data = { 'vim': { 'text': 'Vim', 'order': '1' }, 'emacs': { 'text': 'Emacs', 'order': '2' }, } # type: ProfileFieldData favorite_editor = try_add_realm_custom_profile_field( zulip_realm, "Favorite editor", CustomProfileField.CHOICE, field_data=field_data) birthday = try_add_realm_custom_profile_field( zulip_realm, "Birthday", CustomProfileField.DATE) favorite_website = try_add_realm_custom_profile_field( zulip_realm, "Favorite website", CustomProfileField.URL, hint="Or your personal blog's URL") mentor = try_add_realm_custom_profile_field( zulip_realm, "Mentor", CustomProfileField.USER) github_profile = try_add_realm_default_custom_profile_field( zulip_realm, "github") # Fill in values for Iago and Hamlet hamlet = get_user("*****@*****.**", zulip_realm) do_update_user_custom_profile_data_if_changed( iago, [ { "id": phone_number.id, "value": "+1-234-567-8901" }, { "id": biography.id, "value": "Betrayer of Othello." }, { "id": favorite_food.id, "value": "Apples" }, { "id": favorite_editor.id, "value": "emacs" }, { "id": birthday.id, "value": "2000-1-1" }, { "id": favorite_website.id, "value": "https://zulip.readthedocs.io/en/latest/" }, { "id": mentor.id, "value": [hamlet.id] }, { "id": github_profile.id, "value": 'zulip' }, ]) do_update_user_custom_profile_data_if_changed( hamlet, [ { "id": phone_number.id, "value": "+0-11-23-456-7890" }, { "id": biography.id, "value": "I am:\n* The prince of Denmark\n* Nephew to the usurping Claudius", }, { "id": favorite_food.id, "value": "Dark chocolate" }, { "id": favorite_editor.id, "value": "vim" }, { "id": birthday.id, "value": "1900-1-1" }, { "id": favorite_website.id, "value": "https://blog.zulig.org" }, { "id": mentor.id, "value": [iago.id] }, { "id": github_profile.id, "value": 'zulipbot' }, ]) else: zulip_realm = get_realm("zulip") recipient_streams = [ klass.type_id for klass in Recipient.objects.filter(type=Recipient.STREAM) ] # Extract a list of all users user_profiles = list(UserProfile.objects.filter( is_bot=False)) # type: List[UserProfile] # Create a test realm emoji. IMAGE_FILE_PATH = static_path('images/test-images/checkbox.png') with open(IMAGE_FILE_PATH, 'rb') as fp: check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp) if not options["test_suite"]: # Populate users with some bar data for user in user_profiles: status = UserPresence.ACTIVE # type: int date = timezone_now() client = get_client("website") if user.full_name[0] <= 'H': client = get_client("ZulipAndroid") UserPresence.objects.get_or_create(user_profile=user, client=client, timestamp=date, status=status) user_profiles_ids = [user_profile.id for user_profile in user_profiles] # Create several initial huddles for i in range(options["num_huddles"]): get_huddle(random.sample(user_profiles_ids, random.randint(3, 4))) # Create several initial pairs for personals personals_pairs = [ random.sample(user_profiles_ids, 2) for i in range(options["num_personals"]) ] # Generate a new set of test data. create_test_data() # prepopulate the URL preview/embed data for the links present # in the config.generate_data.json data set. This makes it # possible for populate_db to run happily without Internet # access. with open("zerver/tests/fixtures/docs_url_preview_data.json", "r") as f: urls_with_preview_data = ujson.load(f) for url in urls_with_preview_data: cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME) threads = options["threads"] jobs = [ ] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]] for i in range(threads): count = options["num_messages"] // threads if i < options["num_messages"] % threads: count += 1 jobs.append((count, personals_pairs, options, self.stdout.write, random.randint(0, 10**10))) for job in jobs: generate_and_send_messages(job) if options["delete"]: if options["test_suite"]: # Create test users; the MIT ones are needed to test # the Zephyr mirroring codepaths. testsuite_mit_users = [ ("Fred Sipb (MIT)", "*****@*****.**"), ("Athena Consulting Exchange User (MIT)", "*****@*****.**"), ("Esp Classroom (MIT)", "*****@*****.**"), ] create_users(mit_realm, testsuite_mit_users, tos_version=settings.TOS_VERSION) testsuite_lear_users = [ ("King Lear", "*****@*****.**"), ("Cordelia Lear", "*****@*****.**"), ] create_users(lear_realm, testsuite_lear_users, tos_version=settings.TOS_VERSION) if not options["test_suite"]: # To keep the messages.json fixtures file for the test # suite fast, don't add these users and subscriptions # when running populate_db for the test suite zulip_stream_dict = { "devel": { "description": "For developing" }, "all": { "description": "For **everything**" }, "announce": { "description": "For announcements", 'stream_post_policy': Stream.STREAM_POST_POLICY_ADMINS }, "design": { "description": "For design" }, "support": { "description": "For support" }, "social": { "description": "For socializing" }, "test": { "description": "For testing `code`" }, "errors": { "description": "For errors" }, "sales": { "description": "For sales discussion" } } # type: Dict[str, Dict[str, Any]] # Calculate the maximum number of digits in any extra stream's # number, since a stream with name "Extra Stream 3" could show # up after "Extra Stream 29". (Used later to pad numbers with # 0s). maximum_digits = len(str(options['extra_streams'] - 1)) for i in range(options['extra_streams']): # Pad the number with 0s based on `maximum_digits`. number_str = str(i).zfill(maximum_digits) extra_stream_name = 'Extra Stream ' + number_str zulip_stream_dict[extra_stream_name] = { "description": "Auto-generated extra stream.", } bulk_create_streams(zulip_realm, zulip_stream_dict) # Now that we've created the notifications stream, configure it properly. zulip_realm.notifications_stream = get_stream( "announce", zulip_realm) zulip_realm.save(update_fields=['notifications_stream']) # Add a few default streams for default_stream_name in [ "design", "devel", "social", "support" ]: DefaultStream.objects.create(realm=zulip_realm, stream=get_stream( default_stream_name, zulip_realm)) # Now subscribe everyone to these streams subscribe_users_to_streams(zulip_realm, zulip_stream_dict) # These bots are not needed by the test suite internal_zulip_users_nosubs = [ ("Zulip Commit Bot", "*****@*****.**"), ("Zulip Trac Bot", "*****@*****.**"), ("Zulip Nagios Bot", "*****@*****.**"), ] create_users(zulip_realm, internal_zulip_users_nosubs, bot_type=UserProfile.DEFAULT_BOT) # Mark all messages as read UserMessage.objects.all().update(flags=UserMessage.flags.read) if not options["test_suite"]: # Update pointer of each user to point to the last message in their # UserMessage rows with sender_id=user_profile_id. users = list( UserMessage.objects.filter(message__sender_id=F( 'user_profile_id')).values('user_profile_id').annotate( pointer=Max('message_id'))) for user in users: UserProfile.objects.filter( id=user['user_profile_id']).update( pointer=user['pointer']) create_user_groups() if not options["test_suite"]: # We populate the analytics database here for # development purpose only call_command('populate_analytics_db') self.stdout.write("Successfully populated test database.\n")
import os import re from typing import Optional, Tuple import orjson from django.utils.translation import ugettext as _ from zerver.lib.exceptions import OrganizationAdministratorRequired from zerver.lib.request import JsonableError from zerver.lib.storage import static_path from zerver.lib.upload import upload_backend from zerver.models import Reaction, Realm, RealmEmoji, UserProfile emoji_codes_path = static_path("generated/emoji/emoji_codes.json") if not os.path.exists(emoji_codes_path): # nocoverage # During the collectstatic step of build-release-tarball, # prod-static/serve/generated/emoji won't exist yet. emoji_codes_path = os.path.join( os.path.dirname(__file__), "../../static/generated/emoji/emoji_codes.json", ) with open(emoji_codes_path, "rb") as fp: emoji_codes = orjson.loads(fp.read()) name_to_codepoint = emoji_codes["name_to_codepoint"] codepoint_to_name = emoji_codes["codepoint_to_name"] EMOTICON_CONVERSIONS = emoji_codes["emoticon_conversions"] possible_emoticons = EMOTICON_CONVERSIONS.keys() possible_emoticon_regexes = (re.escape(emoticon)
def handle(self, **options: Any) -> None: if options["percent_huddles"] + options["percent_personals"] > 100: self.stderr.write( "Error! More than 100% of messages allocated.\n") return # Get consistent data for backend tests. if options["test_suite"]: random.seed(0) with connection.cursor() as cursor: # Sometimes bugs relating to confusing recipient.id for recipient.type_id # or <object>.id for <object>.recipient_id remain undiscovered by the test suite # due to these numbers happening to coincide in such a way that it makes tests # accidentally pass. By bumping the Recipient.id sequence by a large enough number, # we can have those ids in a completely different range of values than object ids, # eliminatng the possibility of such coincidences. cursor.execute("SELECT setval('zerver_recipient_id_seq', 100)") # If max_topics is not set, we set it proportional to the # number of messages. if options["max_topics"] is None: options["max_topics"] = 1 + options["num_messages"] // 100 if options["delete"]: # Start by clearing all the data in our database clear_database() # Create our three default realms # Could in theory be done via zerver.lib.actions.do_create_realm, but # welcome-bot (needed for do_create_realm) hasn't been created yet create_internal_realm() zulip_realm = do_create_realm( string_id="zulip", name="Zulip Dev", emails_restricted_to_domains=False, email_address_visibility=Realm.EMAIL_ADDRESS_VISIBILITY_ADMINS, description= "The Zulip development environment default organization." " It's great for testing!", invite_required=False, plan_type=Realm.SELF_HOSTED, org_type=Realm.ORG_TYPES["business"]["id"], ) RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com") assert zulip_realm.notifications_stream is not None zulip_realm.notifications_stream.name = "Verona" zulip_realm.notifications_stream.description = "A city in Italy" zulip_realm.notifications_stream.save( update_fields=["name", "description"]) if options["test_suite"]: mit_realm = do_create_realm( string_id="zephyr", name="MIT", emails_restricted_to_domains=True, invite_required=False, plan_type=Realm.SELF_HOSTED, org_type=Realm.ORG_TYPES["business"]["id"], ) RealmDomain.objects.create(realm=mit_realm, domain="mit.edu") lear_realm = do_create_realm( string_id="lear", name="Lear & Co.", emails_restricted_to_domains=False, invite_required=False, plan_type=Realm.SELF_HOSTED, org_type=Realm.ORG_TYPES["business"]["id"], ) # Default to allowing all members to send mentions in # large streams for the test suite to keep # mention-related tests simple. zulip_realm.wildcard_mention_policy = Realm.WILDCARD_MENTION_POLICY_MEMBERS zulip_realm.save(update_fields=["wildcard_mention_policy"]) # Create test Users (UserProfiles are automatically created, # as are subscriptions to the ability to receive personals). names = [ ("Zoe", "*****@*****.**"), ("Othello, the Moor of Venice", "*****@*****.**"), ("Iago", "*****@*****.**"), ("Prospero from The Tempest", "*****@*****.**"), ("Cordelia, Lear's daughter", "*****@*****.**"), ("King Hamlet", "*****@*****.**"), ("aaron", "*****@*****.**"), ("Polonius", "*****@*****.**"), ("Desdemona", "*****@*****.**"), ("शिव", "*****@*****.**"), ] # For testing really large batches: # Create extra users with semi realistic names to make search # functions somewhat realistic. We'll still create 1000 users # like Extra222 User for some predicability. num_names = options["extra_users"] num_boring_names = 300 for i in range(min(num_names, num_boring_names)): full_name = f"Extra{i:03} User" names.append((full_name, f"extrauser{i}@zulip.com")) if num_names > num_boring_names: fnames = [ "Amber", "Arpita", "Bob", "Cindy", "Daniela", "Dan", "Dinesh", "Faye", "François", "George", "Hank", "Irene", "James", "Janice", "Jenny", "Jill", "John", "Kate", "Katelyn", "Kobe", "Lexi", "Manish", "Mark", "Matt", "Mayna", "Michael", "Pete", "Peter", "Phil", "Phillipa", "Preston", "Sally", "Scott", "Sandra", "Steve", "Stephanie", "Vera", ] mnames = ["de", "van", "von", "Shaw", "T."] lnames = [ "Adams", "Agarwal", "Beal", "Benson", "Bonita", "Davis", "George", "Harden", "James", "Jones", "Johnson", "Jordan", "Lee", "Leonard", "Singh", "Smith", "Patel", "Towns", "Wall", ] non_ascii_names = [ "Günter", "أحمد", "Magnús", "आशी", "イツキ", "语嫣", "அருண்", "Александр", "José", ] # to imitate emoji insertions in usernames raw_emojis = ["😎", "😂", "🐱👤"] for i in range(num_boring_names, num_names): fname = random.choice(fnames) + str(i) full_name = fname if random.random() < 0.7: if random.random() < 0.3: full_name += " " + random.choice(non_ascii_names) else: full_name += " " + random.choice(mnames) if random.random() < 0.1: full_name += " {} ".format(random.choice(raw_emojis)) else: full_name += " " + random.choice(lnames) email = fname.lower() + "@zulip.com" names.append((full_name, email)) create_users(zulip_realm, names, tos_version=settings.TOS_VERSION) iago = get_user_by_delivery_email("*****@*****.**", zulip_realm) do_change_user_role(iago, UserProfile.ROLE_REALM_ADMINISTRATOR, acting_user=None) iago.is_staff = True iago.save(update_fields=["is_staff"]) desdemona = get_user_by_delivery_email("*****@*****.**", zulip_realm) do_change_user_role(desdemona, UserProfile.ROLE_REALM_OWNER, acting_user=None) shiva = get_user_by_delivery_email("*****@*****.**", zulip_realm) do_change_user_role(shiva, UserProfile.ROLE_MODERATOR, acting_user=None) guest_user = get_user_by_delivery_email("*****@*****.**", zulip_realm) guest_user.role = UserProfile.ROLE_GUEST guest_user.save(update_fields=["role"]) # These bots are directly referenced from code and thus # are needed for the test suite. zulip_realm_bots = [ ("Zulip Error Bot", "*****@*****.**"), ("Zulip Default Bot", "*****@*****.**"), ] for i in range(options["extra_bots"]): zulip_realm_bots.append( (f"Extra Bot {i}", f"extrabot{i}@zulip.com")) create_users(zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT) zoe = get_user_by_delivery_email("*****@*****.**", zulip_realm) zulip_webhook_bots = [ ("Zulip Webhook Bot", "*****@*****.**"), ] # If a stream is not supplied in the webhook URL, the webhook # will (in some cases) send the notification as a PM to the # owner of the webhook bot, so bot_owner can't be None create_users( zulip_realm, zulip_webhook_bots, bot_type=UserProfile.INCOMING_WEBHOOK_BOT, bot_owner=zoe, ) aaron = get_user_by_delivery_email("*****@*****.**", zulip_realm) zulip_outgoing_bots = [ ("Outgoing Webhook", "*****@*****.**"), ] create_users( zulip_realm, zulip_outgoing_bots, bot_type=UserProfile.OUTGOING_WEBHOOK_BOT, bot_owner=aaron, ) outgoing_webhook = get_user("*****@*****.**", zulip_realm) add_service( "outgoing-webhook", user_profile=outgoing_webhook, interface=Service.GENERIC, base_url="http://127.0.0.1:5002", token=generate_api_key(), ) # Add the realm internal bots to each realm. create_if_missing_realm_internal_bots() # Create public streams. signups_stream = Realm.INITIAL_PRIVATE_STREAM_NAME stream_list = [ "Verona", "Denmark", "Scotland", "Venice", "Rome", signups_stream, ] stream_dict: Dict[str, Dict[str, Any]] = { "Denmark": { "description": "A Scandinavian country" }, "Scotland": { "description": "Located in the United Kingdom" }, "Venice": { "description": "A northeastern Italian city" }, "Rome": { "description": "Yet another Italian city", "is_web_public": True }, } bulk_create_streams(zulip_realm, stream_dict) recipient_streams: List[int] = [ Stream.objects.get(name=name, realm=zulip_realm).id for name in stream_list ] # Create subscriptions to streams. The following # algorithm will give each of the users a different but # deterministic subset of the streams (given a fixed list # of users). For the test suite, we have a fixed list of # subscriptions to make sure test data is consistent # across platforms. subscriptions_list: List[Tuple[UserProfile, Recipient]] = [] profiles: Sequence[UserProfile] = ( UserProfile.objects.select_related().filter( is_bot=False).order_by("email")) if options["test_suite"]: subscriptions_map = { "*****@*****.**": ["Verona"], "*****@*****.**": ["Verona"], "*****@*****.**": ["Verona", "Denmark", signups_stream], "*****@*****.**": [ "Verona", "Denmark", "Scotland", signups_stream, ], "*****@*****.**": ["Verona", "Denmark", "Scotland"], "*****@*****.**": ["Verona", "Denmark", "Scotland", "Venice"], "*****@*****.**": ["Verona", "Denmark", "Scotland", "Venice", "Rome"], "*****@*****.**": ["Verona"], "*****@*****.**": [ "Verona", "Denmark", "Venice", signups_stream, ], "*****@*****.**": ["Verona", "Denmark", "Scotland"], } for profile in profiles: email = profile.delivery_email if email not in subscriptions_map: raise Exception( f"Subscriptions not listed for user {email}") for stream_name in subscriptions_map[email]: stream = Stream.objects.get(name=stream_name, realm=zulip_realm) r = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id) subscriptions_list.append((profile, r)) else: num_streams = len(recipient_streams) num_users = len(profiles) for i, profile in enumerate(profiles): # Subscribe to some streams. fraction = float(i) / num_users num_recips = int(num_streams * fraction) + 1 for type_id in recipient_streams[:num_recips]: r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id) subscriptions_list.append((profile, r)) subscriptions_to_add: List[Subscription] = [] event_time = timezone_now() all_subscription_logs: (List[RealmAuditLog]) = [] i = 0 for profile, recipient in subscriptions_list: i += 1 color = STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)] s = Subscription( recipient=recipient, user_profile=profile, is_user_active=profile.is_active, color=color, ) subscriptions_to_add.append(s) log = RealmAuditLog( realm=profile.realm, modified_user=profile, modified_stream_id=recipient.type_id, event_last_message_id=0, event_type=RealmAuditLog.SUBSCRIPTION_CREATED, event_time=event_time, ) all_subscription_logs.append(log) Subscription.objects.bulk_create(subscriptions_to_add) RealmAuditLog.objects.bulk_create(all_subscription_logs) # Create custom profile field data phone_number = try_add_realm_custom_profile_field( zulip_realm, "Phone number", CustomProfileField.SHORT_TEXT, hint="") biography = try_add_realm_custom_profile_field( zulip_realm, "Biography", CustomProfileField.LONG_TEXT, hint="What are you known for?", ) favorite_food = try_add_realm_custom_profile_field( zulip_realm, "Favorite food", CustomProfileField.SHORT_TEXT, hint="Or drink, if you'd prefer", ) field_data: ProfileFieldData = { "vim": { "text": "Vim", "order": "1" }, "emacs": { "text": "Emacs", "order": "2" }, } favorite_editor = try_add_realm_custom_profile_field( zulip_realm, "Favorite editor", CustomProfileField.SELECT, field_data=field_data) birthday = try_add_realm_custom_profile_field( zulip_realm, "Birthday", CustomProfileField.DATE) favorite_website = try_add_realm_custom_profile_field( zulip_realm, "Favorite website", CustomProfileField.URL, hint="Or your personal blog's URL", ) mentor = try_add_realm_custom_profile_field( zulip_realm, "Mentor", CustomProfileField.USER) github_profile = try_add_realm_default_custom_profile_field( zulip_realm, "github") # Fill in values for Iago and Hamlet hamlet = get_user_by_delivery_email("*****@*****.**", zulip_realm) do_update_user_custom_profile_data_if_changed( iago, [ { "id": phone_number.id, "value": "+1-234-567-8901" }, { "id": biography.id, "value": "Betrayer of Othello." }, { "id": favorite_food.id, "value": "Apples" }, { "id": favorite_editor.id, "value": "emacs" }, { "id": birthday.id, "value": "2000-01-01" }, { "id": favorite_website.id, "value": "https://zulip.readthedocs.io/en/latest/" }, { "id": mentor.id, "value": [hamlet.id] }, { "id": github_profile.id, "value": "zulip" }, ], ) do_update_user_custom_profile_data_if_changed( hamlet, [ { "id": phone_number.id, "value": "+0-11-23-456-7890" }, { "id": biography.id, "value": "I am:\n* The prince of Denmark\n* Nephew to the usurping Claudius", }, { "id": favorite_food.id, "value": "Dark chocolate" }, { "id": favorite_editor.id, "value": "vim" }, { "id": birthday.id, "value": "1900-01-01" }, { "id": favorite_website.id, "value": "https://blog.zulig.org" }, { "id": mentor.id, "value": [iago.id] }, { "id": github_profile.id, "value": "zulipbot" }, ], ) else: zulip_realm = get_realm("zulip") recipient_streams = [ klass.type_id for klass in Recipient.objects.filter(type=Recipient.STREAM) ] # Extract a list of all users user_profiles: List[UserProfile] = list( UserProfile.objects.filter(is_bot=False)) # Create a test realm emoji. IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png") with open(IMAGE_FILE_PATH, "rb") as fp: check_add_realm_emoji(zulip_realm, "green_tick", iago, fp) if not options["test_suite"]: # Populate users with some bar data for user in user_profiles: status: int = UserPresence.ACTIVE date = timezone_now() client = get_client("website") if user.full_name[0] <= "H": client = get_client("ZulipAndroid") UserPresence.objects.get_or_create( user_profile=user, realm_id=user.realm_id, client=client, timestamp=date, status=status, ) user_profiles_ids = [user_profile.id for user_profile in user_profiles] # Create several initial huddles for i in range(options["num_huddles"]): get_huddle(random.sample(user_profiles_ids, random.randint(3, 4))) # Create several initial pairs for personals personals_pairs = [ random.sample(user_profiles_ids, 2) for i in range(options["num_personals"]) ] create_alert_words(zulip_realm.id) # Generate a new set of test data. create_test_data() # prepopulate the URL preview/embed data for the links present # in the config.generate_data.json data set. This makes it # possible for populate_db to run happily without Internet # access. with open("zerver/tests/fixtures/docs_url_preview_data.json", "rb") as f: urls_with_preview_data = orjson.loads(f.read()) for url in urls_with_preview_data: cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME) if options["delete"]: if options["test_suite"]: # Create test users; the MIT ones are needed to test # the Zephyr mirroring codepaths. testsuite_mit_users = [ ("Fred Sipb (MIT)", "*****@*****.**"), ("Athena Consulting Exchange User (MIT)", "*****@*****.**"), ("Esp Classroom (MIT)", "*****@*****.**"), ] create_users(mit_realm, testsuite_mit_users, tos_version=settings.TOS_VERSION) testsuite_lear_users = [ ("King Lear", "*****@*****.**"), ("Cordelia, Lear's daughter", "*****@*****.**"), ] create_users(lear_realm, testsuite_lear_users, tos_version=settings.TOS_VERSION) if not options["test_suite"]: # To keep the messages.json fixtures file for the test # suite fast, don't add these users and subscriptions # when running populate_db for the test suite # to imitate emoji insertions in stream names raw_emojis = ["😎", "😂", "🐱👤"] zulip_stream_dict: Dict[str, Dict[str, Any]] = { "devel": { "description": "For developing" }, # ビデオゲーム - VideoGames (japanese) "ビデオゲーム": { "description": "Share your favorite video games! {}".format( raw_emojis[2]) }, "announce": { "description": "For announcements", "stream_post_policy": Stream.STREAM_POST_POLICY_ADMINS, }, "design": { "description": "For design" }, "support": { "description": "For support" }, "social": { "description": "For socializing" }, "test": { "description": "For testing `code`" }, "errors": { "description": "For errors" }, # 조리법 - Recipes (Korean) , Пельмени - Dumplings (Russian) "조리법 " + raw_emojis[0]: { "description": "Everything cooking, from pasta to Пельмени" }, } extra_stream_names = [ "802.11a", "Ad Hoc Network", "Augmented Reality", "Cycling", "DPI", "FAQ", "FiFo", "commits", "Control panel", "desktop", "компьютеры", "Data security", "desktop", "काम", "discussions", "Cloud storage", "GCI", "Vaporware", "Recent Trends", "issues", "live", "Health", "mobile", "空間", "provision", "hidrógeno", "HR", "アニメ", ] # Add stream names and stream descriptions for i in range(options["extra_streams"]): extra_stream_name = random.choice( extra_stream_names) + " " + str(i) # to imitate emoji insertions in stream names if random.random() <= 0.15: extra_stream_name += random.choice(raw_emojis) zulip_stream_dict[extra_stream_name] = { "description": "Auto-generated extra stream.", } bulk_create_streams(zulip_realm, zulip_stream_dict) # Now that we've created the notifications stream, configure it properly. zulip_realm.notifications_stream = get_stream( "announce", zulip_realm) zulip_realm.save(update_fields=["notifications_stream"]) # Add a few default streams for default_stream_name in [ "design", "devel", "social", "support" ]: DefaultStream.objects.create(realm=zulip_realm, stream=get_stream( default_stream_name, zulip_realm)) # Now subscribe everyone to these streams subscribe_users_to_streams(zulip_realm, zulip_stream_dict) create_user_groups() if not options["test_suite"]: # We populate the analytics database here for # development purpose only call_command("populate_analytics_db") threads = options["threads"] jobs: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]] = [] for i in range(threads): count = options["num_messages"] // threads if i < options["num_messages"] % threads: count += 1 jobs.append((count, personals_pairs, options, self.stdout.write, random.randint(0, 10**10))) for job in jobs: generate_and_send_messages(job) if options["delete"]: if not options["test_suite"]: # These bots are not needed by the test suite # Also, we don't want interacting with each other # in dev setup. internal_zulip_users_nosubs = [ ("Zulip Commit Bot", "*****@*****.**"), ("Zulip Trac Bot", "*****@*****.**"), ("Zulip Nagios Bot", "*****@*****.**"), ] create_users(zulip_realm, internal_zulip_users_nosubs, bot_type=UserProfile.DEFAULT_BOT) mark_all_messages_as_read() self.stdout.write("Successfully populated test database.\n")
import os import re import ujson from django.utils.translation import ugettext as _ from typing import Optional, Tuple from zerver.lib.request import JsonableError from zerver.lib.storage import static_path from zerver.lib.upload import upload_backend from zerver.lib.exceptions import OrganizationAdministratorRequired from zerver.models import Reaction, Realm, RealmEmoji, UserProfile with open(static_path("generated/emoji/emoji_codes.json")) as fp: emoji_codes = ujson.load(fp) name_to_codepoint = emoji_codes["name_to_codepoint"] codepoint_to_name = emoji_codes["codepoint_to_name"] EMOTICON_CONVERSIONS = emoji_codes["emoticon_conversions"] possible_emoticons = EMOTICON_CONVERSIONS.keys() possible_emoticon_regexes = (re.escape(emoticon) for emoticon in possible_emoticons) terminal_symbols = ',.;?!()\\[\\] "\'\\n\\t' # from composebox_typeahead.js emoticon_regex = ('(?<![^{0}])(?P<emoticon>('.format(terminal_symbols) + ')|('.join(possible_emoticon_regexes) + '))(?![^{0}])'.format(terminal_symbols)) # Translates emoticons to their colon syntax, e.g. `:smiley:`. def translate_emoticons(text: str) -> str: translated = text