def _alter_field(editor, model, old_field, new_field, old_type, new_type, old_db_params, new_db_params, strict=False): orig_alter_field(editor, model, old_field, new_field, old_type, new_type, old_db_params, new_db_params, strict=False) if isinstance(new_field,JSONField): old_index=getattr(old_field,"db_index_options",None) new_index=getattr(new_field,"db_index_options",None) if new_index!=old_index: if old_index: normal_index,with_path_index=partition(lambda index:bool(index.get("path")),old_index) index_names = editor._constraint_names(model, [old_field.column], index=True) if normal_index else [] if with_path_index: all_indexes=editor._constraint_names(model, index=True) for index_info in with_path_index: path_hash=editor._digest(index_info["path"]) for i,index in enumerate(all_indexes): if index.endswith(path_hash): index_names.append(all_indexes.pop(i)) break for index_name in index_names: editor.execute(editor._delete_constraint_sql(editor.sql_delete_index, model, index_name)) if new_index: for sql in editor._create_jsonb_index_sql(model,new_field): editor.execute(sql)
def send_messages(self, email_messages): if not email_messages: return msg_count = 0 try: client_created = self.open() prepared_messages = [ self.prepare_message(message) for message in email_messages ] pre_send.send_robust(self.__class__, messages=prepared_messages) responses = self.client.emails.send_batch( *prepared_messages, TrackOpens=self.get_option("TRACK_OPENS")) post_send.send_robust(self.__class__, messages=prepared_messages, response=responses) sent, not_sent = partition(lambda x: x["ErrorCode"] != 0, responses) msg_count = len(sent) if not_sent: self.raise_for_response(not_sent) if client_created: self.close() except Exception as exc: on_exception.send_robust(self.__class__, raw_messages=email_messages, exception=exc) if not self.fail_silently: raise return msg_count
def resolve_expression( self, query=None, allow_joins=True, reuse=None, summarize=False, for_save=False, ): expressions = list(self.flatten()) # Split expressions and wrappers. index_expressions, wrappers = partition( lambda e: isinstance(e, self.wrapper_classes), expressions, ) wrapper_types = [type(wrapper) for wrapper in wrappers] if len(wrapper_types) != len(set(wrapper_types)): raise ValueError( "Multiple references to %s can't be used in an indexed " "expression." % ", ".join( [wrapper_cls.__qualname__ for wrapper_cls in self.wrapper_classes] ) ) if expressions[1 : len(wrappers) + 1] != wrappers: raise ValueError( "%s must be topmost expressions in an indexed expression." % ", ".join( [wrapper_cls.__qualname__ for wrapper_cls in self.wrapper_classes] ) ) # Wrap expressions in parentheses if they are not column references. root_expression = index_expressions[1] resolve_root_expression = root_expression.resolve_expression( query, allow_joins, reuse, summarize, for_save, ) if not isinstance(resolve_root_expression, Col): root_expression = Func(root_expression, template="(%(expressions)s)") if wrappers: # Order wrappers and set their expressions. wrappers = sorted( wrappers, key=lambda w: self.wrapper_classes.index(type(w)), ) wrappers = [wrapper.copy() for wrapper in wrappers] for i, wrapper in enumerate(wrappers[:-1]): wrapper.set_source_expressions([wrappers[i + 1]]) # Set the root expression on the deepest wrapper. wrappers[-1].set_source_expressions([root_expression]) self.set_source_expressions([wrappers[0]]) else: # Use the root expression, if there are no wrappers. self.set_source_expressions([root_expression]) return super().resolve_expression( query, allow_joins, reuse, summarize, for_save )
def list(self, request, *args, **kwargs): # cites_to can contain citations or IDs, so split out IDs into separate # cites_to_id parameter if 'cites_to' in request.query_params: request._request.GET = params = request._request.GET.copy() cites_to, cites_to_id = partition(lambda c: c.isdigit(), params.getlist('cites_to')) params.setlist('cites_to', cites_to) params.setlist('cites_to_id', cites_to_id) response = super(CaseDocumentViewSet, self).list(request, *args, **kwargs) if request.accepted_renderer.format == 'csv': response = self.bundle_csv_response(response) return response
def bulk_update(model, instances, update_fields=None): instances, create = partition(lambda obj: obj.pk is None, instances) set_fields = ', '.join( ('%(field)s=update_{table}.column%(i)s' % {'field': field, 'i': i + 2}) for i, field in enumerate(update_fields) ) value_placeholder = '({})'.format(', '.join(['%s'] * (len(update_fields) + 1))) values = ','.join([value_placeholder] * len(instances)) query = ' '.join([ 'UPDATE {table} SET ', set_fields, 'FROM (VALUES ', values, ') update_{table}', 'WHERE {table}.{pk} = update_{table}.column1' ]).format(table=model._meta.db_table, pk=model._meta.pk.get_attname_column()[1]) print(query) params = [] for instance in instances: params.append(instance.pk) for field in update_fields: params.append(getattr(instance, field)) connection.cursor().execute(query, params)
from django.test import TestCase # Create your tests here. from django.utils.functional import partition # list_a = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] k = partition(lambda a: a > 2, range(6)) print(k)
def handle(self, *args, **kwargs): print(datetime.today(), "Building dashboard...") pages = {os.path.basename(x).rstrip('.md'): open(x, 'r').read() for x in glob.glob(Filepaths.BACKUP_DIR.value) if os.path.isfile(x)} graph = Graph() flashcards: List[AnkiFlashcard] = [] full_deck_names = make_anki_request('deckNames') # Create nodes for page_title, page in pages.items(): lines = squeeze(page.split('\n')) if not len(lines): continue for page_type in [Tags.QUESTION.value, Tags.REFERENCE.value, Tags.NOTE.value, Tags.POST.value]: matches = [x for x in lines if f'#{page_type}' in x] content, metadata = partition(lambda x: '::' in x, matches) for line in content: # This node is a single line. graph.add_node(line.strip().lstrip('- '), type=page_type, word_count=len(line.split(' '))) if len(metadata): # This node is a page. graph.add_node(page_title, lines=lines, type=page_type) # Find flashcards bulleted_lines = get_bulleted_lines(lines) num_bulleted_lines = len(bulleted_lines) for i, line in enumerate(bulleted_lines): if f"#{Tags.FLASHCARD.value}" in line: well_formatted_flashcard = re.match(FLASHCARD_FRONT_REGEX, line) if not well_formatted_flashcard: post_slack(f"Flashcard front improperly formatted: {line}") continue front_1, flashcard_uuid, front_2 = well_formatted_flashcard.groups() front = front_1.strip() + " " + front_2.strip() # Handle Cloze-style flashcards is_cloze = re.search(CLOZE_REGEX, front) if is_cloze: front = re.sub(CLOZE_REGEX, lambda x: "{{c1::" + x.group(1) + "}}", front) front = AnkiFlashcard.maybe_download_media_files_and_convert(front) + f"<hr/><div>{page_title}</div>" maybe_deck_name, line = extract_tags(ANKI_DECK_TAG_REGEX, line) deck_name = get_full_deck_name(full_deck_names, maybe_deck_name) flashcards.append(ClozeFlashcard(uuid=flashcard_uuid, front=front, deck=deck_name)) continue is_num_lines_long_enough = i + 1 < num_bulleted_lines if not is_num_lines_long_enough: post_slack(f"Flashcard is missing a back: {line}") continue # Handle two-sided flashcards next_line = bulleted_lines[i + 1] next_line_is_indented = re.match(BULLET_REGEX, line).span()[1] < re.match(BULLET_REGEX, next_line).span()[1] if next_line_is_indented: back = re.match(SANS_BULLET_REGEX, next_line).group(1).strip() front = AnkiFlashcard.maybe_download_media_files_and_convert(front) back = AnkiFlashcard.maybe_download_media_files_and_convert(back) + f"<hr/><div>{page_title}</div>" maybe_deck_name, line = extract_tags(ANKI_DECK_TAG_REGEX, line) deck_name = get_full_deck_name(full_deck_names, maybe_deck_name) flashcards.append(TwoSidedFlashcard(uuid=flashcard_uuid, front=front, back=back, deck=deck_name)) continue post_slack(f"Flashcard improperly formatted (next line not indented): {line}\n{next_line}") try: make_anki_request('sync') except Exception: print("Waiting on sync...") time.sleep(5) try: make_anki_request('sync') except Exception: print("Waiting on sync...") time.sleep(5) make_anki_request('sync') # POST flashcards to AnkiConnect for flashcard in flashcards: query = f"deck:{flashcard.deck} ID:{flashcard.uuid}" anki_ids = make_anki_request('findNotes', query=query) if len(anki_ids) == 0: make_anki_request('addNote', note=flashcard.to_add_note_json()) elif len(anki_ids) == 1: make_anki_request('updateNoteFields', note=flashcard.to_update_note_fields_json(anki_ids[0])) else: raise AssertionError(f"{len(anki_ids)} number of cards returned for query {query}") make_anki_request('sync') num_edges = 0 word_count = 0 num_shares = 0 # Add information to nodes for title, data in graph.nodes(data=True): if 'lines' not in data: continue # Edges links = flatten([parse_links(x) for x in data['lines']]) for link in [x for x in links if graph.has_node(x)]: graph.add_edge(link, title) num_edges += 1 # Word count word_count += sum([len(x.split(' ')) for x in data['lines']]) # TODO FIXME # Shares num_shares += len(flatten([parse_shares(x) for x in data['lines'] if f"{Tags.SHARES.value}::" in x])) # Save num_edges with open(Filepaths.NUM_EDGES_FILEPATH.value, "r+") as f: line = f"\n{datetime.today()} {num_edges}" if line not in f.read(): f.write(line) # Save word count with open(Filepaths.WORD_COUNT_FILEPATH.value, "r+") as f: line = f"\n{datetime.today()} {word_count}" if line not in f.read(): f.write(line) # Save shares with open(Filepaths.NUM_SHARES_FILEPATH.value, "r+") as f: line = f"\n{datetime.today()} {num_shares}" if line not in f.read(): f.write(line) # Count edges for title, data in graph.nodes(data=True): data['num_edges'] = len(graph[title]) # Build index.html nodes = list(graph.nodes(data=True)) questions = [(title, data) for title, data in nodes if data['type'] == Tags.QUESTION.value] references = [(title, data) for title, data in nodes if data['type'] == Tags.REFERENCE.value] notes = [(title, data) for title, data in nodes if data['type'] == Tags.NOTE.value] posts = [(title, data) for title, data in nodes if data['type'] == Tags.POST.value] template = Template(open(Filepaths.DASHBOARD_HTML.value).read()) last_updated = datetime.fromtimestamp(os.path.getmtime(glob.glob(Filepaths.BACKUP_DIR.value)[0])) with open(Filepaths.INDEX_HTML.value, "w") as f: f.write(template.render( questions=questions, references=references, notes=notes, posts=posts, words_metric=get_words_metric(), connections_metric=get_connections_metric(), shares_metric=get_shares_metric(), last_updated=last_updated, last_built=open(Filepaths.LAST_BUILT.value).read() )) print(datetime.today(), "Done building dashboard!")
def bulk_upsert(self, objs, unique_constraint=None, update_fields=None, return_ids=True, batch_size=None): assert batch_size is None or batch_size > 0 assert (not return_ids) or (unique_constraint and update_fields) assert (not update_fields) or (unique_constraint and update_fields) assert (not unique_constraint) or (isinstance( unique_constraint, str) or (isinstance(unique_constraint, tuple))) for parent in self.model._meta.get_parent_list(): if parent._meta.concrete_model is not self.model._meta.concrete_model: raise ValueError( "Can't bulk upsert a multi-table inherited model") if not objs: return objs self._for_write = True fields = self.model._meta.concrete_fields if unique_constraint: if isinstance(unique_constraint, str): unique_constraint = self.model._meta.get_field_by_name(unique_constraint)[ 0] if not unique_constraint.unique: raise ValueError( "The unique_constraint argument must be the name of a unique=True field or a tuple of field names declared as unique_together") if isinstance(unique_constraint, AutoField): raise ValueError( "The unique_constraint argument cannot be an AutoField instance") else: if not self.model._meta.unique_together: raise ValueError( "The model hasn't declared any unique_together index") else: unique_constraint = [ uc for uc in self.model._meta.unique_together if uc == unique_constraint] if not unique_constraint: raise ValueError( "The unique_constraint tuple must be an exact match of an existing unique_together index") else: unique_constraint = unique_constraint[0] if update_fields: _up_fields = list() for upf in update_fields: upf = self.model._meta.get_field_by_name(upf)[0] if upf.unique: raise ValueError( "The update_fields argument must be a list of non unique/AutoField field names") _up_fields.append(upf) update_fields = _up_fields objs = list(objs) self._populate_pk_values(objs) with transaction.atomic(using=self.db, savepoint=False): objs_with_pk, objs_without_pk = partition( lambda o: o.pk is None, objs) if objs_with_pk: self._batched_update(objs_with_pk, fields, batch_size) if objs_without_pk: fields = [ f for f in fields if not isinstance(f, AutoField)] ids = self._batched_upsert(objs_without_pk, fields, unique_constraint=unique_constraint, update_fields=update_fields, batch_size=batch_size) if return_ids: assert len(ids) == len(objs_without_pk) for i, id in enumerate(ids): objs_without_pk[i].pk = id del ids return objs