class Person(models.Model):
"""
Describes a speaker in parliament, including information on names, active parliamentary periods and biographical details
"""
FEMALE = 1
MALE = 2
GENDERS = (
(FEMALE, 'Female'),
(MALE, 'Male'),
)
#### Names
surname = models.TextField()
alt_surnames = ArrayField(models.TextField(), null=True)
first_name = models.TextField()
alt_first_names = ArrayField(models.TextField(), null=True)
title = models.TextField(null=True)
academic_title = models.TextField(null=True)
ortszusatz = models.TextField(null=True)
adel = models.TextField(null=True)
prefix = models.TextField(null=True)
aph_id = models.TextField(null=True)
unique_id = models.TextField(null=True)
clean_name = models.TextField(null=True)
## Parliamentary periods
in_parlperiod = ArrayField(models.IntegerField(), null=True)
active_country = models.ForeignKey(
cities.models.Country,
on_delete=models.SET_NULL,
related_name='person_active',
null=True,
verbose_name="Country in which the person is active in politics")
positions = ArrayField(models.TextField(), null=True)
## Bio
dob = models.DateField(null=True)
year_of_birth = models.IntegerField(null=True)
place_of_birth = models.TextField(null=True)
country_of_birth = models.ForeignKey(cities.models.Country,
on_delete=models.SET_NULL,
related_name='person_birth',
null=True)
date_of_death = models.DateField(null=True)
gender = models.IntegerField(null=True, choices=GENDERS)
family_status = models.TextField(null=True)
religion = models.TextField(null=True)
occupation = models.TextField(null=True)
short_bio = models.TextField(null=True)
party = models.ForeignKey(Party, on_delete=models.PROTECT, null=True)
information_source = models.TextField(default="")
creation_date = models.DateTimeField(auto_now_add=True,
null=True,
verbose_name="Date of entry creation")
def __str__(self):
if self.clean_name:
return self.clean_name
else:
return "{} {}".format(self.first_name, self.surname)
def save(self, *args, **kwargs):
if not self.id and not self.alt_surnames:
surnames_list = self.surname.split(" ")
if len(surnames_list) > 1:
self.alt_surnames = [self.surname, surnames_list[-1]]
else:
self.alt_surnames = [self.surname]
if not self.id and not self.alt_first_names:
firstnames_list = self.first_name.split(" ")
if len(firstnames_list) > 1:
self.alt_first_names = [self.first_name] + firstnames_list
else:
self.alt_first_names = [self.first_name]
if not self.id and not self.clean_name:
self.clean_name = "{} {}".format(self.first_name,
self.surname).strip()
if self.title:
self.clean_name = self.title + " " + self.clean_name
if self.ortszusatz:
self.clean_name = self.clean_name + " ({})".format(
self.ortszusatz)
super(Person, self).save(*args, **kwargs)
class SummaryTransactionMonthView(models.Model):
duh = models.UUIDField(primary_key=True,
help_text="Deterministic Unique Hash")
action_date = models.DateField()
fiscal_year = models.IntegerField()
type = models.TextField()
pulled_from = models.TextField()
recipient_location_country_name = models.TextField()
recipient_location_country_code = models.TextField()
recipient_location_state_code = models.TextField()
recipient_location_county_name = models.TextField()
recipient_location_county_code = models.TextField()
recipient_location_zip5 = models.TextField()
recipient_location_congressional_code = models.TextField()
recipient_location_foreign_province = models.TextField()
pop_country_name = models.TextField()
pop_country_code = models.TextField()
pop_state_code = models.TextField()
pop_county_name = models.TextField()
pop_county_code = models.TextField()
pop_zip5 = models.TextField()
pop_congressional_code = models.TextField()
awarding_agency_id = models.IntegerField()
funding_agency_id = models.IntegerField()
awarding_toptier_agency_name = models.TextField()
funding_toptier_agency_name = models.TextField()
awarding_subtier_agency_name = models.TextField()
funding_subtier_agency_name = models.TextField()
awarding_toptier_agency_abbreviation = models.TextField()
funding_toptier_agency_abbreviation = models.TextField()
awarding_subtier_agency_abbreviation = models.TextField()
funding_subtier_agency_abbreviation = models.TextField()
recipient_hash = models.UUIDField()
recipient_name = models.TextField()
recipient_unique_id = models.TextField()
parent_recipient_unique_id = models.TextField()
business_categories = ArrayField(models.TextField(), default=list)
cfda_number = models.TextField(blank=True, null=True)
cfda_title = models.TextField(blank=True, null=True)
product_or_service_code = models.TextField()
product_or_service_description = models.TextField()
naics_code = models.TextField(blank=True, null=True)
naics_description = models.TextField(blank=True, null=True)
total_obl_bin = models.TextField()
type_of_contract_pricing = models.TextField()
type_set_aside = models.TextField()
extent_competed = models.TextField()
generated_pragmatic_obligation = models.DecimalField(max_digits=23,
decimal_places=2,
null=True,
blank=True)
federal_action_obligation = models.DecimalField(max_digits=23,
decimal_places=2,
blank=True,
null=True)
original_loan_subsidy_cost = models.DecimalField(max_digits=23,
decimal_places=2,
null=True,
blank=True)
face_value_loan_guarantee = models.DecimalField(max_digits=23,
decimal_places=2,
null=True,
blank=True)
counts = models.IntegerField()
class Meta:
managed = False
db_table = 'summary_transaction_month_view'
class SubawardView(models.Model):
subaward = models.OneToOneField(Subaward,
primary_key=True,
on_delete=models.deletion.DO_NOTHING)
keyword_ts_vector = SearchVectorField()
award_ts_vector = SearchVectorField()
recipient_name_ts_vector = SearchVectorField()
latest_transaction_id = models.IntegerField()
last_modified_date = models.DateField()
subaward_number = models.TextField()
amount = models.DecimalField(max_digits=23, decimal_places=2)
total_obl_bin = models.TextField()
description = models.TextField(null=True, blank=True)
fiscal_year = models.IntegerField()
action_date = models.DateField()
award_report_fy_month = models.IntegerField()
award_report_fy_year = models.IntegerField()
award = models.OneToOneField(Award, null=True)
awarding_agency_id = models.IntegerField()
funding_agency_id = models.IntegerField()
awarding_toptier_agency_name = models.TextField()
awarding_subtier_agency_name = models.TextField()
funding_toptier_agency_name = models.TextField()
funding_subtier_agency_name = models.TextField()
awarding_toptier_agency_abbreviation = models.TextField()
funding_toptier_agency_abbreviation = models.TextField()
awarding_subtier_agency_abbreviation = models.TextField()
funding_subtier_agency_abbreviation = models.TextField()
recipient_unique_id = models.TextField()
recipient_name = models.TextField()
dba_name = models.TextField()
parent_recipient_unique_id = models.TextField()
parent_recipient_name = models.TextField()
business_type_code = models.TextField()
business_type_description = models.TextField()
award_type = models.TextField()
prime_award_type = models.TextField()
cfda_id = models.IntegerField()
piid = models.TextField()
fain = models.TextField()
business_categories = ArrayField(models.TextField(), default=list)
prime_recipient_name = models.TextField()
pulled_from = models.TextField()
type_of_contract_pricing = models.TextField()
type_set_aside = models.TextField()
extent_competed = models.TextField()
product_or_service_code = models.TextField()
product_or_service_description = models.TextField()
cfda_number = models.TextField()
cfda_title = models.TextField()
recipient_location_country_code = models.TextField()
recipient_location_country_name = models.TextField()
recipient_location_city_name = models.TextField()
recipient_location_state_code = models.TextField()
recipient_location_state_name = models.TextField()
recipient_location_county_code = models.TextField()
recipient_location_county_name = models.TextField()
recipient_location_zip5 = models.TextField()
recipient_location_street_address = models.TextField()
recipient_location_congressional_code = models.TextField()
pop_country_code = models.TextField()
pop_country_name = models.TextField()
pop_state_code = models.TextField()
pop_state_name = models.TextField()
pop_county_code = models.TextField()
pop_county_name = models.TextField()
pop_city_code = models.TextField()
pop_city_name = models.TextField()
pop_zip5 = models.TextField()
pop_street_address = models.TextField()
pop_congressional_code = models.TextField()
class Meta:
managed = False
db_table = 'subaward_view'
class Submission(TimeStampedModel):
SUBMITTED = "submitted"
RUNNING = "running"
FAILED = "failed"
CANCELLED = "cancelled"
FINISHED = "finished"
SUBMITTING = "submitting"
STATUS_OPTIONS = (
(SUBMITTED, SUBMITTED),
(RUNNING, RUNNING),
(FAILED, FAILED),
(CANCELLED, CANCELLED),
(FINISHED, FINISHED),
(SUBMITTING, SUBMITTING),
)
participant_team = models.ForeignKey(ParticipantTeam,
related_name="submissions")
challenge_phase = models.ForeignKey(ChallengePhase,
related_name="submissions")
created_by = models.ForeignKey(User)
status = models.CharField(max_length=30,
choices=STATUS_OPTIONS,
db_index=True)
is_public = models.BooleanField(default=True)
is_flagged = models.BooleanField(default=False)
submission_number = models.PositiveIntegerField(default=0)
download_count = models.IntegerField(default=0)
output = models.TextField(blank=True, null=True)
submitted_at = models.DateTimeField(auto_now_add=True, db_index=True)
started_at = models.DateTimeField(null=True, blank=True, db_index=True)
completed_at = models.DateTimeField(null=True, blank=True, db_index=True)
when_made_public = models.DateTimeField(null=True, blank=True)
input_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"))
stdout_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True,
)
stderr_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True,
)
submission_result_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True,
)
submission_metadata_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True,
)
execution_time_limit = models.PositiveIntegerField(default=300)
method_name = models.CharField(max_length=1000,
default="",
db_index=True,
blank=True)
method_description = models.TextField(blank=True, default="")
publication_url = models.CharField(max_length=1000, default="", blank=True)
project_url = models.CharField(max_length=1000, default="", blank=True)
is_baseline = models.BooleanField(default=False)
job_name = ArrayField(
models.TextField(null=True, blank=True),
default=[],
blank=True,
null=True,
)
def __str__(self):
return "{}".format(self.id)
class Meta:
app_label = "jobs"
db_table = "submission"
@property
def execution_time(self):
"""Returns the execution time of a submission"""
# if self.self.completed_at and self.started_at:
try:
return (self.completed_at - self.started_at).total_seconds()
except: # noqa: E722
return "None"
# else:
# return None
def save(self, *args, **kwargs):
if not self.pk:
sub_num = Submission.objects.filter(
challenge_phase=self.challenge_phase,
participant_team=self.participant_team,
).aggregate(Max("submission_number"))["submission_number__max"]
if sub_num:
self.submission_number = sub_num + 1
else:
self.submission_number = 1
submissions = Submission.objects.filter(
challenge_phase=self.challenge_phase,
participant_team=self.participant_team,
)
num_submissions_to_ignore = submissions.filter(
status__in=submission_status_to_exclude).count()
successful_count = (self.submission_number -
num_submissions_to_ignore)
if successful_count > self.challenge_phase.max_submissions:
logger.info(
"Checking to see if the successful_count {0} is greater than maximum allowed {1}"
.format(successful_count,
self.challenge_phase.max_submissions))
logger.info(
"The submission request is submitted by user {0} from participant_team {1} "
.format(self.created_by.pk, self.participant_team.pk))
raise PermissionDenied({
"error":
"The maximum number of submissions has been reached"
})
else:
logger.info(
"Submission is below for user {0} form participant_team {1} for challenge_phase {2}"
.format(
self.created_by.pk,
self.participant_team.pk,
self.challenge_phase.pk,
))
total_submissions_done = Submission.objects.filter(
challenge_phase__challenge=self.challenge_phase.challenge,
participant_team=self.participant_team,
challenge_phase=self.challenge_phase,
)
submissions_done_today_count = (total_submissions_done.filter(
submitted_at__gte=timezone.now().replace(
hour=0, minute=0, second=0, microsecond=0)).exclude(
status__in=submission_status_to_exclude).count())
submissions_done_in_month_count = (total_submissions_done.filter(
submitted_at__gte=timezone.now().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)).exclude(
status__in=submission_status_to_exclude).count())
if (self.challenge_phase.max_submissions_per_month -
submissions_done_in_month_count == 0):
logger.info(
"Permission Denied: The maximum number of submission for this month has been reached"
)
raise PermissionDenied({
"error":
"The maximum number of submission for this month has been reached"
})
if (self.challenge_phase.max_submissions_per_day -
submissions_done_today_count == 0):
logger.info(
"Permission Denied: The maximum number of submission for today has been reached"
)
raise PermissionDenied({
"error":
"The maximum number of submission for today has been reached"
})
self.is_public = (True if self.challenge_phase.is_submission_public
else False)
self.status = Submission.SUBMITTED
submission_instance = super(Submission, self).save(*args, **kwargs)
return submission_instance
class Topic(models.Model):
"""
The default topic object. The title is usually set according to the top words
"""
title = models.CharField(max_length=80)
manual_title = models.CharField(max_length=80, null=True)
original_title = models.CharField(max_length=80, null=True)
score = models.FloatField(null=True)
share = models.FloatField(null=True)
growth = models.FloatField(null=True)
run_id = models.ForeignKey('RunStats',
db_index=True,
on_delete=models.CASCADE)
year = models.IntegerField(null=True)
period = models.ForeignKey('TimePeriod',
on_delete=models.SET_NULL,
null=True)
primary_dtopic = models.ManyToManyField('DynamicTopic')
top_words = ArrayField(models.TextField(), null=True)
primary_wg = models.IntegerField(null=True)
wg_prop = models.FloatField(null=True)
ipcc_coverage = models.FloatField(null=True)
ipcc_score = models.FloatField(null=True)
ipcc_share = models.FloatField(null=True)
wg_1 = models.FloatField(null=True)
wg_2 = models.FloatField(null=True)
wg_3 = models.FloatField(null=True)
def relevant_words(self, l, n):
# https://www.aclweb.org/anthology/W14-3110
tts = self.topicterm_set.annotate(
share=F('score') / F('alltopic_score'),
rel=l * Ln('score') + (1 - l) * Ln('share'),
).filter(rel__isnull=False).order_by('-rel')[:n].values(
'term__title', 'rel')
return tts
def create_wordintrusion(self, user):
real_words = self.topicterm_set.order_by('-score')[:5]
scores = np.array(
TopicTerm.objects.filter(topic__run_id=self.run_id).values_list(
'score', flat=True))
q99 = np.quantile(scores, 0.99)
q50 = np.quantile(scores, 0.5)
terms = set(
Term.objects.filter(
topicterm__score__gt=q99,
topicterm__topic__run_id=self.run_id).values_list('pk',
flat=True))
bad_terms = Term.objects.filter(pk__in=terms,
topicterm__score__lt=q50,
topicterm__topic=self)
if bad_terms.exists():
bad_term = bad_terms[random.randint(0, bad_terms.count() - 1)]
else:
bad_term = Term.objects.filter(
topicterm__topic=self).order_by('topicterm__score')[0]
word_intrusion = WordIntrusion(topic=self,
user=user,
intruded_word=bad_term)
word_intrusion.save()
for w in real_words:
word_intrusion.real_words.add(w.term)
def __unicode__(self):
return str(self.title)
def __str__(self):
return str(self.title)
class User(OrderedCollectionPageMixin, AbstractUser):
"""a user who wants to read books"""
username = fields.UsernameField()
email = models.EmailField(unique=True, null=True)
key_pair = fields.OneToOneField(
"KeyPair",
on_delete=models.CASCADE,
blank=True,
null=True,
activitypub_field="publicKey",
related_name="owner",
)
inbox = fields.RemoteIdField(unique=True)
shared_inbox = fields.RemoteIdField(
activitypub_field="sharedInbox",
activitypub_wrapper="endpoints",
deduplication_field=False,
null=True,
)
federated_server = models.ForeignKey(
"FederatedServer",
on_delete=models.PROTECT,
null=True,
blank=True,
)
outbox = fields.RemoteIdField(unique=True, null=True)
summary = fields.HtmlField(null=True, blank=True)
local = models.BooleanField(default=False)
bookwyrm_user = fields.BooleanField(default=True)
localname = CICharField(
max_length=255,
null=True,
unique=True,
validators=[fields.validate_localname],
)
# name is your display name, which you can change at will
name = fields.CharField(max_length=100, null=True, blank=True)
avatar = fields.ImageField(
upload_to="avatars/",
blank=True,
null=True,
activitypub_field="icon",
alt_field="alt_text",
)
preview_image = models.ImageField(upload_to="previews/avatars/",
blank=True,
null=True)
followers_url = fields.CharField(max_length=255,
activitypub_field="followers")
followers = models.ManyToManyField(
"self",
symmetrical=False,
through="UserFollows",
through_fields=("user_object", "user_subject"),
related_name="following",
)
follow_requests = models.ManyToManyField(
"self",
symmetrical=False,
through="UserFollowRequest",
through_fields=("user_subject", "user_object"),
related_name="follower_requests",
)
blocks = models.ManyToManyField(
"self",
symmetrical=False,
through="UserBlocks",
through_fields=("user_subject", "user_object"),
related_name="blocked_by",
)
saved_lists = models.ManyToManyField("List",
symmetrical=False,
related_name="saved_lists",
blank=True)
favorites = models.ManyToManyField(
"Status",
symmetrical=False,
through="Favorite",
through_fields=("user", "status"),
related_name="favorite_statuses",
)
default_post_privacy = models.CharField(max_length=255,
default="public",
choices=fields.PrivacyLevels)
remote_id = fields.RemoteIdField(null=True,
unique=True,
activitypub_field="id")
created_date = models.DateTimeField(auto_now_add=True)
updated_date = models.DateTimeField(auto_now=True)
last_active_date = models.DateTimeField(default=timezone.now)
manually_approves_followers = fields.BooleanField(default=False)
# options to turn features on and off
show_goal = models.BooleanField(default=True)
show_suggested_users = models.BooleanField(default=True)
discoverable = fields.BooleanField(default=False)
# feed options
feed_status_types = ArrayField(
models.CharField(max_length=10, blank=False,
choices=FeedFilterChoices),
size=8,
default=get_feed_filter_choices,
)
# annual summary keys
summary_keys = models.JSONField(null=True)
preferred_timezone = models.CharField(
choices=[(str(tz), str(tz)) for tz in pytz.all_timezones],
default=str(pytz.utc),
max_length=255,
)
preferred_language = models.CharField(
choices=LANGUAGES,
null=True,
blank=True,
max_length=255,
)
deactivation_reason = models.CharField(max_length=255,
choices=DeactivationReason,
null=True,
blank=True)
deactivation_date = models.DateTimeField(null=True, blank=True)
confirmation_code = models.CharField(max_length=32,
default=new_access_code)
name_field = "username"
property_fields = [("following_link", "following")]
field_tracker = FieldTracker(fields=["name", "avatar"])
@property
def confirmation_link(self):
"""helper for generating confirmation links"""
link = site_link()
return f"{link}/confirm-email/{self.confirmation_code}"
@property
def following_link(self):
"""just how to find out the following info"""
return f"{self.remote_id}/following"
@property
def alt_text(self):
"""alt text with username"""
# pylint: disable=consider-using-f-string
return "avatar for {:s}".format(self.localname or self.username)
@property
def display_name(self):
"""show the cleanest version of the user's name possible"""
if self.name and self.name != "":
return self.name
return self.localname or self.username
@property
def deleted(self):
"""for consistent naming"""
return not self.is_active
@property
def unread_notification_count(self):
"""count of notifications, for the templates"""
return self.notification_set.filter(read=False).count()
@property
def has_unread_mentions(self):
"""whether any of the unread notifications are conversations"""
return self.notification_set.filter(
read=False,
notification_type__in=["REPLY", "MENTION", "TAG", "REPORT"],
).exists()
activity_serializer = activitypub.Person
@classmethod
def viewer_aware_objects(cls, viewer):
"""the user queryset filtered for the context of the logged in user"""
queryset = cls.objects.filter(is_active=True)
if viewer and viewer.is_authenticated:
queryset = queryset.exclude(blocks=viewer)
return queryset
def update_active_date(self):
"""this user is here! they are doing things!"""
self.last_active_date = timezone.now()
self.save(broadcast=False, update_fields=["last_active_date"])
def to_outbox(self, filter_type=None, **kwargs):
"""an ordered collection of statuses"""
if filter_type:
filter_class = apps.get_model(f"bookwyrm.{filter_type}",
require_ready=True)
if not issubclass(filter_class, Status):
raise TypeError(
"filter_status_class must be a subclass of models.Status")
queryset = filter_class.objects
else:
queryset = Status.objects
queryset = (queryset.filter(
user=self,
deleted=False,
privacy__in=["public", "unlisted"],
).select_subclasses().order_by("-published_date"))
return self.to_ordered_collection(queryset,
collection_only=True,
remote_id=self.outbox,
**kwargs).serialize()
def to_following_activity(self, **kwargs):
"""activitypub following list"""
remote_id = f"{self.remote_id}/following"
return self.to_ordered_collection(
self.following.order_by("-updated_date").all(),
remote_id=remote_id,
id_only=True,
**kwargs,
)
def to_followers_activity(self, **kwargs):
"""activitypub followers list"""
remote_id = self.followers_url
return self.to_ordered_collection(
self.followers.order_by("-updated_date").all(),
remote_id=remote_id,
id_only=True,
**kwargs,
)
def to_activity(self, **kwargs):
"""override default AP serializer to add context object
idk if this is the best way to go about this"""
if not self.is_active:
return self.remote_id
activity_object = super().to_activity(**kwargs)
activity_object["@context"] = [
"https://www.w3.org/ns/activitystreams",
"https://w3id.org/security/v1",
{
"manuallyApprovesFollowers": "as:manuallyApprovesFollowers",
"schema": "http://schema.org#",
"PropertyValue": "schema:PropertyValue",
"value": "schema:value",
},
]
return activity_object
def save(self, *args, **kwargs):
"""populate fields for new local users"""
created = not bool(self.id)
if not self.local and not re.match(regex.FULL_USERNAME, self.username):
# generate a username that uses the domain (webfinger format)
actor_parts = urlparse(self.remote_id)
self.username = f"{self.username}@{actor_parts.netloc}"
# this user already exists, no need to populate fields
if not created:
if self.is_active:
self.deactivation_date = None
elif not self.deactivation_date:
self.deactivation_date = timezone.now()
super().save(*args, **kwargs)
return
# this is a new remote user, we need to set their remote server field
if not self.local:
super().save(*args, **kwargs)
transaction.on_commit(lambda: set_remote_server.delay(self.id))
return
with transaction.atomic():
# populate fields for local users
link = site_link()
self.remote_id = f"{link}/user/{self.localname}"
self.followers_url = f"{self.remote_id}/followers"
self.inbox = f"{self.remote_id}/inbox"
self.shared_inbox = f"{link}/inbox"
self.outbox = f"{self.remote_id}/outbox"
# an id needs to be set before we can proceed with related models
super().save(*args, **kwargs)
# make users editors by default
try:
self.groups.add(Group.objects.get(name="editor"))
except Group.DoesNotExist:
# this should only happen in tests
pass
# create keys and shelves for new local users
self.key_pair = KeyPair.objects.create(
remote_id=f"{self.remote_id}/#main-key")
self.save(broadcast=False, update_fields=["key_pair"])
self.create_shelves()
def delete(self, *args, **kwargs):
"""deactivate rather than delete a user"""
# pylint: disable=attribute-defined-outside-init
self.is_active = False
# skip the logic in this class's save()
super().save(*args, **kwargs)
@property
def local_path(self):
"""this model doesn't inherit bookwyrm model, so here we are"""
# pylint: disable=consider-using-f-string
return "/user/{:s}".format(self.localname or self.username)
def create_shelves(self):
"""default shelves for a new user"""
shelves = [
{
"name": "To Read",
"identifier": "to-read",
},
{
"name": "Currently Reading",
"identifier": "reading",
},
{
"name": "Read",
"identifier": "read",
},
]
for shelf in shelves:
Shelf(
name=shelf["name"],
identifier=shelf["identifier"],
user=self,
editable=False,
).save(broadcast=False)
class BackgroundObservation1Second(models.Model):
background_observation = models.ForeignKey(BackgroundObservation,
on_delete=models.CASCADE)
utc_year = models.IntegerField('UTC year')
utc_seconds = models.IntegerField('UTC seconds')
utc_msec = models.IntegerField('UTC msec')
tcp_count_dhpu = models.IntegerField()
tcp_count_dpu = models.IntegerField()
dpu_count_prereset = models.IntegerField()
UNDEFINED = 'UD'
CONFMODE = 'CF'
OPMODE = 'OP'
SW_MODE_CHOICES = (
(UNDEFINED, "Undefined"),
(CONFMODE, "Configuration mode"),
(OPMODE, "Operational mode"),
)
sw_mode = models.CharField(max_length=2,
choices=SW_MODE_CHOICES,
default=UNDEFINED)
UNDEFINED = 'UD'
TGFSEARCH = 'TG'
HIGHBACKGROUND = 'HB'
AURORALCAPTURE = 'AC'
SW_SUBMODE_CHOICES = (
(UNDEFINED, "Undefined"),
(TGFSEARCH, "TGF Search mode"),
(HIGHBACKGROUND, "High Background"),
(AURORALCAPTURE, "Auroral capture"),
)
sw_submode = models.CharField(max_length=2,
choices=SW_SUBMODE_CHOICES,
default=UNDEFINED)
dau_data_reduc_factor = models.IntegerField()
led_count_ratemeter = models.IntegerField()
hed_count_ratemeter = models.IntegerField()
dau_total_rate = models.IntegerField()
led_accept_count_rate_short = models.IntegerField()
led_accept_count_rate_long = models.IntegerField()
hed_accept_count_rate_short = models.IntegerField()
hed_accept_count_rate_long = models.IntegerField()
led_calc_background_rate_short = models.IntegerField()
led_calc_background_rate_long = models.IntegerField()
hed_calc_background_rate_short = models.IntegerField()
hed_calc_background_rate_long = models.IntegerField()
led_short_win_thresh_1 = models.IntegerField()
led_short_win_thresh_2 = models.IntegerField()
led_short_win_thresh_3 = models.IntegerField()
led_long_win_thresh = models.IntegerField()
hed_short_win_thresh_1 = models.IntegerField()
hed_short_win_thresh_2 = models.IntegerField()
hed_short_win_thresh_3 = models.IntegerField()
hed_long_win_thresh = models.IntegerField()
dau1_dm_if_1_current_offset = models.IntegerField()
dau1_dm_if_2_current_offset = models.IntegerField()
dau1_dm_if_3_current_offset = models.IntegerField()
dau1_dm_if_4_current_offset = models.IntegerField()
dau2_dm_if_1_current_offset = models.IntegerField()
dau2_dm_if_2_current_offset = models.IntegerField()
dau2_dm_if_3_current_offset = models.IntegerField()
dau2_dm_if_4_current_offset = models.IntegerField()
dau3_dm_if_1_current_offset = models.IntegerField()
dau3_dm_if_2_current_offset = models.IntegerField()
dau3_dm_if_3_current_offset = models.IntegerField()
dau3_dm_if_4_current_offset = models.IntegerField()
dau4_dm_if_1_current_offset = models.IntegerField()
dau4_dm_if_2_current_offset = models.IntegerField()
dau4_dm_if_3_current_offset = models.IntegerField()
dau4_dm_if_4_current_offset = models.IntegerField()
dau1_pmt_if_1_current_offset = models.IntegerField()
dau1_pmt_if_2_current_offset = models.IntegerField()
dau1_pmt_if_3_current_offset = models.IntegerField()
dau2_pmt_if_1_current_offset = models.IntegerField()
dau2_pmt_if_2_current_offset = models.IntegerField()
dau2_pmt_if_3_current_offset = models.IntegerField()
dau3_pmt_if_1_current_offset = models.IntegerField()
dau3_pmt_if_2_current_offset = models.IntegerField()
dau3_pmt_if_3_current_offset = models.IntegerField()
dau4_pmt_if_1_current_offset = models.IntegerField()
dau4_pmt_if_2_current_offset = models.IntegerField()
dau4_pmt_if_3_current_offset = models.IntegerField()
led_pulse_height_bin = ArrayField(
models.IntegerField(),
size=10,
)
hed_pulse_height_bin = ArrayField(
models.IntegerField(),
size=10,
)
led_time_bin_values = ArrayField(
models.IntegerField(),
size=31,
)
hed_time_bin_values = ArrayField(
models.IntegerField(),
size=31,
)
class MyModel(PostgreSQLModel):
field = ArrayField(models.IntegerField(), default=None)
class MyModel(PostgreSQLModel):
field = ArrayField(ArrayField(models.CharField()))
def test_model_field_choices(self):
model_field = ArrayField(
models.IntegerField(choices=((1, 'A'), (2, 'B'))))
form_field = model_field.formfield()
self.assertEqual(form_field.clean('1,2'), [1, 2])
class MyModel(PostgreSQLModel):
field = ArrayField(
models.ManyToManyField('postgres_tests.IntegerArrayModel'))
def test_model_field_formfield_size(self):
model_field = ArrayField(models.CharField(max_length=27), size=4)
form_field = model_field.formfield()
self.assertIsInstance(form_field, SimpleArrayField)
self.assertEqual(form_field.max_length, 4)
def test_model_field_formfield(self):
model_field = ArrayField(models.CharField(max_length=27))
form_field = model_field.formfield()
self.assertIsInstance(form_field, SimpleArrayField)
self.assertIsInstance(form_field.base_field, forms.CharField)
self.assertEqual(form_field.base_field.max_length, 27)
class Party(GenericModerateMixin, models.Model):
club = models.ForeignKey(Club,
verbose_name=_("Club"),
on_delete=models.CASCADE)
name = models.CharField(_("Name"), max_length=255)
slug = models.SlugField(_("Slug"),
max_length=150,
db_index=True,
unique=True)
image = models.ForeignKey("storage.Image",
verbose_name=_("Image"),
on_delete=models.SET_NULL,
null=True,
blank=True)
short_description = RichTextField(_("Short description"),
config_name="basic")
description = RichTextUploadingField(_("Description"), config_name="basic")
city = models.ForeignKey(City,
verbose_name=_("City"),
on_delete=models.CASCADE,
null=True)
address = RichTextUploadingField(_("Address"), config_name="basic")
geo = JSONField(_("Geo"), null=True, blank=True)
start_date = models.DateTimeField(_("Start date"))
end_date = models.DateTimeField(_("End date"))
party_type = models.CharField(_("Type"),
max_length=16,
choices=ComminityTypeChoices.TYPES,
db_index=True)
theme = models.CharField(_("Theme"),
max_length=16,
choices=ThemeChoices.THEMES,
db_index=True)
hash_tags = ArrayField(models.CharField(max_length=255),
verbose_name=_("Hash tags"),
default=list)
likes = models.ManyToManyField("users.User",
verbose_name=_("Likes"),
blank=True,
related_name="like_parties")
views = models.PositiveIntegerField(_("Views"), default=0)
man_cost = models.FloatField(_("Man cost"), default=0)
woman_cost = models.FloatField(_("Woman cost"), default=0)
pair_cost = models.FloatField(_("Pair cost"), default=0)
comments = GenericRelation(Comment)
fields_tracker = FieldTracker(fields=["status"])
@property
def comments_count(self):
return self.comments.filter(is_deleted=False).count()
def __str__(self):
return self.name
class Meta:
verbose_name = _("Party")
verbose_name_plural = _("Parties")
class Pulp2ManifestList(Pulp2to3Content):
"""
Pulp 2to3 detail content model to store pulp 2 ManifestList content details
for Pulp 3 content creation.
"""
digest = models.CharField(max_length=255)
media_type = models.CharField(max_length=80)
schema_version = models.IntegerField()
media_type = models.CharField(max_length=80)
listed_manifests = ArrayField(models.CharField(max_length=255))
pulp2_type = "docker_manifest_list"
checksum_type = "sha256"
class Meta:
unique_together = ("digest", "pulp2content")
default_related_name = "docker_manifest_list_detail_model"
@property
def expected_digests(self):
"""Return expected digests."""
return {self.checksum_type: self.digest.split(":")[1]}
@property
def expected_size(self):
"""Return expected size."""
return
@property
def relative_path_for_content_artifact(self):
"""Return relative path."""
return self.digest
@classmethod
def pre_migrate_content_detail(cls, content_batch):
"""
Pre-migrate Pulp 2 ManifestList content with all the fields needed to create
a Pulp 3 Manifest
Args:
content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content.
"""
pulp2_id_obj_map = {
pulp2content.pulp2_id: pulp2content
for pulp2content in content_batch
}
pulp2_ids = pulp2_id_obj_map.keys()
pulp2_m_content_batch = pulp2_models.ManifestList.objects.filter(
id__in=pulp2_ids)
pulp2m_to_save = [
Pulp2ManifestList(
digest=m.digest,
media_type=MEDIA_TYPE.MANIFEST_LIST,
schema_version=m.schema_version,
listed_manifests=[man.digest for man in m.manifests],
pulp2content=pulp2_id_obj_map[m.id],
) for m in pulp2_m_content_batch
]
cls.objects.bulk_create(pulp2m_to_save,
ignore_conflicts=True,
batch_size=DEFAULT_BATCH_SIZE)
def create_pulp3_content(self):
"""
Create a Pulp 3 Manifest unit for saving it later in a bulk operation.
"""
future_relations = {"man_rel": self.listed_manifests}
return (
Manifest(
digest=self.digest,
media_type=self.media_type,
schema_version=self.schema_version,
),
future_relations,
)
def test_deconstruct(self):
field = ArrayField(models.IntegerField())
name, path, args, kwargs = field.deconstruct()
new = ArrayField(*args, **kwargs)
self.assertEqual(type(new.base_field), type(field.base_field))
self.assertIsNot(new.base_field, field.base_field)
class Pulp2Manifest(Pulp2to3Content):
"""
Pulp 2to3 detail content model to store pulp 2 Manifest content details
for Pulp 3 content creation.
"""
digest = models.CharField(max_length=255)
schema_version = models.IntegerField()
media_type = models.CharField(max_length=80)
blobs = ArrayField(models.CharField(max_length=255))
config_blob = models.CharField(max_length=255, null=True)
pulp2_type = "docker_manifest"
checksum_type = "sha256"
class Meta:
unique_together = ("digest", "pulp2content")
default_related_name = "docker_manifest_detail_model"
@property
def expected_digests(self):
"""Return expected digests."""
return {}
@property
def expected_size(self):
"""Return expected size."""
return
@property
def relative_path_for_content_artifact(self):
"""Return relative path."""
return self.digest
@classmethod
def pre_migrate_content_detail(cls, content_batch):
"""
Pre-migrate Pulp 2 Manifest content with all the fields needed to create a Pulp 3 Manifest
Args:
content_batch(list of Pulp2Content): pre-migrated generic data for Pulp 2 content.
"""
def _get_media_type(schema_version):
"""
Return media_type of the manifest.
"""
return (MEDIA_TYPE.MANIFEST_V2
if schema_version == 2 else MEDIA_TYPE.MANIFEST_V1)
def _get_blobs(layers):
"""
Return list of regular blobs that manifest contains.
"""
blobs = []
for layer in layers:
if layer.layer_type != MEDIA_TYPE.FOREIGN_BLOB:
blobs.append(layer.blob_sum)
return blobs
pulp2_id_obj_map = {
pulp2content.pulp2_id: pulp2content
for pulp2content in content_batch
}
pulp2_ids = pulp2_id_obj_map.keys()
pulp2_m_content_batch = pulp2_models.Manifest.objects.filter(
id__in=pulp2_ids)
pulp2m_to_save = []
for m in pulp2_m_content_batch:
pulp2m_to_save.append(
Pulp2Manifest(
digest=m.digest,
media_type=_get_media_type(m.schema_version),
schema_version=m.schema_version,
config_blob=m.config_layer,
blobs=_get_blobs(m.fs_layers),
pulp2content=pulp2_id_obj_map[m.id],
))
cls.objects.bulk_create(pulp2m_to_save,
ignore_conflicts=True,
batch_size=DEFAULT_BATCH_SIZE)
def create_pulp3_content(self):
"""
Create a Pulp 3 Manifest unit for saving it later in a bulk operation.
"""
future_relations = {
"blob_rel": self.blobs,
"config_blob_rel": self.config_blob
}
return (
Manifest(
digest=self.digest,
media_type=self.media_type,
schema_version=self.schema_version,
),
future_relations,
)
def test_deconstruct_with_size(self):
field = ArrayField(models.IntegerField(), size=3)
name, path, args, kwargs = field.deconstruct()
new = ArrayField(*args, **kwargs)
self.assertEqual(new.size, field.size)
class ComputationalResult(models.Model):
""" Meta-information about the output of a computer process. (Ex Salmon) """
class Meta:
db_table = "computational_results"
base_manager_name = "public_objects"
def __str__(self):
processor_name_str = ""
if self.processor:
processor_name_str = ": " + str(self.processor.name)
return "ComputationalResult " + str(self.pk) + processor_name_str
# Managers
objects = models.Manager()
public_objects = PublicObjectsManager()
commands = ArrayField(models.TextField(), default=list)
processor = models.ForeignKey("Processor", blank=True, null=True, on_delete=models.CASCADE)
samples = models.ManyToManyField("Sample", through="SampleResultAssociation")
# The Organism Index used to process the sample.
organism_index = models.ForeignKey(
"OrganismIndex", blank=True, null=True, on_delete=models.SET_NULL
)
is_ccdl = models.BooleanField(default=True)
# Stats
time_start = models.DateTimeField(blank=True, null=True)
time_end = models.DateTimeField(blank=True, null=True)
# Common Properties
is_public = models.BooleanField(default=True)
created_at = models.DateTimeField(editable=False, default=timezone.now)
last_modified = models.DateTimeField(default=timezone.now)
def save(self, *args, **kwargs):
""" On save, update timestamps """
current_time = timezone.now()
if not self.id:
self.created_at = current_time
self.last_modified = current_time
return super(ComputationalResult, self).save(*args, **kwargs)
# Currently unused because of pre_delete signal for computed files
def remove_computed_files_from_s3(self):
""" Removes all associated computed files from S3.
Use this before deleting a computational result.
"""
for computed_file in self.computedfile_set.all():
computed_file.delete_s3_file()
def get_index_length(self):
""" Pull the index_length from one of the result annotations """
annotations = ComputationalResultAnnotation.objects.filter(result=self)
for annotation_json in annotations:
if "index_length" in annotation_json.data:
return annotation_json.data["index_length"]
return None
def get_quant_sf_file(self):
return (
ComputedFile.objects.filter(
result=self, filename="quant.sf", s3_key__isnull=False, s3_bucket__isnull=False,
)
.order_by("-id")
.first()
)
def test_deconstruct_args(self):
field = ArrayField(models.CharField(max_length=20))
name, path, args, kwargs = field.deconstruct()
new = ArrayField(*args, **kwargs)
self.assertEqual(new.base_field.max_length,
field.base_field.max_length)
class AdministrativeDomain(NOCModel):
"""
Administrative Domain
"""
class Meta(object):
verbose_name = _("Administrative Domain")
verbose_name_plural = _("Administrative Domains")
db_table = "sa_administrativedomain"
app_label = "sa"
ordering = ["name"]
name = models.CharField(_("Name"), max_length=255, unique=True)
parent = models.ForeignKey(
"self", verbose_name="Parent", null=True, blank=True, on_delete=models.CASCADE
)
description = models.TextField(_("Description"), null=True, blank=True)
default_pool = DocumentReferenceField(Pool, null=True, blank=True)
# Biosegmentation settings
bioseg_floating_name_template = models.ForeignKey(
Template, null=True, blank=True, on_delete=models.CASCADE
)
bioseg_floating_parent_segment = DocumentReferenceField(
"inv.NetworkSegment", null=True, blank=True
)
# Integration with external NRI systems
# Reference to remote system object has been imported from
remote_system = DocumentReferenceField(RemoteSystem, null=True, blank=True)
# Object id in remote system
remote_id = models.CharField(max_length=64, null=True, blank=True)
# Object id in BI
bi_id = models.BigIntegerField(unique=True)
labels = ArrayField(models.CharField(max_length=250), blank=True, null=True, default=list)
effective_labels = ArrayField(
models.CharField(max_length=250), blank=True, null=True, default=list
)
_id_cache = cachetools.TTLCache(maxsize=1000, ttl=60)
_bi_id_cache = cachetools.TTLCache(maxsize=1000, ttl=60)
_nested_cache = cachetools.TTLCache(maxsize=1000, ttl=60)
def __str__(self):
return self.name
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_id_cache"), lock=lambda _: id_lock)
def get_by_id(cls, id):
ad = AdministrativeDomain.objects.filter(id=id)[:1]
if ad:
return ad[0]
return None
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_bi_id_cache"), lock=lambda _: id_lock)
def get_by_bi_id(cls, id):
ad = AdministrativeDomain.objects.filter(bi_id=id)[:1]
if ad:
return ad[0]
return None
def iter_changed_datastream(self, changed_fields=None):
if config.datastream.enable_administrativedomain:
yield "administrativedomain", self.id
@cachetools.cached(_path_cache, key=lambda s: s.id, lock=id_lock)
def get_path(self):
"""
Returns list of parent administrative domain ids
:return:
"""
if self.parent:
return self.parent.get_path() + [self.id]
return [self.id]
def get_default_pool(self):
if self.default_pool:
return self.default_pool
if self.parent:
return self.parent.get_default_pool()
return None
def get_nested(self):
"""
Returns list of nested administrative domains
:return:
"""
r = [self]
for d in AdministrativeDomain.objects.filter(parent=self):
r += d.get_nested()
return r
@classmethod
@cachetools.cachedmethod(operator.attrgetter("_nested_cache"), lock=lambda _: id_lock)
def get_nested_ids(cls, administrative_domain):
from django.db import connection
if hasattr(administrative_domain, "id"):
administrative_domain = administrative_domain.id
cursor = connection.cursor()
cursor.execute(
"""
WITH RECURSIVE r AS (
SELECT id, parent_id
FROM sa_administrativedomain
WHERE id = %d
UNION
SELECT ad.id, ad.parent_id
FROM sa_administrativedomain ad JOIN r ON ad.parent_id = r.id
)
SELECT id FROM r
"""
% administrative_domain
)
return [r[0] for r in cursor]
@property
def has_children(self):
return True if AdministrativeDomain.objects.filter(parent=self.id) else False
def get_bioseg_floating_name(self, object) -> Optional[str]:
if self.bioseg_floating_name_template:
return self.bioseg_floating_name_template.render_body(object=object)
if self.parent:
return self.parent.get_bioseg_floating_name(object)
return None
def get_bioseg_floating_parent_segment(self) -> Optional["NetworkSegment"]:
if self.bioseg_floating_parent_segment:
return self.bioseg_floating_parent_segment
if self.parent:
return self.parent.get_bioseg_floating_parent_segment()
return None
@classmethod
def can_set_label(cls, label):
if label.enable_administrativedomain:
return True
return False
@classmethod
def can_expose_label(cls, label):
return False
def test_blank_true(self):
field = ArrayField(models.IntegerField(blank=True, null=True))
# This should not raise a validation error
field.clean([1, None], None)
class RunStats(models.Model):
"""
Hold all meta-information on topic model runs
"""
run_id = models.AutoField(primary_key=True)
##Inputs
ONLINE = "on"
BATCH = "ba"
lda_choices = (
(ONLINE, 'Online'),
(BATCH, 'Batch'),
)
SKLEARN = "sk"
LDA_LIB = "ld"
lda_libs = ((SKLEARN, "Sklearn"), (LDA_LIB, "lda"))
max_features = models.IntegerField(
default=0, help_text='Maximum number of terms (0 = no limit)')
min_freq = models.IntegerField(default=1,
help_text='Minimum frequency of terms')
max_df = MinMaxFloat(default=0.95, min_value=0.0, max_value=1.0)
limit = models.IntegerField(
null=True,
default=0,
help_text=
'Limit model to first x documents (leave as zero for no limit)')
ngram = models.IntegerField(null=True,
default=1,
help_text='Length of feature n_gram')
db = models.BooleanField(
default=True,
help_text=
'Record the results into the database? Or just run the model and record statistics?'
)
fancy_tokenization = models.BooleanField(
default=False,
help_text='tokenize so that multiple word keywords remain whole')
K = models.IntegerField(null=True, help_text='Number of topics')
alpha = models.FloatField(
null=True,
default=0.01,
help_text=
'Concentration parameter of Dirichlet distribution of topics in documents'
' (try higher values in LDA, including > 1). Low (high) values indicate that'
' documents should be composed of few (many) topics. Also called theta.'
' In NMF, this is the regularization term alpha')
beta = models.FloatField(
null=True,
blank=True,
default=None,
help_text=
'Concentration parameter of Dirichlet distribution of words in topics.'
' Low (high) values indicate that topics should be composed of few (many) words.'
' Also called eta. This parameter is not used in NMF')
lda_learning_method = models.CharField(
max_length=2,
choices=lda_choices,
null=True,
default=BATCH,
help_text=
'When using LDA in sklearn, you can choose between batch or online learning'
)
lda_library = models.CharField(
max_length=2,
choices=lda_libs,
null=True,
default=SKLEARN,
help_text=
'you can use sklearn or https://github.com/lda-project/lda for LDA')
top_chain_var = models.FloatField(null=True,
default=0.05,
help_text='Chain var parameter for dtm')
max_iter = models.IntegerField(null=True,
default=200,
help_text='Maximum iterations')
rng_seed = models.IntegerField(
null=True,
help_text=
"seed for random number generator for stochastic estimation of topic model (blei dtm)"
)
fulltext = models.BooleanField(
default=False,
help_text='do analysis on fullText? (dependent on availability)')
citations = models.BooleanField(
default=False, help_text='scale term scores by citations?')
# Additional information
language = models.TextField(
null=True,
help_text=
'language of the documents that have been analyzed (also used for stopword identification)'
)
extra_stopwords = ArrayField(
models.TextField(),
null=True,
help_text=
'list of stopwords that are used additionally to the standard ones')
query = models.ForeignKey(
'scoping.Query',
null=True,
on_delete=models.CASCADE,
help_text='relation to the scoping search object')
psearch = models.ForeignKey(
'parliament.Search',
null=True,
on_delete=models.CASCADE,
help_text='relation to the parliamentary search object')
## Progress
process_id = models.IntegerField(null=True)
start = models.DateTimeField(auto_now_add=True)
batch_count = models.IntegerField(default=0)
last_update = models.DateTimeField(auto_now_add=True)
topic_titles_current = models.NullBooleanField(default=False)
topic_scores_current = models.NullBooleanField(default=False)
topic_year_scores_current = models.NullBooleanField(default=False)
## Time spent
runtime = models.DurationField(null=True)
nmf_time = models.FloatField(default=0)
tfidf_time = models.FloatField(default=0)
db_time = models.FloatField(default=0)
status_choices = ((0, 'Not Started'), (1, 'Running'), (2, 'Interrupted'),
(3, 'Finished'))
status = models.IntegerField(choices=status_choices,
default=0,
help_text='status of the model execution')
parent_run_id = models.IntegerField(null=True, help_text='')
docs_seen = models.IntegerField(null=True)
notes = models.TextField(null=True)
LDA = 'LD'
HLDA = 'HL'
DTM = 'DT'
NMF = 'NM'
BDT = 'BD'
METHOD_CHOICES = ((LDA, 'lda'), (HLDA, 'hlda'), (DTM, 'dnmf'),
(NMF, 'nmf'), (BDT, 'BleiDTM'))
method = models.CharField(
max_length=2,
choices=METHOD_CHOICES,
default=NMF,
)
error = models.FloatField(null=True, default=0)
coherence = models.FloatField(null=True)
errortype = models.TextField(null=True)
exclusivity = models.FloatField(null=True)
empty_topics = models.IntegerField(null=True)
iterations = models.IntegerField(null=True)
max_topics = models.IntegerField(null=True)
term_count = models.IntegerField(null=True)
periods = models.ManyToManyField('TimePeriod')
doc_topic_scaled_score = models.BooleanField(default=False)
dt_threshold = models.FloatField(default=0.0005)
dt_threshold_scaled = models.FloatField(default=0.01)
dyn_win_threshold = models.FloatField(default=0.1)
def save(self, *args, **kwargs):
if not self.parent_run_id:
self.parent_run_id = self.run_id
super(RunStats, self).save(*args, **kwargs)
def dt_matrix(self, path, s_size=0, force_overwrite=False):
'''
Return a sparse doctopic matrix and its row and column ids
'''
# see if the required objects already exist
mpath = f"{path}/run_{self.pk}_s_{s_size}_m.npy"
rpath = f"{path}/run_{self.pk}_s_{s_size}_r_ind.npy"
cpath = f"{path}/run_{self.pk}_s_{s_size}_c_ind.npy"
if os.path.exists(mpath):
m = np.load(mpath)[()]
if os.path.exists(rpath):
r_ind = np.load(rpath)
if os.path.exists(cpath):
c_ind = np.load(cpath)
if not force_overwrite:
print(
"We've already calculated the required matrices!")
return (m, c_ind, r_ind)
if self.method == "DT":
dts = DocDynamicTopic.objects
else:
dts = DocTopic.objects
if self.query:
doc_id_var = 'doc__id'
elif self.psearch:
if self.psearch.search_object_type == parliament.models.Search.PARAGRAPH:
doc_id_var = 'par__id'
elif self.psearch.search_object_type == parliament.models.Search.UTTERANCE:
doc_id_var = 'ut__id'
else:
print("I don't know what type of document I have...")
return
db_matrix = dts.filter(run_id=self.pk, score__gt=self.dt_threshold)
docs = set(db_matrix.values_list(doc_id_var, flat=True))
if s_size > 0:
s_docs = random.sample(docs, s_size)
db_matrix = dts.filter(run_id=stat.pk,
score__gt=0.01,
doc__id__in=s_docs)
vs = list(db_matrix.values('score', doc_id_var, 'topic_id'))
c_ind = np.array(
list(
set(
db_matrix.values_list('topic_id',
flat=True).order_by(doc_id_var))))
r_ind = np.array(
list(
set(
db_matrix.values_list(doc_id_var,
flat=True).order_by(doc_id_var))))
d = [x['score'] for x in vs]
c = [int(np.where(c_ind == x['topic_id'])[0]) for x in vs]
r = [int(np.where(r_ind == x[doc_id_var])[0]) for x in vs]
m = coo_matrix((d, (r, c)), shape=(len(r_ind), len(c_ind)))
np.save(mpath, m)
np.save(rpath, r_ind)
np.save(cpath, c_ind)
return (m, c_ind, r_ind)
def calculate_tsne(self, path, p, s_size=0, force_overwrite=False):
"""
Function applied to RunStats object to calculate dimensionality reduction using TSNE
:param path: Results path
:param p:
:param s_size:
:param force_overwrite: (default: False) Overrides already existing results
:return:
"""
m, c_ind, r_ind = self.dt_matrix(path, s_size)
results_path = f"{path}/run_{self.pk}_s_{s_size}_p_{p}_results.npy"
if os.path.exists(results_path):
tsne_results = np.load(results_path)
if not force_overwrite:
print("We've already calculated the tsne positions")
return tsne_results, r_ind
tsne = mTSNE(n_components=2, verbose=0, perplexity=p, n_jobs=4)
tsne_results = tsne.fit_transform(m.toarray())
np.save(results_path, tsne_results)
return tsne_results, r_ind
class AWSCostEntryLineItemDailySummary(models.Model):
"""A daily aggregation of line items.
This table is aggregated by service, and does not
have a breakdown by resource or tags. The contents of this table
should be considered ephemeral. It will be regularly deleted from
and repopulated.
"""
class Meta:
"""Meta for AWSCostEntryLineItemDailySummary."""
db_table = "reporting_awscostentrylineitem_daily_summary"
managed = False
indexes = [
models.Index(fields=["usage_start"],
name="summary_usage_start_idx"),
models.Index(fields=["product_code"],
name="summary_product_code_idx"),
models.Index(fields=["usage_account_id"],
name="summary_usage_account_id_idx"),
GinIndex(fields=["tags"], name="tags_idx"),
models.Index(fields=["account_alias"],
name="summary_account_alias_idx"),
models.Index(fields=["product_family"],
name="summary_product_family_idx"),
models.Index(fields=["instance_type"],
name="summary_instance_type_idx"),
# A GIN functional index named "aws_summ_usage_pfam_ilike" was created manually
# via RunSQL migration operation
# Function: (upper(product_family) gin_trgm_ops)
# A GIN functional index named "aws_summ_usage_pcode_ilike" was created manually
# via RunSQL migration operation
# Function: (upper(product_code) gin_trgm_ops)
]
uuid = models.UUIDField(primary_key=True)
cost_entry_bill = models.ForeignKey("AWSCostEntryBill",
on_delete=models.CASCADE,
null=True)
# The following fields are used for grouping
usage_start = models.DateField(null=False)
usage_end = models.DateField(null=True)
usage_account_id = models.CharField(max_length=50, null=False)
account_alias = models.ForeignKey("AWSAccountAlias",
on_delete=models.PROTECT,
null=True)
product_code = models.CharField(max_length=50, null=False)
product_family = models.CharField(max_length=150, null=True)
availability_zone = models.CharField(max_length=50, null=True)
region = models.CharField(max_length=50, null=True)
instance_type = models.CharField(max_length=50, null=True)
unit = models.CharField(max_length=63, null=True)
organizational_unit = models.ForeignKey("AWSOrganizationalUnit",
on_delete=models.SET_NULL,
null=True)
# The following fields are aggregates
resource_ids = ArrayField(models.TextField(), null=True)
resource_count = models.IntegerField(null=True)
usage_amount = models.DecimalField(max_digits=24,
decimal_places=9,
null=True)
normalization_factor = models.FloatField(null=True)
normalized_usage_amount = models.FloatField(null=True)
currency_code = models.CharField(max_length=10)
unblended_rate = models.DecimalField(max_digits=24,
decimal_places=9,
null=True)
unblended_cost = models.DecimalField(max_digits=24,
decimal_places=9,
null=True)
markup_cost = models.DecimalField(max_digits=24,
decimal_places=9,
null=True)
blended_rate = models.DecimalField(max_digits=24,
decimal_places=9,
null=True)
blended_cost = models.DecimalField(max_digits=24,
decimal_places=9,
null=True)
public_on_demand_cost = models.DecimalField(max_digits=24,
decimal_places=9,
null=True)
public_on_demand_rate = models.DecimalField(max_digits=24,
decimal_places=9,
null=True)
tax_type = models.TextField(null=True)
tags = JSONField(null=True)
source_uuid = models.UUIDField(unique=False, null=True)
class UniversalAwardView(models.Model):
keyword_ts_vector = SearchVectorField()
award_ts_vector = SearchVectorField()
recipient_name_ts_vector = SearchVectorField()
award = models.OneToOneField(Award, primary_key=True)
category = models.TextField()
type = models.TextField()
type_description = models.TextField()
piid = models.TextField()
fain = models.TextField()
uri = models.TextField()
total_obligation = models.DecimalField(max_digits=23,
decimal_places=2,
blank=True,
null=True)
description = models.TextField()
total_subsidy_cost = models.DecimalField(max_digits=23,
decimal_places=2,
null=True,
blank=True)
total_loan_value = models.DecimalField(max_digits=23,
decimal_places=2,
null=True,
blank=True)
total_obl_bin = models.TextField()
recipient_hash = models.UUIDField()
recipient_id = models.IntegerField()
recipient_name = models.TextField()
recipient_unique_id = models.TextField()
parent_recipient_unique_id = models.TextField()
business_categories = ArrayField(models.TextField(), default=list)
action_date = models.DateField()
fiscal_year = models.IntegerField()
last_modified_date = models.TextField()
period_of_performance_start_date = models.DateField()
period_of_performance_current_end_date = models.DateField()
date_signed = models.DateField()
ordering_period_end_date = models.DateField(null=True)
original_loan_subsidy_cost = models.DecimalField(max_digits=23,
decimal_places=2,
null=True,
blank=True)
face_value_loan_guarantee = models.DecimalField(max_digits=23,
decimal_places=2,
null=True,
blank=True)
awarding_agency_id = models.IntegerField()
funding_agency_id = models.IntegerField()
awarding_toptier_agency_name = models.TextField()
funding_toptier_agency_name = models.TextField()
awarding_subtier_agency_name = models.TextField()
funding_subtier_agency_name = models.TextField()
awarding_toptier_agency_code = models.TextField()
funding_toptier_agency_code = models.TextField()
awarding_subtier_agency_code = models.TextField()
funding_subtier_agency_code = models.TextField()
recipient_location_country_code = models.TextField()
recipient_location_country_name = models.TextField()
recipient_location_state_code = models.TextField()
recipient_location_county_code = models.TextField()
recipient_location_county_name = models.TextField()
recipient_location_zip5 = models.TextField()
recipient_location_congressional_code = models.TextField()
pop_country_code = models.TextField()
pop_country_name = models.TextField()
pop_state_code = models.TextField()
pop_county_code = models.TextField()
pop_county_name = models.TextField()
pop_city_code = models.TextField()
pop_zip5 = models.TextField()
pop_congressional_code = models.TextField()
cfda_number = models.TextField()
sai_number = models.TextField()
pulled_from = models.TextField()
type_of_contract_pricing = models.TextField()
extent_competed = models.TextField()
type_set_aside = models.TextField()
product_or_service_code = models.TextField()
product_or_service_description = models.TextField()
naics_code = models.TextField()
naics_description = models.TextField()
class Meta:
managed = False
db_table = 'universal_award_matview'
class Project(models.Model):
name = models.CharField(help_text="Points to project name",
verbose_name="Name",
max_length=1024)
url_slug = models.CharField(help_text="URL slug for project",
verbose_name="URL slug",
unique=True,
max_length=1024,
null=True,
blank=True)
image = models.ImageField(help_text="Project image",
verbose_name="Image",
null=True,
blank=True,
upload_to=project_image_path)
thumbnail_image = models.ImageField(
help_text="Image to show on project card",
verbose_name="Thumbnail image",
null=True,
blank=True,
upload_to=project_image_path)
status = models.ForeignKey('ProjectStatus',
help_text="Points to project's status",
verbose_name="Project Status",
related_name="project_status",
on_delete=models.PROTECT)
start_date = models.DateTimeField(
help_text="Points to start date of the project",
verbose_name="Start Date",
null=True,
blank=True)
end_date = models.DateTimeField(
help_text="Points to end date of the project",
verbose_name="End Date",
null=True,
blank=True)
# created_at value automatically added when new project is created
created_at = models.DateTimeField(
help_text="Points to creation date of the project",
verbose_name="Created At",
auto_now_add=True)
# updated_at value automatically changes every time there is change in project object.
updated_at = models.DateTimeField(
help_text="Points to time when project was updated",
verbose_name="Updated At",
auto_now=True)
short_description = models.CharField(
help_text="Points to short description about the project",
verbose_name="Short Description",
null=True,
blank=True,
max_length=240)
description = models.CharField(
help_text="Points to detailed description about the project",
verbose_name="Description",
null=True,
blank=True,
max_length=4800)
country = models.CharField(help_text="Points to a country of the project",
verbose_name="Country",
max_length=512,
null=True,
blank=True)
city = models.CharField(help_text="Points to a city of the project",
verbose_name="City",
max_length=512,
null=True,
blank=True)
collaborators_welcome = models.BooleanField(
help_text="If collaborators are welcome or not for the project",
verbose_name="Collaborators welcome",
default=False)
skills = models.ManyToManyField(
Skill,
related_name="project_skills",
help_text="Points to all skills project persist or required",
verbose_name="Skills",
blank=True)
helpful_connections = ArrayField(models.CharField(max_length=264),
blank=True,
null=True,
size=10)
is_draft = models.BooleanField(
help_text="Whether project is public or just a private draft",
verbose_name="Is Draft?",
default=False)
website = models.CharField(help_text="Website",
verbose_name="Project's website",
max_length=256,
null=True,
blank=True)
rating = models.PositiveSmallIntegerField(
help_text=
"The larger the number, the more to the top this project will be displayed",
verbose_name="Rating (1-100)",
default=100)
class Meta:
app_label = "organization"
verbose_name = "Project"
verbose_name_plural = "Projects"
ordering = ['-rating', '-id']
def __str__(self):
return "(%d) %s" % (self.pk, self.name)
class AwardMatview(models.Model):
generated_unique_award_id = models.TextField(
primary_key=True, db_column='generated_unique_award_id')
latest_transaction = models.ForeignKey(
to='awards.TransactionMatview',
to_field='generated_unique_transaction_id',
db_column='latest_transaction_unique_id',
related_query_name='latest_transaction')
action_date = models.TextField()
agency_id = models.TextField()
assistance_type = models.TextField()
awarding_agency_abbr = models.TextField()
awarding_agency_code = models.TextField()
awarding_agency_id = models.TextField()
awarding_agency_name = models.TextField()
awarding_office_code = models.TextField()
awarding_office_name = models.TextField()
awarding_sub_tier_agency_abbr = models.TextField()
awarding_sub_tier_agency_c = models.TextField()
awarding_sub_tier_agency_n = models.TextField()
base_and_all_options_value = models.DecimalField(max_digits=23,
decimal_places=2)
business_categories = ArrayField(models.TextField())
business_funds_indicator = models.TextField()
business_types = models.TextField()
business_types_description = models.TextField()
category = models.TextField()
certified_date = models.DateTimeField()
cfda_number = models.TextField()
cfda_objectives = models.TextField()
cfda_title = models.TextField()
clinger_cohen_act_pla_desc = models.TextField()
clinger_cohen_act_planning = models.TextField()
commercial_item_acqui_desc = models.TextField()
commercial_item_acquisitio = models.TextField()
commercial_item_test_desc = models.TextField()
commercial_item_test_progr = models.TextField()
consolidated_contract = models.TextField()
consolidated_contract_desc = models.TextField()
contract_award_type_desc = models.TextField()
cost_or_pricing_data = models.TextField()
cost_or_pricing_data_desc = models.TextField()
date_signed = models.TextField()
construction_wage_rate_req = models.TextField()
construction_wage_rat_desc = models.TextField()
description = models.TextField()
dod_claimant_prog_cod_desc = models.TextField()
dod_claimant_program_code = models.TextField()
domestic_or_foreign_e_desc = models.TextField()
domestic_or_foreign_entity = models.TextField()
evaluated_preference = models.TextField()
evaluated_preference_desc = models.TextField()
extent_compete_description = models.TextField()
extent_competed = models.TextField()
fain = models.TextField()
fair_opportunity_limi_desc = models.TextField()
fair_opportunity_limited_s = models.TextField()
fed_biz_opps = models.TextField()
fed_biz_opps_description = models.TextField()
fiscal_year = models.TextField()
foreign_funding = models.TextField()
foreign_funding_desc = models.TextField()
funding_agency_abbr = models.TextField()
funding_agency_code = models.TextField()
funding_agency_id = models.TextField()
funding_agency_name = models.TextField()
funding_office_code = models.TextField()
funding_office_name = models.TextField()
funding_sub_tier_agency_abbr = models.TextField()
funding_sub_tier_agency_co = models.TextField()
funding_sub_tier_agency_na = models.TextField()
idv_type = models.TextField()
idv_type_description = models.TextField()
information_technolog_desc = models.TextField()
information_technology_com = models.TextField()
interagency_contract_desc = models.TextField()
interagency_contracting_au = models.TextField()
last_modified_date = models.DateTimeField()
major_program = models.TextField()
multi_year_contract = models.TextField()
multi_year_contract_desc = models.TextField()
multiple_or_single_aw_desc = models.TextField()
multiple_or_single_award_i = models.TextField()
naics = models.TextField()
naics_description = models.TextField()
number_of_offers_received = models.TextField()
officer_1_amount = models.TextField()
officer_1_name = models.TextField()
officer_2_amount = models.TextField()
officer_2_name = models.TextField()
officer_3_amount = models.TextField()
officer_3_name = models.TextField()
officer_4_amount = models.TextField()
officer_4_name = models.TextField()
officer_5_amount = models.TextField()
officer_5_name = models.TextField()
other_than_full_and_o_desc = models.TextField()
other_than_full_and_open_c = models.TextField()
parent_award_piid = models.TextField()
parent_recipient_unique_id = models.TextField()
period_of_performance_current_end_date = models.DateTimeField()
period_of_performance_start_date = models.DateTimeField()
piid = models.TextField()
pop_city_name = models.TextField()
pop_code = models.TextField()
pop_congressional_code = models.TextField()
pop_country_code = models.TextField()
pop_country_name = models.TextField()
pop_county_code = models.TextField()
pop_county_name = models.TextField()
pop_foreign_province = models.TextField()
pop_state_code = models.TextField()
pop_state_name = models.TextField()
pop_zip5 = models.TextField()
potential_total_value_of_award = models.DecimalField(max_digits=23,
decimal_places=2)
price_evaluation_adjustmen = models.TextField()
product_or_service_co_desc = models.TextField()
product_or_service_code = models.TextField()
program_acronym = models.TextField()
program_system_or_equ_desc = models.TextField()
program_system_or_equipmen = models.TextField()
pulled_from = models.TextField()
purchase_card_as_paym_desc = models.TextField()
purchase_card_as_payment_m = models.TextField()
recipient_location_address_line1 = models.TextField()
recipient_location_address_line2 = models.TextField()
recipient_location_address_line3 = models.TextField()
recipient_location_city_code = models.TextField()
recipient_location_city_name = models.TextField()
recipient_location_congressional_code = models.TextField()
recipient_location_country_code = models.TextField()
recipient_location_country_name = models.TextField()
recipient_location_county_code = models.TextField()
recipient_location_county_name = models.TextField()
recipient_location_foreign_city_name = models.TextField()
recipient_location_foreign_postal_code = models.TextField()
recipient_location_foreign_province = models.TextField()
recipient_location_state_code = models.TextField()
recipient_location_state_name = models.TextField()
recipient_location_zip5 = models.TextField()
recipient_name = models.TextField()
recipient_unique_id = models.TextField()
record_type = models.IntegerField()
referenced_idv_agency_desc = models.TextField()
referenced_idv_agency_iden = models.TextField()
sai_number = models.TextField()
sea_transportation = models.TextField()
sea_transportation_desc = models.TextField()
labor_standards = models.TextField()
labor_standards_descrip = models.TextField()
small_business_competitive = models.TextField()
solicitation_identifier = models.TextField()
solicitation_procedur_desc = models.TextField()
solicitation_procedures = models.TextField()
subaward_count = models.IntegerField()
subcontracting_plan = models.TextField()
subcontracting_plan_desc = models.TextField()
total_obligation = models.DecimalField(max_digits=23, decimal_places=2)
total_outlay = models.DecimalField(max_digits=23, decimal_places=2)
total_subaward_amount = models.DecimalField(max_digits=23,
decimal_places=2)
total_subsidy_cost = models.DecimalField(max_digits=23,
decimal_places=2,
null=True,
blank=True)
total_loan_value = models.DecimalField(max_digits=23,
decimal_places=2,
null=True,
blank=True)
type = models.TextField()
type_description = models.TextField()
type_of_contract_pric_desc = models.TextField()
type_of_contract_pricing = models.TextField()
type_of_idc = models.TextField()
type_of_idc_description = models.TextField()
type_set_aside = models.TextField()
type_set_aside_description = models.TextField()
uri = models.TextField()
materials_supplies_article = models.TextField()
materials_supplies_descrip = models.TextField()
class Meta:
managed = False
db_table = 'award_matview'
class Perimeter(models.Model):
"""
Represents a single application perimeter for an Aid.
Each perimeter represents a location (e.g a region, a department, a
commune…) and contains data about where it is located.
E.g: the commune of Montpellier is located in Hérault, Occitanie, France, Europe.
Since nothing is simple when administration is involved, some perimeters
e.g epcis can be spread over several departments / regions.
Drainage basins are another edge case, since they are independant from any
existing legal borders.
https://fr.wikipedia.org/wiki/Bassin_hydrographique
"""
SCALES_TUPLE = (
(1, 'commune', 'Commune'),
(5, 'epci', 'EPCI'),
(8, 'basin', 'Bassin hydrographique'),
(10, 'department', 'Département'),
(15, 'region', 'Région'),
(16, 'overseas', 'Outre-mer'),
(17, 'mainland', 'Métropole'),
(18, 'adhoc', 'Ad-hoc'),
(20, 'country', 'Pays'),
(25, 'continent', 'Continent'),
)
SCALES = Choices(*SCALES_TUPLE)
scale = models.PositiveIntegerField(_('Scale'), choices=SCALES)
name = models.CharField(_('Name'), max_length=128)
code = models.CharField(
_('Code'),
max_length=16,
help_text=_('Internal usage only, not relevant for Ad-hoc perimeters'))
contained_in = models.ManyToManyField('geofr.Perimeter',
verbose_name=_('Contained in'),
related_name='contains',
blank=True)
manually_created = models.BooleanField(_('Manually created'),
default=False)
is_visible_to_users = models.BooleanField(
_('The perimeter is visible to users'), default=True)
continent = models.CharField(_('Continent'), max_length=2, default='EU')
country = models.CharField('Pays', max_length=3,
default='FRA') # ISO_3166-3 codes
regions = ArrayField( # Array of region codes (INSEE COG)
verbose_name='Régions',
base_field=models.CharField(max_length=2),
default=list,
blank=True)
departments = ArrayField( # Array of depts codes (INSEE COG)
verbose_name='Départements',
base_field=models.CharField(max_length=3),
default=list,
blank=True)
epci = models.CharField(
_('EPCI'),
max_length=32, # INSEE COG
blank=True)
basin = models.CharField(
_('Drainage basin'),
max_length=32, # Sandre code
blank=True)
zipcodes = ArrayField(verbose_name=_('Zip codes'),
base_field=models.CharField(max_length=8),
null=True,
blank=True)
is_overseas = models.BooleanField(verbose_name=_('Is overseas?'),
null=True)
# This field is used to store the name without accent, for indexing.
# We recently updated the perimeter search to ignore accents, and at first
# we were using postgres' unaccent extension.
# We ran into an issue where the trigram index was not being used anymore
# bumping the querytime from a few ms to more than 300ms.
# Since it's kinda hard to add an index on an unaccent expression, it's
# just easier to store an index the unaccented version of the name.
unaccented_name = models.CharField(
_('Name without accent (for indexing purpose)'), max_length=128)
date_created = models.DateTimeField(_('Date created'),
default=timezone.now)
date_updated = models.DateTimeField(_('Date updated'), auto_now=True)
class Meta:
verbose_name = _('Perimeter')
verbose_name_plural = _('Perimeters')
unique_together = (('scale', 'code'), )
indexes = [
GinIndex(name='name_trgm',
fields=['name'],
opclasses=['gin_trgm_ops']),
GinIndex(name='unaccented_name_trgm',
fields=['unaccented_name'],
opclasses=['gin_trgm_ops']),
]
def __str__(self):
if not self.scale:
return ''
if self.scale == self.SCALES.commune and self.zipcodes:
_str = '{} ({} – {})'.format(self.name, self.get_scale_display(),
', '.join(self.zipcodes))
elif self.scale <= self.SCALES.region:
_str = '{} ({})'.format(self.name, self.get_scale_display())
else:
_str = self.name
return _str
@property
def id_slug(self):
return '{}-{}'.format(self.id, slugify(self.name))
def save(self, *args, **kwargs):
self.unaccented_name = remove_accents(self.name)
return super().save(*args, **kwargs)
class Component(models.Model): outline = models.ForeignKey(CourseOutline, related_name='components', on_delete=models.CASCADE) name = models.CharField(blank=True, max_length=200) outcomes = ArrayField(models.IntegerField()) weight = models.FloatField()
class SpeakerRole(models.Model):
"""
Describes role of the speaker :model:`parliament.Person` in parliament
"""
name = models.TextField()
alt_names = ArrayField(models.TextField(), null=True)
