class CallLater(models.Model): name = models.CharField(max_length=64, default='', editable=True, verbose_name=u'additional lookup field') time_to_run = models.DateTimeField(default=timezone.now) time_to_stop = models.DateTimeField(null=True, blank=True) function = PickledObjectField() args = PickledObjectField(null=True) kwargs = PickledObjectField(null=True) repeat = models.PositiveIntegerField(default=1) every = models.DurationField(null=True, blank=True) when_check_if_failed = models.DateTimeField( default=far_future_fail_timeout) retries = models.PositiveIntegerField(default=3) timeout_retries = models.PositiveIntegerField(default=2) problem = models.BooleanField(default=False) def save(self, *args, **kwargs): # (<django.db.models.fields.PositiveIntegerField>,) is returned if self.repeat not set # I assume i must save() model before default value is given if self.every is None and type( self.repeat) is int and self.repeat is not 1: raise ValidationError( 'you must set a repeat time (via every=[timedelta]) if you want a function called many times' ' (each time after current time + repeat') super(CallLater, self).save(*args, **kwargs) def check_individual(self): preprocess_instance(self, timezone.now())
class LayersConfiguration(Configuration): ds_a = models.ForeignKey(GeoDataSource, on_delete=models.CASCADE, related_name='+', null=True) ds_b = models.ForeignKey(GeoDataSource, on_delete=models.CASCADE, related_name='+', null=True) popup_data = PickledObjectField(null=True) colours = PickledObjectField(null=True) def __init__(self, name, description, pk_a, pk_b, popup, colours): super().__init__() self.ds_a = pk_a self.ds_b = pk_b self.set_colours(colours) self.set_popup_data(popup) self.set_name(name) self.set_description(description) def set_colours(self, colours): self.colours = colours def set_popup_data(self, popup_data): self.popup_data = popup_data
class Task(Model): """The queued task, persisted in the database (so it can be polled for status) """ taskid = CharField(max_length=36, primary_key=True) function_name = CharField(max_length=255) args = PickledObjectField() kwargs = PickledObjectField() return_value = PickledObjectField(null=True) error = TextField(blank=True, null=True) queued = DateTimeField(blank=True, null=True) started = DateTimeField(blank=True, null=True) finished = DateTimeField(blank=True, null=True) status = CharField(max_length=1, choices=STATUS_CHOICES, default=Status.QUEUED) class Meta: db_table = 'django_ztaskq_task' @classmethod def run_task(cls, task_id, logger): try: task = cls.objects.get(pk=task_id) except ObjectDoesNotExist, e: logger.info('Could not get task with id %s:\n%s' % (task_id, e)) return task.run(logger)
class AuthRecord(models.Model): # These two fields define a unique AuthRecord: the name of the # provider and the identifier the provider uses to identify the # user, which if possible should be stable across changes in # screen names. provider = models.CharField(max_length=32, db_index=True) uid = models.CharField(max_length=128) # The Django User associated with the provider-uid pair. user = models.ForeignKey(User, related_name="singlesignon", db_index=True) # Profile information returned by the most recent OAuth callback, etc. auth_token = PickledObjectField() profile = PickledObjectField() # General metadata. created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) class Meta: verbose_name = "Authentication Record" unique_together = (("provider", "uid"), ) # don't add any ordering because it causes mysql filesort on joins def __unicode__(self): return self.provider + " " + self.uid[ 0:10] + " -> " + self.user.username
class TestingModel(models.Model): pickle_field = PickledObjectField() compressed_pickle_field = PickledObjectField(compress=True) default_pickle_field = PickledObjectField(default=(D1, S1, T1, L1)) callable_pickle_field = PickledObjectField(default=date.today) non_copying_field = PickledObjectField(copy=False, default=TestCopyDataType('boom!'))
class Task(Model): taskid = CharField(max_length=36, primary_key=True) function_name = CharField(max_length=255) args = PickledObjectField() kwargs = PickledObjectField() return_value = PickledObjectField(null=True) error = TextField(blank=True, null=True) queued = DateTimeField(blank=True, null=True) started = DateTimeField(blank=True, null=True) finished = DateTimeField(blank=True, null=True) status = CharField(max_length=1, choices=STATUS_CHOICES, default=Status.QUEUED) def save(self, *args, **kwargs): if not self.queued: self.queued = datetime.datetime.utcnow() super(Task, self).save(*args, **kwargs) class Meta: db_table = 'django_ztaskq_task' @classmethod def run_task(cls, task_id): try: task = cls.objects.get(pk=task_id) except Exception, e: logger.info('Could not get task with id %s:\n%s' % (task_id, e)) return task.run()
class Audit(models.Model): date = models.DateTimeField(auto_now=True) in_progress = models.BooleanField(default=True) validated = models.BooleanField(default=False) election_type = models.CharField(max_length=16) audit_type = models.CharField(max_length=16) risk_limit = models.FloatField() random_seed = models.CharField(max_length=128, blank=True, null=True) random_seed_time = models.DateTimeField() n_winners = models.IntegerField(default=1) max_polls = models.IntegerField() polled_ballots = models.IntegerField(default=0) preliminary_count = models.FileField() shuffled = PickledObjectField(default=list) vote_count = PickledObjectField(default=dict) accum_recount = PickledObjectField(default=dict) max_p_value = models.FloatField(default=1) def _update_accum_recounted(self, recount, save=True): for c in recount: self.accum_recount[c] += recount[c] if save: self.save() def get_df(self, path): df = pd.read_csv(path) if self.election_type == utils.DHONDT: df['party'] = df['party'].fillna('') return df def get_grouped(self, df): if self.election_type == utils.DHONDT: group = df.groupby('party') else: # self.election_type == utils.SIMPLE_MAJORITY or self.election_type == utils.SUPER_MAJORITY group = df.groupby('candidate') return group.sum()['votes'].sort_values(ascending=False).to_dict() def add_polled_ballots(self, recount_df, save=True): vote_recount = self.get_grouped(recount_df) self._update_accum_recounted(vote_recount, save=False) self.polled_ballots += sum(vote_recount.values()) if self.audit_type == utils.BALLOT_POLLING: self.shuffled = self.shuffled[sum(vote_recount.values()):] else: # self.audit_type == utils.COMPARISON table_count = len(recount_df['table'].unique()) self.shuffled = self.shuffled[table_count:] if save: self.save()
class ClusterConfiguration(Configuration): cluster_strategy = PickledObjectField(null=True) ds = models.ForeignKey(GeoDataSource, on_delete=models.CASCADE, related_name='+', null=True) col_a = models.CharField(max_length=50, null=True) col_b = models.CharField(max_length=50, null=True) centroids = PickledObjectField(null=True) labels = PickledObjectField(null=True) def set_strategy(self, strategy): self.cluster_strategy = strategy def set_ds(self, ds): self.ds = ds def set_cols(self, col_a, col_b): self.col_a = col_a self.col_b = col_b def get_col_a(self): return self.col_a def get_col_b(self): return self.col_b def get_strategy(self): return self.cluster_strategy.get_details() def get_ds(self): return self.ds def get_centroids(self): return self.centroids def get_labels(self): return self.labels # en este caso, el resultado de apply será el de aplicar el algoritmo elegido sobre el ds # guardado y devolver la info para mostrar los datos en el front --> visualización def apply(self): pass def clusterize(self): results = self.cluster_strategy.clusterize(self.ds, self.col_a, self.col_b) return results
class Task(models.Model): STATUS_CHOICES = Choices( ('pending', 'Pending'), ('running', 'Running'), ('success', 'Success'), ('failed', 'Failed') ) id = models.CharField(max_length=32, primary_key=True, editable=False) name = models.CharField(max_length=100, editable=False) #TODO Is this needed..? func = models.CharField(max_length=256, help_text='Reference to task function') hook = models.CharField(max_length=256, null=True, help_text='Function to call after task completes (passed Task instance)') args = PickledObjectField(null=True, protocol=-1, help_text='Positional arguments provided to function') kwargs = PickledObjectField(null=True, protocol=-1, help_text='Keyword arguments provided to function') result = PickledObjectField(null=True, protocol=-1, help_text="Return value of task function") group = models.CharField(max_length=100, editable=False, null=True, help_text="Task group, so results for related tasks can be grouped together") cluster_type = models.CharField(max_length=80, null=True, help_text='Allows specifying which clusters this task can run on') created_time = models.DateTimeField(editable=False, help_text='Time task was first created') start_time = models.DateTimeField(editable=False, help_text='Start time of most recent task execution') duration = models.IntegerField(editable=False, help_text='Duration of most recent task execution') status = models.CharField(choices=STATUS_CHOICES, default=STATUS_CHOICES.pending, editable=False) attempt_count = models.IntegerField(default=0) objects = TaskManager() def group_result(self, failures=False): if self.group: return self.objects.get_group_results(self.group, failures) def group_count(self, failures=False): if self.group: return self.objects.get_group_count(self.group, failures) def time_taken(self): return (self.end_time - self.start_time).total_seconds() @property def short_result(self): return truncatechars(self.result, 100) def __str__(self): return f"{self.name or self.id}" class Meta: app_label = "django_q" ordering = ["-stopped"]
class ConfigOption(DiffModel, UniqueNameMixin): owner = models.ForeignKey('db.Owner', related_name='config_options', on_delete=models.CASCADE) key = models.CharField(max_length=256) secret = EncryptedPickledObjectField(null=True, blank=True) value = PickledObjectField(null=True, blank=True) class Meta: app_label = 'db' unique_together = (('owner', 'key'), ) def __str__(self): return self.unique_name @cached_property def unique_name(self): return '{} <{}>'.format( self.owner, self.key) if self.owner else '{}'.format(self.key) @property def is_secret(self): return self.secret is not None @property def is_value(self): return self.value is not None
class FrontSetting(models.Model): key = models.CharField(max_length=50) value = PickledObjectField(editable=False) def update_setting(self, data): self.value = data self.save()
class GeoLinkedFile(GeoDataSource): source_a = models.ForeignKey(GeoDataSource, on_delete=models.CASCADE, related_name='+', null=True) source_b = models.ForeignKey(IDataSource, on_delete=models.CASCADE, related_name='+', null=True) link_strategy = PickledObjectField(null=True) def set_def_name(self): self.name = self.source_a.get_name() + self.source_b.get_name() def set_lstrategy(self, strategy): self.link_strategy = strategy def set_sources(self, sa, sb): self.source_a = sa self.source_b = sb def link(self): self.dataset = self.link_strategy.link(self.source_a, self.source_b) def get_details(self): geo_linked_details = { 'source_a': self.source_a.get_name(), 'source_b': self.source_b.get_name() } geo_linked_details.update(super().get_details()) geo_linked_details.update(self.link_strategy.get_details()) return geo_linked_details
class Policy(SharedKey, Name, Tags, PolicyLookup): """ A Policy is a loosely defined data structure. That represents a policy of a policy set. Policies may be shared across sets. Their semantic meaning may be determined by their shared key and they may be categorized by their tags. A policy has a range of possible values, anything from True/False to a number range or anything else that can be selected and have meaning. The range is serialized by the values attribute. Classes that have PolicySet attributes, namely ConfigEntity instances, should store the actual selected value of each Policy in a separate data structure ConfigEntity instances store policy settings in ConfigEntity.selections.policy_sets. See that attribute to understand how policy value selections are stored. """ schema = models.CharField(max_length=100, null=True) objects = GeoInheritanceManager() policies = models.ManyToManyField('Policy', default=lambda: []) # Pickle the set of values into a single string field # The allowed values of the policy. This should be anything that can be serialized and represented on the client values = PickledObjectField() def update_or_create_policy(self, policy_config): child_policy = Policy.objects.update_or_create( key=policy_config['key'], schema='%s__%s' % (self.schema, policy_config['key']) if self.schema else policy_config['key'], defaults=dict( name=policy_config['name'], description=policy_config.get('description', None), values=policy_config. get('values', {}) ))[0] if policy_config.get('policies', None) and len(policy_config['policies']) > 0: child_policy.policies.add(*map(lambda child_policy_config: child_policy.update_or_create_policy(child_policy_config), policy_config['policies'])) return child_policy class Meta(object): app_label = 'main'
def backwards(self, orm): # Deleting model 'FormValue' db.delete_table('form_designer_formvalue') # Adding field 'FormLog.data' db.add_column('form_designer_formlog', 'data', self.gf('picklefield.fields.PickledObjectField')( null=True, blank=True), keep_default=False) from form_designer.models import FormLog from picklefield import PickledObjectField tmp_data = PickledObjectField(null=True, blank=True) tmp_data.contribute_to_class(FormLog, 'data') if not db.dry_run: for log in FormLog.objects.all(): log.data = log.get_data() raise Exception(log.data) log.save() # Deleting field 'FormLog.created_by' db.delete_column('form_designer_formlog', 'created_by_id')
class SettySettings(models.Model): # 190 chars or there is a key length error in mysql 5.6 name = models.CharField(max_length=190, primary_key=True) app_name = models.CharField(max_length=190, choices=APP_CHOICES, blank=True) value = PickledObjectField() type = models.CharField(max_length=8, choices=TypeChoices.ALL_CHOICES) created_time = models.DateTimeField(auto_now_add=True) updated_time = models.DateTimeField(auto_now=True) @property def value_unpacked(self): return str(self.value) @value_unpacked.setter def value_unpacked(self, value): self.value = value def __str__(self): return '{}={}'.format(self.name, self.value_unpacked) class Meta: verbose_name = 'Setty Settings' verbose_name_plural = 'Setty Settings'
def forwards(self, orm): # Adding model 'FormValue' db.create_table('form_designer_formvalue', ( ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), ('form_log', self.gf('django.db.models.fields.related.ForeignKey')(related_name='values', to=orm['form_designer.FormLog'])), ('field_name', self.gf('django.db.models.fields.SlugField')(max_length=255, db_index=True)), ('value', self.gf('picklefield.fields.PickledObjectField')(null=True, blank=True)), )) db.send_create_signal('form_designer', ['FormValue']) # Adding field 'FormLog.created_by' db.add_column('form_designer_formlog', 'created_by', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'], null=True, blank=True), keep_default=False) db.rename_column('form_designer_formlog', 'data', 'tmp_data') from form_designer.models import FormLog from picklefield import PickledObjectField tmp_data = PickledObjectField(null=True, blank=True) tmp_data.contribute_to_class(FormLog, 'tmp_data') for log in FormLog.objects.all(): log.set_data(log.tmp_data) log.save() # Deleting field 'FormLog.data' db.delete_column('form_designer_formlog', 'tmp_data')
class GlobalCounter(models.Model): # グローバル変数を格納しておくモデル # 固定パラメータ detector_strides = PickledObjectField(default=(5, 7, 9)) static_th = models.FloatField(default=10) norep_std_th = models.FloatField(default=13) norep_ent_th = models.FloatField(default=1.0) inrep_std_th = models.FloatField(default=13) inrep_ent_th = models.FloatField(default=1.1) lastsix_ent_th = models.FloatField(default=1.1) history_num = models.IntegerField(default=9) # アップデートするパラメータ in_time = models.IntegerField(default=0) out_time = models.IntegerField(default=0) cooldown_in_time = models.IntegerField(default=0) cooldown_out_time = models.IntegerField(default=0) global_counter = models.IntegerField(default=0) winner_stride = models.IntegerField(default=0) cur_state = models.IntegerField(default=State.NO_REP) in_frame_num = models.IntegerField(default=-1) actions_counter = models.IntegerField(default=0) def update(self, id: int, params: dict): GlobalCounter.objects.filter(id=id).update(**params)
class BaseTransformer(models.Model): """ BaseTransformer abstract model - parent class for TextUnitTransformer and DocumentTransformer # TODO: Document/TextUnit interface for transformer # TODO: Discuss stronger "typing" of transformers (e.g., as CHOICE field or further normalization) """ # Transformer name name = models.CharField(max_length=1024, db_index=True, unique=True) # Transformer version, for version-tracked transformers version = models.CharField(max_length=1024, db_index=True) # Name for TextUnitVector - matches TextUnitVector vector_name = models.CharField(max_length=1024, db_index=True) # Serialized model object model_object = PickledObjectField(compress=True) # Active/valid model field used to invalidate or de-activate models is_active = models.BooleanField(default=True, db_index=True) class Meta: abstract = True unique_together = (("name", "version"), ) ordering = ('name', 'vector_name') def __str__(self): return "{} (name={}, version={}, class_name={}" \ .format(self._meta.model.__name__, self.name, self.version, self.vector_name)
class LocalCounter(models.Model): # 各カウンターのメンバ変数を格納しておくモデル global_counter = models.ForeignKey(GlobalCounter, on_delete=models.CASCADE) interval = models.IntegerField() rep_count = models.IntegerField(default=0) frame_residue = models.IntegerField(default=0) count_array = PickledObjectField() ent_arr = PickledObjectField() std_arr = PickledObjectField() label_array = PickledObjectField() def update(self, global_counter: GlobalCounter, interval: int, params: dict): LocalCounter.objects.filter(global_counter_id=global_counter.id, interval=interval).update(**params)
class Option(models.Model): site = models.ForeignKey(Site, related_name='options', verbose_name=_('Site')) key = models.CharField(max_length=255, verbose_name=_('Key')) value = PickledObjectField(null=True, default=True, verbose_name=_('Value'), editable=VALUE_EDITABLE) autoload = models.BooleanField(default=False, verbose_name=_('Autoload')) updated_at = models.DateTimeField(auto_now=True, verbose_name=_('Update at')) created_at = models.DateTimeField(auto_now_add=True, verbose_name=_('Create at')) expires_at = models.DateTimeField(blank=True, null=True, verbose_name=_('Expires at')) # override default manager objects = OptionManager() all = models.Manager() on_site = objects def __unicode__(self): return u"{0.site.name:s}:{0.key:s}".format(self) class Meta: unique_together = ('site', 'key',) verbose_name = _('Option') verbose_name_plural = _('Options')
class Ticket(models.Model): TYPE_STRIPE = 'stripe' TYPE_PAYPAL = 'paypal' TYPES = ((TYPE_STRIPE, 'Stripe'), (TYPE_PAYPAL, 'PayPal')) STATUS_INPROGRESS = 1 STATUS_FAILED = 2 STATUS_COMPLETED = 3 STATUS = ((STATUS_INPROGRESS, 'Inprogress'), (STATUS_FAILED, 'Failed'), (STATUS_COMPLETED, 'Completed')) objects = TicketManager() competition = models.ForeignKey(Competition, related_name='tickets') user = models.ForeignKey(Account, related_name='tickets') method = models.CharField(choices=TYPES, max_length=10) paymentID = models.CharField(max_length=255) status = models.IntegerField(choices=STATUS) data = PickledObjectField() message_log = models.CharField(max_length=255, null=True, blank=True) created = models.DateTimeField(auto_now_add=True) @property def payment_method(self): for key, val in self.TYPES: if key == self.method: return val return '' @property def payment_status(self): for key, val in self.STATUS: if key == self.status: return val return ''
class WaitedFile(models.Model): path = models.CharField(max_length=255, unique=True, db_index=True) task_id = models.CharField(max_length=128, db_index=True, null=True, blank=True) task = PickledObjectField(null=True, editable=False) description = models.CharField(max_length=255, null=True, blank=True) @classmethod def exists(cls, path): """Returns opened file or None. `path` is relative to WAITER_ROOT. Won't open a path leading outside of WAITER_ROOT. """ abs_path = check_abspath(path) # Pre-fetch objects for deletion to avoid minor race condition relevant = [o.id for o in cls.objects.filter(path=path)] if isfile(abs_path): cls.objects.filter(id__in=relevant).delete() return True else: return False @classmethod def can_order(cls, path): return (cls.objects.filter(path=path).exists() or cls.exists(path) or cls.objects.count() < WAITER_MAX_QUEUE) def is_stale(self): if self.task is None: # Race; just let the other task roll. return False if self.task.status not in (u'PENDING', u'STARTED', u'SUCCESS', u'RETRY'): return True return False @classmethod def order(cls, path, task_creator, description=None): """ Returns an URL for the user to follow. If the file is ready, returns download URL. If not, starts preparing it and returns waiting URL. task_creator: function taking a path and generating the file; description: a string or string proxy with a description for user; """ already = cls.exists(path) if not already: waited, created = cls.objects.get_or_create(path=path) if created or waited.is_stale(): waited.task = task_creator(check_abspath(path)) waited.task_id = waited.task.task_id waited.description = description waited.save() return reverse("waiter", args=[path]) return join(WAITER_URL, path)
class TwitterCache(models.Model): account = models.CharField(max_length=100, db_index=True, unique=True) tweets = PickledObjectField(default=list) updated = ModificationDateTimeField() objects = TwitterCacheManager() def __unicode__(self): return u'Tweets from @' + self.account
class LocationToPartnerMapping(models.Model): mapping = PickledObjectField(null=True) def update(self, mapping): self.objects.all().delete() self.objects.create(mapping=mapping) @classmethod def get_mapping(cls): return FormattedKeyDict(cls.objects.first().mapping)
class Result(base_model.BaseModel): query = models.ForeignKey(Query, on_delete=models.CASCADE) engine = models.CharField(max_length=100) result = PickledObjectField() objects = ResultManager() class Meta: verbose_name = _('result') verbose_name_plural = _('results')
class GrandProfitAbilityReport(models.Model): created = models.DateTimeField(auto_now=True) created_for = models.DateField() data = PickledObjectField(default=dict) type = models.IntegerField(default=0, null=False) from_server = models.IntegerField(default=0, null=False) from_id = models.IntegerField(default=0, null=False) def __str__(self): return f'{self.from_server},{self.from_id},{self.created},{self.created_for},{self.type}'
class Intersection(models.Model): """ Describes an intersection from on geographic table to another. The tree is a Token object defined for a subclass that can embed 0 to many more Tokens recursively. The tree describes an intersection query completely or can use parameters in the form {param_name} to provide spaces that are filled in by field values of subclasses """ objects = GeoInheritanceManager() # Holds a default Token tree in memory. We don't store this in the database since it is constant. default_tree = None, # IF the customization of the default Token tree is needed, it is stored in the database here tree = PickledObjectField(default=None, null=True) # Class constant indicating the join type # This helps the front-end determine what kind of UI to show join_type_key = None # This is set True to indicate that the Intersection is a template instance that is shared among # instances, such as Behaviors, that never update the Intersection is_template = models.BooleanField(default=False) # feature_behavior is implicitly created by Django, since FeatureBehavior has a toOne @property def join_type(self): if not self.join_type_key: raise Exception("Class %s does not define a join_type_key" % self.__class__.__name__) return JoinType.objects.get(key=self.join_type_key) @property def subclassed(self): """ Return the subclassed version of the Intersection. If not yet persisted the intersection instance will be a subclass or null :return: """ return Intersection.objects.get_subclass(id=self.id) if\ self.id else\ self @property def feature_behavior(self): """ This is justed used by the API to back-dirty the feature_behavior from the Intersection. It should probably be handled exclusively in the front- end :return: """ if self.is_template: # Multiples possible, so return None return None return list(self.featurebehavior_set.all())[0] class Meta(object): abstract = False app_label = 'main'
class Constance(models.Model): key = models.CharField(max_length=255, unique=True) value = PickledObjectField(null=True, blank=True) class Meta: verbose_name = _('constance') verbose_name_plural = _('constances') db_table = 'constance_config' def __unicode__(self): return self.key
def forwards(self, orm): from picklefield import PickledObjectField tmp_data = PickledObjectField(null=True, blank=True) tmp_data.contribute_to_class(orm['form_designer.FormLog'], 'tmp_data') for log in orm['form_designer.FormLog'].objects.all(): log.set_data(log.tmp_data) log.save() # Deleting field 'FormLog.data' db.delete_column('form_designer_formlog', 'tmp_data')
class Constance(models.Model): key = models.TextField() value = PickledObjectField() class Meta: verbose_name = _('constance') verbose_name_plural = _('constances') db_table = 'constance_config' def __unicode__(self): return self.key