class Reader(db_wrapper.Model):
user = peewee.ForeignKeyField(UserAccount, backref='readers')
book = peewee.ForeignKeyField(Book, backref='readers')
status = peewee.CharField(max_length=20, default='unread')
rating = peewee.SmallIntegerField(null=True)
added_date = peewee.DateTimeField(default=lambda: arrow.utcnow().naive)
read_date = peewee.DateTimeField(null=True)
meta = postgres_ext.BinaryJSONField(default=lambda: {})
class Book(db_wrapper.Model):
isbn = peewee.BigIntegerField(unique=True, null=True)
title = peewee.TextField()
authors = peewee.TextField()
added_date = peewee.DateTimeField(default=lambda: arrow.utcnow().naive)
meta = postgres_ext.BinaryJSONField(default=lambda: {})
class Meta:
indexes = ((('title', 'authors'), True), )
class FeedSettings(peewee.Model):
retailer = peewee.ForeignKeyField(Retailer, related_name='feed_settings')
active = peewee.BooleanField(default=True)
url = peewee.TextField()
format = peewee.CharField()
mapper = peewee.CharField()
index_key = peewee.CharField(max_length=3, default='rus')
params = postgres_ext.BinaryJSONField(default={})
priority = peewee.IntegerField(default=5)
comment = peewee.TextField()
default_gender = peewee.IntegerField(default=None)
default_category = peewee.IntegerField(default=None)
class Meta:
database = db
schema = 'store'
class Vulnerability(BaseModel):
artifact = peewee.ForeignKeyField(
Artifact,
help_text='Affected artifact.')
provider = peewee.CharField(
help_text='Tool providing the information.')
reference = peewee.CharField(
help_text='Vulnerability identifier used by the provider.')
details = postgres_ext.BinaryJSONField(
help_text='Extra information provided about the vulnerability.')
last_seen = peewee.DateTimeField(
default=datetime.datetime.utcnow,
help_text=('Timestamp of the latest analysis detecting '
'this vulnerability.'))
class Meta:
indexes = (
(('artifact', 'provider', 'reference'), True), # Unique Index
)
class PostgresMaster(PostgresModel):
### PEEWEE FIELDS ###
id = peewee.IntegerField(primary_key=True)
artists = postgres_ext.BinaryJSONField(null=True)
genres = postgres_ext.ArrayField(peewee.TextField, null=True)
main_release_id = peewee.IntegerField(null=True)
styles = postgres_ext.ArrayField(peewee.TextField, null=True)
title = peewee.TextField()
year = peewee.IntegerField(null=True)
### PEEWEE META ###
class Meta:
db_table = 'masters'
### PUBLIC METHODS ###
@classmethod
def bootstrap(cls):
cls.drop_table(True)
cls.create_table()
cls.bootstrap_pass_one()
@classmethod
def from_element(cls, element):
data = cls.tags_to_fields(element)
return cls(**data)
@classmethod
def bootstrap_pass_one(cls):
PostgresModel.bootstrap_pass_one(
model_class=cls,
xml_tag='master',
name_attr='title',
skip_without=['title'],
)
class Retailer(peewee.Model):
name = peewee.CharField()
description = peewee.CharField(null=True)
picture = peewee.CharField(null=True)
active = peewee.BooleanField(default=True)
ad_model = peewee.CharField(null=True,
verbose_name=u'c/a') #TODO: to choices
legal_info = peewee.TextField(null=True)
countries = peewee.ManyToManyField(Country,
backref='retailers',
through_model=DefferedThroughRetailer)
currency = peewee.ForeignKeyField(Currency)
sm_accounts_white = postgres_ext.ArrayField(peewee.IntegerField, null=True)
sm_accounts_black = postgres_ext.ArrayField(peewee.IntegerField, null=True)
#NOTE: deprecated, will be removed
datafeed_url = peewee.CharField(null=True)
index_key = peewee.CharField(max_length=3, default='rus')
url_key_mode = peewee.CharField(max_length=1, default='F')
datafeed_params = postgres_ext.BinaryJSONField(default={})
international = peewee.BooleanField(default=False)
class Meta:
database = db
schema = 'store'
table_alias = 'r'
@property
def feed(self):
return self.feed_settings.first()
@property
def feeds(self):
return [
fs for fs in self.feed_settings.where(FeedSettings.active == True)
]
class PostgresEntity(PostgresModel):
### CLASS VARIABLES ###
_strip_pattern = re.compile(r'(\(\d+\)|[^(\w\s)]+)')
class BootstrapPassTwoWorker(multiprocessing.Process):
def __init__(self, entity_type, indices):
multiprocessing.Process.__init__(self)
self.entity_type = entity_type
self.indices = indices
def run(self):
proc_number = self.name.split('-')[-1]
corpus = {}
total = len(self.indices)
for i, entity_id in enumerate(self.indices):
with PostgresEntity._meta.database.execution_context():
progress = float(i) / total
try:
PostgresEntity.bootstrap_pass_two_single(
entity_type=self.entity_type,
entity_id=entity_id,
annotation=proc_number,
corpus=corpus,
progress=progress,
)
except:
print(
'ERROR:',
self.entity_type,
entity_id,
proc_number,
)
traceback.print_exc()
class BootstrapPassThreeWorker(multiprocessing.Process):
def __init__(self, entity_type, indices):
multiprocessing.Process.__init__(self)
self.entity_type = entity_type
self.indices = indices
def run(self):
proc_name = self.name
total = len(self.indices)
for i, entity_id in enumerate(self.indices):
with PostgresEntity._meta.database.execution_context():
progress = float(i) / total
try:
PostgresEntity.bootstrap_pass_three_single(
entity_type=self.entity_type,
entity_id=entity_id,
annotation=proc_name,
progress=progress,
)
except:
print('ERROR:', self.entity_type, entity_id, proc_name)
traceback.print_exc()
### PEEWEE FIELDS ###
entity_id = peewee.IntegerField(index=False)
entity_type = peewee.IntegerField(index=False)
name = peewee.TextField(index=True)
relation_counts = postgres_ext.BinaryJSONField(null=True, index=False)
metadata = postgres_ext.BinaryJSONField(null=True, index=False)
entities = postgres_ext.BinaryJSONField(null=True, index=False)
search_content = postgres_ext.TSVectorField(index=True)
### PEEWEE META ###
class Meta:
db_table = 'entities'
primary_key = peewee.CompositeKey('entity_type', 'entity_id')
### PUBLIC METHODS ###
@classmethod
def bootstrap(cls):
cls.drop_table(True)
cls.create_table()
cls.bootstrap_pass_one()
cls.bootstrap_pass_two()
@classmethod
def bootstrap_pass_one(cls):
PostgresModel.bootstrap_pass_one(
cls,
'artist',
id_attr='entity_id',
name_attr='name',
skip_without=['name'],
)
PostgresModel.bootstrap_pass_one(
cls,
'label',
id_attr='entity_id',
name_attr='name',
skip_without=['name'],
)
@classmethod
def get_entity_iterator(cls, entity_type, pessimistic=False):
if not pessimistic:
id_query = cls.select(peewee.fn.Max(cls.entity_id))
id_query = id_query.where(cls.entity_type == entity_type)
max_id = id_query.scalar()
for i in range(1, max_id + 1):
query = cls.select().where(
cls.entity_id == i,
cls.entity_type == entity_type,
)
if not query.count():
continue
document = query.get()
yield document
else:
id_query = cls.select(cls.entity_id)
id_query = id_query.where(cls.entity_type == entity_type)
for entity in id_query:
entity_id = entity.entity_id
entity = cls.select().where(
cls.entity_id == entity_id,
cls.entity_type == entity_type,
).get()
yield entity
@classmethod
def get_indices(cls, entity_type, pessimistic=False):
indices = []
if not pessimistic:
maximum_id = cls.select(
peewee.fn.Max(cls.entity_id)).where(
cls.entity_type == entity_type
).scalar()
step = maximum_id // multiprocessing.cpu_count()
for start in range(0, maximum_id, step):
stop = start + step
indices.append(range(start, stop))
else:
query = cls.select(cls.entity_id)
query = query.where(cls.entity_type == entity_type)
query = query.order_by(cls.entity_id)
query = query.tuples()
all_ids = tuple(_[0] for _ in query)
ratio = [1] * multiprocessing.cpu_count()
for chunk in sequencetools.partition_sequence_by_ratio_of_lengths(
all_ids, ratio):
indices.append(chunk)
return indices
@classmethod
def bootstrap_pass_two(cls, pessimistic=False):
entity_type = 1
indices = cls.get_indices(entity_type, pessimistic=pessimistic)
workers = [cls.BootstrapPassTwoWorker(entity_type, x)
for x in indices]
for worker in workers:
worker.start()
for worker in workers:
worker.join()
for worker in workers:
worker.terminate()
entity_type = 2
indices = cls.get_indices(entity_type, pessimistic=pessimistic)
workers = [cls.BootstrapPassTwoWorker(entity_type, x)
for x in indices]
for worker in workers:
worker.start()
for worker in workers:
worker.join()
for worker in workers:
worker.terminate()
@classmethod
def bootstrap_pass_three(cls, pessimistic=False):
entity_type = 1
indices = cls.get_indices(entity_type, pessimistic=pessimistic)
workers = [cls.BootstrapPassThreeWorker(entity_type, x)
for x in indices]
for worker in workers:
worker.start()
for worker in workers:
worker.join()
for worker in workers:
worker.terminate()
entity_type = 2
indices = cls.get_indices(entity_type, pessimistic=pessimistic)
workers = [cls.BootstrapPassThreeWorker(entity_type, _)
for _ in indices]
for worker in workers:
worker.start()
for worker in workers:
worker.join()
for worker in workers:
worker.terminate()
@classmethod
def bootstrap_pass_two_single(
cls,
entity_type,
entity_id,
annotation='',
corpus=None,
progress=None,
):
skipped_template = u'{} (Pass 2) {:.3%} [{}]\t[SKIPPED] (id:{}) [{:.8f}]: {}'
changed_template = u'{} (Pass 2) {:.3%} [{}]\t (id:{}) [{:.8f}]: {}'
query = cls.select().where(
cls.entity_id == entity_id,
cls.entity_type == entity_type,
)
if not query.count():
return
document = query.get()
corpus = corpus or {}
with systemtools.Timer(verbose=False) as timer:
changed = document.resolve_references(corpus)
if not changed:
message = skipped_template.format(
cls.__name__.upper(),
progress,
annotation,
(document.entity_type, document.entity_id),
timer.elapsed_time,
document.name,
)
print(message)
return
document.save()
message = changed_template.format(
cls.__name__.upper(),
progress,
annotation,
(document.entity_type, document.entity_id),
timer.elapsed_time,
document.name,
)
print(message)
@classmethod
def bootstrap_pass_three_single(
cls,
entity_type,
entity_id,
annotation='',
progress=None,
):
import discograph
query = cls.select(
cls.entity_id,
cls.entity_type,
cls.name,
cls.relation_counts,
).where(
cls.entity_id == entity_id,
cls.entity_type == entity_type,
)
if not query.count():
return
document = query.get()
entity_id = document.entity_id
where_clause = (
(discograph.PostgresRelation.entity_one_id == entity_id) &
(discograph.PostgresRelation.entity_one_type == entity_type)
)
where_clause |= (
(discograph.PostgresRelation.entity_two_id == entity_id) &
(discograph.PostgresRelation.entity_two_type == entity_type)
)
query = discograph.PostgresRelation.select().where(where_clause)
relation_counts = {}
for relation in query:
if relation.role not in relation_counts:
relation_counts[relation.role] = set()
key = (
relation.entity_one_type,
relation.entity_one_id,
relation.entity_two_type,
relation.entity_two_id,
)
relation_counts[relation.role].add(key)
for role, keys in relation_counts.items():
relation_counts[role] = len(keys)
if not relation_counts:
return
document.relation_counts = relation_counts
document.save()
message_pieces = [
cls.__name__.upper(),
progress,
annotation,
(document.entity_type, document.entity_id),
document.name,
len(relation_counts),
]
template = u'{} (Pass 3) {:.3%} [{}]\t(id:{}) {}: {}'
message = template.format(*message_pieces)
print(message)
@classmethod
def element_to_names(cls, names):
result = {}
if names is None or not len(names):
return result
for name in names:
name = name.text
if not name:
continue
result[name] = None
return result
@classmethod
def element_to_names_and_ids(cls, names_and_ids):
result = {}
if names_and_ids is None or not len(names_and_ids):
return result
for i in range(0, len(names_and_ids), 2):
discogs_id = int(names_and_ids[i].text)
name = names_and_ids[i + 1].text
result[name] = discogs_id
return result
@classmethod
def element_to_parent_label(cls, parent_label):
result = {}
if parent_label is None or parent_label.text is None:
return result
name = parent_label.text.strip()
if not name:
return result
result[name] = None
return result
@classmethod
def element_to_sublabels(cls, sublabels):
result = {}
if sublabels is None or not len(sublabels):
return result
for sublabel in sublabels:
name = sublabel.text
if name is None:
continue
name = name.strip()
if not name:
continue
result[name] = None
return result
@classmethod
def fixup_search_content(cls):
template = 'FIXUP ({}:{}): {} -> {}'
for document in cls.get_entity_iterator(entity_type=1):
document.search_content = cls.string_to_tsvector(document.name)
document.save()
message = template.format(
document.entity_type,
document.entity_id,
document.name,
document.search_content,
)
print(message)
for document in cls.get_entity_iterator(entity_type=2):
document.search_content = cls.string_to_tsvector(document.name)
document.save()
message = template.format(
document.entity_type,
document.entity_id,
document.name,
document.search_content,
)
print(message)
@classmethod
def from_element(cls, element):
data = cls.tags_to_fields(element)
return cls(**data)
@classmethod
def preprocess_data(cls, data, element):
data['metadata'] = {}
data['entities'] = {}
for key in (
'aliases',
'groups',
'members',
'parent_label',
'sublabels',
):
if key in data:
data['entities'][key] = data.pop(key)
for key in (
'contact_info',
'name_variations',
'profile',
'real_name',
'urls',
):
if key in data:
data['metadata'][key] = data.pop(key)
if 'name' in data and data.get('name'):
search_content = data.get('name')
search_content = search_content.lower()
search_content = stringtools.strip_diacritics(search_content)
data['search_content'] = peewee.fn.to_tsvector(search_content)
if element.tag == 'artist':
data['entity_type'] = 1
elif element.tag == 'label':
data['entity_type'] = 2
return data
def resolve_references(self, corpus):
changed = False
if not self.entities:
return changed
if self.entity_type == 1:
entity_type = 1
for section in ('aliases', 'groups', 'members'):
if section not in self.entities:
continue
for entity_name in self.entities[section].keys():
key = (entity_type, entity_name)
self.update_corpus(corpus, key)
if key in corpus:
self.entities[section][entity_name] = corpus[key]
changed = True
elif self.entity_type == 2:
entity_type = 2
for section in ('parent_label', 'sublabels'):
if section not in self.entities:
continue
for entity_name in self.entities[section].keys():
key = (entity_type, entity_name)
self.update_corpus(corpus, key)
if key in corpus:
self.entities[section][entity_name] = corpus[key]
changed = True
return changed
def roles_to_relation_count(self, roles):
count = 0
relation_counts = self.relation_counts or {}
for role in roles:
if role == 'Alias':
count += len(self.entities.get('aliases', ()))
elif role == 'Member Of':
count += len(self.entities.get('groups', ()))
count += len(self.entities.get('members', ()))
elif role == 'Sublabel Of':
count += len(self.entities.get('parent_label', ()))
count += len(self.entities.get('sublabels', ()))
else:
count += relation_counts.get(role, 0)
return count
@classmethod
def search_multi(cls, entity_keys):
artist_ids, label_ids = [], []
for entity_type, entity_id in entity_keys:
if entity_type == 1:
artist_ids.append(entity_id)
elif entity_type == 2:
label_ids.append(entity_id)
if artist_ids and label_ids:
where_clause = (
(
(cls.entity_type == 1) &
(cls.entity_id.in_(artist_ids))
) | (
(cls.entity_type == 2) &
(cls.entity_id.in_(label_ids))
)
)
elif artist_ids:
where_clause = (
(cls.entity_type == 1) &
(cls.entity_id.in_(artist_ids))
)
else:
where_clause = (
(cls.entity_type == 2) &
(cls.entity_id.in_(label_ids))
)
return cls.select().where(where_clause)
@classmethod
def search_text(cls, search_string):
search_string = search_string.lower()
search_string = stringtools.strip_diacritics(search_string)
search_string = ','.join(search_string.split())
query = PostgresEntity.raw("""
SELECT entity_type,
entity_id,
name,
ts_rank_cd(search_content, query, 63) AS rank
FROM entities,
to_tsquery(%s) query
WHERE query @@ search_content
ORDER BY rank DESC
LIMIT 100
""", search_string)
return query
@classmethod
def string_to_tsvector(cls, string):
string = string.lower()
string = stringtools.strip_diacritics(string)
string = cls._strip_pattern.sub('', string)
tsvector = peewee.fn.to_tsvector(string)
return tsvector
def structural_roles_to_entity_keys(self, roles):
entity_keys = set()
if self.entity_type == 1:
entity_type = 1
if 'Alias' in roles:
for section in ('aliases',):
if section not in self.entities:
continue
for entity_id in self.entities[section].values():
if not entity_id:
continue
entity_keys.add((entity_type, entity_id))
if 'Member Of' in roles:
for section in ('groups', 'members'):
if section not in self.entities:
continue
for entity_id in self.entities[section].values():
if not entity_id:
continue
entity_keys.add((entity_type, entity_id))
elif self.entity_type == 2:
entity_type = 2
if 'Sublabel Of' in roles:
for section in ('parent_label', 'sublabels'):
if section not in self.entities:
continue
for entity_id in self.entities[section].values():
if not entity_id:
continue
entity_keys.add((entity_type, entity_id))
return entity_keys
def structural_roles_to_relations(self, roles):
from discograph import PostgresRelation
relations = {}
if self.entity_type == 1:
entity_type = 1
role = 'Alias'
if role in roles and 'aliases' in self.entities:
for entity_id in self.entities['aliases'].values():
if not entity_id:
continue
ids = sorted((entity_id, self.entity_id))
relation = PostgresRelation(
entity_one_type=entity_type,
entity_one_id=ids[0],
entity_two_type=entity_type,
entity_two_id=ids[1],
role=role,
)
relations[relation.link_key] = relation
role = 'Member Of'
if role in roles:
if 'groups' in self.entities:
for entity_id in self.entities['groups'].values():
if not entity_id:
continue
relation = PostgresRelation(
entity_one_type=entity_type,
entity_one_id=self.entity_id,
entity_two_type=entity_type,
entity_two_id=entity_id,
role=role,
)
relations[relation.link_key] = relation
if 'members' in self.entities:
for entity_id in self.entities['members'].values():
if not entity_id:
continue
relation = PostgresRelation(
entity_one_type=entity_type,
entity_one_id=entity_id,
entity_two_type=entity_type,
entity_two_id=self.entity_id,
role=role,
)
relations[relation.link_key] = relation
elif self.entity_type == 2 and 'Sublabel Of' in roles:
entity_type = 2
role = 'Sublabel Of'
if 'parent_label' in self.entities:
for entity_id in self.entities['parent_label'].values():
if not entity_id:
continue
relation = PostgresRelation(
entity_one_type=entity_type,
entity_one_id=self.entity_id,
entity_two_type=entity_type,
entity_two_id=entity_id,
role=role,
)
relations[relation.link_key] = relation
if 'sublabels' in self.entities:
for entity_id in self.entities['sublabels'].values():
if not entity_id:
continue
relation = PostgresRelation(
entity_one_type=entity_type,
entity_one_id=entity_id,
entity_two_type=entity_type,
entity_two_id=self.entity_id,
role=role,
)
relations[relation.link_key] = relation
return relations
@classmethod
def update_corpus(cls, corpus, key):
if key in corpus:
return
entity_type, entity_name = key
query = cls.select().where(
cls.entity_type == entity_type,
cls.name == entity_name,
)
if query.count():
corpus[key] = query.get().entity_id
### PUBLIC PROPERTIES ###
@property
def entity_key(self):
return (self.entity_type, self.entity_id)
@property
def json_entity_key(self):
entity_type, entity_id = self.entity_key
if entity_type == 1:
return 'artist-{}'.format(self.entity_id)
elif entity_type == 2:
return 'label-{}'.format(self.entity_id)
raise ValueError(self.entity_key)
@property
def size(self):
if self.entity_type == 1:
members = self.entities.get('members', ())
elif self.entity_type == 2:
members = self.entities.get('sublabels', ())
return len(members)
class PostgresRelation(PostgresModel):
### CLASS VARIABLES ###
class EntityType(datastructuretools.Enumeration):
ARTIST = 1
LABEL = 2
class BootstrapWorker(multiprocessing.Process):
corpus = {}
def __init__(self, indices):
multiprocessing.Process.__init__(self)
self.indices = indices
def run(self):
proc_name = self.name
for release_id in self.indices:
try:
PostgresRelation.bootstrap_pass_one_inner(
release_id,
self.corpus,
annotation=proc_name,
)
except:
traceback.print_exc()
aggregate_roles = (
'Compiled By',
'Curated By',
'DJ Mix',
'Hosted By',
'Presenter',
)
word_pattern = re.compile('\s+')
### PEEWEE FIELDS ###
entity_one_type = peewee.IntegerField(index=False)
entity_one_id = peewee.IntegerField(index=False)
entity_two_type = peewee.IntegerField(index=False)
entity_two_id = peewee.IntegerField(index=False)
role = peewee.CharField(index=False)
releases = postgres_ext.BinaryJSONField(null=True, index=False)
### PEEWEE META ###
class Meta:
db_table = 'relations'
primary_key = peewee.CompositeKey(
'entity_one_type',
'entity_one_id',
'entity_two_type',
'entity_two_id',
'role',
)
indexes = (
(('entity_one_type', 'entity_one_id', 'entity_two_type',
'entity_two_id', 'role'), True),
(('entity_two_type', 'entity_two_id', 'entity_one_type',
'entity_one_id', 'role'), True),
)
### PRIVATE METHODS ###
@classmethod
def _as_artist_credits(cls, companies):
artists = []
for company in companies:
artist = {
'name': company['name'],
'id': company['id'],
'roles': [{
'name': company['entity_type_name']
}],
}
artists.append(artist)
return artists
### PUBLIC METHODS ###
def as_json(self):
data = {
'key': self.link_key,
'role': self.role,
'source': self.json_entity_one_key,
'target': self.json_entity_two_key,
}
if hasattr(self, 'distance'):
data['distance'] = self.distance
if hasattr(self, 'pages'):
data['pages'] = tuple(sorted(self.pages))
return data
@classmethod
def bootstrap(cls):
cls.drop_table(True)
cls.create_table()
cls.bootstrap_pass_one()
@classmethod
def bootstrap_pass_one(cls, pessimistic=False):
import discograph
indices = discograph.PostgresRelease.get_indices(pessimistic)
workers = [cls.BootstrapWorker(_) for _ in indices]
for worker in workers:
worker.start()
for worker in workers:
worker.join()
for worker in workers:
worker.terminate()
@classmethod
def bootstrap_pass_one_inner(cls, release_id, corpus, annotation=''):
import discograph
database = cls._meta.database
with database.execution_context(with_transaction=False):
release_cls = discograph.PostgresRelease
query = release_cls.select().where(release_cls.id == release_id)
if not query.count():
return
document = query.get()
relations = cls.from_release(document)
print('{} (Pass 1) [{}]\t(id:{})\t[{}] {}'.format(
cls.__name__.upper(),
annotation,
document.id,
len(relations),
document.title,
))
for relation in relations:
instance, created = cls.create_or_get(
entity_one_type=relation['entity_one_type'],
entity_one_id=relation['entity_one_id'],
entity_two_type=relation['entity_two_type'],
entity_two_id=relation['entity_two_id'],
role=relation['role'],
)
if created:
instance.releases = {}
instance.random = random.random()
if 'release_id' in relation:
release_id = relation['release_id']
year = relation.get('year')
if not instance.releases:
instance.releases = {}
instance.releases[release_id] = year
instance.save()
@classmethod
def from_release(cls, release):
import discograph
triples = set()
artists, labels, is_compilation = cls.get_release_setup(release)
triples.update(
cls.get_artist_label_relations(
artists,
labels,
is_compilation,
))
aggregate_roles = {}
if is_compilation:
iterator = itertools.product(labels, release.extra_artists)
else:
iterator = itertools.product(artists, release.extra_artists)
for entity_two, credit in iterator:
for role in credit['roles']:
role = role['name']
if role not in discograph.CreditRole.all_credit_roles:
continue
elif role in cls.aggregate_roles:
if role not in aggregate_roles:
aggregate_roles[role] = []
aggregate_credit = (cls.EntityType.ARTIST, credit['id'])
aggregate_roles[role].append(aggregate_credit)
continue
entity_one = (cls.EntityType.ARTIST, credit['id'])
triples.add((entity_one, role, entity_two))
if is_compilation:
iterator = itertools.product(labels, release.companies)
else:
iterator = itertools.product(artists, release.companies)
for entity_one, company in iterator:
role = company['entity_type_name']
if role not in discograph.CreditRole.all_credit_roles:
continue
entity_two = (cls.EntityType.LABEL, company['id'])
triples.add((entity_one, role, entity_two))
all_track_artists = set()
for track in release.tracklist:
track_artists = set((cls.EntityType.ARTIST, _['id'])
for _ in track.get('artists', ()))
all_track_artists.update(track_artists)
if not track.get('extra_artists'):
continue
track_artists = track_artists or artists or labels
iterator = itertools.product(track_artists, track['extra_artists'])
for entity_two, credit in iterator:
for role in credit.get('roles', ()):
role = role['name']
if role not in discograph.CreditRole.all_credit_roles:
continue
entity_one = (cls.EntityType.ARTIST, credit['id'])
triples.add((entity_one, role, entity_two))
for role, aggregate_artists in aggregate_roles.items():
iterator = itertools.product(all_track_artists, aggregate_artists)
for track_artist, aggregate_artist in iterator:
entity_one = aggregate_artist
entity_two = track_artist
triples.add((entity_one, role, entity_two))
triples = sorted(triples)
relations = cls.from_triples(triples, release=release)
return relations
@classmethod
def get_artist_label_relations(cls, artists, labels, is_compilation):
triples = set()
iterator = itertools.product(artists, labels)
if is_compilation:
role = 'Compiled On'
else:
role = 'Released On'
for artist, label in iterator:
triples.add((artist, role, label))
return triples
@classmethod
def get_random(cls, roles=None):
n = random.random()
where_clause = (cls.random > n)
if roles:
where_clause &= (cls.role.in_(roles))
query = cls.select().where(where_clause).order_by(
cls.random, cls.role).limit(1)
print('Query:', query)
while not query.count():
n = random.random()
where_clause = (cls.random > n)
if roles:
where_clause &= (cls.role.in_(roles))
query = cls.select()
query = query.where(where_clause)
query = query.order_by(cls.random)
query = query.limit(1)
print('Query:', query)
return query.get()
@classmethod
def get_release_setup(cls, release):
is_compilation = False
artists = set(
(cls.EntityType.ARTIST, _['id']) for _ in release.artists)
labels = set((cls.EntityType.LABEL, _.get('id'))
for _ in release.labels if _.get('id'))
if len(artists) == 1 and release.artists[0]['name'] == 'Various':
is_compilation = True
artists.clear()
for track in release.tracklist:
artists.update((cls.EntityType.ARTIST, _['id'])
for _ in track.get('artists', ()))
#for format_ in release.formats:
# for description in format_.get('descriptions', ()):
# if description == 'Compilation':
# is_compilation = True
# break
return artists, labels, is_compilation
@classmethod
def from_triples(cls, triples, release=None):
relations = []
for entity_one, role, entity_two in triples:
entity_one_type, entity_one_id = entity_one
entity_two_type, entity_two_id = entity_two
relation = dict(
entity_one_id=entity_one_id,
entity_one_type=entity_one_type,
entity_two_id=entity_two_id,
entity_two_type=entity_two_type,
role=role,
)
if release is not None:
relation['release_id'] = release.id
if release.release_date is not None:
relation['year'] = release.release_date.year
relations.append(relation)
return relations
@classmethod
def search(
cls,
entity_id,
entity_type=1,
roles=None,
year=None,
query_only=False,
):
where_clause = ((cls.entity_one_id == entity_id) &
(cls.entity_one_type == entity_type))
where_clause |= ((cls.entity_two_id == entity_id) &
(cls.entity_two_type == entity_type))
if roles:
where_clause &= (cls.role.in_(roles))
if year is not None:
year_clause = cls.year.is_null(True)
if isinstance(year, int):
year_clause |= cls.year == year
else:
year_clause |= cls.year.between(year[0], year[1])
where_clause &= year_clause
query = cls.select().where(where_clause)
if query_only:
return query
return list(query)
@classmethod
def search_multi(cls, entity_keys, roles=None):
assert entity_keys
artist_ids, label_ids = [], []
for entity_type, entity_id in entity_keys:
if entity_type == 1:
artist_ids.append(entity_id)
elif entity_type == 2:
label_ids.append(entity_id)
if artist_ids:
artist_where_clause = (((cls.entity_one_type == 1) &
(cls.entity_one_id.in_(artist_ids))) |
((cls.entity_two_type == 1) &
(cls.entity_two_id.in_(artist_ids))))
if label_ids:
label_where_clause = (((cls.entity_one_type == 2) &
(cls.entity_one_id.in_(label_ids))) |
((cls.entity_two_type == 2) &
(cls.entity_two_id.in_(label_ids))))
if artist_ids and label_ids:
where_clause = artist_where_clause | label_where_clause
elif artist_ids:
where_clause = artist_where_clause
elif label_ids:
where_clause = label_where_clause
if roles:
where_clause &= (cls.role.in_(roles))
query = cls.select().where(where_clause)
relations = {}
for relation in query:
relations[relation.link_key] = relation
return relations
@classmethod
def search_bimulti(
cls,
lh_entities,
rh_entities,
roles=None,
year=None,
verbose=True,
):
def build_query(lh_type, lh_ids, rh_type, rh_ids):
where_clause = cls.entity_one_type == lh_type
where_clause &= cls.entity_two_type == rh_type
where_clause &= cls.entity_one_id.in_(lh_ids)
where_clause &= cls.entity_two_id.in_(rh_ids)
if roles:
where_clause &= cls.role.in_(roles)
if year is not None:
year_clause = cls.year.is_null(True)
if isinstance(year, int):
year_clause |= cls.year == year
else:
year_clause |= cls.year.between(year[0], year[1])
where_clause &= year_clause
query = cls.select().where(where_clause)
return query
lh_artist_ids = []
lh_label_ids = []
rh_artist_ids = []
rh_label_ids = []
for entity_type, entity_id in lh_entities:
if entity_type == 1:
lh_artist_ids.append(entity_id)
else:
lh_label_ids.append(entity_id)
for entity_type, entity_id in rh_entities:
if entity_type == 1:
rh_artist_ids.append(entity_id)
else:
rh_label_ids.append(entity_id)
queries = []
if lh_artist_ids:
lh_type, lh_ids = 1, lh_artist_ids
if rh_artist_ids:
rh_type, rh_ids = 1, rh_artist_ids
query = build_query(lh_type, lh_ids, rh_type, rh_ids)
queries.append(query)
if rh_label_ids:
rh_type, rh_ids = 2, rh_label_ids
query = build_query(lh_type, lh_ids, rh_type, rh_ids)
queries.append(query)
if lh_label_ids:
lh_type, lh_ids = 2, lh_label_ids
if rh_artist_ids:
rh_type, rh_ids = 1, rh_artist_ids
query = build_query(lh_type, lh_ids, rh_type, rh_ids)
queries.append(query)
if rh_label_ids:
rh_type, rh_ids = 2, rh_label_ids
query = build_query(lh_type, lh_ids, rh_type, rh_ids)
queries.append(query)
relations = []
for query in queries:
#print(query)
relations.extend(query)
relations = {relation.link_key: relation for relation in relations}
return relations
### PUBLIC PROPERTIES ###
@property
def entity_one_key(self):
return (self.entity_one_type, self.entity_one_id)
@property
def entity_two_key(self):
return (self.entity_two_type, self.entity_two_id)
@property
def json_entity_one_key(self):
if self.entity_one_type == 1:
return 'artist-{}'.format(self.entity_one_id)
elif self.entity_one_type == 2:
return 'label-{}'.format(self.entity_one_id)
raise ValueError(self.entity_one_key)
@property
def json_entity_two_key(self):
if self.entity_two_type == 1:
return 'artist-{}'.format(self.entity_two_id)
elif self.entity_two_type == 2:
return 'label-{}'.format(self.entity_two_id)
raise ValueError(self.entity_two_key)
@property
def link_key(self):
source = self.json_entity_one_key
target = self.json_entity_two_key
role = self.word_pattern.sub('-', self.role).lower()
pieces = [
source,
role,
target,
]
return '-'.join(str(_) for _ in pieces)
class Product(peewee.Model):
REMAP_FIELDS = {}
retailer = peewee.ForeignKeyField(Retailer)
offer = peewee.CharField()
datafeed_id = peewee.IntegerField() #TODO: to FK
active = peewee.BooleanField(default=True)
available = peewee.BooleanField(default=True)
name = peewee.CharField(null=True)
type_prefix = peewee.CharField(null=True)
vendor = peewee.CharField(null=True)
model = peewee.CharField(null=True)
description = peewee.CharField(null=True)
currency = peewee.CharField(null=True)
price = peewee.DecimalField(20, 4, null=True)
old_price = peewee.DecimalField(20, 4, null=True)
url = peewee.CharField(null=True)
code = peewee.CharField(null=True)
category = peewee.IntegerField(null=True) #TODO: to FK
gender = peewee.IntegerField() #TODO: to FK
categories_raw = peewee.CharField(null=True)
market_category = peewee.CharField(null=True)
country_of_origin = peewee.CharField(null=True)
barcode = peewee.CharField(null=True)
params = postgres_ext.BinaryJSONField(null=True)
substitution_product_id = peewee.IntegerField(null=True)
likes_count = peewee.IntegerField(null=True, default=0)
class Meta:
database = db
schema = 'store'
@classmethod
def load_from_source(cls, source):
remaped = {cls.REMAP_FIELDS.get(k, k): v for k, v in source.items()}
instance = shortcuts.dict_to_model(cls, remaped, ignore_unknown=True)
is_new = not cls.filter(id=instance.id).exists()
return is_new, instance
@staticmethod
def get_random_recommendation_for(ids):
Recommendation = Product.alias()
ProductRecommendation = Product.original_product_binds.rel_model
return (Recommendation.select().join(
ProductRecommendation,
on=(Recommendation.id == ProductRecommendation.recommendation)
).join(Product, on=(
Product.id == ProductRecommendation.original)).join(Retailer).join(
Picture, on=(Picture.product == Recommendation.id
)).where((Product.id << ids)
& (Product.active == True)
& (Recommendation.active == True)
& (Retailer.active == True)
& (Picture.active == True)
& (Picture.main == True)).order_by(
fn.Random()).first())
@property
def main_picture(self):
return self.pictures.filter(main=True, active=True).first()
@staticmethod
def get_main_pictures(products):
return dict(
Picture.get_for_products([p['id'] for p in products]).tuples())
class Datafeed(peewee.Model):
STATUS_COMPLETED = 'c'
STATUS_WAITING = 'w'
STATUS_ERROR = 'e'
STATUS_DATAFEED_DOWNLOADED = 'd'
STATUS_DATAFEED_PROCESSED = 'p'
STATUS_PICTURES_DOWNLOADED = 'g'
STATUS_INDEX_CREATED = 'i'
STATUS_INDEX_MERGED = 'm'
STATUSES = (
('waiting', STATUS_WAITING),
('download', STATUS_DATAFEED_DOWNLOADED),
('process', STATUS_DATAFEED_PROCESSED),
('pictures', STATUS_PICTURES_DOWNLOADED),
('index', STATUS_INDEX_CREATED),
('merged', STATUS_INDEX_MERGED),
('completed', STATUS_COMPLETED),
('error', STATUS_ERROR),
)
retailer = peewee.ForeignKeyField(Retailer, related_name='datafeeds')
issued = peewee.DateTimeField(null=True)
imported = peewee.DateTimeField(null=True, default=datetime.datetime.now)
status = peewee.CharField(default=STATUS_WAITING, max_length=1)
log = postgres_ext.BinaryJSONField()
class Meta:
database = db
schema = 'store'
@property
def settings(self):
return self.retailer.feed
@property
def format(self):
return self.settings.format
@property
def parser(self):
return self.format.partition('.')[0]
@property
def mapper(self):
return self.settings.mapper
@property
def mapper_params(self):
return self.settings.params
@property
def index_key(self):
return self.settings.index_key
@property
def url(self):
return self.settings.url
@property
def feeds(self):
return self.retailer.feeds
def activate(self):
with db.transaction():
# sql.DEACTIVATE_PRODUCTS
Product.update({
Product.active: False
}).where((Product.retailer == self.retailer)
& (Product.datafeed_id != self.id)).execute()
Picture.raw(sql.DEACTIVATE_PICTURES, self.retailer_id,
self.id).execute()
def get_feed(self, feed_id):
return self.retailer.feed_settings.where(
FeedSettings.id == feed_id).first()
class PostgresRelease(PostgresModel):
### CLASS VARIABLES ###
_artists_mapping = {}
_companies_mapping = {}
_tracks_mapping = {}
class BootstrapPassTwoWorker(multiprocessing.Process):
def __init__(self, indices):
multiprocessing.Process.__init__(self)
self.indices = indices
def run(self):
proc_name = self.name
corpus = {}
total = len(self.indices)
for i, release_id in enumerate(self.indices):
with PostgresRelease._meta.database.execution_context():
progress = float(i) / total
try:
PostgresRelease.bootstrap_pass_two_single(
release_id=release_id,
annotation=proc_name,
corpus=corpus,
progress=progress,
)
except:
print('ERROR:', release_id, proc_name)
traceback.print_exc()
### PEEWEE FIELDS ###
id = peewee.IntegerField(primary_key=True)
artists = postgres_ext.BinaryJSONField(null=True, index=False)
companies = postgres_ext.BinaryJSONField(null=True, index=False)
country = peewee.TextField(null=True, index=False)
extra_artists = postgres_ext.BinaryJSONField(null=True, index=False)
formats = postgres_ext.BinaryJSONField(null=True, index=False)
genres = postgres_ext.ArrayField(peewee.TextField, null=True, index=False)
identifiers = postgres_ext.BinaryJSONField(null=True, index=False)
labels = postgres_ext.BinaryJSONField(null=True, index=False)
master_id = peewee.IntegerField(null=True, index=False)
notes = peewee.TextField(null=True, index=False)
release_date = peewee.DateTimeField(null=True, index=False)
styles = postgres_ext.ArrayField(peewee.TextField, null=True, index=False)
title = peewee.TextField(index=False)
tracklist = postgres_ext.BinaryJSONField(null=True, index=False)
### PEEWEE META ###
class Meta:
db_table = 'releases'
### PUBLIC METHODS ###
@classmethod
def bootstrap(cls):
cls.drop_table(True)
cls.create_table()
cls.bootstrap_pass_one()
cls.bootstrap_pass_two()
@classmethod
def bootstrap_pass_one(cls):
PostgresModel.bootstrap_pass_one(
model_class=cls,
xml_tag='release',
name_attr='title',
skip_without=['title'],
)
@classmethod
def get_indices(cls, pessimistic=False):
indices = []
if not pessimistic:
maximum_id = cls.select(
peewee.fn.Max(cls.id)).scalar()
step = maximum_id // multiprocessing.cpu_count()
for start in range(0, maximum_id, step):
stop = start + step
indices.append(range(start, stop))
else:
query = cls.select(cls.id)
query = query.order_by(cls.id)
query = query.tuples()
all_ids = tuple(_[0] for _ in query)
ratio = [1] * (multiprocessing.cpu_count() * 2)
for chunk in sequencetools.partition_sequence_by_ratio_of_lengths(
all_ids, ratio):
indices.append(chunk)
return indices
@classmethod
def get_release_iterator(cls, pessimistic=False):
if not pessimistic:
maximum_id = cls.select(peewee.fn.Max(cls.id)).scalar()
for i in range(1, maximum_id + 1):
query = cls.select().where(cls.id == i)
if not query.count():
continue
document = query.get()
yield document
else:
id_query = cls.select(cls.id)
for release in id_query:
release_id = release.id
release = cls.select().where(cls.id == release_id).get()
yield release
@classmethod
def bootstrap_pass_two(cls, pessimistic=False):
indices = cls.get_indices(pessimistic=pessimistic)
workers = [cls.BootstrapPassTwoWorker(x) for x in indices]
for worker in workers:
worker.start()
for worker in workers:
worker.join()
for worker in workers:
worker.terminate()
@classmethod
def bootstrap_pass_two_single(
cls,
release_id,
annotation='',
corpus=None,
progress=None,
):
skipped_template = u'{} (Pass 2) {:.3%} [{}]\t[SKIPPED] (id:{}) [{:.8f}]: {}'
changed_template = u'{} (Pass 2) {:.3%} [{}]\t (id:{}) [{:.8f}]: {}'
query = cls.select().where(cls.id == release_id)
if not query.count():
return
document = query.get()
with systemtools.Timer(verbose=False) as timer:
changed = document.resolve_references(corpus)
if not changed:
message = skipped_template.format(
cls.__name__.upper(),
progress,
annotation,
document.id,
timer.elapsed_time,
document.title,
)
print(message)
return
document.save()
message = changed_template.format(
cls.__name__.upper(),
progress,
annotation,
document.id,
timer.elapsed_time,
document.title,
)
print(message)
@classmethod
def element_to_artist_credits(cls, element):
result = []
if element is None or not len(element):
return result
for subelement in element:
data = cls.tags_to_fields(
subelement,
ignore_none=True,
mapping=cls._artists_mapping,
)
result.append(data)
return result
@classmethod
def element_to_company_credits(cls, element):
result = []
if element is None or not len(element):
return result
for subelement in element:
data = cls.tags_to_fields(
subelement,
ignore_none=True,
mapping=cls._companies_mapping,
)
result.append(data)
return result
@classmethod
def element_to_formats(cls, element):
result = []
if element is None or not len(element):
return result
for subelement in element:
document = {
'name': subelement.get('name'),
'quantity': subelement.get('qty'),
}
if subelement.get('text'):
document['text'] = subelement.get('text')
if len(subelement):
subelement = subelement[0]
descriptions = Bootstrapper.element_to_strings(subelement)
document['descriptions'] = descriptions
result.append(document)
return result
@classmethod
def element_to_identifiers(cls, element):
result = []
if element is None or not len(element):
return result
for subelement in element:
data = {
'description': subelement.get('description'),
'type': subelement.get('type'),
'value': subelement.get('value'),
}
result.append(data)
return result
@classmethod
def element_to_label_credits(cls, element):
result = []
if element is None or not len(element):
return result
for subelement in element:
data = {
'catalog_number': subelement.get('catno'),
'name': subelement.get('name'),
}
result.append(data)
return result
@classmethod
def element_to_roles(cls, element):
def from_text(text):
name = ''
current_buffer = ''
details = []
had_detail = False
bracket_depth = 0
for character in text:
if character == '[':
bracket_depth += 1
if bracket_depth == 1 and not had_detail:
name = current_buffer
current_buffer = ''
had_detail = True
elif 1 < bracket_depth:
current_buffer += character
elif character == ']':
bracket_depth -= 1
if not bracket_depth:
details.append(current_buffer)
current_buffer = ''
else:
current_buffer += character
else:
current_buffer += character
if current_buffer and not had_detail:
name = current_buffer
name = name.strip()
detail = ', '.join(_.strip() for _ in details)
result = {'name': name}
if detail:
result['detail'] = detail
return result
credit_roles = []
if element is None or not element.text:
return credit_roles or None
current_text = ''
bracket_depth = 0
for character in element.text:
if character == '[':
bracket_depth += 1
elif character == ']':
bracket_depth -= 1
elif not bracket_depth and character == ',':
current_text = current_text.strip()
if current_text:
credit_roles.append(from_text(current_text))
current_text = ''
continue
current_text += character
current_text = current_text.strip()
if current_text:
credit_roles.append(from_text(current_text))
return credit_roles or None
@classmethod
def element_to_tracks(cls, element):
result = []
if element is None or not len(element):
return result
for subelement in element:
data = cls.tags_to_fields(
subelement,
ignore_none=True,
mapping=cls._tracks_mapping,
)
result.append(data)
return result
@classmethod
def from_element(cls, element):
data = cls.tags_to_fields(element)
data['id'] = int(element.get('id'))
return cls(**data)
def resolve_references(self, corpus, spuriously=False):
import discograph
changed = False
spurious_id = 0
for entry in self.labels:
name = entry['name']
entity_key = (2, name)
if not spuriously:
discograph.PostgresEntity.update_corpus(corpus, entity_key)
if entity_key in corpus:
entry['id'] = corpus[entity_key]
changed = True
elif spuriously:
spurious_id -= 1
corpus[entity_key] = spurious_id
entry['id'] = corpus[entity_key]
changed = True
return changed
class UserAccount(db_wrapper.Model):
email = peewee.TextField(unique=True)
password_hash = peewee.TextField()
ctime = peewee.DateTimeField(default=lambda: arrow.utcnow().naive)
profile = postgres_ext.BinaryJSONField(default=lambda: {})
settings = postgres_ext.BinaryJSONField(default=lambda: {})