class PostFile(FileLinkMixin, db.Model):
__tablename__ = "post_files"
id = db.Column(db.Integer, primary_key=True)
post_id = db.Column(db.Integer, db.ForeignKey('post.id', ondelete='CASCADE'), index=True, nullable=False)
post = db.relationship('Post')
file_id = db.Column(db.Integer, db.ForeignKey('file.id', ondelete="CASCADE"), index=True, nullable=False)
file = db.relationship('File', lazy='joined')
class Redirect(db.Model):
__tablename__ = 'redirect'
id = db.Column(db.Integer, primary_key=True)
nid = db.Column(db.Integer)
old_url = db.Column(db.String(250),
nullable=False,
unique=True,
index=True)
new_url = db.Column(db.String(250))
def __str__(self):
if self.nid:
target = "nid %s" % self.nid
else:
target = self.new_url
return u'<Redirect from %s to %s>' % (self.old_url, target)
@classmethod
def object_for_nid(cls, nid):
for cls in resource_slugs.itervalues():
if hasattr(cls, 'nid'):
obj = cls.query.filter(cls.nid == nid).first()
if obj:
return obj
@classmethod
def for_url(cls, old_url):
dest = None
nid = None
if old_url.endswith("/"):
old_url = old_url[:-1]
# check for /node/1234
match = re.match('^/node/(\d+)$', old_url)
if match:
nid = match.group(1)
else:
redirect = cls.query.filter(cls.old_url == old_url).first()
if redirect:
if redirect.new_url:
dest = redirect.new_url
elif redirect.nid:
nid = redirect.nid
if nid:
# lookup based on nid
obj = cls.object_for_nid(nid)
if obj:
dest = obj.url
if dest and not dest.startswith('http') and not dest.startswith('/'):
dest = '/' + dest
return dest
class Role(db.Model, RoleMixin):
__tablename__ = "role"
id = db.Column(db.Integer(), primary_key=True)
name = db.Column(db.String(80), unique=True)
description = db.Column(db.String(255))
def __unicode__(self):
return unicode(self.name)
class Page(db.Model):
""" A basic CMS page. """
__tablename__ = 'page'
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String, nullable=False)
slug = db.Column(db.String, nullable=False, unique=True, index=True)
body = db.Column(db.Text)
created_at = db.Column(db.DateTime(timezone=True),
index=True,
unique=False,
nullable=False,
server_default=func.now())
updated_at = db.Column(db.DateTime(timezone=True),
server_default=func.now(),
onupdate=func.current_timestamp())
files = db.relationship("PageFile", lazy='joined')
show_files = db.Column(db.Boolean,
nullable=False,
default=True,
server_default=sql.expression.true())
featured = db.Column(db.Boolean(),
default=False,
server_default=sql.expression.false(),
nullable=False,
index=True)
@validates('slug')
def validate_slug(self, key, value):
return value.strip('/')
def to_dict(self, include_related=False):
tmp = serializers.model_to_dict(self, include_related=include_related)
return tmp
class SoundcloudTrackRetry(db.Model):
__tablename__ = "soundcloud_track_retry"
id = db.Column(db.Integer, primary_key=True)
created_at = db.Column(
db.DateTime(timezone=True),
nullable=False,
default=datetime.utcnow
)
updated_at = db.Column(
db.DateTime(timezone=True),
nullable=False,
default=datetime.utcnow,
onupdate=func.current_timestamp()
)
soundcloud_track_id = db.Column(
db.Integer,
db.ForeignKey('soundcloud_track.id'),
nullable=False,
unique=False
)
soundcloud_track = db.relationship('SoundcloudTrack', backref=backref('retries', lazy='joined'), lazy=True)
class Post(db.Model):
__tablename__ = 'post'
id = db.Column(db.Integer, primary_key=True)
title = db.Column(db.String, nullable=False)
slug = db.Column(db.String, nullable=False, unique=True, index=True)
featured = db.Column(db.Boolean(), default=False, server_default=sql.expression.false(), nullable=False, index=True)
body = db.Column(db.Text)
date = db.Column(db.DateTime(timezone=True), index=True, unique=False, nullable=False, server_default=func.now())
files = db.relationship("PostFile", lazy='joined')
created_at = db.Column(db.DateTime(timezone=True), index=True, unique=False, nullable=False, server_default=func.now())
updated_at = db.Column(db.DateTime(timezone=True), server_default=func.now(), onupdate=func.current_timestamp())
@validates('slug')
def validate_slug(self, key, value):
return value.strip('/')
def __unicode__(self):
return unicode(self.title)
class Organisation(db.Model):
__tablename__ = "organisation"
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(255), nullable=False)
domain = db.Column(db.String(100), nullable=False)
paid_subscriber = db.Column(db.Boolean)
created_at = db.Column(db.DateTime(timezone=True),
default=datetime.datetime.now)
# when does this subscription expire?
expiry = db.Column(db.Date(), default=one_year_later)
contact = db.Column(db.String(255))
# premium committee subscriptions
subscriptions = db.relationship('Committee',
secondary='organisation_committee',
passive_deletes=True)
def subscribed_to_committee(self, committee):
""" Does this organisation have an active subscription to `committee`? """
return not self.has_expired() and (committee in self.subscriptions)
def has_expired(self):
return (self.expiry
is not None) and (datetime.date.today() > self.expiry)
def __unicode__(self):
return unicode(self.name)
def to_dict(self, include_related=False):
tmp = serializers.model_to_dict(self, include_related=include_related)
# send subscriptions back as a dict
subscription_dict = {}
if tmp.get('subscriptions'):
for committee in tmp['subscriptions']:
subscription_dict[committee['id']] = committee.get('name')
tmp['subscriptions'] = subscription_dict
# set 'has_expired' flag as appropriate
tmp['has_expired'] = self.has_expired()
return tmp
class EmailTemplate(db.Model):
__tablename__ = 'email_template'
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(100), nullable=False)
description = db.Column(db.String(1024))
subject = db.Column(db.String(100))
body = db.Column(db.Text)
created_at = db.Column(db.DateTime(timezone=True),
index=True,
unique=False,
nullable=False,
server_default=func.now())
updated_at = db.Column(db.DateTime(timezone=True),
server_default=func.now(),
onupdate=func.current_timestamp())
@property
def utm_campaign(self):
return re.sub(r'[^a-z0-9 -]+', '', self.name.lower()).replace(' ', '-')
class SoundcloudTrack(db.Model):
"""
- Tracks where uri and state is null are either busy being uploaded,
or failed to upload to SoundCloud.
- Tracks where state is 'failed' reflect that soundcloud
failed to process the track.
"""
__tablename__ = "soundcloud_track"
id = db.Column(db.Integer, primary_key=True)
created_at = db.Column(db.DateTime(timezone=True),
nullable=False,
default=datetime.utcnow)
updated_at = db.Column(db.DateTime(timezone=True),
nullable=False,
default=datetime.utcnow,
onupdate=func.current_timestamp())
file_id = db.Column(db.Integer,
db.ForeignKey('file.id'),
nullable=False,
unique=True)
file = db.relationship('File',
backref=backref('soundcloud_track',
uselist=False,
lazy='joined'),
lazy=True)
# Soundcloud resource URI for the track (i.e. https://api.soundcloud...id)
uri = db.Column(db.String())
# Last known value of SoundCloud's opinion of the track state
state = db.Column(db.String())
def __str__(self):
return unicode(self).encode('utf-8')
def __unicode__(self):
return u'<SoundcloudTrack id=%d>' % self.id
@classmethod
def new_from_file(cls, client, file):
if db.session.query(
cls.id).filter(cls.file_id == file.id).scalar() is not None:
logging.info(
"File already started being uploaded to Soundcloud: %s" % file)
db.session.rollback()
return
# Immediately create the SoundcloudTrack to indicate that work
# has started for this track and may be in progress.
# Potential concurrent runs can rely on an exception here to avoid
# uploading the same file twice.
soundcloud_track = cls(file=file)
db.session.add(soundcloud_track)
db.session.commit()
with file.open() as file_handle:
logging.info("Uploading to SoundCloud: %s" % file)
track = client.post('/tracks',
track={
'title':
file.title,
'description':
cls._html_description(file),
'sharing':
'public',
'asset_data':
file_handle,
'license':
'cc-by',
'artwork_data':
open(SOUNDCLOUD_ARTWORK_PATH, 'rb'),
'genre':
file.event_files[0].event.type,
'tag_list':
file.event_files[0].event.type,
'downloadable':
'true',
'streamable':
'true',
'feedable':
'true',
})
logging.info("Done uploading to SoundCloud: %s" % file)
file_handle.close()
soundcloud_track.uri = track.uri
soundcloud_track.state = track.state
db.session.commit()
@staticmethod
def _html_description(file):
"""
HTML description for presentation on Soundcloud.
staticmethod because it's needed before the instance exists.
"""
return 'Sound recording from:<br>' + \
'<br>'.join(
"<a href='%s'>%s</a>" % (ef.event.url, ef.event.title)
for ef in file.event_files
)
def sync_state(self, client):
track = client.get(self.uri)
if track.state != self.state:
self.state = track.state
db.session.commit()
logger.info("SoundCloud track %s state is now [%s]" %
(self, self.state))
@classmethod
def upload_files(cls, client):
q = cls.get_unstarted_query()
logging.info("Audio files yet to be uploaded to SoundCloud: %d" %
cls.get_unstarted_count(q))
batch = cls.get_unstarted_batch(q)
# Rollback this transaction - it was just to gather candidates for upload
db.session.rollback()
logging.info("Uploading %d files to SoundCloud" % len(batch))
for file in batch:
cls.new_from_file(client, file)
@classmethod
def sync(cls):
client = Client(client_id=app.config['SOUNDCLOUD_APP_KEY_ID'],
client_secret=app.config['SOUNDCLOUD_APP_KEY_SECRET'],
username=app.config['SOUNDCLOUD_USERNAME'],
password=app.config['SOUNDCLOUD_PASSWORD'])
cls.upload_files(client)
cls.sync_upload_state(client)
@classmethod
def get_unstarted_query(cls):
"""
Get audio files for which there's no SoundcloudTrack.
Order by id as a hacky way to roughly get the latest files first
"""
# Query files that aren't connected to events so we can ignore them for
# now - it's not clear that they're actually visible - they might well
# have been mistaken uploads that shouldn't suddenly appear on PMG's
# public soundcloud profile.
q_files_with_meetings = db.session.query(File.id) \
.outerjoin(EventFile) \
.filter(EventFile.file_id == None) \
.filter(File.file_mime.like('audio/%'))
return db.session.query(File) \
.outerjoin(cls) \
.filter(cls.file_id == None) \
.filter(File.file_mime.like('audio/%')) \
.filter(~File.id.in_(q_files_with_meetings)) \
.order_by(desc(File.id))
@staticmethod
def get_unstarted_count(q):
return q.count()
@staticmethod
def get_unstarted_batch(q):
return q.limit(app.config['MAX_SOUNDCLOUD_BATCH']).all()
@classmethod
def sync_upload_state(cls, client):
tracks = db.session.query(cls) \
.filter(cls.state.in_(UNFINISHED_STATES)) \
.order_by(cls.created_at).all()
for track in tracks:
track.sync_state(client)
class User(db.Model, UserMixin):
__tablename__ = "user"
id = db.Column(db.Integer, primary_key=True)
email = db.Column(db.String(255), unique=True, nullable=False, index=True)
name = db.Column(db.String(255), nullable=True)
password = db.Column(db.String(255), default='', server_default='', nullable=False)
active = db.Column(db.Boolean(), default=True, server_default=sql.expression.true())
confirmed_at = db.Column(db.DateTime(timezone=True))
last_login_at = db.Column(db.DateTime(timezone=True))
current_login_at = db.Column(db.DateTime(timezone=True))
last_login_ip = db.Column(db.String(100))
current_login_ip = db.Column(db.String(100))
login_count = db.Column(db.Integer)
subscribe_daily_schedule = db.Column(db.Boolean(), default=False)
created_at = db.Column(db.DateTime(timezone=True), index=True, unique=False, nullable=False, server_default=func.now())
updated_at = db.Column(db.DateTime(timezone=True), index=True, unique=False, nullable=False, server_default=func.now(), onupdate=func.current_timestamp())
# when does this subscription expire?
expiry = db.Column(db.Date(), default=one_year_later)
organisation_id = db.Column(db.Integer, db.ForeignKey('organisation.id'))
organisation = db.relationship('Organisation', backref='users', lazy=False, foreign_keys=[organisation_id])
# premium committee subscriptions, in addition to any that the user's organisation might have
subscriptions = db.relationship('Committee', secondary='user_committee', passive_deletes=True)
# committees that the user chooses to follow
following = db.relationship('Committee', secondary='user_following', passive_deletes=True)
# alerts for changes to committees
committee_alerts = db.relationship('Committee', secondary='user_committee_alerts', passive_deletes=True, lazy='joined')
roles = db.relationship('Role', secondary='roles_users', backref=db.backref('users', lazy='dynamic'))
def __unicode__(self):
return unicode(self.email)
def is_confirmed(self):
return self.confirmed_at is not None
def has_expired(self):
return (self.expiry is not None) and (datetime.date.today() > self.expiry)
def update_current_login(self):
now = datetime.datetime.utcnow()
if self.current_login_at.replace(tzinfo=None) + datetime.timedelta(hours=1) < now:
self.current_login_at = now
db.session.commit()
def subscribed_to_committee(self, committee):
""" Does this user have an active subscription to `committee`? """
# admin users have access to everything
if self.has_role('editor'):
return True
# inactive users should go away
if not self.active:
return False
# expired users should go away
if self.has_expired():
return False
# first see if this user has a subscription
if committee in self.subscriptions:
return True
# now check if our organisation has access
return self.organisation and self.organisation.subscribed_to_committee(committee)
def gets_alerts_for(self, committee):
from ..models.resources import Committee
if not isinstance(committee, Committee):
committee = Committee.query.get(committee)
return committee in self.committee_alerts
def follows(self, committee):
from ..models.resources import Committee
if not isinstance(committee, Committee):
committee = Committee.query.get(committee)
return committee in self.following
def get_followed_committee_meetings(self):
from ..models.resources import CommitteeMeeting
following = CommitteeMeeting.committee_id.in_([f.id for f in self.following])
return CommitteeMeeting.query.filter(following).order_by(desc(CommitteeMeeting.date))
def follow_committee(self, committee):
from ..models.resources import Committee
if not isinstance(committee, Committee):
committee = Committee.query.get(committee)
self.following.append(committee)
def unfollow_committee(self, committee):
from ..models.resources import Committee
if not isinstance(committee, Committee):
committee = Committee.query.get(committee)
self.following.remove(committee)
@validates('organisation')
def validate_organisation(self, key, org):
if org:
self.expiry = org.expiry
return org
@validates('email')
def validate_email(self, key, email):
if email:
email = email.lower()
if not self.organisation and email:
user_domain = email.split("@")[-1]
self.organisation = Organisation.query.filter_by(domain=user_domain).first()
return email
def to_dict(self, include_related=False):
tmp = serializers.model_to_dict(self, include_related=include_related)
tmp.pop('password')
tmp.pop('last_login_ip')
tmp.pop('current_login_ip')
tmp.pop('last_login_at')
tmp.pop('current_login_at')
tmp.pop('confirmed_at')
tmp.pop('login_count')
tmp['has_expired'] = self.has_expired()
# send committee alerts back as a dict
alerts_dict = {}
if tmp.get('committee_alerts'):
for committee in tmp['committee_alerts']:
alerts_dict[committee['id']] = committee.get('name')
tmp['committee_alerts'] = alerts_dict
return tmp
)
app.extensions.get('mail').send(msg)
def subscribe_to_newsletter(user):
""" Add this user to the sharpspring PMG Monitor newsletter mailing list
"""
if app.config.get('SHARPSPRING_API_SECRET'):
from pmg.sharpspring import Sharpspring
Sharpspring().subscribeToList(user, '310799364')
roles_users = db.Table(
'roles_users',
db.Column(
'user_id',
db.Integer(),
db.ForeignKey('user.id')),
db.Column(
'role_id',
db.Integer(),
db.ForeignKey('role.id')))
organisation_committee = db.Table(
'organisation_committee',
db.Column(
'organisation_id',
db.Integer(),
db.ForeignKey('organisation.id', ondelete='CASCADE')),
db.Column(
'committee_id',
db.Integer(),
class SavedSearch(db.Model):
""" A search saved by a user that they get email
alerts about.
"""
__tablename__ = 'saved_search'
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer,
db.ForeignKey('user.id', ondelete='CASCADE'),
nullable=False)
user = db.relationship('User', backref='saved_searches', lazy=True)
# search terms
search = db.Column(db.String(255), nullable=False)
# only search for some items?
content_type = db.Column(db.String(255))
# only search linked to a committee?
committee_id = db.Column(db.Integer,
db.ForeignKey('committee.id', ondelete='CASCADE'))
committee = db.relationship('Committee', lazy=True)
# The last time an alert was sent. We compare new search results to this to determine
# if they're fresh
last_alerted_at = db.Column(db.DateTime(timezone=True),
nullable=False,
server_default=func.now())
created_at = db.Column(db.DateTime(timezone=True),
index=True,
unique=False,
nullable=False,
server_default=func.now())
updated_at = db.Column(db.DateTime(timezone=True),
server_default=func.now(),
onupdate=func.current_timestamp())
def check_and_send_alert(self):
""" Check if there are new items for this search and send an
alert if there are.
"""
hits = self.find_new_hits()
if hits:
log.info("Found %d new results for saved search %s" %
(len(hits), self))
self.send_alert(hits)
def send_alert(self, hits):
""" Send an email alert for the search results in +hits+.
NOTE: this commits the database session, to prevent later errors from causing
us to send duplicate emails.
"""
# we embed this into the actual email template
html = render_template('saved_search_alert.html',
search=self,
results=hits)
send_sendgrid_email(
subject="New matches for your search '%s'" % self.search,
from_name="PMG Notifications",
from_email="*****@*****.**",
recipient_users=[self.user],
html=html,
utm_campaign='searchalert',
)
# save that we sent this alert
self.last_alerted_at = arrow.utcnow().datetime
db.session.commit()
def find_new_hits(self):
from pmg.search import Search
# find hits updated since the last time we did this search
search = Search().search(self.search,
document_type=self.content_type,
committee=self.committee_id)
if 'hits' not in search:
log.warn("Error doing search for %s: %s" % (self, search))
return
timestamp = self.last_alerted_at.astimezone(pytz.utc)
hits = search['hits']['hits']
hits = [
h for h in hits
if arrow.get(h['_source']['updated_at']).datetime >= timestamp
]
return hits
def url(self, **kwargs):
params = {'q': self.search}
if self.content_type:
params['filter[type]'] = self.content_type
if self.committee_id:
params['filter[committee]'] = self.committee_id
params.update(kwargs)
return url_for('search', **params)
@property
def friendly_content_type(self):
from pmg.search import Search
if self.content_type:
return Search.friendly_data_types[self.content_type]
def __repr__(self):
return u'<SavedSearch id=%s user=%s>' % (self.id, self.user)
@classmethod
def send_all_alerts(cls):
""" Find saved searches with new content and send the email alerts.
"""
log.info("Sending all alerts")
for alert in SavedSearch.query.all():
alert.check_and_send_alert()
log.info("Sending alerts finished")
@classmethod
def find(cls, user, q, content_type=None, committee_id=None):
return cls.query.filter(cls.user == user, cls.search == q.lower(),
cls.content_type == content_type,
cls.committee_id == committee_id).first()
@classmethod
def find_or_create(cls, user, q, content_type=None, committee_id=None):
search = cls.find(user, q, content_type, committee_id)
if not search:
search = cls(user=user,
search=q.lower(),
content_type=content_type,
committee_id=committee_id)
search.last_alerted_at = arrow.utcnow().datetime
db.session.add(search)
return search
class SoundcloudTrack(db.Model):
"""
- Tracks where uri and state is null are either busy being uploaded,
or failed to upload to SoundCloud.
- Tracks where state is 'failed' reflect that soundcloud
failed to process the track.
"""
__tablename__ = "soundcloud_track"
id = db.Column(db.Integer, primary_key=True)
created_at = db.Column(
db.DateTime(timezone=True),
nullable=False,
default=datetime.utcnow
)
updated_at = db.Column(
db.DateTime(timezone=True),
nullable=False,
default=datetime.utcnow,
onupdate=func.current_timestamp()
)
file_id = db.Column(
db.Integer,
db.ForeignKey('file.id'),
nullable=False,
unique=True
)
file = db.relationship('File', backref=backref('soundcloud_track', uselist=False, lazy='joined'), lazy=True)
# Soundcloud resource URI for the track (i.e. https://api.soundcloud...id)
uri = db.Column(db.String())
# Last known value of SoundCloud's opinion of the track state
state = db.Column(db.String())
def __str__(self):
return unicode(self).encode('utf-8')
def __unicode__(self):
return u'<SoundcloudTrack id=%d>' % self.id
@classmethod
def new_from_file(cls, client, file):
if db.session.query(cls.id).filter(cls.file_id == file.id).scalar() is not None:
logging.info("File already started being uploaded to Soundcloud: %s" % file)
db.session.rollback()
return
# Immediately create the SoundcloudTrack to indicate that work
# has started for this track and may be in progress.
# Potential concurrent runs can rely on an exception here to avoid
# uploading the same file twice.
soundcloud_track = cls(file=file)
db.session.add(soundcloud_track)
db.session.commit()
soundcloud_track._upload(client)
def _upload(self, client):
with self.file.open() as file_handle:
logging.info("Uploading to SoundCloud: %s" % self.file)
track = client.post('/tracks', track={
'title': self.file.title,
'description': SoundcloudTrack._html_description(self.file),
'sharing': 'public',
'asset_data': file_handle,
'license': 'cc-by',
'artwork_data': open(SOUNDCLOUD_ARTWORK_PATH, 'rb'),
'genre': self.file.event_files[0].event.type,
'tag_list': self.file.event_files[0].event.type,
'downloadable': 'true',
'streamable': 'true',
'feedable': 'true',
})
logging.info("Done uploading to SoundCloud: %s" % self.file)
file_handle.close()
self.uri = track.uri
self.state = track.state
db.session.commit()
@staticmethod
def _html_description(file):
"""
HTML description for presentation on Soundcloud.
staticmethod because it's needed before the instance exists.
"""
return 'Sound recording from:<br>' + \
'<br>'.join(
"<a href='%s'>%s</a>" % (ef.event.url, ef.event.title)
for ef in file.event_files
)
def sync_state(self, client):
logging.info("Checking state of %s", self.uri)
track = client.get(self.uri)
if track.state != self.state:
self.state = track.state
db.session.commit()
logger.info("SoundCloud track %s state is now [%s]" % (self, self.state))
@classmethod
def upload_files(cls, client):
q = cls.get_unstarted_query()
logging.info("Audio files yet to be uploaded to SoundCloud: %d"
% cls.get_unstarted_count(q))
batch = cls.get_unstarted_batch(q)
# Rollback this transaction - it was just to gather candidates for upload
db.session.rollback()
logging.info("Uploading %d files to SoundCloud" % len(batch))
for file in batch:
cls.new_from_file(client, file)
@classmethod
def sync(cls):
client = Client(client_id=app.config['SOUNDCLOUD_APP_KEY_ID'],
client_secret=app.config['SOUNDCLOUD_APP_KEY_SECRET'],
username=app.config['SOUNDCLOUD_USERNAME'],
password=app.config['SOUNDCLOUD_PASSWORD'])
cls.upload_files(client)
cls.sync_upload_state(client)
cls.handle_failed(client)
@classmethod
def get_unstarted_query(cls):
"""
Get audio files for which there's no SoundcloudTrack.
Order by id as a hacky way to roughly get the latest files first
"""
# Query files that aren't connected to events so we can ignore them for
# now - it's not clear that they're actually visible - they might well
# have been mistaken uploads that shouldn't suddenly appear on PMG's
# public soundcloud profile.
q_files_with_meetings = db.session.query(File.id) \
.outerjoin(EventFile) \
.filter(EventFile.file_id == None) \
.filter(File.file_mime.like('audio/%'))
return db.session.query(File) \
.outerjoin(cls) \
.filter(cls.file_id == None) \
.filter(File.file_mime.like('audio/%')) \
.filter(~File.id.in_(q_files_with_meetings)) \
.order_by(desc(File.id))
@staticmethod
def get_unstarted_count(q):
return q.count()
@staticmethod
def get_unstarted_batch(q):
return q.limit(app.config['MAX_SOUNDCLOUD_BATCH']).all()
@classmethod
def sync_upload_state(cls, client):
tracks = db.session.query(cls) \
.filter(cls.state.in_(UNFINISHED_STATES)) \
.order_by(cls.created_at).all()
for track in tracks:
track.sync_state(client)
@classmethod
def handle_failed(cls, client):
for track_id, retries in db.session.query(cls.id,
func.count(SoundcloudTrackRetry.id).label('retries'))\
.filter(cls.state == 'failed')\
.outerjoin(SoundcloudTrackRetry)\
.group_by(cls.id)\
.order_by('retries')\
.limit(app.config['MAX_SOUNDCLOUD_BATCH']):
if retries <= app.config['MAX_SOUNDCLOUD_RETRIES']:
soundcloud_track = db.session.query(cls).get(track_id)
# Sometimes tracks apparently go from failed to finished. Yeah.
try:
soundcloud_track.sync_state(client)
except HTTPError:
logging.info("HTTP Error checking state of failed SoundCloud upload")
if soundcloud_track.state == 'failed':
soundcloud_track.retry_upload(client)
def retry_upload(self, client):
logging.info("Retrying failed soundcloud track %r" % self)
retry = SoundcloudTrackRetry(soundcloud_track=self)
db.session.add(retry)
db.session.commit()
try:
client.delete(self.uri)
except HTTPError as delete_result:
# Handle brokenness at SoundCloud where deleting a track
# results with an HTTP 500 response yet the track
# is gone (HTTP 404) when you try to GET it.
if delete_result.response.status_code == 500:
try:
client.get(self.uri)
# If the delete gves a 500 and the GET is successful, we're
# not sure it was deleted so don't continue the retry, just
# let the next retry try to delete again and continue if
# it's finished deleting by then.
logging.info(("Tried to delete but track %s but apparently " +\
"it's still there") % self.uri)
return
except HTTPError as get_result:
if get_result.response.status_code != 404:
raise Exception(("Can't tell if track %s that we attempted " +\
"to delete is still there.") % self.uri)
elif delete_result.response.status_code == 404:
logging.info(("Track %s was already missing from SoundCloud when " +\
"to retry %r") % (self.uri, self))
elif delete_result.response.status_code != 200:
raise Exception(("Unexpected result when deleting %s from " +\
"SoundCloud") % self)
# If we get here we expect that we've successfully deleted
# the failed track from SoundCloud.
# Indicate that we've started uploading
self.state = None
db.session.commit()
self._upload(client)
