def task_stations(self, grab, task):
stations = grab.xpath_list('//div[@class="dirlist"]')
if grab.xpath_exists('//span[contains(text(), "show more")]'):
yield self.new_stations_task(genre=task.genre,
start_index=task.last + len(stations))
for dirlist in stations:
info = dirlist.xpath('./div[1]/a[1]')[0]
url, name = info.get('href'), info.get('name')
stream = dirlist.xpath('./div[@class="dirtype"]/text()')[0]
bitrate = dirlist.xpath('./div[@class="dirbitrate"]/text()')[0]
stream = model.get_or_create(model.Stream,
name=stream)
bitrate = model.get_or_create(model.Bitrate,
name=bitrate)
station = model.get_or_create(model.Station,
name=name,
url=url,
stream=stream,
bitrate=bitrate)
station.genres.append(task.genre)
if stations:
session.commit()
def grab_all_categories():
for category_name, subcategories in get_categories():
print category_name, subcategories
category_record = model.get_or_create(model.Category,
name=category_name,
parent=None,
site=model.free_lance_ru)
for subcategory_name in subcategories:
model.get_or_create(model.Category,
name=subcategory_name,
parent=category_record,
site=model.free_lance_ru)
def grab_all_categories():
for category_name, subcategories in get_categories():
print category_name, subcategories
category_record = model.get_or_create(
model.Category,
name=category_name,
parent=None,
site=model.free_lance_ru
)
for subcategory_name in subcategories:
model.get_or_create(
model.Category,
name=subcategory_name,
parent=category_record,
site=model.free_lance_ru
)
def update_relations(model, query, params):
"""Updates related fields of a model that are present in ``params``.
`model`
An elixir model instance that will have its relations updated.
`query`
An sqlalchemy Query instance that evalutes to all instances that
should be updated.
`params`
A dictionary with two keys ``add`` and ``remove``. Both of them
contains a list of items that should be added or removed from
such a relation.
"""
fields = []
cols = model.get_columns()
relations = model.get_relations()
for col in set(relations).intersection(params.keys()):
submodel = cols[col].property.mapper.class_
from_request = params[col]
# Let's add new columns to the relation being managed.
for subparams in from_request.get('add', ()):
if 'id' in subparams:
subinst = submodel.get_by(id=subparams.pop('id'))
else:
vssubparams = _validate_field_list(submodel.__name__,
subparams, subparams.keys())
subinst = get_or_create(submodel, **vssubparams)[0]
for instance in query:
getattr(instance, col).append(subinst)
# Now is the time to handle relations being removed from a field
# of the instance. We'll do nothing here if there's no id param
for subparams in from_request.get('remove', ()):
try:
remove = subparams.pop('__delete__')
except KeyError:
remove = False
if 'id' in subparams:
subinst = submodel.get_by(id=subparams['id'])
else:
vssubparams = _validate_field_list(submodel.__name__,
subparams, subparams.keys())
subinst = submodel.get_by(**vssubparams)
for instance in query:
field = getattr(instance, col)
field.remove(subinst)
if remove:
subinst.delete()
fields.append(col)
return fields
def send_metric():
from model import Metrics, Client, get_or_create
if request.method == 'POST':
data = request.json
if data.get('client') and data.get('value'):
client = get_or_create(db.session, Client, name=data['client'])
Metrics.create_metric(db.session, client.id, data['value'])
return 'OK', 200
return 'FAIL', 500
def task_subgenres(self, grab, task):
subgenres = grab.xpath_list('//li[@class="secgen"]/a/text()')
for subgenre in subgenres:
subgenre_record = model.get_or_create(model.Genre,
name=subgenre,
parent=task.genre)
yield self.new_stations_task(genre=subgenre_record,
start_index=0)
def task_genres(self, grab, task):
genres = grab.xpath_list('//li[@class="prigen"]/a/text()')
for genre in genres:
genre_record = model.get_or_create(model.Genre,
name=genre,
parent=None)
grab = self.create_grab_instance()
grab.setup(url='http://www.shoutcast.com/genre.jsp',
post=dict(genre=genre))
yield Task(name='subgenres',
genre=genre_record,
grab=grab)
yield self.new_stations_task(genre=genre_record,
start_index=0)
# Getting the list of relations that will be added later
cols = model.get_columns()
relations = model.get_relations()
# Looking for what we're going to set to the model right now
props = set(cols.keys()).intersection(params.keys()).difference(relations)
instance = model(**dict([(i, params[i]) for i in props]))
# Handling relations, a single level is allowed
for col in set(relations).intersection(params.keys()):
submodel = cols[col].property.mapper.class_
subvalidator = getattr(CONFIG['validators'], submodel.__name__)
for subparams in params[col]:
subparams = subvalidator.to_python(subparams)
subinst = get_or_create(submodel, **subparams)[0]
getattr(instance, col).append(subinst)
session.add(instance)
session.commit()
# We return a ok message and the just created instance id
return dumps({'status': 'ok', 'message': 'You rock!', 'id': instance.id})
def build_search_param(model, fname, relation, operation, value):
"""Translates an operation described as a string to a valid
sqlalchemy query parameter.
This takes, for example, the operation ``gt`` and converts it to
something like this: ``field > value``.