def handle(self, *args, **options):
basedir = options['directories'][0]
cas = BasicCAS()
for audioFile in AudioFile.objects.all():
destination = join(basedir, audioFile.map())
# probably upgrade or is the same
if islink(destination): unlink(destination)
print destination
cas.link_out(audioFile.ref, destination)
def handle(self, *args, **options):
basedir = options['directories'][0]
cas = BasicCAS()
for audioFile in AudioFile.objects.all():
destination = join(basedir, audioFile.map() )
# probably upgrade or is the same
if islink(destination): unlink(destination)
print destination
cas.link_out(audioFile.ref,destination)
class PeerCrawler(Job):
description = """
Crawl another crates server, downloading and adding files
that are not on this crates server
"""
max_workers = 2
def __init__(self,peer):
self.cas = BasicCAS()
self.peer = peer
self.urlobj = urlparse(peer.url)
self.deserialize = get_deserializer('json')
def _build_url(self,**kwargs):
return urlunparse( self.urlobj._replace(**kwargs) )
def enumerate_tasks(self):
url = self._build_url(path="dump/AudioFiles")
json = get(url).text
return self.deserialize(json)
def process_task(self,item):
# N.B. item is not a simple django model object, but behaves like one
# this relationship is lost by the serialiser (currently)
item.object.ref = item.object.pk
# remember peer for stats/filtering
item.object.peer = self.peer
# TODO: can I just do item.exists() or something?
if AudioFile.objects.filter(ref=item.object.ref).exists():
raise TaskSkipped()
# dl audio file into CAS via ref (may already have, even if not index if index is destroyed)
if not self.cas.has(item.object.ref):
url = self._build_url(path="cas/%s" % item.object.ref)
generator = get(url,stream=True).iter_content(chunk_size=8192)
ref = self.cas.insert_generator(generator)
else:
ref = item.object.ref
if item.object.cover_art_ref and not self.cas.has(item.object.cover_art_ref):
url = self._build_url(path="cas/%s" % item.object.cover_art_ref)
generator = get(url,stream=True).iter_content(chunk_size=8192)
self.cas.insert_generator(generator)
if ref != item.object.ref:
self.cas.delete(ref)
raise IOError(ref)
item.object.save(ref)
assert ref == item.object.ref # network/disk/h4x0r problem? Just exit now. Later throw/catch/delete and warn or something.
class PeerCrawler(Job):
description = """
Crawl another crates server, downloading and adding files
that are not on this crates server
"""
max_workers = 2
def __init__(self, peer):
self.cas = BasicCAS()
self.peer = peer
self.urlobj = urlparse(peer.url)
self.deserialize = get_deserializer('json')
def _build_url(self, **kwargs):
return urlunparse(self.urlobj._replace(**kwargs))
def enumerate_tasks(self):
url = self._build_url(path="dump/AudioFiles")
json = get(url).text
return self.deserialize(json)
def process_task(self, item):
# N.B. item is not a simple django model object, but behaves like one
# this relationship is lost by the serialiser (currently)
item.object.ref = item.object.pk
# remember peer for stats/filtering
item.object.peer = self.peer
# TODO: can I just do item.exists() or something?
if AudioFile.objects.filter(ref=item.object.ref).exists():
raise TaskSkipped()
# dl audio file into CAS via ref (may already have, even if not index if index is destroyed)
if not self.cas.has(item.object.ref):
url = self._build_url(path="cas/%s" % item.object.ref)
generator = get(url, stream=True).iter_content(chunk_size=8192)
ref = self.cas.insert_generator(generator)
else:
ref = item.object.ref
if item.object.cover_art_ref and not self.cas.has(
item.object.cover_art_ref):
url = self._build_url(path="cas/%s" % item.object.cover_art_ref)
generator = get(url, stream=True).iter_content(chunk_size=8192)
self.cas.insert_generator(generator)
if ref != item.object.ref:
self.cas.delete(ref)
raise IOError(ref)
item.object.save(ref)
assert ref == item.object.ref # network/disk/h4x0r problem? Just exit now. Later throw/catch/delete and warn or something.
def __init__(self,peer):
self.cas = BasicCAS()
self.peer = peer
self.urlobj = urlparse(peer.url)
self.deserialize = get_deserializer('json')
def __init__(self, peer):
self.cas = BasicCAS()
self.peer = peer
self.urlobj = urlparse(peer.url)
self.deserialize = get_deserializer('json')
