def handle(self, *args, **options): basedir = options['directories'][0] cas = BasicCAS() for audioFile in AudioFile.objects.all(): destination = join(basedir, audioFile.map()) # probably upgrade or is the same if islink(destination): unlink(destination) print destination cas.link_out(audioFile.ref, destination)
def handle(self, *args, **options): basedir = options['directories'][0] cas = BasicCAS() for audioFile in AudioFile.objects.all(): destination = join(basedir, audioFile.map() ) # probably upgrade or is the same if islink(destination): unlink(destination) print destination cas.link_out(audioFile.ref,destination)
class PeerCrawler(Job): description = """ Crawl another crates server, downloading and adding files that are not on this crates server """ max_workers = 2 def __init__(self,peer): self.cas = BasicCAS() self.peer = peer self.urlobj = urlparse(peer.url) self.deserialize = get_deserializer('json') def _build_url(self,**kwargs): return urlunparse( self.urlobj._replace(**kwargs) ) def enumerate_tasks(self): url = self._build_url(path="dump/AudioFiles") json = get(url).text return self.deserialize(json) def process_task(self,item): # N.B. item is not a simple django model object, but behaves like one # this relationship is lost by the serialiser (currently) item.object.ref = item.object.pk # remember peer for stats/filtering item.object.peer = self.peer # TODO: can I just do item.exists() or something? if AudioFile.objects.filter(ref=item.object.ref).exists(): raise TaskSkipped() # dl audio file into CAS via ref (may already have, even if not index if index is destroyed) if not self.cas.has(item.object.ref): url = self._build_url(path="cas/%s" % item.object.ref) generator = get(url,stream=True).iter_content(chunk_size=8192) ref = self.cas.insert_generator(generator) else: ref = item.object.ref if item.object.cover_art_ref and not self.cas.has(item.object.cover_art_ref): url = self._build_url(path="cas/%s" % item.object.cover_art_ref) generator = get(url,stream=True).iter_content(chunk_size=8192) self.cas.insert_generator(generator) if ref != item.object.ref: self.cas.delete(ref) raise IOError(ref) item.object.save(ref) assert ref == item.object.ref # network/disk/h4x0r problem? Just exit now. Later throw/catch/delete and warn or something.
class PeerCrawler(Job): description = """ Crawl another crates server, downloading and adding files that are not on this crates server """ max_workers = 2 def __init__(self, peer): self.cas = BasicCAS() self.peer = peer self.urlobj = urlparse(peer.url) self.deserialize = get_deserializer('json') def _build_url(self, **kwargs): return urlunparse(self.urlobj._replace(**kwargs)) def enumerate_tasks(self): url = self._build_url(path="dump/AudioFiles") json = get(url).text return self.deserialize(json) def process_task(self, item): # N.B. item is not a simple django model object, but behaves like one # this relationship is lost by the serialiser (currently) item.object.ref = item.object.pk # remember peer for stats/filtering item.object.peer = self.peer # TODO: can I just do item.exists() or something? if AudioFile.objects.filter(ref=item.object.ref).exists(): raise TaskSkipped() # dl audio file into CAS via ref (may already have, even if not index if index is destroyed) if not self.cas.has(item.object.ref): url = self._build_url(path="cas/%s" % item.object.ref) generator = get(url, stream=True).iter_content(chunk_size=8192) ref = self.cas.insert_generator(generator) else: ref = item.object.ref if item.object.cover_art_ref and not self.cas.has( item.object.cover_art_ref): url = self._build_url(path="cas/%s" % item.object.cover_art_ref) generator = get(url, stream=True).iter_content(chunk_size=8192) self.cas.insert_generator(generator) if ref != item.object.ref: self.cas.delete(ref) raise IOError(ref) item.object.save(ref) assert ref == item.object.ref # network/disk/h4x0r problem? Just exit now. Later throw/catch/delete and warn or something.
def __init__(self,peer): self.cas = BasicCAS() self.peer = peer self.urlobj = urlparse(peer.url) self.deserialize = get_deserializer('json')
def __init__(self, peer): self.cas = BasicCAS() self.peer = peer self.urlobj = urlparse(peer.url) self.deserialize = get_deserializer('json')