class Recording(BackendItem): logCategory = 'dvbd_store' def __init__(self,store, id,parent_id, file,title, date,duration, mimetype): self.store = store self.id = 'recording.%s' % id self.parent_id = parent_id self.real_id = id path = unicode(file) # make sure path is an absolute local path (and not an URL) if path.startswith("file://"): path = path[7:] self.location = FilePath(path) self.title = unicode(title) self.mimetype = str(mimetype) self.date = datetime.fromtimestamp(int(date)) self.duration = int(duration) try: self.size = self.location.getsize() except Exception, msg: self.size = 0 self.bitrate = 0 self.url = self.store.urlbase + str(self.id)
def render_POST(self, request): context = self._context(request) form, status, appstruct = self._validate_form(request) if status == 'ok': filename = context['renderer'](appstruct) fileobj = FilePath(filename) request.setHeader('content-disposition', 'attachment; filename="output.pdf"') request.setHeader('content-length', str(fileobj.getsize())) request.setHeader('content-type', 'application/pdf') producer = NoRangeStaticProducer(request, fileobj.open('r')) producer.start() return NOT_DONE_YET data['renderer_form'] = form.render(appstruct) self.render_template(data, request) return NOT_DONE_YET
def _get_new(self): """ Get new requests to process. This will define the `main loop` of the FileChecker service. """ # Check ACTIVE accounts to get files with running traffic captures active_requests = yield self._get_requests("ACTIVE") if active_requests: log.info("FILECHECKER:: Checking pcap files of ACTIVE accounts.") for request in active_requests: username = request[0] ip_addr = request[6] # Create path k = "{}_{}".format(username, ip_addr) pcap_file = "{}.pcap".format(k) pcap_file = os.path.join(self.pcaps_path, pcap_file) log.debug("FILECHECKER:: Checking {}".format(pcap_file)) # First limit should be zero if self.reached_limits.get(k) is None: self.reached_limits[k] = 0 try: fp = FilePath(pcap_file) fsize = fp.getsize() # Check each size limit only once if self.reached_limits[k] + 1 < len(self.size_limit): if fsize > int( self.size_limit[self.reached_limits[k] + 1]): text = "File {} has reached the limit of "\ "{} bytes!".format( pcap_file, self.size_limit[self.reached_limits[k]+1] ) log.info("FILECHECKER:: {}".format(text)) yield self.slackbot.post(text, self.slack_channel) self.reached_limits[k] = self.reached_limits[k] + 1 else: log.debug("FILECHECKER:: File {} has reached the last" " size limit of {} bytes!".format( pcap_file, self.size_limit[self.reached_limits[k]])) except Exception as e: log.debug("FILECHECKER:: Could'nt get size: {}".format(e)) else: log.debug("FILECHECKER:: No ACTIVE accounts - Keep waiting.")
def get_index(self, path): index = {} for root, dirs, files in os.walk(path): index[root] = {} for f in files: path = os.path.join(root, f) filepath = FilePath(path) try: index[path] = { 'mtime': filepath.getModificationTime(), 'size': filepath.getsize(), } except OSError: # file could be a broken symlink, or deleted mid-scan continue return index
def getDirectory(self, path='/'): self.fs = yield FilePath(path) if not self.fs.getPermissions(): defer.returnValue(False) files = [] for f in self.fs.listdir(): if f == '/': continue fp = path+f fs = FilePath(fp) # dont follow symlinks if fs.realpath().path != fp: continue perm = None isdir = fs.isdir() size = fs.getsize() modified = datetime.utcfromtimestamp(fs.getModificationTime()) df = DiscoveredFile( resource_id=self.data['resource_id'], file_path=path, file_name=f, file_isdir=isdir, file_size=size, file_modified=modified, file_perm=perm ) print '[%s] LIST %s.' % (self.data['resource_name'], fp if not fp.endswith('.') else fp) files.append(df) defer.returnValue(files)
class Recording(BackendItem): logCategory = 'dvbd_store' def __init__(self, store, id, parent_id, file, title, date, duration, mimetype): BackendItem.__init__(self) self.store = store self.id = 'recording.%s' % id self.parent_id = parent_id self.real_id = id path = str(file) # make sure path is an absolute local path (and not an URL) if path.startswith('file://'): path = path[7:] self.location = FilePath(path) self.title = str(title) self.mimetype = str(mimetype) self.date = datetime.fromtimestamp(int(date)) self.duration = int(duration) try: self.size = self.location.getsize() except Exception as msg: self.size = 0 self.bitrate = 0 self.url = self.store.urlbase + str(self.id) def get_children(self, start=0, end=0): return [] def get_child_count(self): return 0 def get_item(self, parent_id=None): self.debug(f'Recording get_item {self.id} @ {self.parent_id}') # create item item = DIDLLite.VideoBroadcast(self.id, self.parent_id) item.date = self.date item.title = self.title # add http resource res = DIDLLite.Resource(self.url, f'http-get:*:{self.mimetype}:*') if self.size > 0: res.size = self.size if self.duration > 0: res.duration = str(self.duration) if self.bitrate > 0: res.bitrate = str(self.bitrate) item.res.append(res) # add internal resource res = DIDLLite.Resource( 'file://' + urllib.parse.quote(self.get_path()), f'internal:{self.store.server.coherence.hostname}:' f'{self.mimetype}:*', ) if self.size > 0: res.size = self.size if self.duration > 0: res.duration = str(self.duration) if self.bitrate > 0: res.bitrate = str(self.bitrate) item.res.append(res) return item def get_id(self): return self.id def get_name(self): return self.title def get_url(self): return self.url def get_path(self): return self.location.path
class Recording(BackendItem): logCategory = 'dvbd_store' def __init__(self,store, id,parent_id, file,title, date,duration, mimetype): self.store = store self.id = 'recording.%s' % id self.parent_id = parent_id self.real_id = id self.location = FilePath(unicode(file)) self.title = unicode(title) self.mimetype = str(mimetype) self.date = datetime.fromtimestamp(int(date)) self.duration = int(duration) self.size = self.location.getsize() self.bitrate = 0 self.url = self.store.urlbase + str(self.id) def get_children(self, start=0, end=0): return [] def get_child_count(self): return 0 def get_item(self, parent_id=None): self.debug("Recording get_item %r @ %r" %(self.id,self.parent_id)) # create item item = DIDLLite.VideoBroadcast(self.id,self.parent_id) item.date = self.date item.title = self.title # add http resource res = DIDLLite.Resource(self.url, 'http-get:*:%s:*' % self.mimetype) if self.size > 0: res.size = self.size if self.duration > 0: res.duration = str(self.duration) if self.bitrate > 0: res.bitrate = str(bitrate) item.res.append(res) # add internal resource res = DIDLLite.Resource('file://'+ urllib.quote(self.get_path()), 'internal:%s:%s:*' % (self.store.server.coherence.hostname,self.mimetype)) if self.size > 0: res.size = self.size if self.duration > 0: res.duration = str(self.duration) if self.bitrate > 0: res.bitrate = str(bitrate) item.res.append(res) return item def get_id(self): return self.id def get_name(self): return self.title def get_url(self): return self.url def get_path(self): return self.location.path
class FSItem(BackendItem): logCategory = 'fs_item' def __init__( self, object_id, parent, path, mimetype, urlbase, UPnPClass, update=False, store=None, ): BackendItem.__init__(self) self.id = object_id self.parent = parent if parent: parent.add_child(self, update=update) if mimetype == 'root': self.location = str(path) else: if mimetype == 'item' and path is None: path = os.path.join(parent.get_realpath(), str(self.id)) # self.location = FilePath(unicode(path)) self.location = FilePath(path) self.mimetype = mimetype if urlbase[-1] != '/': urlbase += '/' self.url = urlbase + str(self.id) self.store = store if parent is None: parent_id = -1 else: parent_id = parent.get_id() self.item = UPnPClass(object_id, parent_id, self.get_name()) if isinstance(self.item, Container): self.item.childCount = 0 self.child_count = 0 self.children = [] self.sorted = False self.caption = None if mimetype in ['directory', 'root']: self.update_id = 0 self.get_url = lambda: self.url # self.item.searchable = True # self.item.searchClass = 'object' if (isinstance(self.location, FilePath) and self.location.isdir() is True): self.check_for_cover_art() if getattr(self, 'cover', None): _, ext = os.path.splitext(self.cover) ''' add the cover image extension to help clients not reacting on the mimetype ''' self.item.albumArtURI = ''.join( (urlbase, str(self.id), '?cover', str(ext))) else: self.get_url = lambda: self.url if self.mimetype.startswith('audio/'): if getattr(parent, 'cover', None): _, ext = os.path.splitext(parent.cover) ''' add the cover image extension to help clients not reacting on the mimetype ''' self.item.albumArtURI = ''.join( (urlbase, str(self.id), '?cover', ext)) _, host_port, _, _, _ = urlsplit(urlbase) if host_port.find(':') != -1: host, port = tuple(host_port.split(':')) else: host = host_port try: size = self.location.getsize() except Exception: size = 0 if (self.store.server and self.store.server.coherence.config.get( 'transcoding', 'no') == 'yes'): if self.mimetype in ( 'application/ogg', 'audio/ogg', 'audio/x-wav', 'audio/x-m4a', 'application/x-flac', ): new_res = Resource( self.url + '/transcoded.mp3', f'http-get:*:{"audio/mpeg"}:*', ) new_res.size = None # self.item.res.append(new_res) if mimetype != 'item': res = Resource( 'file://' + quote(self.get_path(), encoding='utf-8'), f'internal:{host}:{self.mimetype}:*', ) res.size = size self.item.res.append(res) if mimetype != 'item': res = Resource(self.url, f'http-get:*:{self.mimetype}:*') else: res = Resource(self.url, 'http-get:*:*:*') res.size = size self.item.res.append(res) ''' if this item is of type audio and we want to add a transcoding rule for it, this is the way to do it: create a new Resource object, at least a 'http-get' and maybe an 'internal' one too for transcoding to wav this looks like that res = Resource( url_for_transcoded audio, 'http-get:*:audio/x-wav:%s'% ';'.join( ['DLNA.ORG_PN=JPEG_TN']+simple_dlna_tags)) res.size = None self.item.res.append(res) ''' if (self.store.server and self.store.server.coherence.config.get( 'transcoding', 'no') == 'yes'): if self.mimetype in ( 'audio/mpeg', 'application/ogg', 'audio/ogg', 'audio/x-wav', 'audio/x-m4a', 'audio/flac', 'application/x-flac', ): dlna_pn = 'DLNA.ORG_PN=LPCM' dlna_tags = simple_dlna_tags[:] # dlna_tags[1] = 'DLNA.ORG_OP=00' dlna_tags[2] = 'DLNA.ORG_CI=1' new_res = Resource( self.url + '?transcoded=lpcm', f'http-get:*:{"audio/L16;rate=44100;channels=2"}:' f'{";".join([dlna_pn] + dlna_tags)}', ) new_res.size = None # self.item.res.append(new_res) if self.mimetype != 'audio/mpeg': new_res = Resource( self.url + '?transcoded=mp3', f'http-get:*:{"audio/mpeg"}:*', ) new_res.size = None # self.item.res.append(new_res) ''' if this item is an image and we want to add a thumbnail for it we have to follow these rules: create a new Resource object, at least a 'http-get' and maybe an 'internal' one too for an JPG this looks like that res = Resource(url_for_thumbnail, 'http-get:*:image/jpg:%s'% ';'.join( ['DLNA.ORG_PN=JPEG_TN']+simple_dlna_tags)) res.size = size_of_thumbnail self.item.res.append(res) and for a PNG the Resource creation is like that res = Resource(url_for_thumbnail, 'http-get:*:image/png:%s'% ';'.join( simple_dlna_tags+['DLNA.ORG_PN=PNG_TN'])) if not hasattr(self.item, 'attachments'): self.item.attachments = {} self.item.attachments[key] = utils.StaticFile( filename_of_thumbnail) ''' if (self.mimetype in ('image/jpeg', 'image/png') or self.mimetype.startswith('video/')): try: filename, mimetype, dlna_pn = _find_thumbnail( self.get_path()) except NoThumbnailFound: pass except Exception: self.warning(traceback.format_exc()) else: dlna_tags = simple_dlna_tags[:] dlna_tags[ 3] = 'DLNA.ORG_FLAGS=00f00000000000000000000000000000' hash_from_path = str(id(filename)) new_res = Resource( self.url + '?attachment=' + hash_from_path, f'http-get:*:{mimetype}:' f'{";".join([dlna_pn] + dlna_tags)}', ) new_res.size = os.path.getsize(filename) self.item.res.append(new_res) if not hasattr(self.item, 'attachments'): self.item.attachments = {} self.item.attachments[hash_from_path] = utils.StaticFile( filename) if self.mimetype.startswith('video/'): # check for a subtitles file caption, _ = os.path.splitext(self.get_path()) caption = caption + '.srt' if os.path.exists(caption): hash_from_path = str(id(caption)) mimetype = 'smi/caption' new_res = Resource( self.url + '?attachment=' + hash_from_path, f'http-get:*:{mimetype}:{"*"}', ) new_res.size = os.path.getsize(caption) self.caption = new_res.data self.item.res.append(new_res) if not hasattr(self.item, 'attachments'): self.item.attachments = {} self.item.attachments[hash_from_path] = utils.StaticFile( caption, defaultType=mimetype, ) try: # FIXME: getmtime is deprecated in Twisted 2.6 self.item.date = datetime.fromtimestamp( self.location.getmtime()) except Exception: self.item.date = None def rebuild(self, urlbase): # print('rebuild', self.mimetype) if self.mimetype != 'item': return # print('rebuild for', self.get_path()) mimetype, _ = mimetypes.guess_type(self.get_path(), strict=False) if mimetype is None: return self.mimetype = mimetype # print('rebuild', self.mimetype) UPnPClass = classChooser(self.mimetype) self.item = UPnPClass(self.id, self.parent.id, self.get_name()) if getattr(self.parent, 'cover', None): _, ext = os.path.splitext(self.parent.cover) # add the cover image extension to help # clients not reacting on the mimetype self.item.albumArtURI = ''.join( (urlbase, str(self.id), '?cover', ext)) _, host_port, _, _, _ = urlsplit(urlbase) if host_port.find(':') != -1: host, port = tuple(host_port.split(':')) else: host = host_port res = Resource( 'file://' + quote(self.get_path()), f'internal:{host}:{self.mimetype}:*', ) try: res.size = self.location.getsize() except Exception: res.size = 0 self.item.res.append(res) res = Resource(self.url, f'http-get:*:{self.mimetype}:*') try: res.size = self.location.getsize() except Exception: res.size = 0 self.item.res.append(res) try: # FIXME: getmtime is deprecated in Twisted 2.6 self.item.date = datetime.fromtimestamp(self.location.getmtime()) except Exception: self.item.date = None self.parent.update_id += 1 def check_for_cover_art(self): ''' let's try to find in the current directory some jpg file, or png if the jpg search fails, and take the first one that comes around ''' try: jpgs = [ i.path for i in self.location.children() if i.splitext()[1] in ('.jpg', '.JPG') ] try: self.cover = jpgs[0] except IndexError: pngs = [ i.path for i in self.location.children() if i.splitext()[1] in ('.png', '.PNG') ] try: self.cover = pngs[0] except IndexError: return except UnicodeDecodeError: self.warning( f'UnicodeDecodeError - there is something wrong with a ' + f'file located in {self.location.path}') def remove(self): # print('FSItem remove', self.id, self.get_name(), self.parent) if self.parent: self.parent.remove_child(self) del self.item def add_child(self, child, update=False): self.children.append(child) self.child_count += 1 if isinstance(self.item, Container): self.item.childCount += 1 if update: self.update_id += 1 self.sorted = False def remove_child(self, child): # print(f'remove_from {self.id:d} ({self.get_name()}) ' # f'child {child.id:d} ({child.get_name()})') if child in self.children: self.child_count -= 1 if isinstance(self.item, Container): self.item.childCount -= 1 self.children.remove(child) self.update_id += 1 self.sorted = False def get_children(self, start=0, request_count=0): if not self.sorted: self.children.sort(key=_natural_key) self.sorted = True if request_count == 0: return self.children[start:] else: return self.children[start:request_count] def get_child_count(self): return self.child_count def get_id(self): return self.id def get_update_id(self): if hasattr(self, 'update_id'): return self.update_id else: return None def get_path(self): if self.mimetype in ['directory', 'root']: return None if isinstance(self.location, FilePath): return self.location.path else: return self.location def get_realpath(self): if isinstance(self.location, FilePath): return self.location.path else: return self.location def set_path(self, path=None, extension=None): if path is None: path = self.get_path() if extension is not None: path, old_ext = os.path.splitext(path) path = ''.join((path, extension)) if isinstance(self.location, FilePath): self.location = FilePath(path) else: self.location = path def get_name(self): if isinstance(self.location, FilePath): name = self.location.basename() else: name = self.location return name def get_cover(self): if self.cover: return self.cover try: return self.parent.cover except AttributeError: return None def get_parent(self): return self.parent def get_item(self): return self.item def get_xml(self): return self.item.toString() def __repr__(self): return ('id: ' + str(self.id) + ' @ ' + str(self.get_name().encode('ascii', 'xmlcharrefreplace')))
class FSItem(BackendItem): logCategory = 'fs_item' def __init__(self, object_id, parent, path, mimetype, urlbase, UPnPClass,update=False): self.id = object_id self.parent = parent if parent: parent.add_child(self,update=update) if mimetype == 'root': self.location = unicode(path) else: if mimetype == 'item' and path is None: path = os.path.join(parent.get_path(),unicode(self.id)) #self.location = FilePath(unicode(path)) self.location = FilePath(path) self.mimetype = mimetype if urlbase[-1] != '/': urlbase += '/' self.url = urlbase + str(self.id) if parent == None: parent_id = -1 else: parent_id = parent.get_id() self.item = UPnPClass(object_id, parent_id, self.get_name()) if isinstance(self.item, Container): self.item.childCount = 0 self.child_count = 0 self.children = [] if mimetype in ['directory','root']: self.update_id = 0 self.get_url = lambda : self.url self.get_path = lambda : None #self.item.searchable = True #self.item.searchClass = 'object' if(isinstance(self.location,FilePath) and self.location.isdir() == True): self.check_for_cover_art() if hasattr(self, 'cover'): _,ext = os.path.splitext(self.cover) """ add the cover image extension to help clients not reacting on the mimetype """ self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext)) else: self.get_url = lambda : self.url if self.mimetype.startswith('audio/'): if hasattr(parent, 'cover'): _,ext = os.path.splitext(parent.cover) """ add the cover image extension to help clients not reacting on the mimetype """ self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext)) _,host_port,_,_,_ = urlsplit(urlbase) if host_port.find(':') != -1: host,port = tuple(host_port.split(':')) else: host = host_port try: size = self.location.getsize() except: size = 0 if mimetype != 'item': res = Resource('file://'+ urllib.quote(self.get_path()), 'internal:%s:%s:*' % (host,self.mimetype)) res.size = size self.item.res.append(res) if mimetype != 'item': res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype) else: res = Resource(self.url, 'http-get:*:*:*') res.size = size self.item.res.append(res) """ if this item is an image and we want to add a thumbnail for it we have to follow these rules: create a new Resource object, at least a 'http-get' and maybe an 'internal' one too for an JPG this looks like that res = Resource(url_for_thumbnail, 'http-get:*:image/jpg:%s'% ';'.join(simple_dlna_tags+('DLNA.ORG_PN=JPEG_TN',))) res.size = size_of_thumbnail self.item.res.append(res) and for a PNG the Resource creation is like that res = Resource(url_for_thumbnail, 'http-get:*:image/png:%s'% ';'.join(simple_dlna_tags+('DLNA.ORG_PN=PNG_TN',))) if not hasattr(self.item, 'attachments'): self.item.attachments = {} self.item.attachments[key] = utils.StaticFile(filename_of_thumbnail) """ if self.mimetype in ('image/jpeg', 'image/png'): path = self.get_path() thumbnail = os.path.join(os.path.dirname(path),'.thumbs',os.path.basename(path)) if os.path.exists(thumbnail): mimetype,_ = mimetypes.guess_type(thumbnail, strict=False) if mimetype in ('image/jpeg','image/png'): if mimetype == 'image/jpeg': dlna_pn = 'DLNA.ORG_PN=JPEG_TN' else: dlna_pn = 'DLNA.ORG_PN=PNG_TN' hash_from_path = str(id(thumbnail)) new_res = Resource(self.url+'?attachment='+hash_from_path, 'http-get:*:%s:%s' % (mimetype, ';'.join(simple_dlna_tags+(dlna_pn,)))) new_res.size = os.path.getsize(thumbnail) self.item.res.append(new_res) if not hasattr(self.item, 'attachments'): self.item.attachments = {} self.item.attachments[hash_from_path] = utils.StaticFile(urllib.quote(thumbnail)) try: # FIXME: getmtime is deprecated in Twisted 2.6 self.item.date = datetime.fromtimestamp(self.location.getmtime()) except: self.item.date = None def rebuild(self, urlbase): #print "rebuild", self.mimetype if self.mimetype != 'item': return #print "rebuild for", self.get_path() mimetype,_ = mimetypes.guess_type(self.get_path(),strict=False) if mimetype == None: return self.mimetype = mimetype #print "rebuild", self.mimetype UPnPClass = classChooser(self.mimetype) self.item = UPnPClass(self.id, self.parent.id, self.get_name()) if hasattr(self.parent, 'cover'): _,ext = os.path.splitext(self.parent.cover) """ add the cover image extension to help clients not reacting on the mimetype """ self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext)) _,host_port,_,_,_ = urlsplit(urlbase) if host_port.find(':') != -1: host,port = tuple(host_port.split(':')) else: host = host_port res = Resource('file://'+urllib.quote(self.get_path()), 'internal:%s:%s:*' % (host,self.mimetype)) try: res.size = self.location.getsize() except: res.size = 0 self.item.res.append(res) res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype) try: res.size = self.location.getsize() except: res.size = 0 self.item.res.append(res) try: # FIXME: getmtime is deprecated in Twisted 2.6 self.item.date = datetime.fromtimestamp(self.location.getmtime()) except: self.item.date = None self.parent.update_id += 1 def check_for_cover_art(self): """ let's try to find in the current directory some jpg file, or png if the jpg search fails, and take the first one that comes around """ try: jpgs = [i.path for i in self.location.children() if i.splitext()[1] in ('.jpg', '.JPG')] try: self.cover = jpgs[0] except IndexError: pngs = [i.path for i in self.location.children() if i.splitext()[1] in ('.png', '.PNG')] try: self.cover = pngs[0] except IndexError: return except UnicodeDecodeError: self.warning("UnicodeDecodeError - there is something wrong with a file located in %r", self.location.path) def remove(self): #print "FSItem remove", self.id, self.get_name(), self.parent if self.parent: self.parent.remove_child(self) del self.item def add_child(self, child, update=False): self.children.append(child) self.child_count += 1 if isinstance(self.item, Container): self.item.childCount += 1 if update == True: self.update_id += 1 def remove_child(self, child): #print "remove_from %d (%s) child %d (%s)" % (self.id, self.get_name(), child.id, child.get_name()) if child in self.children: self.child_count -= 1 if isinstance(self.item, Container): self.item.childCount -= 1 self.children.remove(child) self.update_id += 1 def get_children(self,start=0,request_count=0): if request_count == 0: return self.children[start:] else: return self.children[start:request_count] def get_child_count(self): return self.child_count def get_id(self): return self.id def get_update_id(self): if hasattr(self, 'update_id'): return self.update_id else: return None def get_path(self): if isinstance( self.location,FilePath): return self.location.path else: self.location def set_path(self,path=None,extension=None): if path is None: path = self.get_path() if extension is not None: path,old_ext = os.path.splitext(path) path = ''.join((path,extension)) if isinstance( self.location,FilePath): self.location = FilePath(path) else: self.location = path def get_name(self): if isinstance( self.location,FilePath): name = self.location.basename().decode("utf-8", "replace") else: name = self.location.decode("utf-8", "replace") return name def get_cover(self): try: return self.cover except: try: return self.parent.cover except: return '' def get_parent(self): return self.parent def get_item(self): return self.item def get_xml(self): return self.item.toString() def __repr__(self): return 'id: ' + str(self.id) + ' @ ' + self.get_name().encode('ascii','xmlcharrefreplace')
def startCopying(self, msg): def _done(ign, src, dst, f): log.msg("File copied successfully %(src)s -> %(dst)s" % { 'src': src, 'dst': dst }) # self.publish('file-transfer-ack', fileTransferReq, id=msg.id.getValue(), state=0, table=msg.table.getValue()) self.amqp.send_message(exchange="rb", routing_key="file-transfer-ack", msg=json.dumps({ 'id': msg['id'], 'state': 0, 'table': msg['table'], })) def _err_done(reason, src, dst, sendreq=None): log.msg( "File transfer error %(src)s -> %(dst)s. Reason: %(reason)s" % { 'src': src, 'dst': dst, 'reason': reason.getErrorMessage() }, logLevel=logging.ERROR) if sendreq: #self.publish('file-transfer-ack', fileTransferReq, id=msg.id.getValue(), state=1, table=msg.table.getValue()) self.amqp.send_message(exchange="rb", routing_key="file-transfer-ack", msg=json.dumps({ 'id': msg['id'], 'state': 1, 'table': msg['table'], })) return defer.fail(reason) if self.working > self.QUEUESIZE: self.queue.put(msg) return self.incpendingqueue(None, msg=msg) srcfile = msg['src'] dstfile = msg['dst'] s = FilePath(srcfile) d = FilePath(dstfile) if s.exists() and d.exists(): if s.getsize() == d.getsize(): log.msg("Ignore coping, files have the same size: %s -> %s" % (srcfile, dstfile)) self.decpendingqueue(None, msg=msg) #self.publish('file-transfer-ack', fileTransferReq, id=msg.id.getValue(), state=0, table=msg.table.getValue()) self.amqp.send_message(exchange="rb", routing_key="file-transfer-ack", msg=json.dumps({ 'id': msg['id'], 'state': 0, 'table': msg['table'], })) return src = FileSender() try: f = open(srcfile, 'r') except Exception, e: log.msg(str(e), logLevel=logging.ERROR) # f=None return _err_done(Failure(e), srcfile, dstfile, 'err').addBoth(self.decpendingqueue, msg=msg)
def startCopying(self, msg): def _done(ign, src, dst, f): log.msg("File copied successfully %(src)s -> %(dst)s" % { 'src': src, 'dst': dst }) self.publish('file-transfer-ack', fileTransferReq, id=msg.id.getValue(), state=0, table=msg.table.getValue()) def _err_done(reason, src, dst, sendreq=None): log.msg( "File transfer error %(src)s -> %(dst)s. Reason: %(reason)s" % { 'src': src, 'dst': dst, 'reason': reason.getErrorMessage() }, logLevel=logging.ERROR) if sendreq: self.publish('file-transfer-ack', fileTransferReq, id=msg.id.getValue(), state=1, table=msg.table.getValue()) return defer.fail(reason) if self.working > self.QUEUESIZE: self.queue.put(msg) return self.incpendingqueue(None, msg=msg) srcfile = msg.src.getValue() dstfile = msg.dst.getValue() s = FilePath(srcfile) d = FilePath(dstfile) if s.exists() and d.exists(): if s.getsize() == d.getsize(): log.msg("Ignore coping, files have the same size: %s -> %s" % (srcfile, dstfile)) self.decpendingqueue("SAME_SIZE", msg=msg) self.publish('file-transfer-ack', fileTransferReq, id=msg.id.getValue(), state=0, table=msg.table.getValue()) return src = FileSender() if self.conf.get('transport', 'CHUNK_SIZE_POWER'): src.CHUNK_SIZE = 2**int( self.conf.get('transport', 'CHUNK_SIZE_POWER')) try: f = open(srcfile, 'r') except Exception, e: log.msg(str(e), logLevel=logging.ERROR) # f=None return _err_done(Failure(e), srcfile, dstfile, 'err').addBoth(self.decpendingqueue, msg=msg)
import os import re import sys from datetime import datetime, timedelta from twisted.python.filepath import FilePath date = datetime.now() - timedelta(days=1) datestr = date.strftime("%Y-%m-%d") os.system("wget -c http://eve-central.com/dumps/%s.dump.gz" % (datestr,)) os.system("gunzip %s.dump.gz" % (datestr,)) transactionsFilePath = FilePath("%s.dump" % (datestr,)) #transactionsFilePath = FilePath("transactions.csv") size = float(transactionsFilePath.getsize()) fp = transactionsFilePath.open("r") db = MySQLdb.connect("localhost", "root", db="frisky-iskies") c = db.cursor() c.execute("truncate table transactions") c.execute("alter table transactions disable keys") durationRe = re.compile(r"\D.*") with open("transactions.csv") as csvfile: csvreader = csv.reader(csvfile) csvreader.next() i = 0 for row in csvreader: