class Recording(BackendItem):
logCategory = 'dvbd_store'
def __init__(self,store,
id,parent_id,
file,title,
date,duration,
mimetype):
self.store = store
self.id = 'recording.%s' % id
self.parent_id = parent_id
self.real_id = id
path = unicode(file)
# make sure path is an absolute local path (and not an URL)
if path.startswith("file://"):
path = path[7:]
self.location = FilePath(path)
self.title = unicode(title)
self.mimetype = str(mimetype)
self.date = datetime.fromtimestamp(int(date))
self.duration = int(duration)
try:
self.size = self.location.getsize()
except Exception, msg:
self.size = 0
self.bitrate = 0
self.url = self.store.urlbase + str(self.id)
def render_POST(self, request):
context = self._context(request)
form, status, appstruct = self._validate_form(request)
if status == 'ok':
filename = context['renderer'](appstruct)
fileobj = FilePath(filename)
request.setHeader('content-disposition', 'attachment; filename="output.pdf"')
request.setHeader('content-length', str(fileobj.getsize()))
request.setHeader('content-type', 'application/pdf')
producer = NoRangeStaticProducer(request, fileobj.open('r'))
producer.start()
return NOT_DONE_YET
data['renderer_form'] = form.render(appstruct)
self.render_template(data, request)
return NOT_DONE_YET
def _get_new(self):
"""
Get new requests to process. This will define the `main loop` of
the FileChecker service.
"""
# Check ACTIVE accounts to get files with running traffic captures
active_requests = yield self._get_requests("ACTIVE")
if active_requests:
log.info("FILECHECKER:: Checking pcap files of ACTIVE accounts.")
for request in active_requests:
username = request[0]
ip_addr = request[6]
# Create path
k = "{}_{}".format(username, ip_addr)
pcap_file = "{}.pcap".format(k)
pcap_file = os.path.join(self.pcaps_path, pcap_file)
log.debug("FILECHECKER:: Checking {}".format(pcap_file))
# First limit should be zero
if self.reached_limits.get(k) is None:
self.reached_limits[k] = 0
try:
fp = FilePath(pcap_file)
fsize = fp.getsize()
# Check each size limit only once
if self.reached_limits[k] + 1 < len(self.size_limit):
if fsize > int(
self.size_limit[self.reached_limits[k] + 1]):
text = "File {} has reached the limit of "\
"{} bytes!".format(
pcap_file,
self.size_limit[self.reached_limits[k]+1]
)
log.info("FILECHECKER:: {}".format(text))
yield self.slackbot.post(text, self.slack_channel)
self.reached_limits[k] = self.reached_limits[k] + 1
else:
log.debug("FILECHECKER:: File {} has reached the last"
" size limit of {} bytes!".format(
pcap_file,
self.size_limit[self.reached_limits[k]]))
except Exception as e:
log.debug("FILECHECKER:: Could'nt get size: {}".format(e))
else:
log.debug("FILECHECKER:: No ACTIVE accounts - Keep waiting.")
def get_index(self, path):
index = {}
for root, dirs, files in os.walk(path):
index[root] = {}
for f in files:
path = os.path.join(root, f)
filepath = FilePath(path)
try:
index[path] = {
'mtime': filepath.getModificationTime(),
'size': filepath.getsize(),
}
except OSError:
# file could be a broken symlink, or deleted mid-scan
continue
return index
def getDirectory(self, path='/'):
self.fs = yield FilePath(path)
if not self.fs.getPermissions():
defer.returnValue(False)
files = []
for f in self.fs.listdir():
if f == '/':
continue
fp = path+f
fs = FilePath(fp)
# dont follow symlinks
if fs.realpath().path != fp:
continue
perm = None
isdir = fs.isdir()
size = fs.getsize()
modified = datetime.utcfromtimestamp(fs.getModificationTime())
df = DiscoveredFile(
resource_id=self.data['resource_id'],
file_path=path,
file_name=f,
file_isdir=isdir,
file_size=size,
file_modified=modified,
file_perm=perm
)
print '[%s] LIST %s.' % (self.data['resource_name'], fp if not fp.endswith('.') else fp)
files.append(df)
defer.returnValue(files)
class Recording(BackendItem):
logCategory = 'dvbd_store'
def __init__(self, store, id, parent_id, file, title, date, duration,
mimetype):
BackendItem.__init__(self)
self.store = store
self.id = 'recording.%s' % id
self.parent_id = parent_id
self.real_id = id
path = str(file)
# make sure path is an absolute local path (and not an URL)
if path.startswith('file://'):
path = path[7:]
self.location = FilePath(path)
self.title = str(title)
self.mimetype = str(mimetype)
self.date = datetime.fromtimestamp(int(date))
self.duration = int(duration)
try:
self.size = self.location.getsize()
except Exception as msg:
self.size = 0
self.bitrate = 0
self.url = self.store.urlbase + str(self.id)
def get_children(self, start=0, end=0):
return []
def get_child_count(self):
return 0
def get_item(self, parent_id=None):
self.debug(f'Recording get_item {self.id} @ {self.parent_id}')
# create item
item = DIDLLite.VideoBroadcast(self.id, self.parent_id)
item.date = self.date
item.title = self.title
# add http resource
res = DIDLLite.Resource(self.url, f'http-get:*:{self.mimetype}:*')
if self.size > 0:
res.size = self.size
if self.duration > 0:
res.duration = str(self.duration)
if self.bitrate > 0:
res.bitrate = str(self.bitrate)
item.res.append(res)
# add internal resource
res = DIDLLite.Resource(
'file://' + urllib.parse.quote(self.get_path()),
f'internal:{self.store.server.coherence.hostname}:'
f'{self.mimetype}:*',
)
if self.size > 0:
res.size = self.size
if self.duration > 0:
res.duration = str(self.duration)
if self.bitrate > 0:
res.bitrate = str(self.bitrate)
item.res.append(res)
return item
def get_id(self):
return self.id
def get_name(self):
return self.title
def get_url(self):
return self.url
def get_path(self):
return self.location.path
class Recording(BackendItem):
logCategory = 'dvbd_store'
def __init__(self,store,
id,parent_id,
file,title,
date,duration,
mimetype):
self.store = store
self.id = 'recording.%s' % id
self.parent_id = parent_id
self.real_id = id
self.location = FilePath(unicode(file))
self.title = unicode(title)
self.mimetype = str(mimetype)
self.date = datetime.fromtimestamp(int(date))
self.duration = int(duration)
self.size = self.location.getsize()
self.bitrate = 0
self.url = self.store.urlbase + str(self.id)
def get_children(self, start=0, end=0):
return []
def get_child_count(self):
return 0
def get_item(self, parent_id=None):
self.debug("Recording get_item %r @ %r" %(self.id,self.parent_id))
# create item
item = DIDLLite.VideoBroadcast(self.id,self.parent_id)
item.date = self.date
item.title = self.title
# add http resource
res = DIDLLite.Resource(self.url, 'http-get:*:%s:*' % self.mimetype)
if self.size > 0:
res.size = self.size
if self.duration > 0:
res.duration = str(self.duration)
if self.bitrate > 0:
res.bitrate = str(bitrate)
item.res.append(res)
# add internal resource
res = DIDLLite.Resource('file://'+ urllib.quote(self.get_path()), 'internal:%s:%s:*' % (self.store.server.coherence.hostname,self.mimetype))
if self.size > 0:
res.size = self.size
if self.duration > 0:
res.duration = str(self.duration)
if self.bitrate > 0:
res.bitrate = str(bitrate)
item.res.append(res)
return item
def get_id(self):
return self.id
def get_name(self):
return self.title
def get_url(self):
return self.url
def get_path(self):
return self.location.path
class FSItem(BackendItem):
logCategory = 'fs_item'
def __init__(
self,
object_id,
parent,
path,
mimetype,
urlbase,
UPnPClass,
update=False,
store=None,
):
BackendItem.__init__(self)
self.id = object_id
self.parent = parent
if parent:
parent.add_child(self, update=update)
if mimetype == 'root':
self.location = str(path)
else:
if mimetype == 'item' and path is None:
path = os.path.join(parent.get_realpath(), str(self.id))
# self.location = FilePath(unicode(path))
self.location = FilePath(path)
self.mimetype = mimetype
if urlbase[-1] != '/':
urlbase += '/'
self.url = urlbase + str(self.id)
self.store = store
if parent is None:
parent_id = -1
else:
parent_id = parent.get_id()
self.item = UPnPClass(object_id, parent_id, self.get_name())
if isinstance(self.item, Container):
self.item.childCount = 0
self.child_count = 0
self.children = []
self.sorted = False
self.caption = None
if mimetype in ['directory', 'root']:
self.update_id = 0
self.get_url = lambda: self.url
# self.item.searchable = True
# self.item.searchClass = 'object'
if (isinstance(self.location, FilePath)
and self.location.isdir() is True):
self.check_for_cover_art()
if getattr(self, 'cover', None):
_, ext = os.path.splitext(self.cover)
''' add the cover image extension to help clients
not reacting on the mimetype '''
self.item.albumArtURI = ''.join(
(urlbase, str(self.id), '?cover', str(ext)))
else:
self.get_url = lambda: self.url
if self.mimetype.startswith('audio/'):
if getattr(parent, 'cover', None):
_, ext = os.path.splitext(parent.cover)
''' add the cover image extension to help clients
not reacting on the mimetype '''
self.item.albumArtURI = ''.join(
(urlbase, str(self.id), '?cover', ext))
_, host_port, _, _, _ = urlsplit(urlbase)
if host_port.find(':') != -1:
host, port = tuple(host_port.split(':'))
else:
host = host_port
try:
size = self.location.getsize()
except Exception:
size = 0
if (self.store.server and self.store.server.coherence.config.get(
'transcoding', 'no') == 'yes'):
if self.mimetype in (
'application/ogg',
'audio/ogg',
'audio/x-wav',
'audio/x-m4a',
'application/x-flac',
):
new_res = Resource(
self.url + '/transcoded.mp3',
f'http-get:*:{"audio/mpeg"}:*',
)
new_res.size = None
# self.item.res.append(new_res)
if mimetype != 'item':
res = Resource(
'file://' + quote(self.get_path(), encoding='utf-8'),
f'internal:{host}:{self.mimetype}:*',
)
res.size = size
self.item.res.append(res)
if mimetype != 'item':
res = Resource(self.url, f'http-get:*:{self.mimetype}:*')
else:
res = Resource(self.url, 'http-get:*:*:*')
res.size = size
self.item.res.append(res)
''' if this item is of type audio and we want to add a transcoding
rule for it, this is the way to do it:
create a new Resource object, at least a 'http-get'
and maybe an 'internal' one too
for transcoding to wav this looks like that
res = Resource(
url_for_transcoded audio,
'http-get:*:audio/x-wav:%s'% ';'.join(
['DLNA.ORG_PN=JPEG_TN']+simple_dlna_tags))
res.size = None
self.item.res.append(res)
'''
if (self.store.server and self.store.server.coherence.config.get(
'transcoding', 'no') == 'yes'):
if self.mimetype in (
'audio/mpeg',
'application/ogg',
'audio/ogg',
'audio/x-wav',
'audio/x-m4a',
'audio/flac',
'application/x-flac',
):
dlna_pn = 'DLNA.ORG_PN=LPCM'
dlna_tags = simple_dlna_tags[:]
# dlna_tags[1] = 'DLNA.ORG_OP=00'
dlna_tags[2] = 'DLNA.ORG_CI=1'
new_res = Resource(
self.url + '?transcoded=lpcm',
f'http-get:*:{"audio/L16;rate=44100;channels=2"}:'
f'{";".join([dlna_pn] + dlna_tags)}',
)
new_res.size = None
# self.item.res.append(new_res)
if self.mimetype != 'audio/mpeg':
new_res = Resource(
self.url + '?transcoded=mp3',
f'http-get:*:{"audio/mpeg"}:*',
)
new_res.size = None
# self.item.res.append(new_res)
''' if this item is an image and we want to add a thumbnail for it
we have to follow these rules:
create a new Resource object, at least a 'http-get'
and maybe an 'internal' one too
for an JPG this looks like that
res = Resource(url_for_thumbnail,
'http-get:*:image/jpg:%s'% ';'.join(
['DLNA.ORG_PN=JPEG_TN']+simple_dlna_tags))
res.size = size_of_thumbnail
self.item.res.append(res)
and for a PNG the Resource creation is like that
res = Resource(url_for_thumbnail,
'http-get:*:image/png:%s'% ';'.join(
simple_dlna_tags+['DLNA.ORG_PN=PNG_TN']))
if not hasattr(self.item, 'attachments'):
self.item.attachments = {}
self.item.attachments[key] = utils.StaticFile(
filename_of_thumbnail)
'''
if (self.mimetype in ('image/jpeg', 'image/png')
or self.mimetype.startswith('video/')):
try:
filename, mimetype, dlna_pn = _find_thumbnail(
self.get_path())
except NoThumbnailFound:
pass
except Exception:
self.warning(traceback.format_exc())
else:
dlna_tags = simple_dlna_tags[:]
dlna_tags[
3] = 'DLNA.ORG_FLAGS=00f00000000000000000000000000000'
hash_from_path = str(id(filename))
new_res = Resource(
self.url + '?attachment=' + hash_from_path,
f'http-get:*:{mimetype}:'
f'{";".join([dlna_pn] + dlna_tags)}',
)
new_res.size = os.path.getsize(filename)
self.item.res.append(new_res)
if not hasattr(self.item, 'attachments'):
self.item.attachments = {}
self.item.attachments[hash_from_path] = utils.StaticFile(
filename)
if self.mimetype.startswith('video/'):
# check for a subtitles file
caption, _ = os.path.splitext(self.get_path())
caption = caption + '.srt'
if os.path.exists(caption):
hash_from_path = str(id(caption))
mimetype = 'smi/caption'
new_res = Resource(
self.url + '?attachment=' + hash_from_path,
f'http-get:*:{mimetype}:{"*"}',
)
new_res.size = os.path.getsize(caption)
self.caption = new_res.data
self.item.res.append(new_res)
if not hasattr(self.item, 'attachments'):
self.item.attachments = {}
self.item.attachments[hash_from_path] = utils.StaticFile(
caption,
defaultType=mimetype,
)
try:
# FIXME: getmtime is deprecated in Twisted 2.6
self.item.date = datetime.fromtimestamp(
self.location.getmtime())
except Exception:
self.item.date = None
def rebuild(self, urlbase):
# print('rebuild', self.mimetype)
if self.mimetype != 'item':
return
# print('rebuild for', self.get_path())
mimetype, _ = mimetypes.guess_type(self.get_path(), strict=False)
if mimetype is None:
return
self.mimetype = mimetype
# print('rebuild', self.mimetype)
UPnPClass = classChooser(self.mimetype)
self.item = UPnPClass(self.id, self.parent.id, self.get_name())
if getattr(self.parent, 'cover', None):
_, ext = os.path.splitext(self.parent.cover)
# add the cover image extension to help
# clients not reacting on the mimetype
self.item.albumArtURI = ''.join(
(urlbase, str(self.id), '?cover', ext))
_, host_port, _, _, _ = urlsplit(urlbase)
if host_port.find(':') != -1:
host, port = tuple(host_port.split(':'))
else:
host = host_port
res = Resource(
'file://' + quote(self.get_path()),
f'internal:{host}:{self.mimetype}:*',
)
try:
res.size = self.location.getsize()
except Exception:
res.size = 0
self.item.res.append(res)
res = Resource(self.url, f'http-get:*:{self.mimetype}:*')
try:
res.size = self.location.getsize()
except Exception:
res.size = 0
self.item.res.append(res)
try:
# FIXME: getmtime is deprecated in Twisted 2.6
self.item.date = datetime.fromtimestamp(self.location.getmtime())
except Exception:
self.item.date = None
self.parent.update_id += 1
def check_for_cover_art(self):
''' let's try to find in the current directory some jpg file,
or png if the jpg search fails, and take the first one
that comes around
'''
try:
jpgs = [
i.path for i in self.location.children()
if i.splitext()[1] in ('.jpg', '.JPG')
]
try:
self.cover = jpgs[0]
except IndexError:
pngs = [
i.path for i in self.location.children()
if i.splitext()[1] in ('.png', '.PNG')
]
try:
self.cover = pngs[0]
except IndexError:
return
except UnicodeDecodeError:
self.warning(
f'UnicodeDecodeError - there is something wrong with a ' +
f'file located in {self.location.path}')
def remove(self):
# print('FSItem remove', self.id, self.get_name(), self.parent)
if self.parent:
self.parent.remove_child(self)
del self.item
def add_child(self, child, update=False):
self.children.append(child)
self.child_count += 1
if isinstance(self.item, Container):
self.item.childCount += 1
if update:
self.update_id += 1
self.sorted = False
def remove_child(self, child):
# print(f'remove_from {self.id:d} ({self.get_name()}) '
# f'child {child.id:d} ({child.get_name()})')
if child in self.children:
self.child_count -= 1
if isinstance(self.item, Container):
self.item.childCount -= 1
self.children.remove(child)
self.update_id += 1
self.sorted = False
def get_children(self, start=0, request_count=0):
if not self.sorted:
self.children.sort(key=_natural_key)
self.sorted = True
if request_count == 0:
return self.children[start:]
else:
return self.children[start:request_count]
def get_child_count(self):
return self.child_count
def get_id(self):
return self.id
def get_update_id(self):
if hasattr(self, 'update_id'):
return self.update_id
else:
return None
def get_path(self):
if self.mimetype in ['directory', 'root']:
return None
if isinstance(self.location, FilePath):
return self.location.path
else:
return self.location
def get_realpath(self):
if isinstance(self.location, FilePath):
return self.location.path
else:
return self.location
def set_path(self, path=None, extension=None):
if path is None:
path = self.get_path()
if extension is not None:
path, old_ext = os.path.splitext(path)
path = ''.join((path, extension))
if isinstance(self.location, FilePath):
self.location = FilePath(path)
else:
self.location = path
def get_name(self):
if isinstance(self.location, FilePath):
name = self.location.basename()
else:
name = self.location
return name
def get_cover(self):
if self.cover:
return self.cover
try:
return self.parent.cover
except AttributeError:
return None
def get_parent(self):
return self.parent
def get_item(self):
return self.item
def get_xml(self):
return self.item.toString()
def __repr__(self):
return ('id: ' + str(self.id) + ' @ ' +
str(self.get_name().encode('ascii', 'xmlcharrefreplace')))
class FSItem(BackendItem):
logCategory = 'fs_item'
def __init__(self, object_id, parent, path, mimetype, urlbase, UPnPClass,update=False):
self.id = object_id
self.parent = parent
if parent:
parent.add_child(self,update=update)
if mimetype == 'root':
self.location = unicode(path)
else:
if mimetype == 'item' and path is None:
path = os.path.join(parent.get_path(),unicode(self.id))
#self.location = FilePath(unicode(path))
self.location = FilePath(path)
self.mimetype = mimetype
if urlbase[-1] != '/':
urlbase += '/'
self.url = urlbase + str(self.id)
if parent == None:
parent_id = -1
else:
parent_id = parent.get_id()
self.item = UPnPClass(object_id, parent_id, self.get_name())
if isinstance(self.item, Container):
self.item.childCount = 0
self.child_count = 0
self.children = []
if mimetype in ['directory','root']:
self.update_id = 0
self.get_url = lambda : self.url
self.get_path = lambda : None
#self.item.searchable = True
#self.item.searchClass = 'object'
if(isinstance(self.location,FilePath) and
self.location.isdir() == True):
self.check_for_cover_art()
if hasattr(self, 'cover'):
_,ext = os.path.splitext(self.cover)
""" add the cover image extension to help clients not reacting on
the mimetype """
self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext))
else:
self.get_url = lambda : self.url
if self.mimetype.startswith('audio/'):
if hasattr(parent, 'cover'):
_,ext = os.path.splitext(parent.cover)
""" add the cover image extension to help clients not reacting on
the mimetype """
self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext))
_,host_port,_,_,_ = urlsplit(urlbase)
if host_port.find(':') != -1:
host,port = tuple(host_port.split(':'))
else:
host = host_port
try:
size = self.location.getsize()
except:
size = 0
if mimetype != 'item':
res = Resource('file://'+ urllib.quote(self.get_path()), 'internal:%s:%s:*' % (host,self.mimetype))
res.size = size
self.item.res.append(res)
if mimetype != 'item':
res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype)
else:
res = Resource(self.url, 'http-get:*:*:*')
res.size = size
self.item.res.append(res)
""" if this item is an image and we want to add a thumbnail for it
we have to follow these rules:
create a new Resource object, at least a 'http-get'
and maybe an 'internal' one too
for an JPG this looks like that
res = Resource(url_for_thumbnail,
'http-get:*:image/jpg:%s'% ';'.join(simple_dlna_tags+('DLNA.ORG_PN=JPEG_TN',)))
res.size = size_of_thumbnail
self.item.res.append(res)
and for a PNG the Resource creation is like that
res = Resource(url_for_thumbnail,
'http-get:*:image/png:%s'% ';'.join(simple_dlna_tags+('DLNA.ORG_PN=PNG_TN',)))
if not hasattr(self.item, 'attachments'):
self.item.attachments = {}
self.item.attachments[key] = utils.StaticFile(filename_of_thumbnail)
"""
if self.mimetype in ('image/jpeg', 'image/png'):
path = self.get_path()
thumbnail = os.path.join(os.path.dirname(path),'.thumbs',os.path.basename(path))
if os.path.exists(thumbnail):
mimetype,_ = mimetypes.guess_type(thumbnail, strict=False)
if mimetype in ('image/jpeg','image/png'):
if mimetype == 'image/jpeg':
dlna_pn = 'DLNA.ORG_PN=JPEG_TN'
else:
dlna_pn = 'DLNA.ORG_PN=PNG_TN'
hash_from_path = str(id(thumbnail))
new_res = Resource(self.url+'?attachment='+hash_from_path,
'http-get:*:%s:%s' % (mimetype, ';'.join(simple_dlna_tags+(dlna_pn,))))
new_res.size = os.path.getsize(thumbnail)
self.item.res.append(new_res)
if not hasattr(self.item, 'attachments'):
self.item.attachments = {}
self.item.attachments[hash_from_path] = utils.StaticFile(urllib.quote(thumbnail))
try:
# FIXME: getmtime is deprecated in Twisted 2.6
self.item.date = datetime.fromtimestamp(self.location.getmtime())
except:
self.item.date = None
def rebuild(self, urlbase):
#print "rebuild", self.mimetype
if self.mimetype != 'item':
return
#print "rebuild for", self.get_path()
mimetype,_ = mimetypes.guess_type(self.get_path(),strict=False)
if mimetype == None:
return
self.mimetype = mimetype
#print "rebuild", self.mimetype
UPnPClass = classChooser(self.mimetype)
self.item = UPnPClass(self.id, self.parent.id, self.get_name())
if hasattr(self.parent, 'cover'):
_,ext = os.path.splitext(self.parent.cover)
""" add the cover image extension to help clients not reacting on
the mimetype """
self.item.albumArtURI = ''.join((urlbase,str(self.id),'?cover',ext))
_,host_port,_,_,_ = urlsplit(urlbase)
if host_port.find(':') != -1:
host,port = tuple(host_port.split(':'))
else:
host = host_port
res = Resource('file://'+urllib.quote(self.get_path()), 'internal:%s:%s:*' % (host,self.mimetype))
try:
res.size = self.location.getsize()
except:
res.size = 0
self.item.res.append(res)
res = Resource(self.url, 'http-get:*:%s:*' % self.mimetype)
try:
res.size = self.location.getsize()
except:
res.size = 0
self.item.res.append(res)
try:
# FIXME: getmtime is deprecated in Twisted 2.6
self.item.date = datetime.fromtimestamp(self.location.getmtime())
except:
self.item.date = None
self.parent.update_id += 1
def check_for_cover_art(self):
""" let's try to find in the current directory some jpg file,
or png if the jpg search fails, and take the first one
that comes around
"""
try:
jpgs = [i.path for i in self.location.children() if i.splitext()[1] in ('.jpg', '.JPG')]
try:
self.cover = jpgs[0]
except IndexError:
pngs = [i.path for i in self.location.children() if i.splitext()[1] in ('.png', '.PNG')]
try:
self.cover = pngs[0]
except IndexError:
return
except UnicodeDecodeError:
self.warning("UnicodeDecodeError - there is something wrong with a file located in %r", self.location.path)
def remove(self):
#print "FSItem remove", self.id, self.get_name(), self.parent
if self.parent:
self.parent.remove_child(self)
del self.item
def add_child(self, child, update=False):
self.children.append(child)
self.child_count += 1
if isinstance(self.item, Container):
self.item.childCount += 1
if update == True:
self.update_id += 1
def remove_child(self, child):
#print "remove_from %d (%s) child %d (%s)" % (self.id, self.get_name(), child.id, child.get_name())
if child in self.children:
self.child_count -= 1
if isinstance(self.item, Container):
self.item.childCount -= 1
self.children.remove(child)
self.update_id += 1
def get_children(self,start=0,request_count=0):
if request_count == 0:
return self.children[start:]
else:
return self.children[start:request_count]
def get_child_count(self):
return self.child_count
def get_id(self):
return self.id
def get_update_id(self):
if hasattr(self, 'update_id'):
return self.update_id
else:
return None
def get_path(self):
if isinstance( self.location,FilePath):
return self.location.path
else:
self.location
def set_path(self,path=None,extension=None):
if path is None:
path = self.get_path()
if extension is not None:
path,old_ext = os.path.splitext(path)
path = ''.join((path,extension))
if isinstance( self.location,FilePath):
self.location = FilePath(path)
else:
self.location = path
def get_name(self):
if isinstance( self.location,FilePath):
name = self.location.basename().decode("utf-8", "replace")
else:
name = self.location.decode("utf-8", "replace")
return name
def get_cover(self):
try:
return self.cover
except:
try:
return self.parent.cover
except:
return ''
def get_parent(self):
return self.parent
def get_item(self):
return self.item
def get_xml(self):
return self.item.toString()
def __repr__(self):
return 'id: ' + str(self.id) + ' @ ' + self.get_name().encode('ascii','xmlcharrefreplace')
def startCopying(self, msg):
def _done(ign, src, dst, f):
log.msg("File copied successfully %(src)s -> %(dst)s" % {
'src': src,
'dst': dst
})
# self.publish('file-transfer-ack', fileTransferReq, id=msg.id.getValue(), state=0, table=msg.table.getValue())
self.amqp.send_message(exchange="rb",
routing_key="file-transfer-ack",
msg=json.dumps({
'id': msg['id'],
'state': 0,
'table': msg['table'],
}))
def _err_done(reason, src, dst, sendreq=None):
log.msg(
"File transfer error %(src)s -> %(dst)s. Reason: %(reason)s" %
{
'src': src,
'dst': dst,
'reason': reason.getErrorMessage()
},
logLevel=logging.ERROR)
if sendreq:
#self.publish('file-transfer-ack', fileTransferReq, id=msg.id.getValue(), state=1, table=msg.table.getValue())
self.amqp.send_message(exchange="rb",
routing_key="file-transfer-ack",
msg=json.dumps({
'id': msg['id'],
'state': 1,
'table': msg['table'],
}))
return defer.fail(reason)
if self.working > self.QUEUESIZE:
self.queue.put(msg)
return
self.incpendingqueue(None, msg=msg)
srcfile = msg['src']
dstfile = msg['dst']
s = FilePath(srcfile)
d = FilePath(dstfile)
if s.exists() and d.exists():
if s.getsize() == d.getsize():
log.msg("Ignore coping, files have the same size: %s -> %s" %
(srcfile, dstfile))
self.decpendingqueue(None, msg=msg)
#self.publish('file-transfer-ack', fileTransferReq, id=msg.id.getValue(), state=0, table=msg.table.getValue())
self.amqp.send_message(exchange="rb",
routing_key="file-transfer-ack",
msg=json.dumps({
'id': msg['id'],
'state': 0,
'table': msg['table'],
}))
return
src = FileSender()
try:
f = open(srcfile, 'r')
except Exception, e:
log.msg(str(e), logLevel=logging.ERROR)
# f=None
return _err_done(Failure(e), srcfile, dstfile,
'err').addBoth(self.decpendingqueue, msg=msg)
def startCopying(self, msg):
def _done(ign, src, dst, f):
log.msg("File copied successfully %(src)s -> %(dst)s" % {
'src': src,
'dst': dst
})
self.publish('file-transfer-ack',
fileTransferReq,
id=msg.id.getValue(),
state=0,
table=msg.table.getValue())
def _err_done(reason, src, dst, sendreq=None):
log.msg(
"File transfer error %(src)s -> %(dst)s. Reason: %(reason)s" %
{
'src': src,
'dst': dst,
'reason': reason.getErrorMessage()
},
logLevel=logging.ERROR)
if sendreq:
self.publish('file-transfer-ack',
fileTransferReq,
id=msg.id.getValue(),
state=1,
table=msg.table.getValue())
return defer.fail(reason)
if self.working > self.QUEUESIZE:
self.queue.put(msg)
return
self.incpendingqueue(None, msg=msg)
srcfile = msg.src.getValue()
dstfile = msg.dst.getValue()
s = FilePath(srcfile)
d = FilePath(dstfile)
if s.exists() and d.exists():
if s.getsize() == d.getsize():
log.msg("Ignore coping, files have the same size: %s -> %s" %
(srcfile, dstfile))
self.decpendingqueue("SAME_SIZE", msg=msg)
self.publish('file-transfer-ack',
fileTransferReq,
id=msg.id.getValue(),
state=0,
table=msg.table.getValue())
return
src = FileSender()
if self.conf.get('transport', 'CHUNK_SIZE_POWER'):
src.CHUNK_SIZE = 2**int(
self.conf.get('transport', 'CHUNK_SIZE_POWER'))
try:
f = open(srcfile, 'r')
except Exception, e:
log.msg(str(e), logLevel=logging.ERROR)
# f=None
return _err_done(Failure(e), srcfile, dstfile,
'err').addBoth(self.decpendingqueue, msg=msg)
import os
import re
import sys
from datetime import datetime, timedelta
from twisted.python.filepath import FilePath
date = datetime.now() - timedelta(days=1)
datestr = date.strftime("%Y-%m-%d")
os.system("wget -c http://eve-central.com/dumps/%s.dump.gz" % (datestr,))
os.system("gunzip %s.dump.gz" % (datestr,))
transactionsFilePath = FilePath("%s.dump" % (datestr,))
#transactionsFilePath = FilePath("transactions.csv")
size = float(transactionsFilePath.getsize())
fp = transactionsFilePath.open("r")
db = MySQLdb.connect("localhost", "root", db="frisky-iskies")
c = db.cursor()
c.execute("truncate table transactions")
c.execute("alter table transactions disable keys")
durationRe = re.compile(r"\D.*")
with open("transactions.csv") as csvfile:
csvreader = csv.reader(csvfile)
csvreader.next()
i = 0
for row in csvreader:
