def setUp(self): self.tempdir = tempfile.mkdtemp() self.collection = get_jmutube_collection() self.storage = get_jmutube_storage() self.user = User.objects.create(username='******') self.storage.storage_system.sync_files(self.user) self.delete_files = []
def execute(self): storage = get_jmutube_storage() logging.info("CRASS sorter starting") for file in os.listdir(settings.JMUTUBE_CRASS_INCOMING_FOLDER): age = time.time() - os.path.getmtime(os.path.join(settings.JMUTUBE_CRASS_INCOMING_FOLDER, file)) if age < 600: logging.debug("CRASS skipping file %s with age %s" % (file, age)) continue logging.debug("Processing %s" % file) match = Job.file_re.match(file) if match: dt = datetime( int(match.group("y")), int(match.group("m")), int(match.group("d")), int(match.group("h")), int(match.group("n")), int(match.group("s")), ) schedules = Schedule.objects.filter( computer__mac_address__iexact=match.group("mac"), start_time__lte=dt, end_time__gte=dt ) if not schedules: logging.debug("no schedule found") continue schedule = schedules[0] title = "%s %s (%s)" % (schedule.computer.building, schedule.computer.room, dt) newname = os.path.join( schedule.user.username, "video", (schedule.computer.building + " " + schedule.computer.room + " " + file[13:-4]).replace(" ", "_") + ".mp4", ) if not storage.file_exists(newname): # add mapping log entry Mapping.objects.create(source_file=file, target_file=newname, user=schedule.user) # move file shutil.move( os.path.join(settings.JMUTUBE_CRASS_INCOMING_FOLDER, file), storage.storage_system.path(newname) ) # add file entry record = storage.storage_system.create_record_for_file(schedule.user, newname, "video") # add tags wrapper = OwnedWrapper.objects.get_for_object(user=schedule.user, object=record) Tag.objects.add_tag(wrapper, '"%s %s"' % (schedule.computer.building, schedule.computer.room)) Tag.objects.add_tag(wrapper, "CRASS") Tag.objects.add_tag(wrapper, str(dt.year)) Tag.objects.add_tag(wrapper, '"Week %s"' % dt.isocalendar()[1]) Tag.objects.add_tag( wrapper, ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"][dt.weekday()], ) logging.debug("-> %s" % newname) else: logging.debug("target file already exists") else: logging.debug("invalid file")
def execute(self): storage = get_jmutube_storage() logging.info('CRASS sorter starting') for file in os.listdir(settings.JMUTUBE_CRASS_INCOMING_FOLDER): age = time.time() - os.path.getmtime(os.path.join(settings.JMUTUBE_CRASS_INCOMING_FOLDER, file)) if age < 600: logging.debug('CRASS skipping file %s with age %s' % (file, age)) continue logging.debug('Processing %s' % file) match = Job.file_re.match(file) if match: dt = datetime(int(match.group('y')), int(match.group('m')), int(match.group('d')), int(match.group('h')), int(match.group('n')), int(match.group('s'))) schedules = Schedule.objects.filter(computer__mac_address__iexact=match.group('mac'), start_time__lte=dt, end_time__gte=dt) if not schedules: logging.debug("no schedule found") continue schedule = schedules[0] title = '%s %s (%s)' % (schedule.computer.building, schedule.computer.room, dt) newname = os.path.join(schedule.user.username, 'video', (schedule.computer.building + ' ' + schedule.computer.room + ' ' + file[13:-4]).replace(' ', '_') + '.mp4') if not storage.file_exists(newname): # add mapping log entry Mapping.objects.create(source_file=file, target_file=newname, user=schedule.user) # move file shutil.move(os.path.join(settings.JMUTUBE_CRASS_INCOMING_FOLDER, file), storage.storage_system.path(newname)) # add file entry record = storage.storage_system.create_record_for_file(schedule.user, newname, 'video') # add tags wrapper = OwnedWrapper.objects.get_for_object(user=schedule.user, object=record) Tag.objects.add_tag(wrapper, '"%s %s"' % (schedule.computer.building, schedule.computer.room)) Tag.objects.add_tag(wrapper, 'CRASS') Tag.objects.add_tag(wrapper, str(dt.year)) Tag.objects.add_tag(wrapper, '"Week %s"' % dt.isocalendar()[1]) Tag.objects.add_tag(wrapper, ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'][dt.weekday()]) logging.debug("-> %s" % newname) # cleanup stray xml files left over by the Crestron boxes xml_fn = os.path.join( settings.JMUTUBE_CRASS_INCOMING_FOLDER, file[:-4] + '.xml' ) if os.path.exists(xml_fn): os.unlink(xml_fn) else: logging.debug("target file already exists") else: logging.debug("invalid file")
def handle(self, *args, **options): file = options.get('file') if not file: print "No input file specified" return if not os.path.isfile(file): print "File not found." return clean = options.get('clean', False) _data = simplejson.load(open(file)) storage = get_jmutube_storage() collection = get_jmutube_collection() if clean: def remove(query): print "Before: ", query.count() query.delete() print "After: ", query.count() print "Deleting existing records and media" remove(Media.objects.filter(storage=storage)) remove(Record.objects.filter(collection=collection)) print "Deleting presentations" remove(Presentation.objects.filter(source='jmutube')) return def data(model): dataset = filter(lambda d: d['model'] == model, _data) # need to convert unicode keys to regular strings for data in dataset: data['fields'] = dict((str(k), v) for k, v in data['fields'].iteritems()) return dataset # create user groups groups = dict() print "Creating groups" for d in data('auth.group'): group, created = Group.objects.get_or_create(name=d['fields']['name']) groups[d['pk']] = group print 'c' if created else '.', print # create users users = dict() facstaffgroup = get_jmutube_facultystaff_group() print "Creating users" for d in data('auth.user'): g = d['fields'].pop('groups') u = d['fields'].pop('username') d['fields'].pop('user_permissions') user, created = User.objects.get_or_create(username=u, defaults=d['fields']) users[d['pk']] = user user.groups.add(facstaffgroup) for gid in g: user.groups.add(groups[gid]) print 'c' if created else '.', print # create impersonations print "Creating Impersonations" for d in data('impersonate.impersonation'): g = d['fields'].pop('groups') u = d['fields'].pop('users') imp, created = Impersonation.objects.get_or_create(group=groups[d['fields']['group']]) for uid in u: imp.users.add(users[uid]) for gid in g: imp.groups.add(groups[gid]) print 'c' if created else '.', print # create crass computers computers = dict() print "Creating CRASS computers" for d in data('crass.computer'): computer, created = Computer.objects.get_or_create(building=d['fields']['building'], room=d['fields']['room'], defaults=dict(mac_address=d['fields']['mac_address'])) computers[d['pk']] = computer print 'c' if created else '.', print # create crass schedules print "Creating CRASS schedules" for d in data('crass.schedule'): schedule, created = Schedule.objects.get_or_create(start_time=d['fields']['start_time'], end_time=d['fields']['end_time'], computer=computers[d['fields']['computer']], user=users[d['fields']['user']]) print 'c' if created else '.', print # create crass mappings print "Creating CRASS mappings" for d in data('crass.mapping'): mapping, created = Mapping.objects.get_or_create(time_stamp=d['fields']['time_stamp'], source_file=d['fields']['source_file'], target_file=d['fields']['target_file'], user=users[d['fields']['user']]) print 'c' if created else '.', print # create records and media print "Creating records and media" title_field = Field.objects.get(name='title', standard__prefix='dc') records = dict() files = dict() allmedia = dict() for m in Media.objects.filter(storage=storage): allmedia[m.url] = m.record for d in data('repository.file'): files[d['pk']] = d['fields']['file'] mimetype = MIME_TYPES.get(os.path.splitext(d['fields']['file'])[1], 'application/octet-stream') url = '%s/%s/%s' % (users[d['fields']['user']].username, d['fields']['type'], d['fields']['file']) record = allmedia.get(url) if record: records[d['pk']] = record print '.', continue record = Record.objects.create(owner=users[d['fields']['user']], name=d['fields']['title'], source='jmutube-%s' % d['fields']['type']) records[d['pk']] = record media = Media.objects.create(record=record, url=url, storage=storage, mimetype=mimetype) CollectionItem.objects.create(collection=collection, record=record) FieldValue.objects.create(record=record, field=title_field, value=d['fields']['title'], order=1) Tag.objects.add_tag(OwnedWrapper.objects.get_for_object(user=users[d['fields']['user']], object=record), 'JMUtube') print 'c', print # create presentations print "Creating presentations" presentations = dict() for d in data('repository.playlist'): presentation, created = Presentation.objects.get_or_create(source='jmutube', owner=users[d['fields']['user']], title=d['fields']['title']) presentations[d['pk']] = presentation print 'c' if created else '.', print # create presentation redirects print "Creating presentation redirects" site = Site.objects.get(id=settings.SITE_ID) def create_redirect(old_path, new_path): redirect, created = Redirect.objects.get_or_create(site=site, old_path=old_path, defaults=dict(new_path=new_path)) if not created and redirect.new_path != new_path: redirect.new_path = new_path redirect.save() return redirect, created for oldid, presentation in presentations.iteritems(): redirect, created1 = create_redirect( reverse('jmutube-playlist-rss-feed-legacy', args=(presentation.owner.username, oldid)), reverse('jmutube-playlist-rss-feed', args=(presentation.owner.username, presentation.id))) redirect, created2 = create_redirect( reverse('jmutube-playlist-rss-feed-legacy', args=(presentation.owner.username, oldid)) + '?player=jmutube', reverse('jmutube-playlist-rss-feed', args=(presentation.owner.username, presentation.id)) + '?player=jmutube') redirect, created3 = create_redirect( reverse('jmutube-playlist-play-legacy', args=(presentation.owner.username, oldid)), reverse('jmutube-playlist-play', args=(presentation.owner.username, presentation.id))) print 'c' if created1 or created2 or created3 else '.', print print "Creating file redirects" for oldid, record in records.iteritems(): redirect, created1 = create_redirect( reverse('jmutube-single-file-rss-feed-legacy', args=(presentation.owner.username, files[oldid])), reverse('jmutube-single-file-rss-feed', args=(presentation.owner.username, record.id, record.name)) ) redirect, created2 = create_redirect( reverse('jmutube-single-file-rss-feed-legacy', args=(presentation.owner.username, files[oldid])) + '?player=jmutube', reverse('jmutube-single-file-rss-feed', args=(presentation.owner.username, record.id, record.name)) + '?player=jmutube' ) print 'c' if created1 or created2 else '.', print # create presentation items print "Creating presentation items" for d in data('repository.playlistitem'): item, created = PresentationItem.objects.get_or_create( type=d['fields']['delivery'], order=d['pk'], presentation=presentations[d['fields']['playlist']], record=records[d['fields']['file']] ) print 'c' if created else '.', print # create tags print "Creating tags" tags = dict() for d in data('tagging.tag'): tags[d['pk']] = d['fields']['name'] for d in data('tagging.taggeditem'): record = records.get(d['fields']['object_id']) if record: Tag.objects.add_tag(OwnedWrapper.objects.get_for_object(user=record.owner, object=record), '"%s"' % tags[d['fields']['tag']]) print '-', else: print 'X', print
def handle(self, *args, **options): file = options.get('file') if not file: print "No input file specified" return if not os.path.isfile(file): print "File not found." return clean = options.get('clean', False) _data = simplejson.load(open(file)) storage = get_jmutube_storage() collection = get_jmutube_collection() if clean: def remove(query): print "Before: ", query.count() query.delete() print "After: ", query.count() print "Deleting existing records and media" remove(Media.objects.filter(storage=storage)) remove(Record.objects.filter(collection=collection)) print "Deleting presentations" remove(Presentation.objects.filter(source='jmutube')) return def data(model): dataset = filter(lambda d: d['model'] == model, _data) # need to convert unicode keys to regular strings for data in dataset: data['fields'] = dict( (str(k), v) for k, v in data['fields'].iteritems()) return dataset # create user groups groups = dict() print "Creating groups" for d in data('auth.group'): group, created = Group.objects.get_or_create( name=d['fields']['name']) groups[d['pk']] = group print 'c' if created else '.', print # create users users = dict() facstaffgroup = get_jmutube_facultystaff_group() print "Creating users" for d in data('auth.user'): g = d['fields'].pop('groups') u = d['fields'].pop('username') d['fields'].pop('user_permissions') user, created = User.objects.get_or_create(username=u, defaults=d['fields']) users[d['pk']] = user user.groups.add(facstaffgroup) for gid in g: user.groups.add(groups[gid]) print 'c' if created else '.', print # create impersonations print "Creating Impersonations" for d in data('impersonate.impersonation'): g = d['fields'].pop('groups') u = d['fields'].pop('users') imp, created = Impersonation.objects.get_or_create( group=groups[d['fields']['group']]) for uid in u: imp.users.add(users[uid]) for gid in g: imp.groups.add(groups[gid]) print 'c' if created else '.', print # create crass computers computers = dict() print "Creating CRASS computers" for d in data('crass.computer'): computer, created = Computer.objects.get_or_create( building=d['fields']['building'], room=d['fields']['room'], defaults=dict(mac_address=d['fields']['mac_address'])) computers[d['pk']] = computer print 'c' if created else '.', print # create crass schedules print "Creating CRASS schedules" for d in data('crass.schedule'): schedule, created = Schedule.objects.get_or_create( start_time=d['fields']['start_time'], end_time=d['fields']['end_time'], computer=computers[d['fields']['computer']], user=users[d['fields']['user']]) print 'c' if created else '.', print # create crass mappings print "Creating CRASS mappings" for d in data('crass.mapping'): mapping, created = Mapping.objects.get_or_create( time_stamp=d['fields']['time_stamp'], source_file=d['fields']['source_file'], target_file=d['fields']['target_file'], user=users[d['fields']['user']]) print 'c' if created else '.', print # create records and media print "Creating records and media" title_field = Field.objects.get(name='title', standard__prefix='dc') records = dict() files = dict() allmedia = dict() for m in Media.objects.filter(storage=storage): allmedia[m.url] = m.record for d in data('repository.file'): files[d['pk']] = d['fields']['file'] mimetype = MIME_TYPES.get( os.path.splitext(d['fields']['file'])[1], 'application/octet-stream') url = '%s/%s/%s' % (users[d['fields']['user']].username, d['fields']['type'], d['fields']['file']) record = allmedia.get(url) if record: records[d['pk']] = record print '.', continue record = Record.objects.create(owner=users[d['fields']['user']], name=d['fields']['title'], source='jmutube-%s' % d['fields']['type']) records[d['pk']] = record media = Media.objects.create(record=record, url=url, storage=storage, mimetype=mimetype) CollectionItem.objects.create(collection=collection, record=record) FieldValue.objects.create(record=record, field=title_field, value=d['fields']['title'], order=1) Tag.objects.add_tag( OwnedWrapper.objects.get_for_object( user=users[d['fields']['user']], object=record), 'JMUtube') print 'c', print # create presentations print "Creating presentations" presentations = dict() for d in data('repository.playlist'): presentation, created = Presentation.objects.get_or_create( source='jmutube', owner=users[d['fields']['user']], title=d['fields']['title']) presentations[d['pk']] = presentation print 'c' if created else '.', print # create presentation redirects print "Creating presentation redirects" site = Site.objects.get(id=settings.SITE_ID) def create_redirect(old_path, new_path): redirect, created = Redirect.objects.get_or_create( site=site, old_path=old_path, defaults=dict(new_path=new_path)) if not created and redirect.new_path != new_path: redirect.new_path = new_path redirect.save() return redirect, created for oldid, presentation in presentations.iteritems(): redirect, created1 = create_redirect( reverse('jmutube-playlist-rss-feed-legacy', args=(presentation.owner.username, oldid)), reverse('jmutube-playlist-rss-feed', args=(presentation.owner.username, presentation.id))) redirect, created2 = create_redirect( reverse('jmutube-playlist-rss-feed-legacy', args=(presentation.owner.username, oldid)) + '?player=jmutube', reverse('jmutube-playlist-rss-feed', args=(presentation.owner.username, presentation.id)) + '?player=jmutube') redirect, created3 = create_redirect( reverse('jmutube-playlist-play-legacy', args=(presentation.owner.username, oldid)), reverse('jmutube-playlist-play', args=(presentation.owner.username, presentation.id))) print 'c' if created1 or created2 or created3 else '.', print print "Creating file redirects" for oldid, record in records.iteritems(): redirect, created1 = create_redirect( reverse('jmutube-single-file-rss-feed-legacy', args=(presentation.owner.username, files[oldid])), reverse('jmutube-single-file-rss-feed', args=(presentation.owner.username, record.id, record.name))) redirect, created2 = create_redirect( reverse('jmutube-single-file-rss-feed-legacy', args=(presentation.owner.username, files[oldid])) + '?player=jmutube', reverse('jmutube-single-file-rss-feed', args=(presentation.owner.username, record.id, record.name)) + '?player=jmutube') print 'c' if created1 or created2 else '.', print # create presentation items print "Creating presentation items" for d in data('repository.playlistitem'): item, created = PresentationItem.objects.get_or_create( type=d['fields']['delivery'], order=d['pk'], presentation=presentations[d['fields']['playlist']], record=records[d['fields']['file']]) print 'c' if created else '.', print # create tags print "Creating tags" tags = dict() for d in data('tagging.tag'): tags[d['pk']] = d['fields']['name'] for d in data('tagging.taggeditem'): record = records.get(d['fields']['object_id']) if record: Tag.objects.add_tag( OwnedWrapper.objects.get_for_object(user=record.owner, object=record), '"%s"' % tags[d['fields']['tag']]) print '-', else: print 'X', print
def execute(self): def getElement(dom, *tagname): result = dom for t in tagname: result = result.getElementsByTagName(t)[0] return result def move_or_remove(file, sourcedir, targetdir): if os.path.exists(os.path.join(targetdir, file)): logging.debug("Relay: File %s exists in %s" % (file, targetdir)) os.remove(os.path.join(sourcedir, file)) else: shutil.move(os.path.join(sourcedir, file), targetdir) logging.info("Relay import starting") storage = get_jmutube_storage() files = filter(lambda f: f.endswith('.xml'), os.listdir(settings.JMUTUBE_RELAY_INCOMING_FOLDER)) regex = re.compile('[^0-9a-z]+', flags=re.IGNORECASE) for file in files: age = time.time() - os.path.getmtime(os.path.join(settings.JMUTUBE_RELAY_INCOMING_FOLDER, file)) if age < 600: logging.debug('Relay skipping file %s with age %s' % (file, age)) continue try: dom = parse(os.path.join(settings.JMUTUBE_RELAY_INCOMING_FOLDER, file)) title = getElement(dom, 'presentation', 'title').firstChild.data presenter = getElement(dom, 'presentation', 'presenter', 'userName').firstChild.data user = User.objects.get(username=presenter) files = [f.getAttribute('name') for f in getElement(dom, 'presentation', 'outputFiles', 'fileList').getElementsByTagName('file')] if len(files) > 1: logging.info("Relay importing presentation %s" % file) # Handle presentation outfile = regex.sub('_', os.path.splitext(file)[0]) outfile = make_unique(os.path.join(presenter, 'presentations', outfile + '.zip')) outfilename = os.path.basename(outfile) # create zipped version camrec = False zipfilename = os.path.join(settings.JMUTUBE_RELAY_INCOMING_FOLDER, outfilename) zip = zipfile.ZipFile(zipfilename, 'w', zipfile.ZIP_DEFLATED) for f in files: zip.write(os.path.join(settings.JMUTUBE_RELAY_INCOMING_FOLDER, f), f.encode('ascii')) camrec = camrec or f.endswith('.camrec') zip.write(os.path.join(settings.JMUTUBE_RELAY_INCOMING_FOLDER, file), file) zip.close() # create folder and move everything over outdir = storage.storage_system.path(outfile + '.content') if not os.path.exists(outdir): os.mkdir(outdir) for f in files: move_or_remove(f, settings.JMUTUBE_RELAY_INCOMING_FOLDER, outdir) move_or_remove(file, settings.JMUTUBE_RELAY_INCOMING_FOLDER, outdir) try: shutil.move(zipfilename, storage.storage_system.path(outfile)) except: logging.error("Cannot move ZIP file %s" % zipfilename) os.remove(zipfilename) # create entry point html = filter(lambda f: f.endswith(".htm") or f.endswith(".html"), os.listdir(outdir)) if len(html) == 1 and not html in ('default.htm', 'default.html', 'index.htm', 'index.html'): shutil.copy(os.path.join(outdir, html[0]), os.path.join(outdir, 'index.html')) # add file entry logging.info('Relay creating record for %s: %s (presentations)' % (user, outfile)) record = storage.storage_system.create_record_for_file(user, outfile, 'presentations') else: logging.info("Relay importing single file %s" % file) # Handle single file upload name, ext = os.path.splitext(files[0]) outfile = regex.sub('_', name) + ext outfile = make_unique(os.path.join(presenter, 'video', outfile)) outfilename = os.path.basename(outfile) camrec = outfile.endswith('.camrec') try: logging.info('Relay creating record for %s: %s (video)' % (user, outfile)) record = storage.storage_system.create_record_for_file(user, outfile, 'video') except Exception, e: logging.error("Cannot create record - unsupported file type? [%s]" % e) continue try: shutil.move(os.path.join(settings.JMUTUBE_RELAY_INCOMING_FOLDER, files[0]), storage.storage_system.path(outfile)) except Exception, e: logging.error("Cannot move file %s to %s [%s]" % (files[0], outfile, e)) record.delete() continue try: os.remove(os.path.join(settings.JMUTUBE_RELAY_INCOMING_FOLDER, file)) except Exception, e: logging.error("Cannot remove file %s [%s]" % (file, e)) continue
def execute(self): storage = get_jmutube_storage() logging.info('CRASS sorter starting') for file in os.listdir(settings.JMUTUBE_CRASS_INCOMING_FOLDER): age = time.time() - os.path.getmtime( os.path.join(settings.JMUTUBE_CRASS_INCOMING_FOLDER, file)) if age < 600: logging.debug('CRASS skipping file %s with age %s' % (file, age)) continue logging.debug('Processing %s' % file) match = Job.file_re.match(file) if match: dt = datetime(int(match.group('y')), int(match.group('m')), int(match.group('d')), int(match.group('h')), int(match.group('n')), int(match.group('s'))) schedules = Schedule.objects.filter( computer__mac_address__iexact=match.group('mac'), start_time__lte=dt, end_time__gte=dt) if not schedules: logging.debug("no schedule found") continue schedule = schedules[0] title = '%s %s (%s)' % (schedule.computer.building, schedule.computer.room, dt) newname = os.path.join( schedule.user.username, 'video', (schedule.computer.building + ' ' + schedule.computer.room + ' ' + file[13:-4]).replace(' ', '_') + '.mp4') if not storage.file_exists(newname): # add mapping log entry Mapping.objects.create(source_file=file, target_file=newname, user=schedule.user) # move file shutil.move( os.path.join(settings.JMUTUBE_CRASS_INCOMING_FOLDER, file), storage.storage_system.path(newname)) # add file entry record = storage.storage_system.create_record_for_file( schedule.user, newname, 'video') # add tags wrapper = OwnedWrapper.objects.get_for_object( user=schedule.user, object=record) Tag.objects.add_tag( wrapper, '"%s %s"' % (schedule.computer.building, schedule.computer.room)) Tag.objects.add_tag(wrapper, 'CRASS') Tag.objects.add_tag(wrapper, str(dt.year)) Tag.objects.add_tag(wrapper, '"Week %s"' % dt.isocalendar()[1]) Tag.objects.add_tag(wrapper, [ 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday' ][dt.weekday()]) logging.debug("-> %s" % newname) else: logging.debug("target file already exists") else: logging.debug("invalid file")