def goto(cc=CFG_SITE_NAME, p='', f='', sf='', so='d', docname='', format=''): """ Redirect the user to the latest record in the given collection, optionally within the specified pattern and field. If docname and format are specified, redirect the user to the corresponding docname and format. If docname it is not specified, but there is only a single bibdoc attached to the record will redirect to that one. """ recids = perform_request_search(cc=cc, p=p, f=f, sf=sf, so=so) if recids: ## We shall take the last recid. This is the last one recid = recids[-1] url = '/%s/%s' % (CFG_SITE_RECORD, recid) if format: bibrecdocs = BibRecDocs(recid) if not docname: if len(bibrecdocs.get_bibdoc_names()) == 1: docname = bibrecdocs.get_bibdoc_names()[0] else: return url try: bibdoc = BibRecDocs(recid).get_bibdoc(docname) except InvenioBibDocFileError: return url try: bibdocfile = bibdoc.get_file(format=format) return bibdocfile.get_url() except InvenioBibDocFileError: return url return url
def goto(cc=CFG_SITE_NAME, p='', f='', sf='', so='d', docname='', format=''): """ Redirect the user to the latest record in the given collection, optionally within the specified pattern and field. If docname and format are specified, redirect the user to the corresponding docname and format. If docname it is not specified, but there is only a single bibdoc attached to the record will redirect to that one. """ recids = perform_request_search(cc=cc, p=p, f=f, sf=sf, so=so) if recids: ## We shall take the last recid. This is the last one recid = recids[-1] url = '/%s/%s' % (CFG_SITE_RECORD, recid) if format: bibrecdocs = BibRecDocs(recid) if not docname: if len(bibrecdocs.get_bibdoc_names()) == 1: docname = bibrecdocs.get_bibdoc_names()[0] else: return url try: bibdoc = BibRecDocs(recid).get_bibdoc(docname) except InvenioBibDocFileError: return url try: bibdocfile = bibdoc.get_file(format=format) return bibdocfile.get_url() except InvenioBibDocFileError: return url return url
def goto(type, document='', number=0, lang='en', modif=0): today = time.strftime('%Y-%m-%d') if type == 'SSR': ## We would like a CERN Staff Rules and Regulations recids = perform_request_search(cc='Staff Rules and Regulations', f="925__a:1996-01-01->%s 925__b:%s->9999-99-99" % (today, today)) recid = recids[-1] reportnumber = get_fieldvalues(recid, '037__a')[0] edition = int(reportnumber[-2:]) ## e.g. CERN-STAFF-RULES-ED08 return BibRecDocs(recid).get_bibdoc(make_cern_ssr_docname(lang, edition, modif)).get_file('.pdf').get_url() elif type == "OPER-CIRC": recids = perform_request_search(cc="Operational Circulars", p="reportnumber=\"CERN-OPER-CIRC-%s-*\"" % number, sf="925__a") recid = recids[-1] documents = {} bibrecdocs = BibRecDocs(recid) for docname in bibrecdocs.get_bibdoc_names(): ldocname = docname.lower() if 'implementation' in ldocname: _register_document(documents, docname, 'implementation_en') elif 'application' in ldocname: _register_document(documents, docname, 'implementation_fr') elif 'archiving' in ldocname: _register_document(documents, docname, 'archiving_en') elif 'archivage' in ldocname: _register_document(documents, docname, 'archiving_fr') elif 'annexe' in ldocname or 'annexes_fr' in ldocname: _register_document(documents, docname, 'annex_fr') elif 'annexes_en' in ldocname or 'annex' in ldocname: _register_document(documents, docname, 'annex_en') elif '_en_' in ldocname or '_eng_' in ldocname or '_angl_' in ldocname: _register_document(documents, docname, 'en') elif '_fr_' in ldocname: _register_document(documents, docname, 'fr') return bibrecdocs.get_bibdoc(documents[document]).get_file('.pdf').get_url() elif type == 'ADMIN-CIRC': recids = perform_request_search(cc="Administrative Circulars", p="reportnumber=\"CERN-ADMIN-CIRC-%s-*\"" % number, sf="925__a") recid = recids[-1] documents = {} bibrecdocs = BibRecDocs(recid) for docname in bibrecdocs.get_bibdoc_names(): ldocname = docname.lower() if 'implementation' in ldocname: _register_document(documents, docname, 'implementation-en') elif 'application' in ldocname: _register_document(documents, docname, 'implementation-fr') elif 'archiving' in ldocname: _register_document(documents, docname, 'archiving-en') elif 'archivage' in ldocname: _register_document(documents, docname, 'archiving-fr') elif 'annexe' in ldocname or 'annexes_fr' in ldocname: _register_document(documents, docname, 'annex-fr') elif 'annexes_en' in ldocname or 'annex' in ldocname: _register_document(documents, docname, 'annex-en') elif '_en_' in ldocname or '_eng_' in ldocname or '_angl_' in ldocname: _register_document(documents, docname, 'en') elif '_fr_' in ldocname: _register_document(documents, docname, 'fr') return bibrecdocs.get_bibdoc(documents[document]).get_file('.pdf').get_url()
def test_BibRecDocs(self): """bibdocfile - BibRecDocs functions""" my_bibrecdoc = BibRecDocs(2) #add bibdoc my_bibrecdoc.add_new_file(CFG_PREFIX + '/lib/webtest/invenio/test.jpg', 'Main', 'img_test', False, 'test add new file', 'test', '.jpg') my_bibrecdoc.add_bibdoc(doctype='Main', docname='file', never_fail=False) self.assertEqual(len(my_bibrecdoc.list_bibdocs()), 3) my_added_bibdoc = my_bibrecdoc.get_bibdoc('file') #add bibdocfile in empty bibdoc my_added_bibdoc.add_file_new_version(CFG_PREFIX + '/lib/webtest/invenio/test.gif', \ description= 'added in empty bibdoc', comment=None, format=None, flags=['PERFORM_HIDE_PREVIOUS']) #propose unique docname self.assertEqual(my_bibrecdoc.propose_unique_docname('file'), 'file_2') #has docname self.assertEqual(my_bibrecdoc.has_docname_p('file'), True) #merge 2 bibdocs my_bibrecdoc.merge_bibdocs('img_test', 'file') self.assertEqual( len(my_bibrecdoc.get_bibdoc("img_test").list_all_files()), 2) #check file exists self.assertEqual( my_bibrecdoc.check_file_exists(CFG_PREFIX + '/lib/webtest/invenio/test.jpg'), True) #get bibdoc names self.assertEqual( my_bibrecdoc.get_bibdoc_names('Main')[0], '0104007_02') self.assertEqual(my_bibrecdoc.get_bibdoc_names('Main')[1], 'img_test') #get total size self.assertEqual(my_bibrecdoc.get_total_size(), 1647591) #get total size latest version self.assertEqual(my_bibrecdoc.get_total_size_latest_version(), 1647591) #display value = my_bibrecdoc.display(docname='img_test', version='', doctype='', ln='en', verbose=0, display_hidden=True) self.assert_("<small><b>Main</b>" in value) #get xml 8564 value = my_bibrecdoc.get_xml_8564() self.assert_('/' + CFG_SITE_RECORD + '/2/files/img_test.jpg</subfield>' in value) #check duplicate docnames self.assertEqual(my_bibrecdoc.check_duplicate_docnames(), True)
def check_records(records): for record in records: ## Stupid hack because bibcheck filters does not work as expected if record_get_field_value(record, '980', code='b') == "Hindawi": record.warn("Working on this record") recdoc = BibRecDocs(int(record.record_id)) doc = recdoc.get_bibdoc(recdoc.get_bibdoc_names()[0]) try: xml_file = open(doc.get_file("xml").get_full_path()) except: record.warn("No document can be found") continue xml2 = xml.dom.minidom.parseString(xml_file.read()) subject = get_value_in_tag(xml2, "subject") if subject in ["Editorial", "Erratum", "Corrigendum", "Addendum","Letter to the Editor"]: field = record_get_field_value(record, '980', code='c') if field: if field in ['ERRATUM', 'ADDENDUM', 'EDITORIAL','CORRIGENDUM', 'LETTER TO THE EDITOR']: for position, value in record.iterfield('980__c'): record.amend_field(position, subject.upper()) break else: for position, value in record.iterfield('980__%'): record.add_subfield(position, 'c', subject.upper()) break else: for position, value in record.iterfield('980__%'): record.add_subfield(position, 'c', subject.upper()) break elif subject not in ["Review Article","Research Article","Retraction"]: raise Exception("This subject: %s does not exit in SCOAP3 system" % (subject,))
def test_BibRecDocs(self): """bibdocfile - BibRecDocs functions""" my_bibrecdoc = BibRecDocs(2) #add bibdoc my_bibrecdoc.add_new_file(CFG_PREFIX + '/lib/webtest/invenio/test.jpg', 'Main', 'img_test', False, 'test add new file', 'test', '.jpg') my_bibrecdoc.add_bibdoc(doctype='Main', docname='file', never_fail=False) self.assertEqual(len(my_bibrecdoc.list_bibdocs()), 3) my_added_bibdoc = my_bibrecdoc.get_bibdoc('file') #add bibdocfile in empty bibdoc my_added_bibdoc.add_file_new_version(CFG_PREFIX + '/lib/webtest/invenio/test.gif', \ description= 'added in empty bibdoc', comment=None, format=None, flags=['PERFORM_HIDE_PREVIOUS']) #propose unique docname self.assertEqual(my_bibrecdoc.propose_unique_docname('file'), 'file_2') #has docname self.assertEqual(my_bibrecdoc.has_docname_p('file'), True) #merge 2 bibdocs my_bibrecdoc.merge_bibdocs('img_test', 'file') self.assertEqual(len(my_bibrecdoc.get_bibdoc("img_test").list_all_files()), 2) #check file exists self.assertEqual(my_bibrecdoc.check_file_exists(CFG_PREFIX + '/lib/webtest/invenio/test.jpg'), True) #get bibdoc names self.assertEqual(my_bibrecdoc.get_bibdoc_names('Main')[0], '0104007_02') self.assertEqual(my_bibrecdoc.get_bibdoc_names('Main')[1],'img_test') #get total size self.assertEqual(my_bibrecdoc.get_total_size(), 1647591) #get total size latest version self.assertEqual(my_bibrecdoc.get_total_size_latest_version(), 1647591) #display value = my_bibrecdoc.display(docname='img_test', version='', doctype='', ln='en', verbose=0, display_hidden=True) self.assert_("<small><b>Main</b>" in value) #get xml 8564 value = my_bibrecdoc.get_xml_8564() self.assert_('/record/2/files/img_test.jpg</subfield>' in value) #check duplicate docnames self.assertEqual(my_bibrecdoc.check_duplicate_docnames(), True)
def test_BibRecDocs(self): """bibdocfile - BibRecDocs functions""" my_bibrecdoc = BibRecDocs(2) # add bibdoc my_bibrecdoc.add_new_file( CFG_PREFIX + "/lib/webtest/invenio/test.jpg", "Main", "img_test", False, "test add new file", "test", ".jpg" ) my_bibrecdoc.add_bibdoc(doctype="Main", docname="file", never_fail=False) self.assertEqual(len(my_bibrecdoc.list_bibdocs()), 3) my_added_bibdoc = my_bibrecdoc.get_bibdoc("file") # add bibdocfile in empty bibdoc my_added_bibdoc.add_file_new_version( CFG_PREFIX + "/lib/webtest/invenio/test.gif", description="added in empty bibdoc", comment=None, docformat=None, flags=["PERFORM_HIDE_PREVIOUS"], ) # propose unique docname self.assertEqual(my_bibrecdoc.propose_unique_docname("file"), "file_2") # has docname self.assertEqual(my_bibrecdoc.has_docname_p("file"), True) # merge 2 bibdocs my_bibrecdoc.merge_bibdocs("img_test", "file") self.assertEqual(len(my_bibrecdoc.get_bibdoc("img_test").list_all_files()), 2) # check file exists self.assertEqual(my_bibrecdoc.check_file_exists(CFG_PREFIX + "/lib/webtest/invenio/test.jpg", ".jpg"), True) # get bibdoc names # we can not rely on the order ! names = set([my_bibrecdoc.get_bibdoc_names("Main")[0], my_bibrecdoc.get_bibdoc_names("Main")[1]]) self.assertTrue("0104007_02" in names) self.assertTrue("img_test" in names) # get total size self.assertEqual(my_bibrecdoc.get_total_size(), 1647591) # get total size latest version self.assertEqual(my_bibrecdoc.get_total_size_latest_version(), 1647591) # display # value = my_bibrecdoc.display(docname='img_test', version='', doctype='', ln='en', verbose=0, display_hidden=True) # self.assert_("<small><b>Main</b>" in value) # get xml 8564 value = my_bibrecdoc.get_xml_8564() self.assert_("/" + CFG_SITE_RECORD + "/2/files/img_test.jpg</subfield>" in value) # check duplicate docnames self.assertEqual(my_bibrecdoc.check_duplicate_docnames(), True)
def fix_recid(recid, logfile): """Fix a given recid.""" print "Upgrading record %s ->" % recid, print >> logfile, "Upgrading record %s:" % recid bibrec = BibRecDocs(recid) print >> logfile, bibrec docnames = bibrec.get_bibdoc_names() try: for docname in docnames: print docname, new_bibdocs = bibrec.fix(docname) new_bibdocnames = [bibdoc.get_docname() for bibdoc in new_bibdocs] if new_bibdocnames: print "(created bibdocs: '%s')" % "', '".join(new_bibdocnames), print >> logfile, "(created bibdocs: '%s')" % "', '".join(new_bibdocnames) except InvenioWebSubmitFileError, e: print >> logfile, BibRecDocs(recid) print "%s -> ERROR", e return False
def fix_recid(recid, logfile): """Fix a given recid.""" print "Upgrading record %s ->" % recid, print >> logfile, "Upgrading record %s:" % recid bibrec = BibRecDocs(recid) print >> logfile, bibrec docnames = bibrec.get_bibdoc_names() try: for docname in docnames: print docname, new_bibdocs = bibrec.fix(docname) new_bibdocnames = [bibdoc.get_docname() for bibdoc in new_bibdocs] if new_bibdocnames: print "(created bibdocs: '%s')" % "', '".join(new_bibdocnames), print >> logfile, "(created bibdocs: '%s')" % "', '".join(new_bibdocnames) except InvenioWebSubmitFileError, e: print >> logfile, BibRecDocs(recid) print "%s -> ERROR", e return False
def check_records(records): for record in records: ## Stupid hack because bibcheck filters does not work as expected if record_get_field_value(record, '980', code='b') == "Hindawi": record.warn("Working on this record") recdoc = BibRecDocs(int(record.record_id)) doc = recdoc.get_bibdoc(recdoc.get_bibdoc_names()[0]) try: xml_file = open(doc.get_file("xml").get_full_path()) except: record.warn("No document can be found") continue xml2 = xml.dom.minidom.parseString(xml_file.read()) subject = get_value_in_tag(xml2, "subject") if subject in [ "Editorial", "Erratum", "Corrigendum", "Addendum", "Letter to the Editor" ]: field = record_get_field_value(record, '980', code='c') if field: if field in [ 'ERRATUM', 'ADDENDUM', 'EDITORIAL', 'CORRIGENDUM', 'LETTER TO THE EDITOR' ]: for position, value in record.iterfield('980__c'): record.amend_field(position, subject.upper()) break else: for position, value in record.iterfield('980__%'): record.add_subfield(position, 'c', subject.upper()) break else: for position, value in record.iterfield('980__%'): record.add_subfield(position, 'c', subject.upper()) break elif subject not in [ "Review Article", "Research Article", "Retraction" ]: raise Exception( "This subject: %s does not exit in SCOAP3 system" % (subject, ))
def test_BibDocs(self): """bibdocfile - BibDocs functions""" #add file my_bibrecdoc = BibRecDocs(2) timestamp1 = datetime(*(time.strptime("2011-10-09 08:07:06", "%Y-%m-%d %H:%M:%S")[:6])) my_bibrecdoc.add_new_file(CFG_PREFIX + '/lib/webtest/invenio/test.jpg', 'Main', 'img_test', False, 'test add new file', 'test', '.jpg', modification_date=timestamp1) my_new_bibdoc = my_bibrecdoc.get_bibdoc("img_test") value = my_bibrecdoc.list_bibdocs() self.assertEqual(len(value), 2) #get total file (bibdoc) self.assertEqual(my_new_bibdoc.get_total_size(), 91750) #get recid self.assertEqual(my_new_bibdoc.bibrec_links[0]["recid"], 2) #change name my_new_bibdoc.change_name(2, 'new_name') #get docname my_bibrecdoc = BibRecDocs(2) self.assertEqual(my_bibrecdoc.get_docname(my_new_bibdoc.id), 'new_name') #get type self.assertEqual(my_new_bibdoc.get_type(), 'Main') #get id self.assert_(my_new_bibdoc.get_id() > 80) #set status my_new_bibdoc.set_status('new status') #get status self.assertEqual(my_new_bibdoc.get_status(), 'new status') #get base directory self.assert_(my_new_bibdoc.get_base_dir().startswith(CFG_BIBDOCFILE_FILEDIR)) #get file number self.assertEqual(my_new_bibdoc.get_file_number(), 1) #add file new version timestamp2 = datetime(*(time.strptime("2010-09-08 07:06:05", "%Y-%m-%d %H:%M:%S")[:6])) my_new_bibdoc.add_file_new_version(CFG_PREFIX + '/lib/webtest/invenio/test.jpg', description= 'the new version', comment=None, docformat=None, flags=["PERFORM_HIDE_PREVIOUS"], modification_date=timestamp2) self.assertEqual(my_new_bibdoc.list_versions(), [1, 2]) #revert timestamp3 = datetime.now() time.sleep(2) # so we can see a difference between now() and the time of the revert my_new_bibdoc.revert(1) self.assertEqual(my_new_bibdoc.list_versions(), [1, 2, 3]) self.assertEqual(my_new_bibdoc.get_description('.jpg', version=3), 'test add new file') #get total size latest version self.assertEqual(my_new_bibdoc.get_total_size_latest_version(), 91750) #get latest version self.assertEqual(my_new_bibdoc.get_latest_version(), 3) #list latest files self.assertEqual(len(my_new_bibdoc.list_latest_files()), 1) self.assertEqual(my_new_bibdoc.list_latest_files()[0].get_version(), 3) #list version files self.assertEqual(len(my_new_bibdoc.list_version_files(1, list_hidden=True)), 1) #display # No Display facility inside of an object ! # value = my_new_bibdoc.display(version='', ln='en', display_hidden=True) # self.assert_('>test add new file<' in value) #format already exist self.assertEqual(my_new_bibdoc.format_already_exists_p('.jpg'), True) #get file self.assertEqual(my_new_bibdoc.get_file('.jpg', version='1').get_version(), 1) #set description my_new_bibdoc.set_description('new description', '.jpg', version=1) #get description self.assertEqual(my_new_bibdoc.get_description('.jpg', version=1), 'new description') #set comment my_new_bibdoc.set_description('new comment', '.jpg', version=1) #get comment self.assertEqual(my_new_bibdoc.get_description('.jpg', version=1), 'new comment') #get history assert len(my_new_bibdoc.get_history()) > 0 #check modification date self.assertEqual(my_new_bibdoc.get_file('.jpg', version=1).md, timestamp1) self.assertEqual(my_new_bibdoc.get_file('.jpg', version=2).md, timestamp2) assert my_new_bibdoc.get_file('.jpg', version=3).md > timestamp3 #delete file my_new_bibdoc.delete_file('.jpg', 2) #list all files self.assertEqual(len(my_new_bibdoc.list_all_files()), 2) #delete file my_new_bibdoc.delete_file('.jpg', 3) #add new format timestamp4 = datetime(*(time.strptime("2012-11-10 09:08:07", "%Y-%m-%d %H:%M:%S")[:6])) my_new_bibdoc.add_file_new_format(CFG_PREFIX + '/lib/webtest/invenio/test.gif', version=None, description=None, comment=None, docformat=None, modification_date=timestamp4) self.assertEqual(len(my_new_bibdoc.list_all_files()), 2) #check modification time self.assertEqual(my_new_bibdoc.get_file('.jpg', version=1).md, timestamp1) self.assertEqual(my_new_bibdoc.get_file('.gif', version=1).md, timestamp4) #change the format name my_new_bibdoc.change_docformat('.gif', '.gif;icon-640') self.assertEqual(my_new_bibdoc.format_already_exists_p('.gif'), False) self.assertEqual(my_new_bibdoc.format_already_exists_p('.gif;icon-640'), True) #delete file my_new_bibdoc.delete_file('.jpg', 1) #delete file my_new_bibdoc.delete_file('.gif;icon-640', 1) #empty bibdoc self.assertEqual(my_new_bibdoc.empty_p(), True) #hidden? self.assertEqual(my_new_bibdoc.hidden_p('.jpg', version=1), False) #hide my_new_bibdoc.set_flag('HIDDEN', '.jpg', version=1) #hidden? self.assertEqual(my_new_bibdoc.hidden_p('.jpg', version=1), True) #add and get icon my_new_bibdoc.add_icon( CFG_PREFIX + '/lib/webtest/invenio/icon-test.gif', modification_date=timestamp4) my_bibrecdoc = BibRecDocs(2) value = my_bibrecdoc.get_bibdoc("new_name") self.assertEqual(value.get_icon().docid, my_new_bibdoc.get_icon().docid) self.assertEqual(value.get_icon().version, my_new_bibdoc.get_icon().version) self.assertEqual(value.get_icon().format, my_new_bibdoc.get_icon().format) #check modification time self.assertEqual(my_new_bibdoc.get_icon().md, timestamp4) #delete icon my_new_bibdoc.delete_icon() #get icon self.assertEqual(my_new_bibdoc.get_icon(), None) #delete my_new_bibdoc.delete() self.assertEqual(my_new_bibdoc.deleted_p(), True) #undelete my_new_bibdoc.undelete(previous_status='', recid=2) #expunging my_new_bibdoc.expunge() my_bibrecdoc.build_bibdoc_list() self.failIf('new_name' in my_bibrecdoc.get_bibdoc_names()) self.failUnless(my_bibrecdoc.get_bibdoc_names())
def process_batch_job(batch_job_file): """ Processes a batch job description dictionary @param batch_job_file: a fullpath to a batch job file @type batch_job_file: string @return: 1 if the process was successfull, 0 if not @rtype; int """ def upload_marcxml_file(marcxml): """ Creates a temporary marcxml file and sends it to bibupload """ xml_filename = 'bibencode_'+ str(batch_job['recid']) + '_' + str(uuid.uuid4()) + '.xml' xml_filename = os.path.join(invenio.config.CFG_TMPSHAREDDIR, xml_filename) xml_file = file(xml_filename, 'w') xml_file.write(marcxml) xml_file.close() targs = ['-c', xml_filename] task_low_level_submission('bibupload', 'bibencode', *targs) #---------# # GENERAL # #---------# _task_write_message("----------- Handling Master -----------") ## Check the validity of the batch file here batch_job = json_decode_file(batch_job_file) ## Sanitise batch description and raise errrors batch_job = sanitise_batch_job(batch_job) ## Check if the record exists if record_exists(batch_job['recid']) < 1: raise Exception("Record not found") recdoc = BibRecDocs(batch_job['recid']) #--------------------# # UPDATE FROM MASTER # #--------------------# ## We want to add new stuff to the video's record, using the master as input if getval(batch_job, 'update_from_master'): found_master = False bibdocs = recdoc.list_bibdocs() for bibdoc in bibdocs: bibdocfiles = bibdoc.list_all_files() for bibdocfile in bibdocfiles: comment = bibdocfile.get_comment() description = bibdocfile.get_description() subformat = bibdocfile.get_subformat() m_comment = getval(batch_job, 'bibdoc_master_comment', comment) m_description = getval(batch_job, 'bibdoc_master_description', description) m_subformat = getval(batch_job, 'bibdoc_master_subformat', subformat) if (comment == m_comment and description == m_description and subformat == m_subformat): found_master = True batch_job['input'] = bibdocfile.get_full_path() ## Get the aspect of the from the record try: ## Assumes pbcore metadata mapping batch_job['aspect'] = get_fieldvalues(124, CFG_BIBENCODE_ASPECT_RATIO_MARC_FIELD)[0] except IndexError: pass break if found_master: break if not found_master: _task_write_message("Video master for record %d not found" % batch_job['recid']) task_update_progress("Video master for record %d not found" % batch_job['recid']) ## Maybe send an email? return 1 ## Clean the job to do no upscaling etc if getval(batch_job, 'assure_quality'): batch_job = clean_job_for_quality(batch_job) global _BATCH_STEPS _BATCH_STEPS = len(batch_job['jobs']) ## Generate the docname from the input filename's name or given name bibdoc_video_docname, bibdoc_video_extension = decompose_file(batch_job['input'])[1:] if not bibdoc_video_extension or getval(batch_job, 'bibdoc_master_extension'): bibdoc_video_extension = getval(batch_job, 'bibdoc_master_extension') if getval(batch_job, 'bibdoc_master_docname'): bibdoc_video_docname = getval(batch_job, 'bibdoc_master_docname') write_message("Creating BibDoc for %s" % bibdoc_video_docname) ## If the bibdoc exists, receive it if bibdoc_video_docname in recdoc.get_bibdoc_names(): bibdoc_video = recdoc.get_bibdoc(bibdoc_video_docname) ## Create a new bibdoc if it does not exist else: bibdoc_video = recdoc.add_bibdoc(docname=bibdoc_video_docname) ## Get the directory auf the newly created bibdoc to copy stuff there bibdoc_video_directory = bibdoc_video.get_base_dir() #--------# # MASTER # #--------# if not getval(batch_job, 'update_from_master'): if getval(batch_job, 'add_master'): ## Generate the right name for the master ## The master should be hidden first an then renamed ## when it is really available ## !!! FIX !!! _task_write_message("Adding %s master to the BibDoc" % bibdoc_video_docname) master_format = compose_format( bibdoc_video_extension, getval(batch_job, 'bibdoc_master_subformat', 'master') ) ## If a file of the same format is there, something is wrong, remove it! ## it might be caused by a previous corrupted submission etc. if bibdoc_video.format_already_exists_p(master_format): bibdoc_video.delete_file(master_format, 1) bibdoc_video.add_file_new_format( batch_job['input'], version=1, description=getval(batch_job, 'bibdoc_master_description'), comment=getval(batch_job, 'bibdoc_master_comment'), docformat=master_format ) #-----------# # JOBS LOOP # #-----------# return_code = 1 global _BATCH_STEP for job in batch_job['jobs']: _task_write_message("----------- Job %s of %s -----------" % (_BATCH_STEP, _BATCH_STEPS)) ## Try to substitute docname with master docname if getval(job, 'bibdoc_docname'): job['bibdoc_docname'] = Template(job['bibdoc_docname']).safe_substitute({'bibdoc_master_docname': bibdoc_video_docname}) #-------------# # TRANSCODING # #-------------# if job['mode'] == 'encode': ## Skip the job if assure_quality is not set and marked as fallback if not getval(batch_job, 'assure_quality') and getval(job, 'fallback'): continue if getval(job, 'profile'): profile = get_encoding_profile(job['profile']) else: profile = None ## We need an extension defined fot the video container bibdoc_video_extension = getval(job, 'extension', getval(profile, 'extension')) if not bibdoc_video_extension: raise Exception("No container/extension defined") ## Get the docname and subformat bibdoc_video_subformat = getval(job, 'bibdoc_subformat') bibdoc_slave_video_docname = getval(job, 'bibdoc_docname', bibdoc_video_docname) ## The subformat is incompatible with ffmpegs name convention ## We do the encoding without and rename it afterwards bibdoc_video_fullpath = compose_file( bibdoc_video_directory, bibdoc_slave_video_docname, bibdoc_video_extension ) _task_write_message("Transcoding %s to %s;%s" % (bibdoc_slave_video_docname, bibdoc_video_extension, bibdoc_video_subformat)) ## We encode now directly into the bibdocs directory encoding_result = encode_video( input_file=batch_job['input'], output_file=bibdoc_video_fullpath, acodec=getval(job, 'audiocodec'), vcodec=getval(job, 'videocodec'), abitrate=getval(job, 'videobitrate'), vbitrate=getval(job, 'audiobitrate'), resolution=getval(job, 'resolution'), passes=getval(job, 'passes', 1), special=getval(job, 'special'), specialfirst=getval(job, 'specialfirst'), specialsecond=getval(job, 'specialsecond'), metadata=getval(job, 'metadata'), width=getval(job, 'width'), height=getval(job, 'height'), aspect=getval(batch_job, 'aspect'), # Aspect for every job profile=getval(job, 'profile'), update_fnc=_task_update_overall_status, message_fnc=_task_write_message ) return_code &= encoding_result ## only on success if encoding_result: ## Rename it, adding the subformat os.rename(bibdoc_video_fullpath, compose_file(bibdoc_video_directory, bibdoc_video_extension, bibdoc_video_subformat, 1, bibdoc_slave_video_docname) ) #bibdoc_video._build_file_list() bibdoc_video.touch() bibdoc_video._sync_to_db() bibdoc_video_format = compose_format(bibdoc_video_extension, bibdoc_video_subformat) if getval(job, 'bibdoc_comment'): bibdoc_video.set_comment(getval(job, 'bibdoc_comment'), bibdoc_video_format) if getval(job, 'bibdoc_description'): bibdoc_video.set_description(getval(job, 'bibdoc_description'), bibdoc_video_format) #------------# # EXTRACTION # #------------# # if there are multiple extraction jobs, all the produced files # with the same name will be in the same bibdoc! Make sure that # you use different subformats or docname templates to avoid # conflicts. if job['mode'] == 'extract': if getval(job, 'profile'): profile = get_extract_profile(job['profile']) else: profile = {} bibdoc_frame_subformat = getval(job, 'bibdoc_subformat') _task_write_message("Extracting frames to temporary directory") tmpdir = invenio.config.CFG_TMPDIR + "/" + str(uuid.uuid4()) os.mkdir(tmpdir) #Move this to the batch description bibdoc_frame_docname = getval(job, 'bibdoc_docname', bibdoc_video_docname) tmpfname = (tmpdir + "/" + bibdoc_frame_docname + '.' + getval(profile, 'extension', getval(job, 'extension', 'jpg'))) extraction_result = extract_frames(input_file=batch_job['input'], output_file=tmpfname, size=getval(job, 'size'), positions=getval(job, 'positions'), numberof=getval(job, 'numberof'), width=getval(job, 'width'), height=getval(job, 'height'), aspect=getval(batch_job, 'aspect'), profile=getval(job, 'profile'), update_fnc=_task_update_overall_status, ) return_code &= extraction_result ## only on success: if extraction_result: ## for every filename in the directorys, create a bibdoc that contains ## all sizes of the frame from the two directories files = os.listdir(tmpdir) for filename in files: ## The docname was altered by BibEncode extract through substitution ## Retrieve it from the filename again bibdoc_frame_docname, bibdoc_frame_extension = os.path.splitext(filename) _task_write_message("Creating new bibdoc for %s" % bibdoc_frame_docname) ## If the bibdoc exists, receive it if bibdoc_frame_docname in recdoc.get_bibdoc_names(): bibdoc_frame = recdoc.get_bibdoc(bibdoc_frame_docname) ## Create a new bibdoc if it does not exist else: bibdoc_frame = recdoc.add_bibdoc(docname=bibdoc_frame_docname) ## The filename including path from tmpdir fname = os.path.join(tmpdir, filename) bibdoc_frame_format = compose_format(bibdoc_frame_extension, bibdoc_frame_subformat) ## Same as with the master, if the format allready exists, ## override it, because something went wrong before if bibdoc_frame.format_already_exists_p(bibdoc_frame_format): bibdoc_frame.delete_file(bibdoc_frame_format, 1) _task_write_message("Adding %s jpg;%s to BibDoc" % (bibdoc_frame_docname, getval(job, 'bibdoc_subformat'))) bibdoc_frame.add_file_new_format( fname, version=1, description=getval(job, 'bibdoc_description'), comment=getval(job, 'bibdoc_comment'), docformat=bibdoc_frame_format) ## Remove the temporary folders _task_write_message("Removing temporary directory") shutil.rmtree(tmpdir) _BATCH_STEP = _BATCH_STEP + 1 #-----------------# # FIX BIBDOC/MARC # #-----------------# _task_write_message("----------- Handling MARCXML -----------") ## Fix the BibDoc for all the videos previously created _task_write_message("Updating BibDoc of %s" % bibdoc_video_docname) bibdoc_video._build_file_list() ## Fix the MARC _task_write_message("Fixing MARC") cli_fix_marc({}, [batch_job['recid']], False) if getval(batch_job, 'collection'): ## Make the record visible by moving in from the collection marcxml = ("<record><controlfield tag=\"001\">%d</controlfield>" "<datafield tag=\"980\" ind1=\" \" ind2=\" \">" "<subfield code=\"a\">%s</subfield></datafield></record>" ) % (batch_job['recid'], batch_job['collection']) upload_marcxml_file(marcxml) #---------------------# # ADD MASTER METADATA # #---------------------# if getval(batch_job, 'add_master_metadata'): _task_write_message("Adding master metadata") pbcore = pbcore_metadata(input_file = getval(batch_job, 'input'), pbcoreIdentifier = batch_job['recid'], aspect_override = getval(batch_job, 'aspect')) marcxml = format(pbcore, CFG_BIBENCODE_PBCORE_MARC_XSLT) upload_marcxml_file(marcxml) #------------------# # ADD MARC SNIPPET # #------------------# if getval(batch_job, 'marc_snippet'): marc_snippet = open(getval(batch_job, 'marc_snippet')) marcxml = marc_snippet.read() marc_snippet.close() upload_marcxml_file(marcxml) #--------------# # DELETE INPUT # #--------------# if getval(batch_job, 'delete_input'): _task_write_message("Deleting input file") # only if successfull if not return_code: # only if input matches pattern if getval(batch_job, 'delete_input_pattern', '') in getval(batch_job, 'input'): try: os.remove(getval(batch_job, 'input')) except OSError: pass #--------------# # NOTIFICATION # #--------------# ## Send Notification emails on errors if not return_code: if getval(batch_job, 'notify_user'): _notify_error_user(getval(batch_job, 'notify_user'), getval(batch_job, 'submission_filename', batch_job['input']), getval(batch_job, 'recid'), getval(batch_job, 'submission_title', "")) _task_write_message("Notify user because of an error") if getval(batch_job, 'notify_admin'): _task_write_message("Notify admin because of an error") if type(getval(batch_job, 'notify_admin') == type(str()) ): _notify_error_admin(batch_job, getval(batch_job, 'notify_admin')) else: _notify_error_admin(batch_job) else: if getval(batch_job, 'notify_user'): _task_write_message("Notify user because of success") _notify_success_user(getval(batch_job, 'notify_user'), getval(batch_job, 'submission_filename', batch_job['input']), getval(batch_job, 'recid'), getval(batch_job, 'submission_title', "")) return 1
def test_BibDocs(self): """bibdocfile - BibDocs functions""" #add file my_bibrecdoc = BibRecDocs(2) timestamp1 = datetime( *(time.strptime("2011-10-09 08:07:06", "%Y-%m-%d %H:%M:%S")[:6])) my_bibrecdoc.add_new_file(CFG_PREFIX + '/lib/webtest/invenio/test.jpg', 'Main', 'img_test', False, 'test add new file', 'test', '.jpg', modification_date=timestamp1) my_new_bibdoc = my_bibrecdoc.get_bibdoc("img_test") value = my_bibrecdoc.list_bibdocs() self.assertEqual(len(value), 2) #get total file (bibdoc) self.assertEqual(my_new_bibdoc.get_total_size(), 91750) #get recid self.assertEqual(my_new_bibdoc.bibrec_links[0]["recid"], 2) #change name my_new_bibdoc.change_name(2, 'new_name') #get docname my_bibrecdoc = BibRecDocs(2) self.assertEqual(my_bibrecdoc.get_docname(my_new_bibdoc.id), 'new_name') #get type self.assertEqual(my_new_bibdoc.get_type(), 'Main') #get id self.assert_(my_new_bibdoc.get_id() > 80) #set status my_new_bibdoc.set_status('new status') #get status self.assertEqual(my_new_bibdoc.get_status(), 'new status') #get base directory self.assert_( my_new_bibdoc.get_base_dir().startswith(CFG_BIBDOCFILE_FILEDIR)) #get file number self.assertEqual(my_new_bibdoc.get_file_number(), 1) #add file new version timestamp2 = datetime( *(time.strptime("2010-09-08 07:06:05", "%Y-%m-%d %H:%M:%S")[:6])) my_new_bibdoc.add_file_new_version(CFG_PREFIX + '/lib/webtest/invenio/test.jpg', description='the new version', comment=None, docformat=None, flags=["PERFORM_HIDE_PREVIOUS"], modification_date=timestamp2) self.assertEqual(my_new_bibdoc.list_versions(), [1, 2]) #revert timestamp3 = datetime.now() time.sleep( 2 ) # so we can see a difference between now() and the time of the revert my_new_bibdoc.revert(1) self.assertEqual(my_new_bibdoc.list_versions(), [1, 2, 3]) self.assertEqual(my_new_bibdoc.get_description('.jpg', version=3), 'test add new file') #get total size latest version self.assertEqual(my_new_bibdoc.get_total_size_latest_version(), 91750) #get latest version self.assertEqual(my_new_bibdoc.get_latest_version(), 3) #list latest files self.assertEqual(len(my_new_bibdoc.list_latest_files()), 1) self.assertEqual(my_new_bibdoc.list_latest_files()[0].get_version(), 3) #list version files self.assertEqual( len(my_new_bibdoc.list_version_files(1, list_hidden=True)), 1) #display # No Display facility inside of an object ! # value = my_new_bibdoc.display(version='', ln='en', display_hidden=True) # self.assert_('>test add new file<' in value) #format already exist self.assertEqual(my_new_bibdoc.format_already_exists_p('.jpg'), True) #get file self.assertEqual( my_new_bibdoc.get_file('.jpg', version='1').get_version(), 1) #set description my_new_bibdoc.set_description('new description', '.jpg', version=1) #get description self.assertEqual(my_new_bibdoc.get_description('.jpg', version=1), 'new description') #set comment my_new_bibdoc.set_description('new comment', '.jpg', version=1) #get comment self.assertEqual(my_new_bibdoc.get_description('.jpg', version=1), 'new comment') #get history assert len(my_new_bibdoc.get_history()) > 0 #check modification date self.assertEqual( my_new_bibdoc.get_file('.jpg', version=1).md, timestamp1) self.assertEqual( my_new_bibdoc.get_file('.jpg', version=2).md, timestamp2) assert my_new_bibdoc.get_file('.jpg', version=3).md > timestamp3 #delete file my_new_bibdoc.delete_file('.jpg', 2) #list all files self.assertEqual(len(my_new_bibdoc.list_all_files()), 2) #delete file my_new_bibdoc.delete_file('.jpg', 3) #add new format timestamp4 = datetime( *(time.strptime("2012-11-10 09:08:07", "%Y-%m-%d %H:%M:%S")[:6])) my_new_bibdoc.add_file_new_format(CFG_PREFIX + '/lib/webtest/invenio/test.gif', version=None, description=None, comment=None, docformat=None, modification_date=timestamp4) self.assertEqual(len(my_new_bibdoc.list_all_files()), 2) #check modification time self.assertEqual( my_new_bibdoc.get_file('.jpg', version=1).md, timestamp1) self.assertEqual( my_new_bibdoc.get_file('.gif', version=1).md, timestamp4) #change the format name my_new_bibdoc.change_docformat('.gif', '.gif;icon-640') self.assertEqual(my_new_bibdoc.format_already_exists_p('.gif'), False) self.assertEqual( my_new_bibdoc.format_already_exists_p('.gif;icon-640'), True) #delete file my_new_bibdoc.delete_file('.jpg', 1) #delete file my_new_bibdoc.delete_file('.gif;icon-640', 1) #empty bibdoc self.assertEqual(my_new_bibdoc.empty_p(), True) #hidden? self.assertEqual(my_new_bibdoc.hidden_p('.jpg', version=1), False) #hide my_new_bibdoc.set_flag('HIDDEN', '.jpg', version=1) #hidden? self.assertEqual(my_new_bibdoc.hidden_p('.jpg', version=1), True) #add and get icon my_new_bibdoc.add_icon(CFG_PREFIX + '/lib/webtest/invenio/icon-test.gif', modification_date=timestamp4) my_bibrecdoc = BibRecDocs(2) value = my_bibrecdoc.get_bibdoc("new_name") self.assertEqual(value.get_icon().docid, my_new_bibdoc.get_icon().docid) self.assertEqual(value.get_icon().version, my_new_bibdoc.get_icon().version) self.assertEqual(value.get_icon().format, my_new_bibdoc.get_icon().format) #check modification time self.assertEqual(my_new_bibdoc.get_icon().md, timestamp4) #delete icon my_new_bibdoc.delete_icon() #get icon self.assertEqual(my_new_bibdoc.get_icon(), None) #delete my_new_bibdoc.delete() self.assertEqual(my_new_bibdoc.deleted_p(), True) #undelete my_new_bibdoc.undelete(previous_status='', recid=2) #expunging my_new_bibdoc.expunge() my_bibrecdoc.build_bibdoc_list() self.failIf('new_name' in my_bibrecdoc.get_bibdoc_names()) self.failUnless(my_bibrecdoc.get_bibdoc_names())
def test_BibDocs(self): """bibdocfile - BibDocs functions""" # add file my_bibrecdoc = BibRecDocs(2) timestamp1 = datetime(*(time.strptime("2011-10-09 08:07:06", "%Y-%m-%d %H:%M:%S")[:6])) my_bibrecdoc.add_new_file( CFG_PREFIX + "/lib/webtest/invenio/test.jpg", "Main", "img_test", False, "test add new file", "test", ".jpg", modification_date=timestamp1, ) my_new_bibdoc = my_bibrecdoc.get_bibdoc("img_test") value = my_bibrecdoc.list_bibdocs() self.assertEqual(len(value), 2) # get total file (bibdoc) self.assertEqual(my_new_bibdoc.get_total_size(), 91750) # get recid self.assertEqual(my_new_bibdoc.get_recid(), 2) # change name my_new_bibdoc.change_name("new_name") # get docname self.assertEqual(my_new_bibdoc.get_docname(), "new_name") # get type self.assertEqual(my_new_bibdoc.get_type(), "Main") # get id self.assert_(my_new_bibdoc.get_id() > 80) # set status my_new_bibdoc.set_status("new status") # get status self.assertEqual(my_new_bibdoc.get_status(), "new status") # get base directory self.assert_(my_new_bibdoc.get_base_dir().startswith(CFG_BIBDOCFILE_FILEDIR)) # get file number self.assertEqual(my_new_bibdoc.get_file_number(), 1) # add file new version timestamp2 = datetime(*(time.strptime("2010-09-08 07:06:05", "%Y-%m-%d %H:%M:%S")[:6])) my_new_bibdoc.add_file_new_version( CFG_PREFIX + "/lib/webtest/invenio/test.jpg", description="the new version", comment=None, format=None, flags=["PERFORM_HIDE_PREVIOUS"], modification_date=timestamp2, ) self.assertEqual(my_new_bibdoc.list_versions(), [1, 2]) # revert timestamp3 = datetime.now() time.sleep(2) # so we can see a difference between now() and the time of the revert my_new_bibdoc.revert(1) self.assertEqual(my_new_bibdoc.list_versions(), [1, 2, 3]) self.assertEqual(my_new_bibdoc.get_description(".jpg", version=3), "test add new file") # get total size latest version self.assertEqual(my_new_bibdoc.get_total_size_latest_version(), 91750) # get latest version self.assertEqual(my_new_bibdoc.get_latest_version(), 3) # list latest files self.assertEqual(len(my_new_bibdoc.list_latest_files()), 1) self.assertEqual(my_new_bibdoc.list_latest_files()[0].get_version(), 3) # list version files self.assertEqual(len(my_new_bibdoc.list_version_files(1, list_hidden=True)), 1) # display value = my_new_bibdoc.display(version="", ln="en", display_hidden=True) self.assert_(">test add new file<" in value) # format already exist self.assertEqual(my_new_bibdoc.format_already_exists_p(".jpg"), True) # get file self.assertEqual(my_new_bibdoc.get_file(".jpg", version="1").get_version(), 1) # set description my_new_bibdoc.set_description("new description", ".jpg", version=1) # get description self.assertEqual(my_new_bibdoc.get_description(".jpg", version=1), "new description") # set comment my_new_bibdoc.set_description("new comment", ".jpg", version=1) # get comment self.assertEqual(my_new_bibdoc.get_description(".jpg", version=1), "new comment") # get history assert len(my_new_bibdoc.get_history()) > 0 # check modification date self.assertEqual(my_new_bibdoc.get_file(".jpg", version=1).md, timestamp1) self.assertEqual(my_new_bibdoc.get_file(".jpg", version=2).md, timestamp2) assert my_new_bibdoc.get_file(".jpg", version=3).md > timestamp3 # delete file my_new_bibdoc.delete_file(".jpg", 2) # list all files self.assertEqual(len(my_new_bibdoc.list_all_files()), 2) # delete file my_new_bibdoc.delete_file(".jpg", 3) # add new format timestamp4 = datetime(*(time.strptime("2012-11-10 09:08:07", "%Y-%m-%d %H:%M:%S")[:6])) my_new_bibdoc.add_file_new_format( CFG_PREFIX + "/lib/webtest/invenio/test.gif", version=None, description=None, comment=None, format=None, modification_date=timestamp4, ) self.assertEqual(len(my_new_bibdoc.list_all_files()), 2) # check modification time self.assertEqual(my_new_bibdoc.get_file(".jpg", version=1).md, timestamp1) self.assertEqual(my_new_bibdoc.get_file(".gif", version=1).md, timestamp4) # delete file my_new_bibdoc.delete_file(".jpg", 1) # delete file my_new_bibdoc.delete_file(".gif", 1) # empty bibdoc self.assertEqual(my_new_bibdoc.empty_p(), True) # hidden? self.assertEqual(my_new_bibdoc.hidden_p(".jpg", version=1), False) # hide my_new_bibdoc.set_flag("HIDDEN", ".jpg", version=1) # hidden? self.assertEqual(my_new_bibdoc.hidden_p(".jpg", version=1), True) # add and get icon my_new_bibdoc.add_icon(CFG_PREFIX + "/lib/webtest/invenio/icon-test.gif", modification_date=timestamp4) value = my_bibrecdoc.list_bibdocs()[1] self.assertEqual(value.get_icon(), my_new_bibdoc.get_icon()) # check modification time self.assertEqual(my_new_bibdoc.get_icon().md, timestamp4) # delete icon my_new_bibdoc.delete_icon() # get icon self.assertEqual(my_new_bibdoc.get_icon(), None) # delete my_new_bibdoc.delete() self.assertEqual(my_new_bibdoc.deleted_p(), True) # undelete my_new_bibdoc.undelete(previous_status="") # expunging my_new_bibdoc.expunge() my_bibrecdoc.build_bibdoc_list() self.failIf("new_name" in my_bibrecdoc.get_bibdoc_names()) self.failUnless(my_bibrecdoc.get_bibdoc_names())
def process_batch_job(batch_job_file): """ Processes a batch job description dictionary @param batch_job_file: a fullpath to a batch job file @type batch_job_file: string @return: 1 if the process was successfull, 0 if not @rtype; int """ def upload_marcxml_file(marcxml): """ Creates a temporary marcxml file and sends it to bibupload """ xml_filename = 'bibencode_' + str(batch_job['recid']) + '_' + str( uuid.uuid4()) + '.xml' xml_filename = os.path.join(invenio.config.CFG_TMPSHAREDDIR, xml_filename) xml_file = file(xml_filename, 'w') xml_file.write(marcxml) xml_file.close() targs = ['-c', xml_filename] task_low_level_submission('bibupload', 'bibencode', *targs) #---------# # GENERAL # #---------# _task_write_message("----------- Handling Master -----------") ## Check the validity of the batch file here batch_job = json_decode_file(batch_job_file) ## Sanitise batch description and raise errrors batch_job = sanitise_batch_job(batch_job) ## Check if the record exists if record_exists(batch_job['recid']) < 1: raise Exception("Record not found") recdoc = BibRecDocs(batch_job['recid']) #--------------------# # UPDATE FROM MASTER # #--------------------# ## We want to add new stuff to the video's record, using the master as input if getval(batch_job, 'update_from_master'): found_master = False bibdocs = recdoc.list_bibdocs() for bibdoc in bibdocs: bibdocfiles = bibdoc.list_all_files() for bibdocfile in bibdocfiles: comment = bibdocfile.get_comment() description = bibdocfile.get_description() subformat = bibdocfile.get_subformat() m_comment = getval(batch_job, 'bibdoc_master_comment', comment) m_description = getval(batch_job, 'bibdoc_master_description', description) m_subformat = getval(batch_job, 'bibdoc_master_subformat', subformat) if (comment == m_comment and description == m_description and subformat == m_subformat): found_master = True batch_job['input'] = bibdocfile.get_full_path() ## Get the aspect of the from the record try: ## Assumes pbcore metadata mapping batch_job['aspect'] = get_fieldvalues( 124, CFG_BIBENCODE_ASPECT_RATIO_MARC_FIELD)[0] except IndexError: pass break if found_master: break if not found_master: _task_write_message("Video master for record %d not found" % batch_job['recid']) task_update_progress("Video master for record %d not found" % batch_job['recid']) ## Maybe send an email? return 1 ## Clean the job to do no upscaling etc if getval(batch_job, 'assure_quality'): batch_job = clean_job_for_quality(batch_job) global _BATCH_STEPS _BATCH_STEPS = len(batch_job['jobs']) ## Generate the docname from the input filename's name or given name bibdoc_video_docname, bibdoc_video_extension = decompose_file( batch_job['input'])[1:] if not bibdoc_video_extension or getval(batch_job, 'bibdoc_master_extension'): bibdoc_video_extension = getval(batch_job, 'bibdoc_master_extension') if getval(batch_job, 'bibdoc_master_docname'): bibdoc_video_docname = getval(batch_job, 'bibdoc_master_docname') write_message("Creating BibDoc for %s" % bibdoc_video_docname) ## If the bibdoc exists, receive it if bibdoc_video_docname in recdoc.get_bibdoc_names(): bibdoc_video = recdoc.get_bibdoc(bibdoc_video_docname) ## Create a new bibdoc if it does not exist else: bibdoc_video = recdoc.add_bibdoc(docname=bibdoc_video_docname) ## Get the directory auf the newly created bibdoc to copy stuff there bibdoc_video_directory = bibdoc_video.get_base_dir() #--------# # MASTER # #--------# if not getval(batch_job, 'update_from_master'): if getval(batch_job, 'add_master'): ## Generate the right name for the master ## The master should be hidden first an then renamed ## when it is really available ## !!! FIX !!! _task_write_message("Adding %s master to the BibDoc" % bibdoc_video_docname) master_format = compose_format( bibdoc_video_extension, getval(batch_job, 'bibdoc_master_subformat', 'master')) ## If a file of the same format is there, something is wrong, remove it! ## it might be caused by a previous corrupted submission etc. if bibdoc_video.format_already_exists_p(master_format): bibdoc_video.delete_file(master_format, 1) bibdoc_video.add_file_new_format( batch_job['input'], version=1, description=getval(batch_job, 'bibdoc_master_description'), comment=getval(batch_job, 'bibdoc_master_comment'), docformat=master_format) #-----------# # JOBS LOOP # #-----------# return_code = 1 global _BATCH_STEP for job in batch_job['jobs']: _task_write_message("----------- Job %s of %s -----------" % (_BATCH_STEP, _BATCH_STEPS)) ## Try to substitute docname with master docname if getval(job, 'bibdoc_docname'): job['bibdoc_docname'] = Template( job['bibdoc_docname']).safe_substitute( {'bibdoc_master_docname': bibdoc_video_docname}) #-------------# # TRANSCODING # #-------------# if job['mode'] == 'encode': ## Skip the job if assure_quality is not set and marked as fallback if not getval(batch_job, 'assure_quality') and getval( job, 'fallback'): continue if getval(job, 'profile'): profile = get_encoding_profile(job['profile']) else: profile = None ## We need an extension defined fot the video container bibdoc_video_extension = getval(job, 'extension', getval(profile, 'extension')) if not bibdoc_video_extension: raise Exception("No container/extension defined") ## Get the docname and subformat bibdoc_video_subformat = getval(job, 'bibdoc_subformat') bibdoc_slave_video_docname = getval(job, 'bibdoc_docname', bibdoc_video_docname) ## The subformat is incompatible with ffmpegs name convention ## We do the encoding without and rename it afterwards bibdoc_video_fullpath = compose_file(bibdoc_video_directory, bibdoc_slave_video_docname, bibdoc_video_extension) _task_write_message( "Transcoding %s to %s;%s" % (bibdoc_slave_video_docname, bibdoc_video_extension, bibdoc_video_subformat)) ## We encode now directly into the bibdocs directory encoding_result = encode_video( input_file=batch_job['input'], output_file=bibdoc_video_fullpath, acodec=getval(job, 'audiocodec'), vcodec=getval(job, 'videocodec'), abitrate=getval(job, 'videobitrate'), vbitrate=getval(job, 'audiobitrate'), resolution=getval(job, 'resolution'), passes=getval(job, 'passes', 1), special=getval(job, 'special'), specialfirst=getval(job, 'specialfirst'), specialsecond=getval(job, 'specialsecond'), metadata=getval(job, 'metadata'), width=getval(job, 'width'), height=getval(job, 'height'), aspect=getval(batch_job, 'aspect'), # Aspect for every job profile=getval(job, 'profile'), update_fnc=_task_update_overall_status, message_fnc=_task_write_message) return_code &= encoding_result ## only on success if encoding_result: ## Rename it, adding the subformat os.rename( bibdoc_video_fullpath, compose_file(bibdoc_video_directory, bibdoc_video_extension, bibdoc_video_subformat, 1, bibdoc_slave_video_docname)) #bibdoc_video._build_file_list() bibdoc_video.touch() bibdoc_video._sync_to_db() bibdoc_video_format = compose_format(bibdoc_video_extension, bibdoc_video_subformat) if getval(job, 'bibdoc_comment'): bibdoc_video.set_comment(getval(job, 'bibdoc_comment'), bibdoc_video_format) if getval(job, 'bibdoc_description'): bibdoc_video.set_description( getval(job, 'bibdoc_description'), bibdoc_video_format) #------------# # EXTRACTION # #------------# # if there are multiple extraction jobs, all the produced files # with the same name will be in the same bibdoc! Make sure that # you use different subformats or docname templates to avoid # conflicts. if job['mode'] == 'extract': if getval(job, 'profile'): profile = get_extract_profile(job['profile']) else: profile = {} bibdoc_frame_subformat = getval(job, 'bibdoc_subformat') _task_write_message("Extracting frames to temporary directory") tmpdir = invenio.config.CFG_TMPDIR + "/" + str(uuid.uuid4()) os.mkdir(tmpdir) #Move this to the batch description bibdoc_frame_docname = getval(job, 'bibdoc_docname', bibdoc_video_docname) tmpfname = ( tmpdir + "/" + bibdoc_frame_docname + '.' + getval(profile, 'extension', getval(job, 'extension', 'jpg'))) extraction_result = extract_frames( input_file=batch_job['input'], output_file=tmpfname, size=getval(job, 'size'), positions=getval(job, 'positions'), numberof=getval(job, 'numberof'), width=getval(job, 'width'), height=getval(job, 'height'), aspect=getval(batch_job, 'aspect'), profile=getval(job, 'profile'), update_fnc=_task_update_overall_status, ) return_code &= extraction_result ## only on success: if extraction_result: ## for every filename in the directorys, create a bibdoc that contains ## all sizes of the frame from the two directories files = os.listdir(tmpdir) for filename in files: ## The docname was altered by BibEncode extract through substitution ## Retrieve it from the filename again bibdoc_frame_docname, bibdoc_frame_extension = os.path.splitext( filename) _task_write_message("Creating new bibdoc for %s" % bibdoc_frame_docname) ## If the bibdoc exists, receive it if bibdoc_frame_docname in recdoc.get_bibdoc_names(): bibdoc_frame = recdoc.get_bibdoc(bibdoc_frame_docname) ## Create a new bibdoc if it does not exist else: bibdoc_frame = recdoc.add_bibdoc( docname=bibdoc_frame_docname) ## The filename including path from tmpdir fname = os.path.join(tmpdir, filename) bibdoc_frame_format = compose_format( bibdoc_frame_extension, bibdoc_frame_subformat) ## Same as with the master, if the format allready exists, ## override it, because something went wrong before if bibdoc_frame.format_already_exists_p( bibdoc_frame_format): bibdoc_frame.delete_file(bibdoc_frame_format, 1) _task_write_message("Adding %s jpg;%s to BibDoc" % (bibdoc_frame_docname, getval(job, 'bibdoc_subformat'))) bibdoc_frame.add_file_new_format( fname, version=1, description=getval(job, 'bibdoc_description'), comment=getval(job, 'bibdoc_comment'), docformat=bibdoc_frame_format) ## Remove the temporary folders _task_write_message("Removing temporary directory") shutil.rmtree(tmpdir) _BATCH_STEP = _BATCH_STEP + 1 #-----------------# # FIX BIBDOC/MARC # #-----------------# _task_write_message("----------- Handling MARCXML -----------") ## Fix the BibDoc for all the videos previously created _task_write_message("Updating BibDoc of %s" % bibdoc_video_docname) bibdoc_video._build_file_list() ## Fix the MARC _task_write_message("Fixing MARC") cli_fix_marc({}, [batch_job['recid']], False) if getval(batch_job, 'collection'): ## Make the record visible by moving in from the collection marcxml = ("<record><controlfield tag=\"001\">%d</controlfield>" "<datafield tag=\"980\" ind1=\" \" ind2=\" \">" "<subfield code=\"a\">%s</subfield></datafield></record>" ) % (batch_job['recid'], batch_job['collection']) upload_marcxml_file(marcxml) #---------------------# # ADD MASTER METADATA # #---------------------# if getval(batch_job, 'add_master_metadata'): _task_write_message("Adding master metadata") pbcore = pbcore_metadata(input_file=getval(batch_job, 'input'), pbcoreIdentifier=batch_job['recid'], aspect_override=getval(batch_job, 'aspect')) marcxml = format(pbcore, CFG_BIBENCODE_PBCORE_MARC_XSLT) upload_marcxml_file(marcxml) #------------------# # ADD MARC SNIPPET # #------------------# if getval(batch_job, 'marc_snippet'): marc_snippet = open(getval(batch_job, 'marc_snippet')) marcxml = marc_snippet.read() marc_snippet.close() upload_marcxml_file(marcxml) #--------------# # DELETE INPUT # #--------------# if getval(batch_job, 'delete_input'): _task_write_message("Deleting input file") # only if successfull if not return_code: # only if input matches pattern if getval(batch_job, 'delete_input_pattern', '') in getval(batch_job, 'input'): try: os.remove(getval(batch_job, 'input')) except OSError: pass #--------------# # NOTIFICATION # #--------------# ## Send Notification emails on errors if not return_code: if getval(batch_job, 'notify_user'): _notify_error_user( getval(batch_job, 'notify_user'), getval(batch_job, 'submission_filename', batch_job['input']), getval(batch_job, 'recid'), getval(batch_job, 'submission_title', "")) _task_write_message("Notify user because of an error") if getval(batch_job, 'notify_admin'): _task_write_message("Notify admin because of an error") if type(getval(batch_job, 'notify_admin') == type(str())): _notify_error_admin(batch_job, getval(batch_job, 'notify_admin')) else: _notify_error_admin(batch_job) else: if getval(batch_job, 'notify_user'): _task_write_message("Notify user because of success") _notify_success_user( getval(batch_job, 'notify_user'), getval(batch_job, 'submission_filename', batch_job['input']), getval(batch_job, 'recid'), getval(batch_job, 'submission_title', "")) return 1
def test_BibDocs(self): """bibdocfile - BibDocs functions""" #add file my_bibrecdoc = BibRecDocs(2) my_bibrecdoc.add_new_file(CFG_PREFIX + '/lib/webtest/invenio/test.jpg', 'Main', 'img_test', False, 'test add new file', 'test', '.jpg') my_new_bibdoc = my_bibrecdoc.get_bibdoc("img_test") value = my_bibrecdoc.list_bibdocs() self.assertEqual(len(value), 2) #get total file (bibdoc) self.assertEqual(my_new_bibdoc.get_total_size(), 91750) #get recid self.assertEqual(my_new_bibdoc.get_recid(), 2) #change name my_new_bibdoc.change_name('new_name') #get docname self.assertEqual(my_new_bibdoc.get_docname(), 'new_name') #get type self.assertEqual(my_new_bibdoc.get_type(), 'Main') #get id self.assert_(my_new_bibdoc.get_id() > 80) #set status my_new_bibdoc.set_status('new status') #get status self.assertEqual(my_new_bibdoc.get_status(), 'new status') #get base directory self.assert_(my_new_bibdoc.get_base_dir().startswith(CFG_WEBSUBMIT_FILEDIR)) #get file number self.assertEqual(my_new_bibdoc.get_file_number(), 1) #add file new version my_new_bibdoc.add_file_new_version(CFG_PREFIX + '/lib/webtest/invenio/test.jpg', description= 'the new version', comment=None, format=None, flags=["PERFORM_HIDE_PREVIOUS"]) self.assertEqual(my_new_bibdoc.list_versions(), [1, 2]) #revert my_new_bibdoc.revert(1) self.assertEqual(my_new_bibdoc.list_versions(), [1, 2, 3]) self.assertEqual(my_new_bibdoc.get_description('.jpg', version=3), 'test add new file') #get total size latest version self.assertEqual(my_new_bibdoc.get_total_size_latest_version(), 91750) #get latest version self.assertEqual(my_new_bibdoc.get_latest_version(), 3) #list latest files self.assertEqual(len(my_new_bibdoc.list_latest_files()), 1) self.assertEqual(my_new_bibdoc.list_latest_files()[0].get_version(), 3) #list version files self.assertEqual(len(my_new_bibdoc.list_version_files(1, list_hidden=True)), 1) #display value = my_new_bibdoc.display(version='', ln='en', display_hidden=True) self.assert_('>test add new file<' in value) #format already exist self.assertEqual(my_new_bibdoc.format_already_exists_p('.jpg'), True) #get file self.assertEqual(my_new_bibdoc.get_file('.jpg', version='1').get_version(), 1) #set description my_new_bibdoc.set_description('new description', '.jpg', version=1) #get description self.assertEqual(my_new_bibdoc.get_description('.jpg', version=1), 'new description') #set comment my_new_bibdoc.set_description('new comment', '.jpg', version=1) #get comment self.assertEqual(my_new_bibdoc.get_description('.jpg', version=1), 'new comment') #get history assert len(my_new_bibdoc.get_history()) > 0 #delete file my_new_bibdoc.delete_file('.jpg', 2) #list all files self.assertEqual(len(my_new_bibdoc.list_all_files()), 2) #delete file my_new_bibdoc.delete_file('.jpg', 3) #add new format my_new_bibdoc.add_file_new_format(CFG_PREFIX + '/lib/webtest/invenio/test.gif', version=None, description=None, comment=None, format=None) self.assertEqual(len(my_new_bibdoc.list_all_files()), 2) #delete file my_new_bibdoc.delete_file('.jpg', 1) #delete file my_new_bibdoc.delete_file('.gif', 1) #empty bibdoc self.assertEqual(my_new_bibdoc.empty_p(), True) #hidden? self.assertEqual(my_new_bibdoc.hidden_p('.jpg', version=1), False) #hide my_new_bibdoc.set_flag('HIDDEN', '.jpg', version=1) #hidden? self.assertEqual(my_new_bibdoc.hidden_p('.jpg', version=1), True) #add and get icon my_new_bibdoc.add_icon( CFG_PREFIX + '/lib/webtest/invenio/icon-test.gif') value = my_bibrecdoc.list_bibdocs()[1] self.assertEqual(value.get_icon(), my_new_bibdoc.get_icon()) #delete icon my_new_bibdoc.delete_icon() #get icon self.assertEqual(my_new_bibdoc.get_icon(), None) #delete my_new_bibdoc.delete() self.assertEqual(my_new_bibdoc.deleted_p(), True) #undelete my_new_bibdoc.undelete(previous_status='') #expunging my_new_bibdoc.expunge() my_bibrecdoc.build_bibdoc_list() self.failIf('new_name' in my_bibrecdoc.get_bibdoc_names()) self.failUnless(my_bibrecdoc.get_bibdoc_names())
def goto(type, document='', number=0, lang='en', modif=0): today = time.strftime('%Y-%m-%d') if type == 'SRR': ## We would like a CERN Staff Rules and Regulations recids = perform_request_search(cc='Staff Rules and Regulations', f="925__a:1996-01-01->%s 925__b:%s->9999-99-99" % (today, today)) recid = recids[-1] reportnumber = get_fieldvalues(recid, '037__a')[0] edition = int(reportnumber[-2:]) ## e.g. CERN-STAFF-RULES-ED08 return BibRecDocs(recid).get_bibdoc(make_cern_ssr_docname(lang, edition, modif)).get_file('.pdf').get_url() elif type == "OPER-CIRC": recids = perform_request_search(cc="Operational Circulars", p="reportnumber:\"CERN-OPER-CIRC-%s-*\"" % number, sf="925__a") recid = recids[-1] documents = {} bibrecdocs = BibRecDocs(recid) for docname in bibrecdocs.get_bibdoc_names(): ldocname = docname.lower() if 'implementation' in ldocname: _register_document(documents, docname, 'implementation-en') elif 'application' in ldocname: _register_document(documents, docname, 'implementation-fr') elif 'archiving' in ldocname: _register_document(documents, docname, 'archiving-en') elif 'archivage' in ldocname: _register_document(documents, docname, 'archiving-fr') elif 'annexe' in ldocname or 'annexes_fr' in ldocname: _register_document(documents, docname, 'annex-fr') elif 'annexes_en' in ldocname or 'annex' in ldocname: _register_document(documents, docname, 'annex-en') elif '_en_' in ldocname or '_eng_' in ldocname or '_angl_' in ldocname: _register_document(documents, docname, 'en') elif '_fr_' in ldocname: _register_document(documents, docname, 'fr') try: return bibrecdocs.get_bibdoc(documents[document]).get_file('.pdf').get_url() except InvenioBibDocFileError: return bibrecdocs.get_bibdoc(documents[document]).get_file('.PDF').get_url() elif type == 'ADMIN-CIRC': recids = perform_request_search(cc="Administrative Circulars", p='reportnumber:"CERN-ADMIN-CIRC-%s-*"' % number, sf="925__a") recid = recids[-1] documents = {} bibrecdocs = BibRecDocs(recid) for docname in bibrecdocs.get_bibdoc_names(): ldocname = docname.lower() if 'implementation' in ldocname: _register_document(documents, docname, 'implementation-en') elif 'application' in ldocname: _register_document(documents, docname, 'implementation-fr') elif 'archiving' in ldocname: _register_document(documents, docname, 'archiving-en') elif 'archivage' in ldocname: _register_document(documents, docname, 'archiving-fr') elif 'annexe' in ldocname or 'annexes_fr' in ldocname: _register_document(documents, docname, 'annex-fr') elif 'annexes_en' in ldocname or 'annex' in ldocname: _register_document(documents, docname, 'annex-en') elif '_en_' in ldocname or '_eng_' in ldocname or '_angl_' in ldocname: _register_document(documents, docname, 'en') elif '_fr_' in ldocname: _register_document(documents, docname, 'fr') try: return bibrecdocs.get_bibdoc(documents[document]).get_file('.pdf').get_url() except InvenioBibDocFileError: return bibrecdocs.get_bibdoc(documents[document]).get_file('.PDF').get_url()
def test_BibDocs(self): """bibdocfile - BibDocs functions""" #add file my_bibrecdoc = BibRecDocs(2) my_bibrecdoc.add_new_file(CFG_PREFIX + '/lib/webtest/invenio/test.jpg', 'Main', 'img_test', False, 'test add new file', 'test', '.jpg') my_new_bibdoc = my_bibrecdoc.get_bibdoc("img_test") value = my_bibrecdoc.list_bibdocs() self.assertEqual(len(value), 2) #get total file (bibdoc) self.assertEqual(my_new_bibdoc.get_total_size(), 91750) #get recid self.assertEqual(my_new_bibdoc.get_recid(), 2) #change name my_new_bibdoc.change_name('new_name') #get docname self.assertEqual(my_new_bibdoc.get_docname(), 'new_name') #get type self.assertEqual(my_new_bibdoc.get_type(), 'Main') #get id self.assert_(my_new_bibdoc.get_id() > 80) #set status my_new_bibdoc.set_status('new status') #get status self.assertEqual(my_new_bibdoc.get_status(), 'new status') #get base directory self.assert_( my_new_bibdoc.get_base_dir().startswith(CFG_WEBSUBMIT_FILEDIR)) #get file number self.assertEqual(my_new_bibdoc.get_file_number(), 1) #add file new version my_new_bibdoc.add_file_new_version(CFG_PREFIX + '/lib/webtest/invenio/test.jpg', description='the new version', comment=None, format=None, flags=["PERFORM_HIDE_PREVIOUS"]) self.assertEqual(my_new_bibdoc.list_versions(), [1, 2]) #revert my_new_bibdoc.revert(1) self.assertEqual(my_new_bibdoc.list_versions(), [1, 2, 3]) self.assertEqual(my_new_bibdoc.get_description('.jpg', version=3), 'test add new file') #get total size latest version self.assertEqual(my_new_bibdoc.get_total_size_latest_version(), 91750) #get latest version self.assertEqual(my_new_bibdoc.get_latest_version(), 3) #list latest files self.assertEqual(len(my_new_bibdoc.list_latest_files()), 1) self.assertEqual(my_new_bibdoc.list_latest_files()[0].get_version(), 3) #list version files self.assertEqual( len(my_new_bibdoc.list_version_files(1, list_hidden=True)), 1) #display value = my_new_bibdoc.display(version='', ln='en', display_hidden=True) self.assert_('>test add new file<' in value) #format already exist self.assertEqual(my_new_bibdoc.format_already_exists_p('.jpg'), True) #get file self.assertEqual( my_new_bibdoc.get_file('.jpg', version='1').get_version(), 1) #set description my_new_bibdoc.set_description('new description', '.jpg', version=1) #get description self.assertEqual(my_new_bibdoc.get_description('.jpg', version=1), 'new description') #set comment my_new_bibdoc.set_description('new comment', '.jpg', version=1) #get comment self.assertEqual(my_new_bibdoc.get_description('.jpg', version=1), 'new comment') #get history assert len(my_new_bibdoc.get_history()) > 0 #delete file my_new_bibdoc.delete_file('.jpg', 2) #list all files self.assertEqual(len(my_new_bibdoc.list_all_files()), 2) #delete file my_new_bibdoc.delete_file('.jpg', 3) #add new format my_new_bibdoc.add_file_new_format(CFG_PREFIX + '/lib/webtest/invenio/test.gif', version=None, description=None, comment=None, format=None) self.assertEqual(len(my_new_bibdoc.list_all_files()), 2) #delete file my_new_bibdoc.delete_file('.jpg', 1) #delete file my_new_bibdoc.delete_file('.gif', 1) #empty bibdoc self.assertEqual(my_new_bibdoc.empty_p(), True) #hidden? self.assertEqual(my_new_bibdoc.hidden_p('.jpg', version=1), False) #hide my_new_bibdoc.set_flag('HIDDEN', '.jpg', version=1) #hidden? self.assertEqual(my_new_bibdoc.hidden_p('.jpg', version=1), True) #add and get icon my_new_bibdoc.add_icon(CFG_PREFIX + '/lib/webtest/invenio/icon-test.gif') value = my_bibrecdoc.list_bibdocs()[1] self.assertEqual(value.get_icon(), my_new_bibdoc.get_icon()) #delete icon my_new_bibdoc.delete_icon() #get icon self.assertEqual(my_new_bibdoc.get_icon(), None) #delete my_new_bibdoc.delete() self.assertEqual(my_new_bibdoc.deleted_p(), True) #undelete my_new_bibdoc.undelete(previous_status='') #expunging my_new_bibdoc.expunge() my_bibrecdoc.build_bibdoc_list() self.failIf('new_name' in my_bibrecdoc.get_bibdoc_names()) self.failUnless(my_bibrecdoc.get_bibdoc_names())