def insert_inst_into_980(curdir,uid):
"""collection handling for institutes"""
user_groups = get_usergroups(uid)
if check_field_exists(curdir,"hgf_9201_"):
if read_file(curdir,"hgf_9201_") == "[]": remove_file(curdir,"hgf_9201_") # delete file in case of empty sequence! TODO: this should not happen and has to be fixed in hgfInstitutes.js
if not check_field_exists(curdir,"hgf_9201_"): #make sure that we have at least one institute
if str(uid) == "1": return #do not add institutes for admin
user_insts = extract_user_institutes("0",user_groups)
if user_insts == []:
email_txt = "%s is not assigned to any institute. This email was generated from Covert_hgf_fields and function insert_inst_into_980" %get_recordid(curdir)
send_email(CFG_SITE_ADMIN_EMAIL, CFG_SITE_ADMIN_EMAIL, "ERROR: no institute assigned", email_txt,header="",html_header="")
return #this should not happen!
jsondict = user_insts #add institute even if no institute chosen to be inserted into 980
else:
jsondict = read_json(curdir,"hgf_9201_")
inst_list = []
list_980 = read_json(curdir,"hgf_980__")
for inst in jsondict:
if {"a":inst["0"]} in list_980: continue
inst_list.append({"a":inst["0"]})
if inst_list == []: return
list_980 += inst_list
#check if users institut in 980, if not take it from user_info
if str(uid) == "1": pass # no adding of institutes into 980 for admin
else:
str_list_980 = [str(i) for i in list_980] #convert list with dicts into list with str(dicts), because python sets use list with strings
intersection_groups = set(str_list_980) & set(user_groups) # user institute not in 980 yet
intersection_vdb = set(["{'a': 'VDB'}", "{'a': 'VDBRELEVANT'}","{'a': 'VDBINPRINT'}"]) & set(str_list_980) # not vdb_relevant
if intersection_groups == set([]) and intersection_vdb == set([]): # # prevent submitting vdb irrelevant stuff for another institute
list_980 += extract_user_institutes("a",user_groups)
write_json(curdir,"hgf_980__",list_980)
def handle_0247(curdir):
""" Handle persistend identifiers in 0247_. This implies to set $2
to source and $a to value. only in case of user input
Note: if we get new PIDs that should be handled we need to adopt
this function!"""
if check_field_exists(curdir,"hgf_0247_"):
listdict_ = read_json(curdir,"hgf_0247_")
else: listdict_ = []
if check_field_exists(curdir,"hgf_0247_a2pat"): # Patent
text = read_file(curdir,"hgf_0247_a2pat")
listdict_.append({"2":"Patent","a":text})
if check_field_exists(curdir,"hgf_0247_a2urn"): # URN
text = read_file(curdir,"hgf_0247_a2urn")
listdict_.append({"2":"URN","a":text})
if check_field_exists(curdir,"hgf_773__a"): # store DOI in both 773__ and in 0247, this is an input field
text = read_file(curdir,"hgf_773__a")
listdict_.append({"2":"doi","a":text})
if (not check_field_exists(curdir,"hgf_773__a") and check_field_exists(curdir,"hgf_773__")): # doi can be stored in 773__ as json array
dict_773 = read_json(curdir,"hgf_773__")
for ent in dict_773: #more then 1 DOI
if not "a" in ent.keys(): continue
listdict_.append({"2":"doi","a":ent["a"]})
if listdict_ == []: return
new_listdict = []
for dict in listdict_:
if dict in new_listdict: continue # remove double entries
new_listdict.append(dict)
write_json(curdir,"hgf_0247_",new_listdict)
#Insert DOI into 773__a only in case no 773__a or 773 json array exist
if check_field_exists(curdir,"773__a"): return #we have a 773__a
if check_field_exists(curdir,"773__"):
listdict_773 = read_json(curdir,"773__")
for ent in listdict_773:
if ent.has_key("a"): return # we have a 773__a
for ent in new_listdict:
if not ent.has_key("2"): continue
if not (ent["2"] == "doi"): continue
# map doi into 773__a
# write DOI in 773__a if we do not yet have one.
# in case of multiple DOIs the first one will win <--> we cannot
# write the 773__ because we do not know if other 773__* fields
# has been inputted and to which belongs the DOI. TODO!
write_file(curdir,"hgf_773__a",ent["a"])
return
def handle_institutes_for_modify(curdir):
"""in case of non-vdb entry we do not have 9201_, but the institutes are stored in 980 as collections. so, prefill the 9201_ from 980"""
_980 = read_json(curdir,"hgf_980__")
jsondict_9201 = []
if check_field_exists(curdir,"hgf_9201_"):
jsondict_9201 = read_json(curdir,"hgf_9201_")
if jsondict_9201 == []:
inst_dict = {"9201_":[]}
for _dict in _980:
if not "a" in _dict.keys(): continue
if len(perform_request_search(p='id:"' + _dict["a"] +'"' ,cc='Institutes')) < 1 : continue #make sure institute exists
inst_dict["9201_"].append({"0":_dict["a"]})
write_json(curdir,"hgf_9201_",inst_dict["9201_"])
def add_jsondict(self,fieldname):
"""add field structure (json structure)"""
marcfield = fieldname.replace("hgf_","")
jsondict = {marcfield:read_json(self.curdir,fieldname)}
for key in jsondict.keys():
# TODO Check if this code is really obsolete
# Should be solved by our new initial ordering of the fields.
#
#if key in self.data.keys():
#datafield already exists
# if isinstance(jsondict[key],list):
# if len(jsondict[key]) != 1: # repeatable field
# self.data[key] = jsondict[key]
# return # jsondict has repeatable field, but we cannot add a single field to repeatable fields --> we delete the single input field and override it with json-structure
# else: #non repeatable
# if len(self.data[key]) != 1: #we have repeatable non-json field-->override with json structure
# self.data[key] = jsondict[key]
# else:
# jsondict[key][0].update(self.data[key][0]) #merge non-json with json structure
# self.data[key] = jsondict[key][0] # add all fields to dictionary
#else:
if isinstance(jsondict[key],list): self.data[key] = jsondict[key] #value is already a list
elif isinstance(jsondict[key],str): #value is string
dta = eval(jsondict[key]) #TODO: eval is evil!
if isinstance(dta,list): self.data[key] = dta #value is a list in a string
else: self.data[key] = [dta] #value is just a string
else: return
def prefill_gender(curdir):
"""prefill gender. normally the radio,checkboxes are prefilled by the Create_modify_interface_hgf, but that field is part of hgf_1001_ technical field, which is by default not a checkbox or radio botton"""
if not check_field_exists(curdir,"hgf_1001_"): return # no author
jsonlist = read_json(curdir,"hgf_1001_")
for jsondict in jsonlist: # there can be only one first author --> len(jsonlist ==1)
if not jsondict.has_key("g"): continue
write_file(curdir,"hgf_1001_g",jsondict["g"])
#delete gender information from technical field and store info in 1001_g
del jsondict["g"]
write_json(curdir,"hgf_1001_",[jsondict])
def prefill_245(curdir):
"""prefill 245__a and 245__f as simple input fields. only 245__h (publication form) into 245__"""
jsonlist = read_json(curdir,"hgf_245__")
pubforms = []
for jsondict in jsonlist:
if jsondict.has_key("h"):
pubformnames = jsondict["h"].split(",") #split multiple pubforms
for i,pubform in enumerate(pubformnames): pubforms.append({"h":pubform,"x":i})
for key in jsondict.keys(): #we have some Input fields
filename = "hgf_245__" + key
# encode jsondict[key] to utf-8 to handle utf-chars.
# TODO Tomek, why does it seem that we need this only here and
# not for any other field?
write_file(curdir,filename,jsondict[key].encode('utf-8'))
write_json(curdir,"hgf_245__",pubforms)
def handle_url(curdir):
if check_field_exists(curdir,"hgf_8564_"):
remove_file(curdir,"hgf_8564_u")
jsondict_list = read_json(curdir,"hgf_8564_")
#only one URL can be submitted/modified. bibedit urls die ;)
for i in jsondict_list:
if not i.has_key("u"): continue # no subfield u detected
if CFG_SITE_URL in i["u"]: continue # skip internal file
write_file(curdir,"hgf_8564_u",i["u"])
remove_file(curdir,"hgf_8564_")
return # write only one URL
if check_field_exists(curdir,"hgf_8564_u"):
text = read_file(curdir,"hgf_8564_u")
if CFG_SITE_URL in text: remove_file(curdir,"hgf_8564_u") #skip internal file
def filter_980(curdir):
if not check_field_exists(curdir,"hgf_980__"): return []
coll_list = read_json(curdir,"hgf_980__")
doctype_tuples = get_docid_docname_alldoctypes()
doctype_collections = []
[doctype_collections.append(tup[0]) for tup in doctype_tuples]
filter_collections = doctype_collections + get_technical_collections()
json_filter_collections = [{"a":collection} for collection in filter_collections]
old_collections = []
for coll in coll_list:
if not coll.has_key("a"): continue #drop all "foreign" collections
if coll in json_filter_collections: continue #drop workflow collections
if len(perform_request_search(p='id:"' + coll["a"] +'"' ,cc='Institutes')) >0: continue #drop institutes
old_collections.append(coll)
return old_collections
def handle_980(curdir):
new_list = []
list_980 = read_json(curdir,"hgf_980__")
doctype = read_file(curdir,"doctype")
old_index = list_980.index({"a":doctype})
list_980.insert(0, list_980.pop(old_index)) #move original doctype to be first entry in 980 list, needed by invenio (more likely a bug)
#remove double entries
for _dict in list_980:
if _dict in new_list: continue # remove double entries
new_list.append(_dict)
if check_field_exists(curdir,"hgf_delete"): new_list.append({"c":"DELETED"}) # user wants to delete this record
new_list = set_restriction(new_list) # #set UNRESTRICTED if 980 collection appears in CFG_PUBLIC_COLLECTIONS
write_json(curdir,"hgf_980__",new_list)
def handle_1001(curdir): """add gender to 1001_ technical field: 1001_ contains a list of a single dict with the name of the first author. gender should be applied to that one (we use gender only for phd-like entries), so we add it to the end of the dict. NOTE: for further processing the newly written technical field must not contain a real JSON structure, but again only this list of a single hash. """ if not check_field_exists(curdir,"hgf_1001_g"): return # no gender set if check_field_exists(curdir,"hgf_1001_"): jsondict = read_json(curdir,"hgf_1001_") gender = read_file(curdir,"hgf_1001_g") jsondict[0]["g"] = gender # 100 contains only one person write_json(curdir,"hgf_1001_",jsondict)
def add_FFT(curdir):
"""
!!!move_files_to_storage, move files to done have to be deleted from websubmit function!!!
add FFT tag into record
if this function is used: the functions stamp_uploaded_files should not be used in the websubmit anymore
"""
if not check_field_exists(curdir,"hgf_file"): return None # no file submitted
fulltext_filename = read_file(curdir,"hgf_file")
fulltext_path = os.path.join(curdir,"files","hgf_file",fulltext_filename)
if not os.path.exists(fulltext_path): return None # no file named in hgf_file in files directory. something is wrong..
if os.path.getsize(fulltext_path) == 0: #check file size
#send email
#first get the url record link
if not check_field_exists(curdir,"SN"): return None # no recid-->something is wrong..
recid = get_recordid(curdir)
rec_url = CFG_SITE_URL + "/record/" + recid
#create email
email_txt = 'Dear Sir or Madam, \n\nAn empty file has been submitted for the record: %s\n\nProbably it was caused, because the file has been deleted from its directory before final submission into %s !!!\nIt is possible, that the record itself is not available, when this email was sent, but it should be processed within minutes. Once this is finished you may add the fulltext by accessing %s and using "modify record" link \n\n' %(rec_url,CFG_SITE_NAME,rec_url)
email_subject = 'File submission incomplete!!!'
#email check
if check_field_exists(curdir,"SuE"): email_to = read_file(curdir,"SuE") # get email from submitter
else: email_to = CFG_SITE_ADMIN_EMAIL # in case somehow no email of submitter exists, send email to admin
send_email(CFG_SITE_ADMIN_EMAIL, email_to, email_subject, email_txt,copy_to_admin=CFG_WEBSUBMIT_COPY_MAILS_TO_ADMIN,header="",html_header="")
return None #cancel file submission (the submitter has already been informed via email), the original submission will be processed.
inst_dict_list = read_json(curdir,"hgf_9201_") #read in institutes
inst_list = []
restriction = "firerole: allow groups 'STAFF'" # staff is always
# add the institutes id and append the external auth info as this
# builds the actual group name we need to allow here.
for inst in inst_dict_list: restriction += ",'" + inst["0"] + ' ['+CFG_EXTERNAL_AUTH_DEFAULT+']' + "'" # TODO: multiple authentifications
filename = read_file(curdir,"hgf_file")
file_path = os.path.join(curdir,"files","hgf_file",filename)
if not check_field_exists(curdir,"rn"): return
rn = read_file(curdir,"rn")
#fill subfields for FFT
fft_dict = {}
fft_dict["a"] = file_path
fft_dict["n"] = rn
fft_dict["r"] = restriction
write_json(curdir,"hgf_FFT__",fft_dict)
def prefill_0247(curdir):
"""prefill URN, Patent"""
if check_field_exists(curdir,"hgf_0247_"): #json structure
jsonlist = read_json(curdir,"hgf_0247_")
for jsondict in jsonlist:
if not jsondict.has_key("2"): continue
if jsondict["2"] == "Patent":
write_file(curdir,"hgf_0247_a2pat",jsondict["a"])
elif jsondict["2"] == "URN":
write_file(curdir,"hgf_0247_a2urn",jsondict["a"])
if check_field_exists(curdir,"hgf_0247_2"):
if check_field_exists(curdir,"hgf_0247_a"):
subfield_2 = read_file(curdir,"hgf_0247_2")
subfield_a = read_file(curdir,"hgf_0247_a")
if subfield_2 == "Patent":
write_file(curdir,"hgf_0247_a2pat",subfield_a)
elif subfield_2 == "URN":
write_file(curdir,"hgf_0247_a2urn",subfield_a)
def handle_245(curdir):
"""245__a: title and 245__f:date -->text input fields
245__h:publication form --> autosuggest
We need to read in 245__ (if exists) and add 245__a and 245__f in json format
"""
date,title = "",""
# Title is special: we have non-structured input fields by default
# where $f (date) needs a special handling plus we have a structured
# input field from the possible token input of media type (AC's
# request) => we have to assemble the structured field from it's
# parts, and then re-store it as structure to a file, then the
# follwoing workflow can transparently handle it as if it was passed
# by a structure in the first place.
# Get unstructured stuff
if check_field_exists(curdir,"hgf_245__a"): title = read_file(curdir,"hgf_245__a")
if check_field_exists(curdir,"hgf_245__f"): date = read_file(curdir,"hgf_245__f")
# Initialize the structure
jsondict = {}
jsondict['245__'] = {}
dict = {}
# Try to get what we have already in the structure as such
if check_field_exists(curdir,"hgf_245__"):
jsondict = read_json(curdir,"hgf_245__")
# in case of multiple publication forms (???, should be non repeatable, but just in case: create seperated comma string)
pubforms = []
for pubform in jsondict:
if 'h' in pubform:
pubforms.append(pubform["h"])
pubstring = ", ".join(pubforms)
if pubstring == "": jsondict = {}
else: jsondict = {"h":pubstring}
# Add unstructured fields, if they exist
if not title == "": jsondict["a"] = title
if not date == "": jsondict["f"] = date
# Write the full structured file
write_json(curdir,"hgf_245__",jsondict)
def insert_3367(curdir):
"""get doctype from authorities and create 3367 and set our ddoctypes into 980 """
doctype = read_file(curdir,"doctype")
access = read_file(curdir,"access") #submission id
subtype = ''
try:
# Check if we have a refinement of the doctype. Usually we have
# this only for talks which could be "Invited" or whatever. If so,
# add it to 3367_$x
subtype = read_file(curdir,"hgf_3367_x")
except:
# Usually, we do not have refinements.
pass
doctype_dict = get_pubtype_info(doctype)
if doctype_dict == {}:
doctype_dict_list = [{"m":doctype}] #no authority
# Run over the dictionary and build up a list of all document types.
# Note that not all document types have to be hgf-types, they may as
# well stem from other vocabularies (DINI/DRIVER...)
else:
doctype_dict_list = handle_list_of_doctype_dict(doctype_dict,access,doctype,subtype)
doctype_dict_list = add_reportdoctype(curdir,doctype, doctype_dict_list) #add intrep doctype
doctype_dict_list = add_journaldoctype(curdir,doctype, doctype_dict_list) #add journal doctype
doctype_dict_list = add_bookdoctype(curdir,doctype, doctype_dict_list) #add book doctype
doctype_dict_list = add_procdoctype(curdir,doctype, doctype_dict_list) #add proc doctype
if check_field_exists(curdir,"hgf_980__"):
list_980 = read_json(curdir,"hgf_980__")
else: list_980 = []
# Only add our own doctypes to 980 (ie collections and not DINI/DRIVER)
for dict in doctype_dict_list:
try:
if {"a":dict["m"]} in list_980: continue
list_980.append({"a":dict["m"]})
except:
pass
write_json(curdir,"hgf_980__",list_980)
write_json(curdir,"hgf_3367_",doctype_dict_list)