def restDOI(DOI, code_srcDir, restbase, sheet_sample, user, runCtx):
message = ''
new_dataset = False
response = None
# check existence
try:
# dsurl = restbase + '/nmr/dataset?doi='+DOI
dsurl = restbase + '/nmr/dataset?id=' + runCtx['datasetId']
rq = urllib.request.Request(dsurl)
# j = json.loads(urllib.request.urlopen(rq, context=ssl._create_unverified_context()).read().decode('utf-8'))
nmCurate = nm_rest(logging, runCtx['sysToken'],
runCtx['curateApiToken'],
runCtx['curateRefreshToken'], rq)
#j = json.loads(nmCurate.urlopen(None).read().encode("utf8")) # no data for GET request
rv = nmCurate.urlopen(None)
j = json.loads(rv.read().decode('utf8')) # no data for GET request
if len(j["data"]) > 0:
if j["data"][0]['doi'] == runCtx['nm_dataset_initial_doi']:
new_dataset = True
response = j["data"][0]
else:
message += 'Expected to find datasetId: ' + runCtx[
'datasetId'] + ', but no data found.'
logging.error(message)
except:
message += 'exception occurred during dataset GET by datasetId: ' + runCtx[
'datasetId'] + '\n'
# message += 'exception occurred during dataset GET by doi\n'
message += 'exception: ' + str(traceback.format_exc()) + '\n'
logging.error('exception: ' + str(traceback.format_exc()))
if message != '':
return (None, message)
# if doi doesn't exist, ds-create
## NOTE: should not create datasets anymore. The datasetId in the job_parameters should already exist
## However, initially, the dataset is mostly empty, so the data must still be set up
dsInfo = response
if new_dataset:
# special case, special issue madeup DOI
if 'ma-SI' in DOI: # DOI from spreadsheet is special issue
logging.info('DOI from spreadsheet: ' + DOI + ' dsInfo: ' +
json.dumps(dsInfo))
DOI = 'unpublished-' + DOI
dsInfo['doi'] = DOI
# generate ds_data for special issue by reading the Excel
dsInfo = specialIssueRest(sheet_sample, dsInfo)
else: # DOI from spreadsheet is standard DOI
# now fetch the metadata using doi-crawler
crawlerDict = mainDOIsoupFirst(DOI, code_srcDir)
# check if doi valid
if len(crawlerDict) == 0:
message += '[DOI Error] Please check the reported DOI, it seems that DOI does not exist.\n'
return (None, message)
dsInfo = mapToRest(dsInfo, crawlerDict)['dsUpdate']
# return response at the end, if response is not None, message will be ''
return (dsInfo, message)
"resultpage":
webBaseUri + '/nm#/OtsuResult?refuri=' + jobDataUriSuffix +
'/' + jobId,
"jobinfo": {
"resultcode": rc
},
"user":
userId
}
}
print('email data: %s' % emaildata)
logging.info('emaildata: ' + json.dumps(emaildata))
rq = urllib.request.Request(emailurl)
logging.info('request created using emailurl')
rq.add_header('Content-Type', 'application/json')
nmEmail = nm_rest(logging, sysToken, emailApiToken, emailRefreshToken,
rq)
#r = urllib2.urlopen(rq, json.dumps(emaildata))
r = nmEmail.urlopen(json.dumps(emaildata).encode("utf8"))
logging.info('sent success email: ' + str(r.getcode()))
except:
logging.info('exception occurred sending run_otsu success email')
logging.info('exception: ' + traceback.format_exc())
else: # send error email
try:
logging.info('emailurl: ' + emailurl)
emaildata = {
"jobid": jobId,
"jobtype": jobType,
"emailtemplatename": "failure",
"emailvars": {
"jobinfo": {
def uploadFilesAndAdjustXMLImageRefs(jobDir, schemaId, xmlId):
imageFiles = [
] # track image files uploaded -- which go into default bucket (returned id is substituted back into XML)
# all other files go into 'inputfiles' bucket with filename {schemaId}/{xmlId}/filename (returned ID is not stored)
# for now let exceptions bubble up to handler in caller
restbase = os.environ['NM_LOCAL_REST_BASE']
webbase = os.environ['NM_WEB_BASE_URI']
curateFilesUrl = restbase + '/nmr/blob'
sysToken = os.environ['NM_AUTH_SYSTEM_TOKEN']
curateApiToken = os.environ['NM_AUTH_API_TOKEN_CURATE']
curateRefreshToken = os.environ['NM_AUTH_API_REFRESH_CURATE']
xmlName = jobDir + '/xml/' + xmlId + '.xml'
xmlTree = ET.parse(xmlName)
updatedXml = False
for f in xmlTree.findall('.//MICROSTRUCTURE/ImageFile/File'):
fn = f.text.split('/')[-1]
imageFiles.append(fn)
fullImageFileName = jobDir + '/' + fn
logging.debug('uploading: ' + fullImageFileName)
dataUri = DataURI.from_file(fullImageFileName)
dataUri = dataUri.replace("\n", "") # remove line feeds
## logging.debug('dataUri: ' + dataUri)
curatefiledata = '{"filename":"' + fn + '","dataUri":"' + dataUri + '"}'
# logging.debug(curatefiledata + ' len is: ' + str(len(curatefiledata)))
curatefiledata = json.loads(curatefiledata)
rq = urllib2.Request(curateFilesUrl)
logging.debug('request created using createFilesUrl')
rq.add_header('Content-Type', 'application/json')
nmCurateFiles = nm_rest(logging, sysToken, curateApiToken,
curateRefreshToken, rq)
r = nmCurateFiles.urlopen(json.dumps(curatefiledata).encode("utf8"))
if r.getcode() == 201:
uploadId = json.loads(r.read())['data']['id']
imageRef = webbase + '/nmr/blob?id=' + uploadId
logging.debug('new image value for XML: ' + imageRef)
f.text = imageRef # update XML node with new image reference
## testing -- raise ValueError('Upload successful. returned id: ' + uploadId) ## for testing
updatedXml = True
else:
raise ValueError('Unexpected return code from image upload (' +
fn + '): ' + str(r.getcode()))
if updatedXml == True:
# update the XML in the file
xmlTree.write(xmlName)
# get a list of all files in jobDir and upload them
dataFiles = os.listdir(jobDir)
for f in dataFiles:
fn = jobDir + '/' + f
if os.path.isfile(
fn
) and f not in imageFiles: # make sure it's a regular file and wasn't already uploaded as an image
dataUri = DataURI.from_file(fn)
dataUri = dataUri.replace("\n", "") # remove line feeds
objFN = schemaId + '/' + xmlId + '/' + f
curatefiledata = '{"filename":"' + objFN + '","bucketName":"curateinput","dataUri":"' + dataUri + '"}'
# logging.debug(curatefiledata + ' len is: ' + str(len(curatefiledata)))
curatefiledata = json.loads(curatefiledata)
rq = urllib2.Request(curateFilesUrl)
logging.debug('request created using createFilesUrl')
rq.add_header('Content-Type', 'application/json')
nmCurateFiles = nm_rest(logging, sysToken, curateApiToken,
curateRefreshToken, rq)
r = nmCurateFiles.urlopen(
json.dumps(curatefiledata).encode("utf8"))
if r.getcode() == 201:
uploadId = json.loads(r.read())['data']['id']
logging.debug('uploaded file ID: ' + uploadId)
## testing - raise ValueError('Upload of input successful. returned id: ' + uploadId) ## for testing
else:
raise ValueError('Unexpected return code from file upload (' +
objFN + '): ' + str(r.getcode()))
def conversion(jobDir, code_srcDir, xsdDir, templateName, user):
restbase = os.environ['NM_LOCAL_REST_BASE']
xsdFilename = xsdDir.split("/")[-1]
# initialize messages
messages = []
# check #1: extension of templateName
if os.path.splitext(templateName)[1] not in {'.xlsx', '.xls'}:
messages = [
'[Upload Error] The Excel template file should have extensions like ".xlsx" or ".xls".'
]
return ('failure', messages)
# get the ID
runEVI(jobDir, code_srcDir, templateName, restbase, user)
# check #2: see if ID conversion is successful
if not os.path.exists(jobDir + '/ID.txt'):
if os.path.exists(jobDir + '/error_message.txt'):
with open(jobDir + '/error_message.txt', 'r+') as f:
error_message = f.read()
# read the error message
messages_raw = error_message.split('\n')
for message in messages_raw:
if len(message.strip()) > 0:
messages.append(message.strip())
return ('failure', messages)
messages += [
'[Conversion Error] Failed to assemble the ID. Please check or contact the administrator!'
]
return ('failure', messages)
else:
with open(jobDir + '/ID.txt', 'r') as f:
ID = f.read()
# kickoff the conversion script
logName = compiler(jobDir, code_srcDir, xsdDir, templateName, restbase)
# check #3: see if there is an error_message.txt generated during conversion
if os.path.exists(jobDir + '/error_message.txt'):
with open(jobDir + '/error_message.txt', 'r+') as f:
error_message = f.read()
# read the error message
messages_raw = error_message.split('\n')
for message in messages_raw:
if len(message.strip()) > 0:
messages.append(message.strip())
# check #4: check the schema validation results
with open(logName, 'r', encoding='utf-8') as f:
vld_log = csv.DictReader(f)
for row in vld_log:
logging.info(str(row))
if ID in row['\ufeffxml directory']:
if "the atomic type 'xs:double'" in row['error']:
messages.append('[XML Schema Validation Error] ' +
row['error'].strip() +
', should be a number.')
else:
messages.append('[XML Schema Validation Error] ' +
row['error'].strip())
if len(messages) > 0:
return ('failure', messages)
# check #5: upload and check if the uploading is successful
try:
sysToken = os.environ['NM_AUTH_SYSTEM_TOKEN']
curateApiToken = os.environ['NM_AUTH_API_TOKEN_CURATE']
curateRefreshToken = os.environ['NM_AUTH_API_REFRESH_CURATE']
# rest call for schemaID
schemaurl = restbase + '/nmr/templates/select?filename=' + xsdFilename
rq = urllib.request.Request(schemaurl)
j = json.loads(
urllib.request.urlopen(
rq, context=ssl._create_unverified_context()).read().decode(
"utf-8"))
schemaId = j["data"][0]["_id"]
# upload input files and image files referenced in xml
uploadFilesAndAdjustXMLImageRefs(jobDir, schemaId, ID)
# curate-insert rest call
with open(jobDir + "/xml/" + ID + ".xml", "r") as f:
content = f.read()
curate_insert_url = restbase + '/nmr/curate'
curate_data = {
"userid": user,
"title": ID + ".xml",
"schemaId": schemaId,
"curatedDataState":
"Valid", # Valid causes ingest to kick of next. Also, it has passed validation.
"ispublished": "false",
"isPublic": "false",
"content": content
}
rq = urllib.request.Request(curate_insert_url)
# logging.info('request created using curate_insert_url')
rq.add_header('Content-Type', 'application/json')
## r = urllib.request.urlopen(rq, json.dumps(curate_data), context=ssl._create_unverified_context())
nmCurateFiles = nm_rest(logging, sysToken, curateApiToken,
curateRefreshToken, rq)
r = nmCurateFiles.urlopen(json.dumps(curate_data).encode("utf8"))
# logging.info('curate insert request posted: ' + str(r.getcode()))
except:
messages.append('exception occurred during curate-insert')
messages.append('exception: ' + str(traceback.format_exc()))
if len(messages) > 0:
return ('failure', messages)
# pass all the checks
return ('success', messages)
def uploadFilesAndAdjustXMLImageRefs(jobDir, schemaId, xmlId, runCtx):
imageFiles = [
] # track image files uploaded -- which go into default bucket (returned id is substituted back into XML)
# all other files go into 'curateinput' bucket with filename {schemaId}/{xmlId}/filename (returned ID is not stored)
# for now let exceptions bubble up to handler in caller
restbase = runCtx['restbase']
webbase = runCtx['webbase']
curateFilesUrl = restbase + '/nmr/blob/create'
blobs = []
datasetId = runCtx['datasetId']
# get a list of all files in jobDir and upload them
dataFiles = os.listdir(jobDir)
for f in dataFiles:
fn = jobDir + '/' + f
# changed following check to save imageFiles into curateinput bucket as well since not doing so causes downstream issue
if os.path.isfile(
fn
): ## and f not in imageFiles: # make sure it's a regular file and wasn't already uploaded as an image
dataUri = DataURI.from_file(fn)
dataUri = dataUri.replace("\n", "") # remove line feeds
## objFN = schemaId + '/' + xmlId + '/' + f # change to datasetid/xmlid/filename
## objFN = datasetId + '/' + xmlId + '/' + f
objFN = f
# curatefiledata = '{"filename":"'+ objFN + '","bucketName":"curateinput", "dataUri":"' + dataUri + '"}'
curatefiledata = '{"filename":"' + objFN + '","dataUri":"' + dataUri + '","originalDatasetId":"' + datasetId + '"}'
# logging.debug(curatefiledata + ' len is: ' + str(len(curatefiledata)))
curatefiledata = json.loads(curatefiledata)
rq = urllib.request.Request(curateFilesUrl)
logging.debug('request created using createFilesUrl')
rq.add_header('Content-Type', 'application/json')
nmCurateFiles = nm_rest(logging, runCtx['sysToken'],
runCtx['curateApiToken'],
runCtx['curateRefreshToken'], rq)
r = nmCurateFiles.urlopen(
json.dumps(curatefiledata).encode("utf8"))
if r.getcode() == 201:
uploadId = json.loads(r.read().decode("utf-8"))['data']['id']
logging.debug('uploaded file ID: ' + uploadId)
content_type = mime.Types.of(objFN)[0].content_type
blob_info = {
'type': 'blob',
'id': uploadId,
'metadata': {
'filename': objFN,
'contentType': content_type
}
}
if runCtx['excelTemplateName'] == objFN:
blob_info['metadata']['is_completed_pnc_template'] = True
blobs.append(blob_info)
## testing - raise ValueError('Upload of input successful. returned id: ' + uploadId) ## for testing
else:
raise ValueError('Unexpected return code from file upload (' +
objFN + '): ' + str(r.getcode()))
# TODO remove this block after testing
#xmlTitleRe = re.compile('^[A-Z]([0-9]+)[_][S]([0-9]+)[_][\S]+[_]\d{4}([.][Xx][Mm][Ll])?$')
#reMatch = xmlTitleRe.match(xmlId) ## TODO handle invalid title i.e. reMatch == None
#if reMatch == None:
# logging.error('xmlId (title) does not match expected format: ' + xmlId + ' (match was None)')
#datasetId = reMatch.group(1)
xmlName = jobDir + '/xml/' + xmlId + '.xml'
xmlTree = etree.parse(xmlName)
updatedXml = False
for f in xmlTree.findall('.//MICROSTRUCTURE/ImageFile/File'):
fn = f.text.split('/')[-1]
blob = None
for b in blobs:
bfn = b['metadata']['filename']
if bfn == fn:
blob = b
if blob == None:
raise ValueError('Unable to match xml: ' + xmlId +
' MICROSTRUCTURE image filename: ' + fn +
' with an uploaded image name.')
else:
imageRef = webbase + '/nmr/blob?id=' + blob['id']
logging.debug('new image value for XML: ' + imageRef)
f.text = imageRef # update XML node with new image reference
## testing -- raise ValueError('Upload successful. returned id: ' + uploadId) ## for testing
updatedXml = True
if updatedXml == True:
# update the XML in the file
xmlTree.write(xmlName)
return blobs
def conversion(jobDir, code_srcDir, xsdDir, templateName, user, datasetId):
restbase = os.environ['NM_LOCAL_REST_BASE']
sysToken = os.environ['NM_AUTH_SYSTEM_TOKEN']
curateApiToken = os.environ['NM_AUTH_API_TOKEN_CURATE']
curateRefreshToken = os.environ['NM_AUTH_API_REFRESH_CURATE']
nm_dataset_initial_doi = os.environ['NM_DATASET_INITIAL_DOI']
webbase = os.environ['NM_WEB_BASE_URI']
xsdFilename = xsdDir.split("/")[-1]
runCtx = {
'nm_dataset_initial_doi': nm_dataset_initial_doi,
'datasetId': datasetId,
'schemaName': xsdFilename,
'schemaId': '',
'restbase': restbase,
'webbase': webbase,
'sysToken': sysToken,
'curateApiToken': curateApiToken,
'curateRefreshToken': curateRefreshToken,
'excelTemplateName': templateName,
'xmlID': '',
'user': user
}
# initialize messages
messages = []
try:
# rest call for schemaID
schemaurl = restbase + '/nmr/templates/select?filename=' + xsdFilename
rq = urllib.request.Request(schemaurl)
j = json.loads(
urllib.request.urlopen(
rq, context=ssl._create_unverified_context()).read().decode(
"utf-8"))
schemaId = j["data"][0]["_id"]
runCtx['schemaId'] = schemaId
except:
messages.append('Exception looking up schema information')
return ('failure', messages)
# check #1: extension of templateName
if os.path.splitext(templateName)[1] not in {'.xlsx', '.xls'}:
messages = [
'[Upload Error] The Excel template file should have extensions like ".xlsx" or ".xls".'
]
return ('failure', messages)
# get the ID
(dsInfo, dsMessage) = runEVI(jobDir, code_srcDir, templateName, restbase,
user, runCtx)
if dsInfo == None:
messages.append(dsMessage)
return ('failure', messages)
# check #2: see if ID conversion is successful
if not os.path.exists(jobDir + '/ID.txt'):
if os.path.exists(jobDir + '/error_message.txt'):
with open(jobDir + '/error_message.txt', 'r+') as f:
error_message = f.read()
# read the error message
messages_raw = error_message.split('\n')
for message in messages_raw:
if len(message.strip()) > 0:
messages.append(message.strip())
return ('failure', messages)
messages += [
'[Conversion Error] Failed to assemble the ID. Please check or contact the administrator!'
]
return ('failure', messages)
else:
with open(jobDir + '/ID.txt', 'r') as f:
ID = f.read()
# kickoff the conversion script
logName = compiler(jobDir, code_srcDir, xsdDir, templateName, restbase,
runCtx)
# check #3: see if there is an error_message.txt generated during conversion
if os.path.exists(jobDir + '/error_message.txt'):
with open(jobDir + '/error_message.txt', 'r+') as f:
error_message = f.read()
# read the error message
messages_raw = error_message.split('\n')
for message in messages_raw:
if len(message.strip()) > 0:
messages.append(message.strip())
# check #4: check the schema validation results
with open(logName, 'r', encoding='utf-8') as f:
vld_log = csv.DictReader(f)
for row in vld_log:
logging.info(str(row))
if ID in row['\ufeffxml directory']:
if "the atomic type 'xs:double'" in row['error']:
messages.append('[XML Schema Validation Error] ' +
row['error'].strip() +
', should be a number.')
else:
messages.append('[XML Schema Validation Error] ' +
row['error'].strip())
if len(messages) > 0:
return ('failure', messages)
# check #5: call the ChemProps API and add in the standardized chemical names, uSMILES and density
try:
xmlName = jobDir + "/xml/" + ID + ".xml"
xmlTree = etree.parse(xmlName)
# collect all MatrixComponent/ChemicalName and FillerComponent/ChemicalName packages
matcomps = xmlTree.findall('.//MatrixComponent')
filcomps = xmlTree.findall('.//FillerComponent')
# for each package, send to ChemProps API
chemprops_api_url = restbase + '/api/v1/chemprops'
# MatrixComponent
for matcomp in matcomps:
chemprops_data = {
"polfil":
"pol",
"ChemicalName":
matcomp.findtext('ChemicalName'),
"Abbreviation":
'' if matcomp.findtext('Abbreviation') is None else
matcomp.findtext('Abbreviation'),
"TradeName":
'' if matcomp.findtext('TradeName') is None else
matcomp.findtext('TradeName'),
"uSMILES":
'' if matcomp.findtext('uSMILES') is None else
matcomp.findtext('uSMILES'),
"nmId":
ID
}
# chemprops_rq = urllib.request.Request(chemprops_api_url)
logging.debug(
'request created for ChemProps using chemprops_api_url')
logging.debug('Searching polymer package: ' +
json.dumps(chemprops_data)) #.encode("utf8"))
# chemprops_rq.add_header('Content-Type','application/json')
# chemprops_search = nm_rest(logging, sysToken, jobApiToken, jobRefreshToken, chemprops_rq)
# r = chemprops_search.urlopen(json.dumps(chemprops_data).encode("utf8"))
r = requests.get(
chemprops_api_url, params=chemprops_data, verify=False
) # requirement for auth removed for ChemProps for now
# if r.getcode() == 200:
if r.status_code == 200:
# result = json.loads(r.read().decode("utf-8"))
result = r.json()
logging.debug('Searching result: ' +
json.dumps(result)) #.encode("utf8"))
# now we modify xml with result
stdCN = matcomp.find('.//StdChemicalName')
if stdCN is None:
stdCN = etree.SubElement(matcomp, 'StdChemicalName')
stdCN.text = result['StandardName']
uSMILES = matcomp.find('.//uSMILES')
if uSMILES is None:
uSMILES = etree.SubElement(matcomp, 'uSMILES')
uSMILES.text = result['uSMILES']
if matcomp.find('Density') is None:
density = etree.SubElement(matcomp, 'Density')
dens_des = etree.SubElement(density, 'description')
dens_des.text = 'inserted from ChemProps, NanoMine'
dens_val = etree.SubElement(density, 'value')
dens_val.text = result['density']
dens_uni = etree.SubElement(density, 'unit')
dens_uni.text = 'g/cm3'
## testing - raise ValueError('Upload of input successful. returned id: ' + uploadId) ## for testing
elif r.status_code == 404:
logging.error('Matrix not found: ' +
json.dumps(chemprops_data))
else:
raise ValueError('Unexpected return code from ChemProps: ' +
str(r.status_code))
# FillerComponent
for filcomp in filcomps:
chemprops_data = {
"polfil":
"fil",
"ChemicalName":
filcomp.findtext('ChemicalName'),
"Abbreviation":
'' if filcomp.findtext('Abbreviation') is None else
filcomp.findtext('Abbreviation'),
"nmId":
ID
}
# chemprops_rq = urllib.request.Request(chemprops_api_url)
logging.debug(
'request created for ChemProps using chemprops_api_url')
logging.debug('Searching filler package: ' +
json.dumps(chemprops_data)) #.encode("utf8"))
# chemprops_rq.add_header('Content-Type','application/json')
# chemprops_search = nm_rest(logging, sysToken, jobApiToken, jobRefreshToken, chemprops_rq)
# r = chemprops_search.urlopen(json.dumps(chemprops_data).encode("utf8"))
r = requests.get(
chemprops_api_url, params=chemprops_data, verify=False
) # requirement for auth removed for ChemProps for now
# if r.getcode() == 200:
if r.status_code == 200:
# result = json.loads(r.read().decode("utf-8"))
result = r.json()
logging.debug('Searching result: ' +
json.dumps(result)) #.encode("utf8"))
# now we modify xml with result
stdCN = filcomp.find('.//StdChemicalName')
if stdCN is None:
stdCN = etree.SubElement(filcomp, 'StdChemicalName')
stdCN.text = result['StandardName']
if filcomp.find('Density') is None:
density = etree.SubElement(filcomp, 'Density')
dens_des = etree.SubElement(density, 'description')
dens_des.text = 'inserted from ChemProps, NanoMine'
dens_val = etree.SubElement(density, 'value')
dens_val.text = str(result['density'])
dens_uni = etree.SubElement(density, 'unit')
dens_uni.text = 'g/cm3'
## testing - raise ValueError('Upload of input successful. returned id: ' + uploadId) ## for testing
elif r.status_code == 404:
logging.error('Filler not found: ' +
json.dumps(chemprops_data))
else:
raise ValueError('Unexpected return code from ChemProps: ' +
str(r.status_code))
# sort modified elements with xsdTraverse module
xsdt = xsdTraverse(xsdDir)
# sort MatrixComponent by xpath PolymerNanocomposite/MATERIALS/Matrix/MatrixComponent
xsdt.sortSubElementsByPath(
xmlTree, 'PolymerNanocomposite/MATERIALS/Matrix/MatrixComponent')
# sort FillerComponent by xpath PolymerNanocomposite/MATERIALS/Filler/FillerComponent
xsdt.sortSubElementsByPath(
xmlTree, 'PolymerNanocomposite/MATERIALS/Filler/FillerComponent')
xmlTree.write(xmlName, encoding="UTF-8", xml_declaration=True)
except:
messages.append('exception occurred during ChemProps-query')
messages.append('exception: ' + str(traceback.format_exc()))
if len(messages) > 0:
return ('failure', messages)
# check #6: call the mf-vf conversion agent and add in the calculated mf/vf
try:
xmlName = jobDir + "/xml/" + ID + ".xml"
mvc = mfvfConvert(xmlName)
mvc.run()
xmlTree = etree.parse(xmlName)
# sort FillerComponent by xpath PolymerNanocomposite/MATERIALS/Filler/FillerComponent
xsdt.sortSubElementsByPath(
xmlTree, 'PolymerNanocomposite/MATERIALS/Filler/FillerComponent')
xmlTree.write(xmlName, encoding="UTF-8", xml_declaration=True)
except:
messages.append(
'exception occurred during mass fraction-volume fraction conversion'
)
messages.append('exception: ' + str(traceback.format_exc()))
if len(messages) > 0:
return ('failure', messages)
# check #7: upload and check if the uploading is successful
try:
blobs = uploadFilesAndAdjustXMLImageRefs(jobDir, runCtx['schemaId'],
ID, runCtx)
# curate-insert rest call
with open(jobDir + "/xml/" + ID + ".xml", "r") as f:
content = f.read()
curate_insert_url = restbase + '/nmr/curate'
curate_data = {
"userid": user,
"title": ID + ".xml",
"schemaId": runCtx['schemaId'],
"datasetId": runCtx['datasetId'],
"curatedDataState":
"Valid", # Valid causes ingest to kick of next. Also, it has passed validation.
"ispublished": "false",
"isPublic": "false",
"content": content
}
rq = urllib.request.Request(curate_insert_url)
rq.add_header('Content-Type', 'application/json')
nmCurateFiles = nm_rest(logging, runCtx['sysToken'],
runCtx['curateApiToken'],
runCtx['curateRefreshToken'], rq)
rv = nmCurateFiles.urlopen(json.dumps(curate_data).encode("utf8"))
response = json.loads(rv.read().decode("utf-8"))['data']
logging.info('curate insert for title: ' + ID + ' datasetId: ' +
runCtx['datasetId'] + ' request posted. Result: ' +
str(rv.getcode()))
except:
messages.append('exception occurred during curate-insert')
messages.append('exception: ' + str(traceback.format_exc()))
if len(messages) > 0:
return ('failure', messages)
# logging.debug('Xml curate Response: ' + str(response))
xmlMongoId = response['_id']
blobs.append({
'type': 'xmldata',
'id': xmlMongoId,
'metadata': {
'contentType': 'application/xml',
'filename': ID + '.xml'
}
})
fileset = {'fileset': ID, 'files': blobs}
filesets = dsInfo.get('filesets')
if filesets == None:
dsInfo['filesets'] = []
dsInfo['filesets'].append(fileset)
response = None # initialize response of the request
# POST ds-create
try:
if not dsInfo[
'userid'] == user: # do not bypass this since updateEx implies admin acccess and OVERWRITE of another user's dataset WOULD occur
raise messages.append('Update of dataset failed. Job user: '******' is not the owner: ' +
dsInfo['userid'])
return ('failure', messages)
ds_update_url = restbase + '/nmr/dataset/updateEx'
rq = urllib.request.Request(ds_update_url)
# logging.info('request created using ds_create_url')
rq.add_header('Content-Type', 'application/json')
nmCurate = nm_rest(logging, runCtx['sysToken'],
runCtx['curateApiToken'],
runCtx['curateRefreshToken'], rq)
rv = nmCurate.urlopen(json.dumps({'dsUpdate': dsInfo}).encode("utf-8"))
response = json.loads(rv.read().decode("utf-8"))['data']
except:
messages.append('exception occurred during dataset-update\n')
messages.append('exception: ' + str(traceback.format_exc()) + '\n')
# assemble the PID
if response is None:
messages.append(
'exception occurred during getting the response of dataset-update\n'
)
if len(messages) != 0:
return ('failure', messages)
# pass all the checks
return ('success', messages)
