# create dictionary of fields for new version
kw_version['instance_id'] = inst_obj.id
# retrieve all files for version
allfiles = list_drs_files(vers_path)
# loop through allfiles and make sure only the instance variable is present (ie GISS is breaking CMOR rules)
files=[]
for f in allfiles:
if var == f.split("_")[0]:
files.append(f)
# add version to database if does not exist yet
v_obj,new = insert_unique(db, Version, **kw_version)
# check if files objects exist already if not save details in list of dictionaries (rows)
# add both tracking-ids and sha256 checksums
if v_obj.filenames()==[]:
rows=[]
for f in files:
checksum=check_hash(v_obj.path+"/"+f,'sha256')
trackid=get_trackid(v_obj.path+"/"+f)
rows.append(dict(filename=f, sha256=checksum, tracking_id=trackid, version_id=v_obj.id))
# add files to database with bulk insert
add_bulk_items(db, VersionFile, rows)
# if some files exist already use insert_unique instead
else:
kw_files['version_id']=v_obj.id
for f in files:
kw_files['filename']=f
kw_files['sha256']=check_hash(v_obj.path+"/"+f,'sha256')
kw_files['tracking_id']=get_trackid(v_obj.path+"/"+f)
insert_unique(db, VersionFile, **kw_files)
fversion=check_version(vers_path+"/"+fpaths[0])
if fversion:
kw_version['version']= fversion
else:
kw_version['version']= "NA"
# add instance to database if does not exist yet
inst_obj,new = insert_unique(db, Instance, **kw_instance)
# create dictionary of fields for new version
kw_version['instance_id'] = inst_obj.id
# add version to database if does not exist yet
v_obj,new = insert_unique(db, Version, **kw_version)
# check if files objects exist already if not add from files dictionary
# add both tracking-ids and checksums, i checksums are "None" calculate sha256
for i,f in enumerate(kw_files):
if f['checksum']=="None":
kw_files[i][ctype]=check_hash(v_obj.path+"/"+f['filename'],ctype)
f.pop('checksum')
else:
kw_files[i][ctype]=f.pop('checksum')
if f['tracking_id']=="":
kw_files[i]['tracking_id']=get_trackid(v_obj.path+"/"+f['filename'])
kw_files[i]['version_id']=v_obj.id
# add files to database with bulk insert
if v_obj.filenames()==[]:
add_bulk_items(db, VersionFile, kw_files)
# if some files exist already use insert_unique instead
else:
for i,f in enumerate(kw_files):
insert_unique(db, VersionFile, **f)