# create dictionary of fields for new version kw_version['instance_id'] = inst_obj.id # retrieve all files for version allfiles = list_drs_files(vers_path) # loop through allfiles and make sure only the instance variable is present (ie GISS is breaking CMOR rules) files=[] for f in allfiles: if var == f.split("_")[0]: files.append(f) # add version to database if does not exist yet v_obj,new = insert_unique(db, Version, **kw_version) # check if files objects exist already if not save details in list of dictionaries (rows) # add both tracking-ids and sha256 checksums if v_obj.filenames()==[]: rows=[] for f in files: checksum=check_hash(v_obj.path+"/"+f,'sha256') trackid=get_trackid(v_obj.path+"/"+f) rows.append(dict(filename=f, sha256=checksum, tracking_id=trackid, version_id=v_obj.id)) # add files to database with bulk insert add_bulk_items(db, VersionFile, rows) # if some files exist already use insert_unique instead else: kw_files['version_id']=v_obj.id for f in files: kw_files['filename']=f kw_files['sha256']=check_hash(v_obj.path+"/"+f,'sha256') kw_files['tracking_id']=get_trackid(v_obj.path+"/"+f) insert_unique(db, VersionFile, **kw_files)
fversion=check_version(vers_path+"/"+fpaths[0]) if fversion: kw_version['version']= fversion else: kw_version['version']= "NA" # add instance to database if does not exist yet inst_obj,new = insert_unique(db, Instance, **kw_instance) # create dictionary of fields for new version kw_version['instance_id'] = inst_obj.id # add version to database if does not exist yet v_obj,new = insert_unique(db, Version, **kw_version) # check if files objects exist already if not add from files dictionary # add both tracking-ids and checksums, i checksums are "None" calculate sha256 for i,f in enumerate(kw_files): if f['checksum']=="None": kw_files[i][ctype]=check_hash(v_obj.path+"/"+f['filename'],ctype) f.pop('checksum') else: kw_files[i][ctype]=f.pop('checksum') if f['tracking_id']=="": kw_files[i]['tracking_id']=get_trackid(v_obj.path+"/"+f['filename']) kw_files[i]['version_id']=v_obj.id # add files to database with bulk insert if v_obj.filenames()==[]: add_bulk_items(db, VersionFile, kw_files) # if some files exist already use insert_unique instead else: for i,f in enumerate(kw_files): insert_unique(db, VersionFile, **f)