def bulkInsertArgoMerge(tableName): profs = argo.loadArgoMerge(dataMode='D', ocean='all') for i in range(len(profs)): mergeDF = mergeTemplate() fname = profs.iloc[i, 0].split('/')[-1] floatID = (fname.split('_')[0])[2:] cycle = (fname.split('_')[1]).split('.')[0] path = cfgv.rep_merge_argo_raw + fname mergeDF, bulkPath = fillMergeDF(mergeDF, path, floatID, cycle) print('%s Argo_Merge profile (index: %d, name: %s)' % (datetime.today(), i, fname)) try: dc.bulkInsert(bulkPath, tableName) finally: removed = False while not removed: try: os.remove(bulkPath) removed = True except: time.sleep(3) removed = False print('%s Done' % datetime.today()) return
def bulkInsertLineage(tableName): dataTitle = 'tax16s' print('%s Inserting Bulk %s into %s.' % (datetime.today(), dataTitle, tableName)) try: bulkPath = cfgv.rep_tax16s_raw + 'diel1.bf100_id99.rain_sig_all.bf100.vs_MarRef.uniq_taxids.csv' dc.bulkInsert(bulkPath, tableName) finally: pass print('%s Done' % datetime.today()) return
def bulkInsert_Darwin_Climatology(tableName, path): try: for month in range(1, 13): print('%s Bulk insert into %s; month: %s.' % (datetime.today(), tableName, calendar.month_name[month])) bulkPath = path % month dc.bulkInsert(bulkPath, tableName) finally: pass print('%s Done' % datetime.today()) return
def bulkInsertLineage(tableName): dataTitle = 'Lineage' print('%s Inserting Bulk %s into %s.' % (datetime.today(), dataTitle, tableName)) try: bulkPath = cfgv.rep_lineage_raw + 'lineages-2018-03-12.csv' dc.bulkInsert(bulkPath, tableName) finally: pass print('%s Done' % datetime.today()) return
def insertHourly(itnum, hour, tableName, nrt): try: bulkPath = '' bulkPath = makeBulkWind(itnum, nrt, hour) dc.bulkInsert(bulkPath, tableName) print('\t %s Bulk %s %7.7d_%2.2dh inserted.' % (datetime.today(), dataTitle, itnum, hour)) finally: if bulkPath != '': os.remove(bulkPath) return
def bulkInsertEpiMicroscopy_HOT(tableName): dataTitle = 'HOT_EpiMicroscopy' print('%s Inserting Bulk %s into %s.' % (datetime.today(), dataTitle, tableName)) try: bulkPath = '' bulkPath = makeBulkEpiMicroscopy_HOT() dc.bulkInsert(bulkPath, tableName) finally: if bulkPath != '': os.remove(bulkPath) print('%s Done' % datetime.today()) return
def bulkInsertKM1314_Cobalamin(tableName): dataTitle = 'KM1314_cobalamin' print('%s Inserting Bulk %s into %s.' % (datetime.today(), dataTitle, tableName)) try: bulkPath = '' bulkPath = makeBulkCobalamin() dc.bulkInsert(bulkPath, tableName) finally: if bulkPath != '': os.remove(bulkPath) print('%s Done' % datetime.today()) return
def bulkInsertLCS(itnumStart, itnumEnd, tableName): dataTitle = 'LCS' for itnum in range(itnumStart, itnumEnd+1): print('%s Inserting Bulk %s %7.7d into %s.' % (datetime.today(), dataTitle, itnum, tableName)) try: bulkPath = '' bulkPath = makeBulkLCS(itnum, nrt) #print('\t %s Bulk %s %7.7d ready.' % (datetime.today(), dataTitle, itnum)) dc.bulkInsert(bulkPath, tableName) finally: if bulkPath != '': os.remove(bulkPath) return
def bulkInsertMacrozooplankton_HOT(tableName): dataTitle = 'HOT_Macrozooplankton' print('%s Inserting Bulk %s into %s.' % (datetime.today(), dataTitle, tableName)) try: bulkPath = '' bulkPath = makeBulkMacrozooplankton_HOT() dc.bulkInsert(bulkPath, tableName) finally: if bulkPath != '': os.remove(bulkPath) print('%s Done' % datetime.today()) return
def bulkInsertESV(tableName, usr, psw): dataTitle = 'ESV' print('%s Inserting Bulk %s into %s.' % (datetime.today(), dataTitle, tableName)) try: bulkPath = '' bulkPath = makeBulkESV() #print('\t %s Bulk %s %7.7d ready.' % (datetime.today(), dataTitle, itnum)) dc.bulkInsert(filePath=bulkPath, tableName=tableName, usr=usr, psw=psw) finally: if bulkPath != '': os.remove(bulkPath) print('%s Done' % datetime.today()) return
def bulkInsertSeaFlowCruises(): dataTitle = 'SeaFlow_Cruises' print('%s Inserting Bulk %s into %s.' % (datetime.today(), dataTitle, tableName)) try: bulkPath = '' bulkPath = cfgv.seaflow_raw + 'SeaFlow_instrument_log.txt' #print('\t %s Bulk %s %7.7d ready.' % (datetime.today(), dataTitle, itnum)) dc.bulkInsert(bulkPath, tableName, '\t') finally: if bulkPath != '': print('File inserted: ' + bulkPath) #os.remove(bulkPath) return
def bulkInsertSeaFlow(tableName): dataTitle = 'SeaFlow' print('%s Inserting Bulk %s into %s.' % (datetime.today(), dataTitle, tableName)) try: bulkPath = '' bulkPath = makeBulkSeaFlow() #print('\t %s Bulk %s %7.7d ready.' % (datetime.today(), dataTitle, itnum)) dc.bulkInsert(bulkPath, tableName) finally: if bulkPath != '': os.remove(bulkPath) print('%s Done' % datetime.today()) return
def bulkInsertCHL(itnumStart, itnumEnd, tableName): dataTitle = 'CHL' ## specific to this data set: data files always start at the first day of the year ## and the increment with 8 day intervals for itnum in range(itnumStart, itnumEnd + 1, 8): print('%s Inserting Bulk %s %7.7d into %s.' % (datetime.today(), dataTitle, itnum, tableName)) try: bulkPath = '' bulkPath = makeBulkCHL(itnum, nrt) dc.bulkInsert(bulkPath, tableName, usr='', psw='') finally: if bulkPath != '': os.remove(bulkPath) return
def bulkInsertBottle_HOT(tableName): dataTitle = 'HOT_Bottle' print('%s Inserting Bulk %s into %s.' % (datetime.today(), dataTitle, tableName)) try: bulkPath = '' stations = ['aloha', 'kahe', 'kaena', 'hale', 'whots50', 'whots52'] dfs = [] for station in stations: dfs.append(makeBulkBottle_HOT(station)) bulkPath = combine(dfs) dc.bulkInsert(bulkPath, tableName) finally: if bulkPath != '': os.remove(bulkPath) print('%s Done' % datetime.today()) return