def bulkInsertArgoMerge(tableName):
profs = argo.loadArgoMerge(dataMode='D', ocean='all')
for i in range(len(profs)):
mergeDF = mergeTemplate()
fname = profs.iloc[i, 0].split('/')[-1]
floatID = (fname.split('_')[0])[2:]
cycle = (fname.split('_')[1]).split('.')[0]
path = cfgv.rep_merge_argo_raw + fname
mergeDF, bulkPath = fillMergeDF(mergeDF, path, floatID, cycle)
print('%s Argo_Merge profile (index: %d, name: %s)' %
(datetime.today(), i, fname))
try:
dc.bulkInsert(bulkPath, tableName)
finally:
removed = False
while not removed:
try:
os.remove(bulkPath)
removed = True
except:
time.sleep(3)
removed = False
print('%s Done' % datetime.today())
return
def bulkInsertLineage(tableName):
dataTitle = 'tax16s'
print('%s Inserting Bulk %s into %s.' %
(datetime.today(), dataTitle, tableName))
try:
bulkPath = cfgv.rep_tax16s_raw + 'diel1.bf100_id99.rain_sig_all.bf100.vs_MarRef.uniq_taxids.csv'
dc.bulkInsert(bulkPath, tableName)
finally:
pass
print('%s Done' % datetime.today())
return
def bulkInsert_Darwin_Climatology(tableName, path):
try:
for month in range(1, 13):
print('%s Bulk insert into %s; month: %s.' %
(datetime.today(), tableName, calendar.month_name[month]))
bulkPath = path % month
dc.bulkInsert(bulkPath, tableName)
finally:
pass
print('%s Done' % datetime.today())
return
def bulkInsertLineage(tableName):
dataTitle = 'Lineage'
print('%s Inserting Bulk %s into %s.' %
(datetime.today(), dataTitle, tableName))
try:
bulkPath = cfgv.rep_lineage_raw + 'lineages-2018-03-12.csv'
dc.bulkInsert(bulkPath, tableName)
finally:
pass
print('%s Done' % datetime.today())
return
def insertHourly(itnum, hour, tableName, nrt):
try:
bulkPath = ''
bulkPath = makeBulkWind(itnum, nrt, hour)
dc.bulkInsert(bulkPath, tableName)
print('\t %s Bulk %s %7.7d_%2.2dh inserted.' %
(datetime.today(), dataTitle, itnum, hour))
finally:
if bulkPath != '':
os.remove(bulkPath)
return
def bulkInsertEpiMicroscopy_HOT(tableName):
dataTitle = 'HOT_EpiMicroscopy'
print('%s Inserting Bulk %s into %s.' % (datetime.today(), dataTitle, tableName))
try:
bulkPath = ''
bulkPath = makeBulkEpiMicroscopy_HOT()
dc.bulkInsert(bulkPath, tableName)
finally:
if bulkPath != '':
os.remove(bulkPath)
print('%s Done' % datetime.today())
return
def bulkInsertKM1314_Cobalamin(tableName):
dataTitle = 'KM1314_cobalamin'
print('%s Inserting Bulk %s into %s.' % (datetime.today(), dataTitle, tableName))
try:
bulkPath = ''
bulkPath = makeBulkCobalamin()
dc.bulkInsert(bulkPath, tableName)
finally:
if bulkPath != '':
os.remove(bulkPath)
print('%s Done' % datetime.today())
return
def bulkInsertLCS(itnumStart, itnumEnd, tableName):
dataTitle = 'LCS'
for itnum in range(itnumStart, itnumEnd+1):
print('%s Inserting Bulk %s %7.7d into %s.' % (datetime.today(), dataTitle, itnum, tableName))
try:
bulkPath = ''
bulkPath = makeBulkLCS(itnum, nrt)
#print('\t %s Bulk %s %7.7d ready.' % (datetime.today(), dataTitle, itnum))
dc.bulkInsert(bulkPath, tableName)
finally:
if bulkPath != '':
os.remove(bulkPath)
return
def bulkInsertMacrozooplankton_HOT(tableName):
dataTitle = 'HOT_Macrozooplankton'
print('%s Inserting Bulk %s into %s.' %
(datetime.today(), dataTitle, tableName))
try:
bulkPath = ''
bulkPath = makeBulkMacrozooplankton_HOT()
dc.bulkInsert(bulkPath, tableName)
finally:
if bulkPath != '':
os.remove(bulkPath)
print('%s Done' % datetime.today())
return
def bulkInsertESV(tableName, usr, psw):
dataTitle = 'ESV'
print('%s Inserting Bulk %s into %s.' %
(datetime.today(), dataTitle, tableName))
try:
bulkPath = ''
bulkPath = makeBulkESV()
#print('\t %s Bulk %s %7.7d ready.' % (datetime.today(), dataTitle, itnum))
dc.bulkInsert(filePath=bulkPath, tableName=tableName, usr=usr, psw=psw)
finally:
if bulkPath != '':
os.remove(bulkPath)
print('%s Done' % datetime.today())
return
def bulkInsertSeaFlowCruises():
dataTitle = 'SeaFlow_Cruises'
print('%s Inserting Bulk %s into %s.' %
(datetime.today(), dataTitle, tableName))
try:
bulkPath = ''
bulkPath = cfgv.seaflow_raw + 'SeaFlow_instrument_log.txt'
#print('\t %s Bulk %s %7.7d ready.' % (datetime.today(), dataTitle, itnum))
dc.bulkInsert(bulkPath, tableName, '\t')
finally:
if bulkPath != '':
print('File inserted: ' + bulkPath)
#os.remove(bulkPath)
return
def bulkInsertSeaFlow(tableName):
dataTitle = 'SeaFlow'
print('%s Inserting Bulk %s into %s.' %
(datetime.today(), dataTitle, tableName))
try:
bulkPath = ''
bulkPath = makeBulkSeaFlow()
#print('\t %s Bulk %s %7.7d ready.' % (datetime.today(), dataTitle, itnum))
dc.bulkInsert(bulkPath, tableName)
finally:
if bulkPath != '':
os.remove(bulkPath)
print('%s Done' % datetime.today())
return
def bulkInsertCHL(itnumStart, itnumEnd, tableName):
dataTitle = 'CHL'
## specific to this data set: data files always start at the first day of the year
## and the increment with 8 day intervals
for itnum in range(itnumStart, itnumEnd + 1, 8):
print('%s Inserting Bulk %s %7.7d into %s.' %
(datetime.today(), dataTitle, itnum, tableName))
try:
bulkPath = ''
bulkPath = makeBulkCHL(itnum, nrt)
dc.bulkInsert(bulkPath, tableName, usr='', psw='')
finally:
if bulkPath != '':
os.remove(bulkPath)
return
def bulkInsertBottle_HOT(tableName):
dataTitle = 'HOT_Bottle'
print('%s Inserting Bulk %s into %s.' % (datetime.today(), dataTitle, tableName))
try:
bulkPath = ''
stations = ['aloha', 'kahe', 'kaena', 'hale', 'whots50', 'whots52']
dfs = []
for station in stations:
dfs.append(makeBulkBottle_HOT(station))
bulkPath = combine(dfs)
dc.bulkInsert(bulkPath, tableName)
finally:
if bulkPath != '':
os.remove(bulkPath)
print('%s Done' % datetime.today())
return
