def importDataset(self):
"""
_importDataset_
Import the Input Dataset contents and inject it into the DB. The
DBSWriter.importDataset should also import the parent Dataset. The
parent importing seems to work with DBS_2_0_8
"""
# //
# // Getting Local and Global DBS URLs
#//
localDBS = getLocalDBSURL()
dbsWriter = DBSWriter(localDBS)
globalDBS = self.dbsUrl
try:
dbsWriter.importDataset(
globalDBS,
self.inputDataset(),
localDBS,
onlyClosed=self.onlyClosedBlocks,
skipNoSiteError=True
)
except Exception, ex:
msg = "Error importing dataset to be processed into local DBS\n"
msg += "Source Dataset: %s\n" % self.inputDataset()
msg += "Source DBS: %s\n" % globalDBS
msg += "Destination DBS: %s\n" % localDBS
msg += str(ex)
logging.error(msg)
raise RuntimeError, msg
def importDataset(self):
"""
_importDataset_
Import the Dataset contents and inject it into the DB.
"""
# //
# // Import the dataset to be processed into the local DBS
#//
localDBS = getLocalDBSURL()
dbsWriter = DBSWriter(localDBS)
globalDBS = getGlobalDBSURL()
try:
dbsWriter.importDataset(
globalDBS,
self.inputDataset(),
localDBS,
self.onlyClosedBlocks
)
except Exception, ex:
msg = "Error importing dataset to be processed into local DBS\n"
msg += "Source Dataset: %s\n" % self.inputDataset()
msg += "Source DBS: %s\n" % globalDBS
msg += "Destination DBS: %s\n" % localDBS
msg += str(ex)
logging.error(msg)
return 1
def updateDataset(self):
"""
_updateDataset_
Look for new fileblocks not in the DB for this dataset and
import them
"""
owner = DatabaseAPI.ownerIndex(self.workflowSpec.workflowName())
if owner == None:
knownBlocks = []
else:
knownBlocks = DatabaseAPI.listKnownFileblocks(owner)
logging.info("knownBlocks: %s" % str(knownBlocks))
# //
# // Create a new splitter from the DBS/DLS containing all
#// current fileblocks and filter out the blocks that are
# //already known.
# //
#//
# //
# // Re Import the dataset to be processed into the local DBS
#// to get any new blocks and files
localDBS = getLocalDBSURL()
dbsWriter = DBSWriter(localDBS)
globalDBS = getGlobalDBSURL()
try:
dbsWriter.importDataset(
globalDBS,
self.inputDataset(),
localDBS,
self.onlyClosedBlocks
)
except Exception, ex:
msg = "Error importing dataset to be processed into local DBS\n"
msg += "Source Dataset: %s\n" % self.inputDataset()
msg += "Source DBS: %s\n" % self.dbsUrl
msg += str(ex)
logging.error(msg)
return 1
if opt == "--dataset":
dataset = arg
if opt == "--source":
source = arg
if opt == "--target":
target = arg
if dataset == None or source == None or target == None :
print usage
sys.exit(1)
print 'Migrating dataset: %s from %s to %s' % (dataset,source,target)
dbsWriter = DBSWriter(target)
try:
dbsWriter.importDataset(
source,
dataset,
target)
except Exception, ex:
msg = "Error importing dataset to be processed into local DBS\n"
msg += "Source Dataset: %s\n" % dataset
msg += "Source DBS: %s\n" % source
msg += "Destination DBS: %s\n" % target
msg += str(ex)
logging.error(msg)
sys.exit(1)
dataset = None
source = None
target = None
for opt, arg in opts:
if opt == "--dataset":
dataset = arg
if opt == "--source":
source = arg
if opt == "--target":
target = arg
if dataset == None or source == None or target == None:
print usage
sys.exit(1)
print 'Migrating dataset: %s from %s to %s' % (dataset, source, target)
dbsWriter = DBSWriter(target)
try:
dbsWriter.importDataset(source, dataset, target)
except Exception, ex:
msg = "Error importing dataset to be processed into local DBS\n"
msg += "Source Dataset: %s\n" % dataset
msg += "Source DBS: %s\n" % source
msg += "Destination DBS: %s\n" % target
msg += str(ex)
logging.error(msg)
sys.exit(1)
