def get_prep_id(dataset):
dbs = dbsClient.DbsApi('https://cmsweb.cern.ch/dbs/prod/global/DBSReader')
dbs_response = dbs.listDatasetArray(dataset = dataset, detail = True)
if not dbs_response:
raise RuntimeError("Invalid dataset: %s" % dataset)
prep_id = dbs_response[0]['prep_id']
logging.debug('Got prep ID: {}'.format(prep_id))
return prep_id
def publishInDBS3(taskname):
"""
"""
def createLogdir(dirname):
"""
Create the directory dirname ignoring erors in case it exists. Exit if
the directory cannot be created.
"""
try:
os.mkdir(dirname)
except OSError as ose:
if ose.errno != 17: #ignore the "Directory already exists error"
print(str(ose))
print("The task worker need to access the '%s' directory" % dirname)
sys.exit(1)
createLogdir('taskLogs')
logger = logging.getLogger(taskname)
logging.basicConfig(filename='taskLogs/'+taskname+'.log', level=logging.INFO, format=config.General.logMsgFormat)
logger.info("Getting files to publish")
toPublish = []
# TODO move from new to done when processed
with open("/tmp/"+taskname+".json") as f:
toPublish = json.load(f)
workflow = taskname
if len(toPublish) == 0:
return "EMPTY"
if not workflow:
logger.info("NO TASKNAME: %s" % toPublish[0])
for k, v in toPublish[0].iteritems():
if k == 'taskname':
logger.info("Starting: %s: %s" % (k, v))
wfnamemsg = "%s: " % (workflow)
user = toPublish[0]["User"]
try:
group = toPublish[0]["Group"]
role = toPublish[0]["Role"]
except:
group = ""
role = ""
if not group or group in ['null']:
group = ""
if not role or role in ['null']:
role = ""
userDN = toPublish[0]["UserDN"]
pnn = toPublish[0]["Destination"]
logger.info(wfnamemsg+" "+user)
READ_PATH = "/DBSReader"
READ_PATH_1 = "/DBSReader/"
# TODO: get user role and group
try:
proxy = Proxy(userDN, group, role, logger)
except:
logger.exception("Failed to retrieve user proxy")
return "FAILED"
oracelInstance = config.General.oracleDB
oracleDB = HTTPRequests(oracelInstance,
proxy,
proxy)
fileDoc = dict()
fileDoc['subresource'] = 'search'
fileDoc['workflow'] = workflow
try:
results = oracleDB.get(task_path,
data=encodeRequest(fileDoc))
except Exception as ex:
logger.error("Failed to get acquired publications from oracleDB for %s: %s" % (workflow, ex))
return "FAILED"
logger.info(results[0]['desc']['columns'])
try:
inputDatasetIndex = results[0]['desc']['columns'].index("tm_input_dataset")
inputDataset = results[0]['result'][inputDatasetIndex]
sourceURLIndex = results[0]['desc']['columns'].index("tm_dbs_url")
sourceURL = results[0]['result'][sourceURLIndex]
publish_dbs_urlIndex = results[0]['desc']['columns'].index("tm_publish_dbs_url")
publish_dbs_url = results[0]['result'][publish_dbs_urlIndex]
#sourceURL = "https://cmsweb.cern.ch/dbs/prod/global/DBSReader"
if not sourceURL.endswith(READ_PATH) and not sourceURL.endswith(READ_PATH_1):
sourceURL += READ_PATH
except Exception:
logger.exception("ERROR")
## When looking up parents may need to look in global DBS as well.
globalURL = sourceURL
globalURL = globalURL.replace('phys01', 'global')
globalURL = globalURL.replace('phys02', 'global')
globalURL = globalURL.replace('phys03', 'global')
globalURL = globalURL.replace('caf', 'global')
pr = os.environ.get("SOCKS5_PROXY")
logger.info(wfnamemsg+"Source API URL: %s" % sourceURL)
sourceApi = dbsClient.DbsApi(url=sourceURL, proxy=pr)
logger.info(wfnamemsg+"Global API URL: %s" % globalURL)
globalApi = dbsClient.DbsApi(url=globalURL, proxy=pr)
WRITE_PATH = "/DBSWriter"
MIGRATE_PATH = "/DBSMigrate"
READ_PATH = "/DBSReader"
if publish_dbs_url.endswith(WRITE_PATH):
publish_read_url = publish_dbs_url[:-len(WRITE_PATH)] + READ_PATH
publish_migrate_url = publish_dbs_url[:-len(WRITE_PATH)] + MIGRATE_PATH
else:
publish_migrate_url = publish_dbs_url + MIGRATE_PATH
publish_read_url = publish_dbs_url + READ_PATH
publish_dbs_url += WRITE_PATH
try:
logger.debug(wfnamemsg+"Destination API URL: %s" % publish_dbs_url)
destApi = dbsClient.DbsApi(url=publish_dbs_url, proxy=pr)
logger.debug(wfnamemsg+"Destination read API URL: %s" % publish_read_url)
destReadApi = dbsClient.DbsApi(url=publish_read_url, proxy=pr)
logger.debug(wfnamemsg+"Migration API URL: %s" % publish_migrate_url)
migrateApi = dbsClient.DbsApi(url=publish_migrate_url, proxy=pr)
except:
logger.exception('Wrong DBS URL %s' % publish_dbs_url)
return "FAILED"
logger.info("inputDataset: %s" % inputDataset)
noInput = len(inputDataset.split("/")) <= 3
# TODO: fix dbs dep
if not noInput:
try:
existing_datasets = sourceApi.listDatasets(dataset=inputDataset, detail=True, dataset_access_type='*')
primary_ds_type = existing_datasets[0]['primary_ds_type']
# There's little chance this is correct, but it's our best guess for now.
# CRAB2 uses 'crab2_tag' for all cases
existing_output = destReadApi.listOutputConfigs(dataset=inputDataset)
except:
logger.exception('Wrong DBS URL %s' % publish_dbs_url)
return "FAILED"
if not existing_output:
msg = "Unable to list output config for input dataset %s." % (inputDataset)
logger.error(wfnamemsg+msg)
global_tag = 'crab3_tag'
else:
global_tag = existing_output[0]['global_tag']
else:
msg = "This publication appears to be for private MC."
logger.info(wfnamemsg+msg)
primary_ds_type = 'mc'
global_tag = 'crab3_tag'
acquisition_era_name = "CRAB"
processing_era_config = {'processing_version': 1, 'description': 'CRAB3_processing_era'}
appName = 'cmsRun'
appVer = toPublish[0]["swversion"]
pset_hash = toPublish[0]['publishname'].split("-")[-1]
gtag = str(toPublish[0]['globaltag'])
if gtag == "None":
gtag = global_tag
try:
if toPublish[0]['acquisitionera'] and not toPublish[0]['acquisitionera'] in ["null"]:
acquisitionera = str(toPublish[0]['acquisitionera'])
else:
acquisitionera = acquisition_era_name
except:
acquisitionera = acquisition_era_name
_, primName, procName, tier = toPublish[0]['outdataset'].split('/')
primds_config = {'primary_ds_name': primName, 'primary_ds_type': primary_ds_type}
msg = "About to insert primary dataset: %s" % (str(primds_config))
logger.debug(wfnamemsg+msg)
destApi.insertPrimaryDataset(primds_config)
msg = "Successfully inserted primary dataset %s." % (primName)
logger.debug(wfnamemsg+msg)
final = {}
failed = []
publish_in_next_iteration = []
published = []
dataset = toPublish[0]['outdataset']
# Find all (valid) files already published in this dataset.
try:
existingDBSFiles = destReadApi.listFiles(dataset=dataset, detail=True)
existingFiles = [f['logical_file_name'] for f in existingDBSFiles]
existingFilesValid = [f['logical_file_name'] for f in existingDBSFiles if f['is_file_valid']]
msg = "Dataset %s already contains %d files" % (dataset, len(existingFiles))
msg += " (%d valid, %d invalid)." % (len(existingFilesValid), len(existingFiles) - len(existingFilesValid))
logger.info(wfnamemsg+msg)
final['existingFiles'] = len(existingFiles)
except Exception as ex:
msg = "Error when listing files in DBS: %s" % (str(ex))
msg += "\n%s" % (str(traceback.format_exc()))
logger.error(wfnamemsg+msg)
return "FAILED"
# check if actions are needed
workToDo = False
for fileTo in toPublish:
if fileTo['lfn'] not in existingFilesValid:
workToDo = True
if not workToDo:
msg = "Nothing uploaded, %s has these files already or not enough files." % (dataset)
logger.info(wfnamemsg+msg)
return "NOTHING TO DO"
acquisition_era_config = {'acquisition_era_name': acquisitionera, 'start_date': 0}
output_config = {'release_version': appVer,
'pset_hash': pset_hash,
'app_name': appName,
'output_module_label': 'o',
'global_tag': global_tag,
}
msg = "Published output config."
logger.debug(wfnamemsg+msg)
dataset_config = {'dataset': dataset,
'processed_ds_name': procName,
'data_tier_name': tier,
'acquisition_era_name': acquisitionera,
'dataset_access_type': 'VALID',
'physics_group_name': 'CRAB3',
'last_modification_date': int(time.time()),
}
msg = "About to insert dataset: %s" % (str(dataset_config))
logger.info(wfnamemsg+msg)
del dataset_config['acquisition_era_name']
# List of all files that must (and can) be published.
dbsFiles = []
dbsFiles_f = []
# Set of all the parent files from all the files requested to be published.
parentFiles = set()
# Set of parent files for which the migration to the destination DBS instance
# should be skipped (because they were not found in DBS).
parentsToSkip = set()
# Set of parent files to migrate from the source DBS instance
# to the destination DBS instance.
localParentBlocks = set()
# Set of parent files to migrate from the global DBS instance
# to the destination DBS instance.
globalParentBlocks = set()
# Loop over all files to publish.
for file_ in toPublish:
logger.info(file_)
# Check if this file was already published and if it is valid.
if file_['lfn'] not in existingFilesValid:
# We have a file to publish.
# Get the parent files and for each parent file do the following:
# 1) Add it to the list of parent files.
# 2) Find the block to which it belongs and insert that block name in
# (one of) the set of blocks to be migrated to the destination DBS.
for parentFile in list(file_['parents']):
if parentFile not in parentFiles:
parentFiles.add(parentFile)
# Is this parent file already in the destination DBS instance?
# (If yes, then we don't have to migrate this block.)
blocksDict = destReadApi.listBlocks(logical_file_name=parentFile)
if not blocksDict:
# No, this parent file is not in the destination DBS instance.
# Maybe it is in the same DBS instance as the input dataset?
blocksDict = sourceApi.listBlocks(logical_file_name=parentFile)
if blocksDict:
# Yes, this parent file is in the same DBS instance as the input dataset.
# Add the corresponding block to the set of blocks from the source DBS
# instance that have to be migrated to the destination DBS.
localParentBlocks.add(blocksDict[0]['block_name'])
else:
# No, this parent file is not in the same DBS instance as input dataset.
# Maybe it is in global DBS instance?
blocksDict = globalApi.listBlocks(logical_file_name=parentFile)
if blocksDict:
# Yes, this parent file is in global DBS instance.
# Add the corresponding block to the set of blocks from global DBS
# instance that have to be migrated to the destination DBS.
globalParentBlocks.add(blocksDict[0]['block_name'])
# If this parent file is not in the destination DBS instance, is not
# the source DBS instance, and is not in global DBS instance, then it
# means it is not known to DBS and therefore we can not migrate it.
# Put it in the set of parent files for which migration should be skipped.
if not blocksDict:
parentsToSkip.add(parentFile)
# If this parent file should not be migrated because it is not known to DBS,
# we remove it from the list of parents in the file-to-publish info dictionary
# (so that when publishing, this "parent" file will not appear as a parent).
if parentFile in parentsToSkip:
msg = "Skipping parent file %s, as it doesn't seem to be known to DBS." % (parentFile)
logger.info(wfnamemsg+msg)
if parentFile in file_['parents']:
file_['parents'].remove(parentFile)
# Add this file to the list of files to be published.
dbsFiles.append(format_file_3(file_))
dbsFiles_f.append(file_)
#print file
published.append(file_['SourceLFN'])
# Print a message with the number of files to publish.
msg = "Found %d files not already present in DBS which will be published." % (len(dbsFiles))
logger.info(wfnamemsg+msg)
# If there are no files to publish, continue with the next dataset.
if len(dbsFiles_f) == 0:
msg = "Nothing to do for this dataset."
logger.info(wfnamemsg+msg)
return "NOTHING TO DO"
# Migrate parent blocks before publishing.
# First migrate the parent blocks that are in the same DBS instance
# as the input dataset.
if localParentBlocks:
msg = "List of parent blocks that need to be migrated from %s:\n%s" % (sourceApi.url, localParentBlocks)
logger.info(wfnamemsg+msg)
statusCode, failureMsg = migrateByBlockDBS3(workflow,
migrateApi,
destReadApi,
sourceApi,
inputDataset,
localParentBlocks
)
if statusCode:
failureMsg += " Not publishing any files."
logger.info(wfnamemsg+failureMsg)
failed.extend([f['SourceLFN'] for f in dbsFiles_f])
failure_reason = failureMsg
published = [x for x in published[dataset] if x not in failed[dataset]]
return "NOTHING TO DO"
# Then migrate the parent blocks that are in the global DBS instance.
if globalParentBlocks:
msg = "List of parent blocks that need to be migrated from %s:\n%s" % (globalApi.url, globalParentBlocks)
logger.info(wfnamemsg+msg)
statusCode, failureMsg = migrateByBlockDBS3(workflow, migrateApi, destReadApi, globalApi, inputDataset, globalParentBlocks)
if statusCode:
failureMsg += " Not publishing any files."
logger.info(wfnamemsg+failureMsg)
failed.extend([f['SourceLFN'] for f in dbsFiles_f])
failure_reason = failureMsg
published = [x for x in published[dataset] if x not in failed[dataset]]
return "NOTHING TO DO"
# Publish the files in blocks. The blocks must have exactly max_files_per_block
# files, unless there are less than max_files_per_block files to publish to
# begin with. If there are more than max_files_per_block files to publish,
# publish as many blocks as possible and leave the tail of files for the next
# PublisherWorker call, unless forced to published.
block_count = 0
count = 0
max_files_per_block = config.General.max_files_per_block
while True:
block_name = "%s#%s" % (dataset, str(uuid.uuid4()))
files_to_publish = dbsFiles[count:count+max_files_per_block]
try:
block_config = {'block_name': block_name, 'origin_site_name': pnn, 'open_for_writing': 0}
msg = "Inserting files %s into block %s." % ([f['logical_file_name']
for f in files_to_publish], block_name)
logger.info(wfnamemsg+msg)
blockDump = createBulkBlock(output_config, processing_era_config,
primds_config, dataset_config,
acquisition_era_config, block_config, files_to_publish)
#logger.debug(wfnamemsg+"Block to insert: %s\n %s" % (blockDump, destApi.__dict__ ))
destApi.insertBulkBlock(blockDump)
block_count += 1
except Exception as ex:
logger.error("Error for files: %s" % [f['SourceLFN'] for f in toPublish])
failed.extend([f['SourceLFN'] for f in toPublish])
msg = "Error when publishing (%s) " % ", ".join(failed)
msg += str(ex)
msg += str(traceback.format_exc())
logger.error(wfnamemsg+msg)
failure_reason = str(ex)
count += max_files_per_block
files_to_publish_next = dbsFiles_f[count:count+max_files_per_block]
if len(files_to_publish_next) < max_files_per_block:
publish_in_next_iteration.extend([f["SourceLFN"] for f in files_to_publish_next])
break
published = [x for x in published if x not in failed + publish_in_next_iteration]
# Fill number of files/blocks published for this dataset.
final['files'] = len(dbsFiles) - len(failed) - len(publish_in_next_iteration)
final['blocks'] = block_count
# Print a publication status summary for this dataset.
msg = "End of publication status for dataset %s:" % (dataset)
msg += " failed (%s) %s" % (len(failed), failed)
msg += ", published (%s) %s" % (len(published), published)
msg += ", publish_in_next_iteration (%s) %s" % (len(publish_in_next_iteration),
publish_in_next_iteration)
msg += ", results %s" % (final)
logger.info(wfnamemsg+msg)
try:
if published:
mark_good(workflow, published, oracleDB, logger)
if failed:
logger.debug("Failed files: %s " % failed)
mark_failed(failed, oracleDB, logger, failure_reason)
except:
logger.exception("Status update failed")
return 0
def publishInDBS3(config, taskname, verbose):
"""
Publish output from one task in DBS
"""
def mark_good(files, crabServer, logger):
"""
Mark the list of files as tranferred
"""
msg = "Marking %s file(s) as published." % len(files)
logger.info(msg)
if dryRun:
logger.info("DryRun: skip marking good file")
return
nMarked = 0
for lfn in files:
data = {}
source_lfn = lfn
docId = getHashLfn(source_lfn)
data['asoworker'] = config.General.asoworker
data['subresource'] = 'updatePublication'
data['list_of_ids'] = [docId]
data['list_of_publication_state'] = ['DONE']
data['list_of_retry_value'] = [1]
data['list_of_failure_reason'] = ['']
try:
result = crabServer.post(api='filetransfers', data=encodeRequest(data))
logger.debug("updated DocumentId: %s lfn: %s Result %s", docId, source_lfn, result)
except Exception as ex:
logger.error("Error updating status for DocumentId: %s lfn: %s", docId, source_lfn)
logger.error("Error reason: %s", ex)
nMarked += 1
if nMarked % 10 == 0:
logger.info('marked %d files', nMarked)
def mark_failed(files, crabServer, logger, failure_reason=""):
"""
Something failed for these files so increment the retry count
"""
msg = "Marking %s file(s) as failed" % len(files)
logger.info(msg)
if dryRun:
logger.debug("DryRun: skip marking failes files")
return
nMarked = 0
for lfn in files:
source_lfn = lfn
docId = getHashLfn(source_lfn)
data = dict()
data['asoworker'] = config.General.asoworker
data['subresource'] = 'updatePublication'
data['list_of_ids'] = [docId]
data['list_of_publication_state'] = ['FAILED']
data['list_of_retry_value'] = [1]
data['list_of_failure_reason'] = [failure_reason]
logger.debug("data: %s ", data)
try:
result = crabServer.post(api='filetransfers', data=encodeRequest(data))
logger.debug("updated DocumentId: %s lfn: %s Result %s", docId, source_lfn, result)
except Exception as ex:
logger.error("Error updating status for DocumentId: %s lfn: %s", docId, source_lfn)
logger.error("Error reason: %s", ex)
nMarked += 1
if nMarked % 10 == 0:
logger.info('marked %d files', nMarked)
def createLogdir(dirname):
"""
Create the directory dirname ignoring erors in case it exists. Exit if
the directory cannot be created.
"""
try:
os.mkdir(dirname)
except OSError as ose:
if ose.errno != 17: #ignore the "Directory already exists error"
print(str(ose))
print("The task worker need to access the '%s' directory" % dirname)
sys.exit(1)
return
def saveSummaryJson(logdir, summary):
"""
Save a publication summary as JSON. Make a new file every time this script runs
:param summary: a summary disctionary. Must at least have key 'taskname'
:param logdir: the directory where to write the summary
:return: the full path name of the written file
"""
taskname = summary['taskname']
counter = 1
summaryFileName = os.path.join(logdir, taskname + '-1.json')
while os.path.exists(summaryFileName):
counter += 1
summaryFileName = os.path.join(logdir, taskname + '-%d.json' % counter)
with open(summaryFileName, 'w') as fd:
json.dump(summary, fd)
return summaryFileName
taskFilesDir = config.General.taskFilesDir
dryRun = config.TaskPublisher.dryRun
username = taskname.split(':')[1].split('_')[0]
logdir = os.path.join(config.General.logsDir, 'tasks', username)
logfile = os.path.join(logdir, taskname + '.log')
createLogdir(logdir)
migrationLogDir = os.path.join(config.General.logsDir, 'migrations')
createLogdir(migrationLogDir)
logger = logging.getLogger(taskname)
logging.basicConfig(filename=logfile, level=logging.INFO, format=config.TaskPublisher.logMsgFormat)
if verbose:
logger.setLevel(logging.DEBUG)
logger.info("Start new iteration on taskname: %s\nGet files to publish", taskname)
# prepare a dummy summary JSON file in case there's nothing to do
nothingToDo = {}
nothingToDo['taskname'] = taskname
nothingToDo['result'] = 'OK'
nothingToDo['reason'] = 'NOTHING TO DO'
nothingToDo['publishedBlocks'] = 0
nothingToDo['failedBlocks'] = 0
nothingToDo['failedBlockDumps'] = []
nothingToDo['publishedFiles'] = 0
nothingToDo['failedFiles'] = 0
nothingToDo['nextIterFiles'] = 0
toPublish = []
# TODO move from new to done when processed
fname = taskFilesDir + taskname + ".json"
with open(fname) as f:
toPublish = json.load(f)
if not toPublish:
logger.info("Empty data file %s", fname)
summaryFileName = saveSummaryJson(logdir, nothingToDo)
return summaryFileName
pnn = toPublish[0]["Destination"]
dataset = toPublish[0]['outdataset']
logger.info("Will publish user files in %s", dataset)
# CRABServer REST API's (see CRABInterface)
try:
instance = config.General.instance
except:
msg = "No instance provided: need to specify config.General.instance in the configuration"
raise ConfigException(msg)
if instance in SERVICE_INSTANCES:
logger.info('Will connect to CRAB service: %s', instance)
restHost = SERVICE_INSTANCES[instance]['restHost']
dbInstance = SERVICE_INSTANCES[instance]['dbInstance']
else:
msg = "Invalid instance value '%s'" % instance
raise ConfigException(msg)
if instance == 'other':
logger.info('Will use restHost and dbInstance from config file')
try:
restHost = config.General.restHost
dbInstance = config.General.dbInstance
except:
msg = "Need to specify config.General.restHost and dbInstance in the configuration"
raise ConfigException(msg)
restURInoAPI = '/crabserver/' + dbInstance
logger.info('Will connect to CRAB Data Base via URL: https://%s/%s', restHost, restURInoAPI)
# CRAB REST API's
crabServer = CRABRest(hostname=restHost, localcert=config.General.serviceCert,
localkey=config.General.serviceKey, retry=3,
userAgent='CRABPublisher')
crabServer.setDbInstance(dbInstance=dbInstance)
data = dict()
data['subresource'] = 'search'
data['workflow'] = taskname
try:
results = crabServer.get(api='task', data=encodeRequest(data))
except Exception as ex:
logger.error("Failed to get acquired publications from oracleDB: %s", ex)
nothingToDo['result'] = 'FAIL'
nothingToDo['reason'] = 'Error contacting CRAB REST'
summaryFileName = saveSummaryJson(logdir, nothingToDo)
return summaryFileName
if verbose:
logger.info(results[0]['desc']['columns'])
try:
inputDatasetIndex = results[0]['desc']['columns'].index("tm_input_dataset")
inputDataset = results[0]['result'][inputDatasetIndex]
sourceURLIndex = results[0]['desc']['columns'].index("tm_dbs_url")
sourceURL = results[0]['result'][sourceURLIndex]
publish_dbs_urlIndex = results[0]['desc']['columns'].index("tm_publish_dbs_url")
publish_dbs_url = results[0]['result'][publish_dbs_urlIndex]
if not sourceURL.endswith("/DBSReader") and not sourceURL.endswith("/DBSReader/"):
sourceURL += "/DBSReader"
except Exception:
logger.exception("ERROR")
# When looking up parents may need to look in global DBS as well.
globalURL = sourceURL
globalURL = globalURL.replace('phys01', 'global')
globalURL = globalURL.replace('phys02', 'global')
globalURL = globalURL.replace('phys03', 'global')
globalURL = globalURL.replace('caf', 'global')
# allow to use a DBS REST host different from cmsweb.cern.ch (which is the
# default inserted by CRAB Client)
sourceURL = sourceURL.replace('cmsweb.cern.ch', config.TaskPublisher.DBShost)
globalURL = globalURL.replace('cmsweb.cern.ch', config.TaskPublisher.DBShost)
publish_dbs_url = publish_dbs_url.replace('cmsweb.cern.ch', config.TaskPublisher.DBShost)
# DBS client relies on X509 env. vars
os.environ['X509_USER_CERT'] = config.General.serviceCert
os.environ['X509_USER_KEY'] = config.General.serviceKey
# create DBS API objects
logger.info("DBS Source API URL: %s", sourceURL)
sourceApi = dbsClient.DbsApi(url=sourceURL)
logger.info("DBS Global API URL: %s", globalURL)
globalApi = dbsClient.DbsApi(url=globalURL)
if publish_dbs_url.endswith('/DBSWriter'):
publish_read_url = publish_dbs_url[:-len('/DBSWriter')] + '/DBSReader'
publish_migrate_url = publish_dbs_url[:-len('/DBSWriter')] + '/DBSMigrate'
else:
publish_migrate_url = publish_dbs_url + '/DBSMigrate'
publish_read_url = publish_dbs_url + '/DBSReader'
publish_dbs_url += '/DBSWriter'
try:
logger.info("DBS Destination API URL: %s", publish_dbs_url)
destApi = dbsClient.DbsApi(url=publish_dbs_url)
logger.info("DBS Destination read API URL: %s", publish_read_url)
destReadApi = dbsClient.DbsApi(url=publish_read_url)
logger.info("DBS Migration API URL: %s", publish_migrate_url)
migrateApi = dbsClient.DbsApi(url=publish_migrate_url)
except Exception:
logger.exception('Wrong DBS URL %s', publish_dbs_url)
nothingToDo['result'] = 'FAIL'
nothingToDo['reason'] = 'Error contacting DBS'
summaryFileName = saveSummaryJson(logdir, nothingToDo)
return summaryFileName
logger.info("inputDataset: %s", inputDataset)
noInput = len(inputDataset.split("/")) <= 3
if not noInput:
try:
existing_datasets = sourceApi.listDatasets(dataset=inputDataset, detail=True, dataset_access_type='*')
primary_ds_type = existing_datasets[0]['primary_ds_type']
# There's little chance this is correct, but it's our best guess for now.
# CRAB2 uses 'crab2_tag' for all cases
existing_output = destReadApi.listOutputConfigs(dataset=inputDataset)
except Exception:
logger.exception('Wrong DBS URL %s', publish_dbs_url)
nothingToDo['result'] = 'FAIL'
nothingToDo['reason'] = 'Error looking up input dataset in DBS'
summaryFileName = saveSummaryJson(logdir, nothingToDo)
return summaryFileName
if not existing_output:
msg = "Unable to list output config for input dataset %s." % (inputDataset)
logger.error(msg)
global_tag = 'crab3_tag'
else:
global_tag = existing_output[0]['global_tag']
else:
msg = "This publication appears to be for private MC."
logger.info(msg)
primary_ds_type = 'mc'
global_tag = 'crab3_tag'
acquisition_era_name = "CRAB"
processing_era_config = {'processing_version': 1, 'description': 'CRAB3_processing_era'}
appName = 'cmsRun'
appVer = toPublish[0]["swversion"]
pset_hash = toPublish[0]['publishname'].split("-")[-1]
gtag = str(toPublish[0]['globaltag'])
if gtag == "None":
gtag = global_tag
try:
if toPublish[0]['acquisitionera'] and not toPublish[0]['acquisitionera'] in ["null"]:
acquisitionera = str(toPublish[0]['acquisitionera'])
else:
acquisitionera = acquisition_era_name
except Exception:
acquisitionera = acquisition_era_name
_, primName, procName, tier = toPublish[0]['outdataset'].split('/')
primds_config = {'primary_ds_name': primName, 'primary_ds_type': primary_ds_type}
msg = "About to insert primary dataset"
logger.debug(msg)
if dryRun:
logger.info("DryRun: skip insertPrimaryDataset")
else:
destApi.insertPrimaryDataset(primds_config)
msg = "Successfully inserted primary dataset %s." % (primName)
logger.info(msg)
final = {}
failed = []
publish_in_next_iteration = []
published = []
# Find all files already published in this dataset.
try:
existingDBSFiles = destReadApi.listFiles(dataset=dataset, detail=True)
existingFiles = [f['logical_file_name'] for f in existingDBSFiles]
existingFilesValid = [f['logical_file_name'] for f in existingDBSFiles if f['is_file_valid']]
msg = "Dataset %s already contains %d files" % (dataset, len(existingFiles))
msg += " (%d valid, %d invalid)." % (len(existingFilesValid), len(existingFiles) - len(existingFilesValid))
logger.info(msg)
final['existingFiles'] = len(existingFiles)
except Exception as ex:
msg = "Error when listing files in DBS: %s" % (str(ex))
msg += "\n%s" % (str(traceback.format_exc()))
logger.error(msg)
nothingToDo['result'] = 'FAIL'
nothingToDo['reason'] = 'Error listing existing files in DBS'
summaryFileName = saveSummaryJson(logdir, nothingToDo)
return summaryFileName
# check if actions are needed
workToDo = False
for fileTo in toPublish:
#print(existingFilesValid)
if fileTo['lfn'] not in existingFiles:
workToDo = True
break
if not workToDo:
msg = "Nothing uploaded, output dataset has these files already."
logger.info(msg)
logger.info('Make sure those files are marked as Done')
# docId is the has of the source LFN i.e. the file in the tmp area at the running site
files = [f['SourceLFN'] for f in toPublish]
mark_good(files, crabServer, logger)
summaryFileName = saveSummaryJson(logdir, nothingToDo)
return summaryFileName
acquisition_era_config = {'acquisition_era_name': acquisitionera, 'start_date': 0}
output_config = {'release_version': appVer,
'pset_hash': pset_hash,
'app_name': appName,
'output_module_label': 'o',
'global_tag': global_tag,
}
dataset_config = {'dataset': dataset,
'processed_ds_name': procName,
'data_tier_name': tier,
'dataset_access_type': 'VALID',
'physics_group_name': 'CRAB3',
'last_modification_date': int(time.time()),
}
logger.info("Output dataset config: %s", str(dataset_config))
# List of all files that must (and can) be published.
dbsFiles = []
dbsFiles_f = []
# Set of all the parent files from all the files requested to be published.
parentFiles = set()
# Set of parent files for which the migration to the destination DBS instance
# should be skipped (because they were not found in DBS).
parentsToSkip = set()
# Set of parent files to migrate from the source DBS instance
# to the destination DBS instance.
localParentBlocks = set()
# Set of parent files to migrate from the global DBS instance
# to the destination DBS instance.
globalParentBlocks = set()
# Loop over all files to publish.
for file_ in toPublish:
if verbose:
logger.info(file_)
# Check if this file was already published and if it is valid.
if file_['lfn'] not in existingFilesValid:
# We have a file to publish.
# Get the parent files and for each parent file do the following:
# 1) Add it to the list of parent files.
# 2) Find the block to which it belongs and insert that block name in
# (one of) the set of blocks to be migrated to the destination DBS.
for parentFile in list(file_['parents']):
if parentFile not in parentFiles:
parentFiles.add(parentFile)
# Is this parent file already in the destination DBS instance?
# (If yes, then we don't have to migrate this block.)
blocksDict = destReadApi.listBlocks(logical_file_name=parentFile)
if not blocksDict:
# No, this parent file is not in the destination DBS instance.
# Maybe it is in the same DBS instance as the input dataset?
blocksDict = sourceApi.listBlocks(logical_file_name=parentFile)
if blocksDict:
# Yes, this parent file is in the same DBS instance as the input dataset.
# Add the corresponding block to the set of blocks from the source DBS
# instance that have to be migrated to the destination DBS.
localParentBlocks.add(blocksDict[0]['block_name'])
else:
# No, this parent file is not in the same DBS instance as input dataset.
# Maybe it is in global DBS instance?
blocksDict = globalApi.listBlocks(logical_file_name=parentFile)
if blocksDict:
# Yes, this parent file is in global DBS instance.
# Add the corresponding block to the set of blocks from global DBS
# instance that have to be migrated to the destination DBS.
globalParentBlocks.add(blocksDict[0]['block_name'])
# If this parent file is not in the destination DBS instance, is not
# the source DBS instance, and is not in global DBS instance, then it
# means it is not known to DBS and therefore we can not migrate it.
# Put it in the set of parent files for which migration should be skipped.
if not blocksDict:
parentsToSkip.add(parentFile)
# If this parent file should not be migrated because it is not known to DBS,
# we remove it from the list of parents in the file-to-publish info dictionary
# (so that when publishing, this "parent" file will not appear as a parent).
if parentFile in parentsToSkip:
msg = "Skipping parent file %s, as it doesn't seem to be known to DBS." % (parentFile)
logger.info(msg)
if parentFile in file_['parents']:
file_['parents'].remove(parentFile)
# Add this file to the list of files to be published.
dbsFiles.append(format_file_3(file_))
dbsFiles_f.append(file_)
#print file
published.append(file_['SourceLFN'])
#published.append(file_['lfn'].replace("/store","/store/temp"))
# Print a message with the number of files to publish.
msg = "Found %d files not already present in DBS which will be published." % (len(dbsFiles))
logger.info(msg)
# If there are no files to publish, continue with the next dataset.
if not dbsFiles_f:
msg = "No file to publish to do for this dataset."
logger.info(msg)
summaryFileName = saveSummaryJson(logdir, nothingToDo)
return summaryFileName
# Migrate parent blocks before publishing.
# First migrate the parent blocks that are in the same DBS instance
# as the input dataset.
if localParentBlocks:
msg = "List of parent blocks that need to be migrated from %s:\n%s" % (sourceApi.url, localParentBlocks)
logger.info(msg)
if dryRun:
logger.info("DryRun: skipping migration request")
else:
statusCode, failureMsg = migrateByBlockDBS3(taskname, migrateApi, destReadApi, sourceApi,
inputDataset, localParentBlocks, migrationLogDir, verbose)
if statusCode:
failureMsg += " Not publishing any files."
logger.info(failureMsg)
summaryFileName = saveSummaryJson(logdir, nothingToDo)
return summaryFileName
# Then migrate the parent blocks that are in the global DBS instance.
if globalParentBlocks:
msg = "List of parent blocks that need to be migrated from %s:\n%s" % (globalApi.url, globalParentBlocks)
logger.info(msg)
if dryRun:
logger.info("DryRun: skipping migration request")
else:
statusCode, failureMsg = migrateByBlockDBS3(taskname, migrateApi, destReadApi, globalApi,
inputDataset, globalParentBlocks, migrationLogDir, verbose)
if statusCode:
failureMsg += " Not publishing any files."
logger.info(failureMsg)
summaryFileName = saveSummaryJson(logdir, nothingToDo)
return summaryFileName
# Publish the files in blocks. The blocks must have exactly max_files_per_block
# files, unless there are less than max_files_per_block files to publish to
# begin with. If there are more than max_files_per_block files to publish,
# publish as many blocks as possible and leave the tail of files for the next
# PublisherWorker call, unless forced to published.
block_count = 0
count = 0
publishedBlocks = 0
failedBlocks = 0
max_files_per_block = config.General.max_files_per_block
dumpList = [] # keep a list of files where blocks which fail publication are dumped
while True:
block_name = "%s#%s" % (dataset, str(uuid.uuid4()))
files_to_publish = dbsFiles[count:count+max_files_per_block]
try:
block_config = {'block_name': block_name, 'origin_site_name': pnn, 'open_for_writing': 0}
if verbose:
msg = "Inserting files %s into block %s." % ([f['logical_file_name']
for f in files_to_publish], block_name)
logger.info(msg)
blockDump = createBulkBlock(output_config, processing_era_config,
primds_config, dataset_config,
acquisition_era_config, block_config, files_to_publish)
#logger.debug("Block to insert: %s\n %s" % (blockDump, destApi.__dict__ ))
if dryRun:
logger.info("DryRun: skip insertBulkBlock")
else:
destApi.insertBulkBlock(blockDump)
block_count += 1
publishedBlocks += 1
except Exception as ex:
#logger.error("Error for files: %s" % [f['SourceLFN'] for f in toPublish])
logger.error("Error for files: %s", [f['lfn'] for f in toPublish])
failed.extend([f['SourceLFN'] for f in toPublish])
#failed.extend([f['lfn'].replace("/store","/store/temp") for f in toPublish])
msg = "Error when publishing (%s) " % ", ".join(failed)
msg += str(ex)
msg += str(traceback.format_exc())
logger.error(msg)
failure_reason = str(ex)
taskFilesDir = config.General.taskFilesDir
fname = os.path.join(taskFilesDir, 'FailedBlocks', 'failed-block-at-%s.txt' % time.time())
with open(fname, 'w') as fd:
fd.write(pprint.pformat(blockDump))
dumpList.append(fname)
failedBlocks += 1
logger.error("FAILING BLOCK DUE TO %s SAVED AS %s", str(ex), fname)
count += max_files_per_block
files_to_publish_next = dbsFiles_f[count:count+max_files_per_block]
if len(files_to_publish_next) < max_files_per_block:
publish_in_next_iteration.extend([f["SourceLFN"] for f in files_to_publish_next])
#publish_in_next_iteration.extend([f["lfn"].replace("/store","/store/temp") for f in files_to_publish_next])
break
published = [x for x in published if x not in failed + publish_in_next_iteration]
# Fill number of files/blocks published for this dataset.
final['files'] = len(dbsFiles) - len(failed) - len(publish_in_next_iteration)
final['blocks'] = block_count
# Print a publication status summary for this dataset.
msg = "End of publication status:"
msg += " failed %s" % len(failed)
if verbose:
msg += ": %s" % failed
msg += ", published %s" % len(published)
if verbose:
msg += ": %s" % published
msg += ", publish_in_next_iteration %s" % len(publish_in_next_iteration)
if verbose:
msg += ": %s" % publish_in_next_iteration
msg += ", results %s" % (final)
logger.info(msg)
try:
if published:
mark_good(published, crabServer, logger)
data['workflow'] = taskname
data['subresource'] = 'updatepublicationtime'
crabServer.post(api='task', data=encodeRequest(data))
if failed:
logger.debug("Failed files: %s ", failed)
mark_failed(failed, crabServer, logger, failure_reason)
except Exception as ex:
logger.exception("Status update failed: %s", ex)
summary = {}
summary['taskname'] = taskname
summary['result'] = 'OK' if not failed else 'FAIL'
summary['reason'] = '' if not failed else 'DBS Publication Failure'
summary['publishedBlocks'] = publishedBlocks
summary['failedBlocks'] = failedBlocks
summary['failedBlockDumps'] = dumpList
summary['publishedFiles'] = len(published)
summary['failedFiles'] = len(failed)
summary['nextIterFiles'] = len(publish_in_next_iteration)
summaryFileName = saveSummaryJson(logdir, summary)
return summaryFileName