def uploadPayload(filenamePrefix, sqliteFile, metaFile, dropboxHost, validationMode, username, password):
"""
_uploadPayload_
Upload a single payload consisting of a sqlite
file and a metadata file to the dropbox.
"""
completeFiles = []
files2delete = []
filenameDB = filenamePrefix + ".db"
filenameTXT = filenamePrefix + ".txt"
filenameTAR = filenamePrefix + ".tar.bz2"
shutil.copy2(sqliteFile['pfn'], filenameDB)
files2delete.append(filenameDB)
# select the right destination db depending
# on whether we are in validation mode
fin = open(metaFile['pfn'])
lines = fin.readlines()
fin.close()
fout = open(filenameTXT, 'w')
if validationMode:
fout.writelines( [ line.replace('prepMetaData ', '', 1) for line in lines if 'prodMetaData ' not in line] )
else:
fout.writelines( [ line.replace('prodMetaData ', '', 1) for line in lines if 'prepMetaData ' not in line] )
fout.close()
files2delete.append(filenameTXT)
os.chmod(filenameDB, stat.S_IREAD | stat.S_IWRITE | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
os.chmod(filenameTXT, stat.S_IREAD | stat.S_IWRITE | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
shutil.copy2(filenameTXT, metaFile['pfn'] + ".uploaded")
uploadStatus = True
try:
upload.uploadTier0Files([filenameDB], username, password)
except:
logging.exception("Something went wrong with the Dropbox upload...")
uploadStatus = False
if uploadStatus:
completeFiles.append(sqliteFile)
completeFiles.append(metaFile)
logging.info(" ==> Upload succeeded for payload %s" % filenamePrefix)
else:
logging.error(" ==> Upload failed for payload %s" % filenamePrefix)
for file2delete in files2delete:
os.remove(file2delete)
return completeFiles
def uploadPayload(filenamePrefix, sqliteFile, metaFile, dropboxHost, validationMode, username, password, serviceProxy):
"""
_uploadPayload_
Upload a single payload consisting of a sqlite
file and a metadata file to the dropbox.
"""
completeFiles = []
files2delete = []
inputCopied = True
filenameDB = filenamePrefix + ".db"
filenameTXT = filenamePrefix + ".txt"
filenameTAR = filenamePrefix + ".tar.bz2"
eosPrefix = "root://eoscms//eos/cms"
sqliteFile['pfn'] = eosPrefix + sqliteFile['lfn']
metaFile['pfn'] = eosPrefix + metaFile['lfn']
command = "export X509_USER_PROXY=%s\n" % serviceProxy
command += "env KRB5CCNAME=/tmp/bla xrdcp -s -f %s %s" % (sqliteFile['pfn'], filenameDB)
p = subprocess.Popen(command, shell = True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = p.communicate()[0]
if p.returncode > 0:
logging.error("Failure during copy from EOS: %s" % output)
logging.error(" ==> Upload failed for payload %s" % filenamePrefix)
inputCopied = False
else:
files2delete.append(filenameDB)
command = "export X509_USER_PROXY=%s\n" % serviceProxy
command += "env KRB5CCNAME=/tmp/bla xrdcp -s -f %s %s" % (metaFile['pfn'], filenameTXT)
p = subprocess.Popen(command, shell = True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = p.communicate()[0]
if p.returncode > 0:
logging.error("Failure during copy from EOS: %s" % output)
logging.error(" ==> Upload failed for payload %s" % filenamePrefix)
inputCopied = False
else:
files2delete.append(filenameTXT)
# select the right destination db depending
# on whether we are in validation mode
if inputCopied:
fin = open(filenameTXT)
lines = fin.readlines()
fin.close()
fout = open(filenameTXT, 'w')
if validationMode:
fout.writelines( [ line.replace('prepMetaData ', '', 1) for line in lines if 'prodMetaData ' not in line] )
else:
fout.writelines( [ line.replace('prodMetaData ', '', 1) for line in lines if 'prepMetaData ' not in line] )
fout.close()
os.chmod(filenameDB, stat.S_IREAD | stat.S_IWRITE | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
os.chmod(filenameTXT, stat.S_IREAD | stat.S_IWRITE | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
if username == None or password == None:
completeFiles.append(sqliteFile)
completeFiles.append(metaFile)
logging.info("No username/password provided for DropBox upload...")
logging.info(" ==> Upload skipped for payload %s" % filenamePrefix)
elif serviceProxy == None:
completeFiles.append(sqliteFile)
completeFiles.append(metaFile)
logging.info("No service proxy provided to access EOS for uploaded record...")
logging.info(" ==> Upload skipped for payload %s" % filenamePrefix)
else:
# needed by the PCL monitoring to know whether we uploaded to prod or validation
command = "export X509_USER_PROXY=%s\n" % serviceProxy
command += "env KRB5CCNAME=/tmp/bla xrdcp -s -f %s %s" % (filenameTXT, metaFile['pfn'] + ".uploaded")
p = subprocess.Popen(command, shell = True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = p.communicate()[0]
if p.returncode > 0:
logging.error("Failure during copy of .uploaded file to EOS: %s" % output)
logging.error(" ==> Upload failed for payload %s" % filenamePrefix)
else:
uploadStatus = True
try:
# normally I would want to evanluate the return code here,
# but as long as the dropbox resturns errors for non-fixable
# problems, I cannot do that (would retry forever)
upload.uploadTier0Files([filenameDB], username, password)
except:
logging.exception("Something went wrong with the Dropbox upload...")
if uploadStatus:
completeFiles.append(sqliteFile)
completeFiles.append(metaFile)
logging.info(" ==> Upload succeeded for payload %s" % filenamePrefix)
else:
logging.error(" ==> Upload failed for payload %s" % filenamePrefix)
for file2delete in files2delete:
os.remove(file2delete)
return completeFiles
def uploadPayload(filenamePrefix, sqliteFile, metaFile, dropboxHost,
validationMode, username, password, serviceProxy):
"""
_uploadPayload_
Upload a single payload consisting of a sqlite
file and a metadata file to the dropbox.
"""
completeFiles = []
files2delete = []
inputCopied = True
filenameDB = filenamePrefix + ".db"
filenameTXT = filenamePrefix + ".txt"
filenameTAR = filenamePrefix + ".tar.bz2"
eosPrefix = "root://eoscms.cern.ch//eos/cms"
sqliteFile['pfn'] = eosPrefix + sqliteFile['lfn']
metaFile['pfn'] = eosPrefix + metaFile['lfn']
command = "export X509_USER_PROXY=%s\n" % serviceProxy
command += "env KRB5CCNAME=/tmp/bla xrdcp -s -f %s %s" % (
sqliteFile['pfn'], filenameDB)
p = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = p.communicate()[0]
if p.returncode > 0:
logging.error("Failure during copy from EOS: %s" % output)
logging.error(" ==> Upload failed for payload %s" % filenamePrefix)
inputCopied = False
else:
files2delete.append(filenameDB)
command = "export X509_USER_PROXY=%s\n" % serviceProxy
command += "env KRB5CCNAME=/tmp/bla xrdcp -s -f %s %s" % (metaFile['pfn'],
filenameTXT)
p = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = p.communicate()[0]
if p.returncode > 0:
logging.error("Failure during copy from EOS: %s" % output)
logging.error(" ==> Upload failed for payload %s" % filenamePrefix)
inputCopied = False
else:
files2delete.append(filenameTXT)
# select the right destination db depending
# on whether we are in validation mode
if inputCopied:
fin = open(filenameTXT)
lines = fin.readlines()
fin.close()
fout = open(filenameTXT, 'w')
if validationMode:
fout.writelines([
line.replace('prepMetaData ', '', 1) for line in lines
if 'prodMetaData ' not in line
])
else:
fout.writelines([
line.replace('prodMetaData ', '', 1) for line in lines
if 'prepMetaData ' not in line
])
fout.close()
os.chmod(
filenameDB,
stat.S_IREAD | stat.S_IWRITE | stat.S_IRUSR | stat.S_IWUSR
| stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
os.chmod(
filenameTXT,
stat.S_IREAD | stat.S_IWRITE | stat.S_IRUSR | stat.S_IWUSR
| stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH | stat.S_IWOTH)
if username == None or password == None:
completeFiles.append(sqliteFile)
completeFiles.append(metaFile)
logging.info("No username/password provided for DropBox upload...")
logging.info(" ==> Upload skipped for payload %s" %
filenamePrefix)
elif serviceProxy == None:
completeFiles.append(sqliteFile)
completeFiles.append(metaFile)
logging.info(
"No service proxy provided to access EOS for uploaded record..."
)
logging.info(" ==> Upload skipped for payload %s" %
filenamePrefix)
else:
# needed by the PCL monitoring to know whether we uploaded to prod or validation
command = "export X509_USER_PROXY=%s\n" % serviceProxy
command += "env KRB5CCNAME=/tmp/bla XRD_WRITERECOVERY=0 xrdcp -s -f %s %s" % (
filenameTXT, metaFile['pfn'] + ".uploaded")
p = subprocess.Popen(command,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = p.communicate()[0]
if p.returncode > 0:
logging.error(
"Failure during copy of .uploaded file to EOS: %s" %
output)
logging.error(" ==> Upload failed for payload %s" %
filenamePrefix)
else:
uploadStatus = True
try:
upload.uploadTier0Files([filenameDB], username, password)
except:
logging.exception(
"Something went wrong with the Dropbox upload...")
uploadStatus = False
if uploadStatus:
completeFiles.append(sqliteFile)
completeFiles.append(metaFile)
logging.info(" ==> Upload succeeded for payload %s" %
filenamePrefix)
else:
logging.error(" ==> Upload failed for payload %s" %
filenamePrefix)
for file2delete in files2delete:
os.remove(file2delete)
return completeFiles