def addToPhEDEx(WFName, fileList):
dsName = getDatasetName(WFName)
# no file to inject for the dataset
if not fileList:
return
# create xml data for injection&subscription
Logger.log('Creating xml file for injection')
xmlData = createXML(WFName, fileList)
# inject data
Logger.log("Injecting to CASTOR: %s" % dsName)
data = request.send(urlInject, {
'data': xmlData,
'node': 'T0_CH_CERN_Export'
})
if data:
# subscribed it so that transfer can start from CASTOR to EOS
Logger.log("Subscribing at EOS : %s" % dsName)
request.send(
urlSubscribe, {
'data': xmlData,
'node': 'T2_CH_CERN',
'group': 'transferops',
'no_mail': 'y',
'comments': 'auto-approved log transfer from CASTOR to EOS'
})
else:
Logger.log('Skipping subscription since injection got failed')
def createXML(WFName, fileInfoList):
dsName = getDatasetName(WFName)
xml = []
xml.append('<data version="2.0">')
xml.append(' <dbs name="Log_Files" dls="dbs">')
xml.append(' <dataset name="%s" is-open="y" is-transient="n">' % dsName)
xml.append(' <block name="%s#01" is-open="y">' % dsName)
for file in fileInfoList:
fileLFN = getLFN(WFName,file['name'])
# if checksum is missing do not add it to xml
if not file['checksum']:
Logger.log('skipping file without checksum: %s' % fileLFN)
continue
xml.append(' <file name="%s" bytes="%s" checksum="adler32:%s"/>' % (fileLFN,file['size'],file['checksum']))
xml.append(' </block>')
xml.append(' </dataset>')
xml.append(' </dbs>')
xml.append('</data>')
return ''.join(xml)
def createXML(WFName, fileInfoList):
dsName = getDatasetName(WFName)
xml = []
xml.append('<data version="2.0">')
xml.append(' <dbs name="Log_Files" dls="dbs">')
xml.append(' <dataset name="%s" is-open="y" is-transient="n">' % dsName)
xml.append(' <block name="%s#01" is-open="y">' % dsName)
for file in fileInfoList:
fileLFN = getLFN(WFName, file['name'])
# if checksum is missing do not add it to xml
if not file['checksum']:
Logger.log('skipping file without checksum: %s' % fileLFN)
continue
xml.append(' <file name="%s" bytes="%s" checksum="adler32:%s"/>' %
(fileLFN, file['size'], file['checksum']))
xml.append(' </block>')
xml.append(' </dataset>')
xml.append(' </dbs>')
xml.append('</data>')
return ''.join(xml)
def addToPhEDEx(WFName, fileList):
dsName = getDatasetName(WFName)
# no file to inject for the dataset
if not fileList:
return
# create xml data for injection&subscription
Logger.log('Creating xml file for injection')
xmlData = createXML(WFName, fileList)
# inject data
Logger.log("Injecting to CASTOR: %s" %dsName)
data = request.send(urlInject, {'data':xmlData,'node':'T0_CH_CERN_Export'})
if data:
# subscribed it so that transfer can start from CASTOR to EOS
Logger.log("Subscribing at EOS : %s" % dsName)
request.send(urlSubscribe, {'data':xmlData,'node':'T2_CH_CERN','group':'transferops','no_mail':'y','comments':'auto-approved log transfer from CASTOR to EOS'})
else:
Logger.log('Skipping subscription since injection got failed')
'data': xmlData,
'node': 'T2_CH_CERN',
'group': 'transferops',
'no_mail': 'y',
'comments': 'auto-approved log transfer from CASTOR to EOS'
})
else:
Logger.log('Skipping subscription since injection got failed')
if __name__ == '__main__':
# Get request object for future PhEDEx calls
request = Request()
# get list of datasets injected before
Logger.log('retrieving list of injected files in %s/%s' % (year, month))
injectedFiles = getAlreadyInjectedFiles()
Logger.log('number of already injected files: %s' % len(injectedFiles))
#list log directories on CASTOR
Logger.log('listing files under %s' % logCastorPath)
cmd = 'nsls -lR --checksum %s' % logCastorPath
process = subprocess.Popen(cmd.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
#process.wait()
Logger.log('listing is completed, parsing the output')
fileList = []
currentWF = None
import time
from Common import Request
from Common import Logger
try: import json
except ImportError: import simplejson as json
urlPhedex = 'https://phedex-dev.cern.ch/phedex/datasvc/json/ops/'
urlDelete = urlPhedex + 'delete'
def getOldDatasets():
datasetList = []
date = time.time() - 60*60*24*2
urlSubscription = urlPhedex + 'subscriptions?node=T2_CH_CERN&dataset=/Log*/*/*&create_since=!%s' % date
result = json.loads(request.send(urlSubscription))
for dataset in result['phedex']['dataset']:
datasetList.append(dataset['name'])
return datasetList
if __name__ == '__main__':
# Get request object for future PhEDEx calls
request = Request()
# get datasets with subscription to EOS older than N months
Logger.log('retrieving list of old datasets'
oldDatasets = getOldDatasets();
Logger.log('number of old datasets: %s' % len(oldDatasets))
episode_g_norm += g_norm
if step % params["hard_update_freq"] == 0:
agent.hard_update_target_network()
# env.render()
state = next_state
if episode_len == params["max_frames_per_episode"]:
done = True
if done:
logger.off()
logger.log(
episode, episode_reward,
episode_loss / episode_len * params["train_period"],
episode_g_norm / episode_len * params["train_period"],
step, beta, episode_len)
episode += 1
obs = env.reset()
state = make_state(state, obs, True)
episode_reward = 0
episode_len = 0
episode_loss = 0
episode_g_norm = 0
logger.on()
else:
# region play
chekpoint = logger.load_weights()
class Database():
def __init__(self, db_file=None, log_file=None, debug=False):
self.dbms = 'unknown'
self.metadata = sqlalchemy.MetaData()
self.schema = {}
self.tables = {}
self.types = {}
self.sources = {}
self.logger = Logger(log_file, debug, 'Database.logger')
def _detect_datatype(self, sample):
""" Supported sqlalchemy types: Boolean, DateTime, Integer, Float, Text
"""
#if not sample:
# #raise NotImplementedError('Expected a sample but got "%s"' % (sample))
# return sqlalchemy.types.String(convert_unicode=True)
#if sample.title() in ['True', 'False']:
# return sqlalchemy.types.Boolean()
#try:
# if int(sample):
# return sqlalchemy.types.Integer()
#except ValueError:
# pass
#try:
# if float(sample):
# return sqlalchemy.types.Float()
#except ValueError:
# pass
#for strf in ['%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S']:
# try:
# if datetime.datetime.strptime(sample, strf):
# return sqlalchemy.types.DateTime()
# except ValueError:
# pass
return sqlalchemy.types.String(convert_unicode=True)
def create_schema(self, table, column_names, samples):
""" samples should be a generator yielding dicts from the csv (e.g. as
returned from DictReader()).
"""
self.sources[table] = samples
if not self.schema.has_key(table):
self.schema[table] = []
for column in column_names:
index = column_names.index(column)
sample = ''
while True:
try:
row = samples.next()
sample = row[column]
if sample != '':
break
if samples.line_num > 100:
break
except StopIteration:
break
if not sample:
self.logger.log('could not find a sample for column: %s' % column, 'DEBUG')
datatype = self._detect_datatype(sample)
while True:
if column.lower() in [name.lower() for name, value in self.schema[table]]:
self.logger.log('found duplicate column "%s"' % (column), 'INFO')
column += '_'
else:
break
self.schema[table].append((column, datatype))
column_names[index] = column
return column_names
def create_table(self, table, primary_key):
if not self.schema.get(table):
raise RuntimeError('this database instance has no schema; please run create_schema first.')
self.tables[table] = sqlalchemy.Table(table, self.metadata)
for column, datatype in self.schema[table]:
if column.lower() == primary_key.lower():
self.tables[table].append_column(sqlalchemy.Column(column, datatype, primary_key=True))
else:
self.tables[table].append_column(sqlalchemy.Column(column, datatype))
try:
self.metadata.create_all(self.engine)
except sqlalchemy.exceptions.OperationalError:
self.logger.log('OperationalError while attempting to create database table!', 'ERROR')
return
class Type(object):
def __init__(self):
pass
orm.mapper(Type, self.tables[table])
self.types[table] = Type
return self.types[table]
def create_session(self):
session = orm.sessionmaker()
session.configure(bind=self.engine)
self.session = session()
return self.session
def commit(self, count=False):
self.logger.log('starting to flush cache to %s database.' % (self.dbms), 'INFO')
self.session.commit()
self.logger.log('finished flushing cache to %s database.' % (self.dbms), 'INFO')
if count:
self.logger.log('total record count is %s.' % (count), 'INFO')
return
data = request.send(urlInject, {'data':xmlData,'node':'T0_CH_CERN_Export'})
if data:
# subscribed it so that transfer can start from CASTOR to EOS
Logger.log("Subscribing at EOS : %s" % dsName)
request.send(urlSubscribe, {'data':xmlData,'node':'T2_CH_CERN','group':'transferops','no_mail':'y','comments':'auto-approved log transfer from CASTOR to EOS'})
else:
Logger.log('Skipping subscription since injection got failed')
if __name__ == '__main__':
# Get request object for future PhEDEx calls
request = Request()
# get list of datasets injected before
Logger.log('retrieving list of injected files in %s/%s' % (year,month))
injectedFiles = getAlreadyInjectedFiles()
Logger.log('number of already injected files: %s' % len(injectedFiles))
#list log directories on CASTOR
Logger.log('listing files under %s' % logCastorPath)
cmd = 'nsls -lR --checksum %s' % logCastorPath
process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
#process.wait()
Logger.log('listing is completed, parsing the output')
fileList = []
currentWF = None
#collect info from castor
for line in process.stdout:
