def addToPhEDEx(WFName, fileList): dsName = getDatasetName(WFName) # no file to inject for the dataset if not fileList: return # create xml data for injection&subscription Logger.log('Creating xml file for injection') xmlData = createXML(WFName, fileList) # inject data Logger.log("Injecting to CASTOR: %s" % dsName) data = request.send(urlInject, { 'data': xmlData, 'node': 'T0_CH_CERN_Export' }) if data: # subscribed it so that transfer can start from CASTOR to EOS Logger.log("Subscribing at EOS : %s" % dsName) request.send( urlSubscribe, { 'data': xmlData, 'node': 'T2_CH_CERN', 'group': 'transferops', 'no_mail': 'y', 'comments': 'auto-approved log transfer from CASTOR to EOS' }) else: Logger.log('Skipping subscription since injection got failed')
def createXML(WFName, fileInfoList): dsName = getDatasetName(WFName) xml = [] xml.append('<data version="2.0">') xml.append(' <dbs name="Log_Files" dls="dbs">') xml.append(' <dataset name="%s" is-open="y" is-transient="n">' % dsName) xml.append(' <block name="%s#01" is-open="y">' % dsName) for file in fileInfoList: fileLFN = getLFN(WFName,file['name']) # if checksum is missing do not add it to xml if not file['checksum']: Logger.log('skipping file without checksum: %s' % fileLFN) continue xml.append(' <file name="%s" bytes="%s" checksum="adler32:%s"/>' % (fileLFN,file['size'],file['checksum'])) xml.append(' </block>') xml.append(' </dataset>') xml.append(' </dbs>') xml.append('</data>') return ''.join(xml)
def createXML(WFName, fileInfoList): dsName = getDatasetName(WFName) xml = [] xml.append('<data version="2.0">') xml.append(' <dbs name="Log_Files" dls="dbs">') xml.append(' <dataset name="%s" is-open="y" is-transient="n">' % dsName) xml.append(' <block name="%s#01" is-open="y">' % dsName) for file in fileInfoList: fileLFN = getLFN(WFName, file['name']) # if checksum is missing do not add it to xml if not file['checksum']: Logger.log('skipping file without checksum: %s' % fileLFN) continue xml.append(' <file name="%s" bytes="%s" checksum="adler32:%s"/>' % (fileLFN, file['size'], file['checksum'])) xml.append(' </block>') xml.append(' </dataset>') xml.append(' </dbs>') xml.append('</data>') return ''.join(xml)
def addToPhEDEx(WFName, fileList): dsName = getDatasetName(WFName) # no file to inject for the dataset if not fileList: return # create xml data for injection&subscription Logger.log('Creating xml file for injection') xmlData = createXML(WFName, fileList) # inject data Logger.log("Injecting to CASTOR: %s" %dsName) data = request.send(urlInject, {'data':xmlData,'node':'T0_CH_CERN_Export'}) if data: # subscribed it so that transfer can start from CASTOR to EOS Logger.log("Subscribing at EOS : %s" % dsName) request.send(urlSubscribe, {'data':xmlData,'node':'T2_CH_CERN','group':'transferops','no_mail':'y','comments':'auto-approved log transfer from CASTOR to EOS'}) else: Logger.log('Skipping subscription since injection got failed')
'data': xmlData, 'node': 'T2_CH_CERN', 'group': 'transferops', 'no_mail': 'y', 'comments': 'auto-approved log transfer from CASTOR to EOS' }) else: Logger.log('Skipping subscription since injection got failed') if __name__ == '__main__': # Get request object for future PhEDEx calls request = Request() # get list of datasets injected before Logger.log('retrieving list of injected files in %s/%s' % (year, month)) injectedFiles = getAlreadyInjectedFiles() Logger.log('number of already injected files: %s' % len(injectedFiles)) #list log directories on CASTOR Logger.log('listing files under %s' % logCastorPath) cmd = 'nsls -lR --checksum %s' % logCastorPath process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) #process.wait() Logger.log('listing is completed, parsing the output') fileList = [] currentWF = None
import time from Common import Request from Common import Logger try: import json except ImportError: import simplejson as json urlPhedex = 'https://phedex-dev.cern.ch/phedex/datasvc/json/ops/' urlDelete = urlPhedex + 'delete' def getOldDatasets(): datasetList = [] date = time.time() - 60*60*24*2 urlSubscription = urlPhedex + 'subscriptions?node=T2_CH_CERN&dataset=/Log*/*/*&create_since=!%s' % date result = json.loads(request.send(urlSubscription)) for dataset in result['phedex']['dataset']: datasetList.append(dataset['name']) return datasetList if __name__ == '__main__': # Get request object for future PhEDEx calls request = Request() # get datasets with subscription to EOS older than N months Logger.log('retrieving list of old datasets' oldDatasets = getOldDatasets(); Logger.log('number of old datasets: %s' % len(oldDatasets))
episode_g_norm += g_norm if step % params["hard_update_freq"] == 0: agent.hard_update_target_network() # env.render() state = next_state if episode_len == params["max_frames_per_episode"]: done = True if done: logger.off() logger.log( episode, episode_reward, episode_loss / episode_len * params["train_period"], episode_g_norm / episode_len * params["train_period"], step, beta, episode_len) episode += 1 obs = env.reset() state = make_state(state, obs, True) episode_reward = 0 episode_len = 0 episode_loss = 0 episode_g_norm = 0 logger.on() else: # region play chekpoint = logger.load_weights()
class Database(): def __init__(self, db_file=None, log_file=None, debug=False): self.dbms = 'unknown' self.metadata = sqlalchemy.MetaData() self.schema = {} self.tables = {} self.types = {} self.sources = {} self.logger = Logger(log_file, debug, 'Database.logger') def _detect_datatype(self, sample): """ Supported sqlalchemy types: Boolean, DateTime, Integer, Float, Text """ #if not sample: # #raise NotImplementedError('Expected a sample but got "%s"' % (sample)) # return sqlalchemy.types.String(convert_unicode=True) #if sample.title() in ['True', 'False']: # return sqlalchemy.types.Boolean() #try: # if int(sample): # return sqlalchemy.types.Integer() #except ValueError: # pass #try: # if float(sample): # return sqlalchemy.types.Float() #except ValueError: # pass #for strf in ['%Y-%m-%dT%H:%M:%S', '%Y-%m-%d %H:%M:%S']: # try: # if datetime.datetime.strptime(sample, strf): # return sqlalchemy.types.DateTime() # except ValueError: # pass return sqlalchemy.types.String(convert_unicode=True) def create_schema(self, table, column_names, samples): """ samples should be a generator yielding dicts from the csv (e.g. as returned from DictReader()). """ self.sources[table] = samples if not self.schema.has_key(table): self.schema[table] = [] for column in column_names: index = column_names.index(column) sample = '' while True: try: row = samples.next() sample = row[column] if sample != '': break if samples.line_num > 100: break except StopIteration: break if not sample: self.logger.log('could not find a sample for column: %s' % column, 'DEBUG') datatype = self._detect_datatype(sample) while True: if column.lower() in [name.lower() for name, value in self.schema[table]]: self.logger.log('found duplicate column "%s"' % (column), 'INFO') column += '_' else: break self.schema[table].append((column, datatype)) column_names[index] = column return column_names def create_table(self, table, primary_key): if not self.schema.get(table): raise RuntimeError('this database instance has no schema; please run create_schema first.') self.tables[table] = sqlalchemy.Table(table, self.metadata) for column, datatype in self.schema[table]: if column.lower() == primary_key.lower(): self.tables[table].append_column(sqlalchemy.Column(column, datatype, primary_key=True)) else: self.tables[table].append_column(sqlalchemy.Column(column, datatype)) try: self.metadata.create_all(self.engine) except sqlalchemy.exceptions.OperationalError: self.logger.log('OperationalError while attempting to create database table!', 'ERROR') return class Type(object): def __init__(self): pass orm.mapper(Type, self.tables[table]) self.types[table] = Type return self.types[table] def create_session(self): session = orm.sessionmaker() session.configure(bind=self.engine) self.session = session() return self.session def commit(self, count=False): self.logger.log('starting to flush cache to %s database.' % (self.dbms), 'INFO') self.session.commit() self.logger.log('finished flushing cache to %s database.' % (self.dbms), 'INFO') if count: self.logger.log('total record count is %s.' % (count), 'INFO') return
data = request.send(urlInject, {'data':xmlData,'node':'T0_CH_CERN_Export'}) if data: # subscribed it so that transfer can start from CASTOR to EOS Logger.log("Subscribing at EOS : %s" % dsName) request.send(urlSubscribe, {'data':xmlData,'node':'T2_CH_CERN','group':'transferops','no_mail':'y','comments':'auto-approved log transfer from CASTOR to EOS'}) else: Logger.log('Skipping subscription since injection got failed') if __name__ == '__main__': # Get request object for future PhEDEx calls request = Request() # get list of datasets injected before Logger.log('retrieving list of injected files in %s/%s' % (year,month)) injectedFiles = getAlreadyInjectedFiles() Logger.log('number of already injected files: %s' % len(injectedFiles)) #list log directories on CASTOR Logger.log('listing files under %s' % logCastorPath) cmd = 'nsls -lR --checksum %s' % logCastorPath process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) #process.wait() Logger.log('listing is completed, parsing the output') fileList = [] currentWF = None #collect info from castor for line in process.stdout: