def __init__(self,
date,
websites=pd.DataFrame(),
init=False,
newSites=False):
configs = Setup.readConfig()
connections = Setup.initDBs()
if newSites != True:
self.database = configs['Database'][
'DS_RetentionListProcesStatus']['Database']
self.server = configs['Database']['DS_RetentionListProcesStatus'][
'Server']
else:
self.database = configs['Database'][
'DS_RetentionListNewSitesProcess']['Database']
self.server = configs['Database'][
'DS_RetentionListNewSitesProcess']['Server']
self.connections = connections
if not websites.empty and init == True:
for k in websites.itertuples():
self.connections[self.server].ExecNoQuery(
" INSERT INTO {database} (ProcesStartDate, Website, \
Initialization,ProcessStatus) \
SELECT '{date}',{websites},'{now}','initialized' "
.format(date=pd.Timestamp(date).date(),
now=pd.Timestamp.now().round('1s'),
database=self.database,
websites=k.siteid))
def __init__(self, date, init=True):
configs = Setup.readConfig()
connections = Setup.initDBs()
self.database = configs['Database']['DS_RetentionlistPreprocesStatus'][
'Database']
self.server = configs['Database']['DS_RetentionlistPreprocesStatus'][
'Server']
self.connections = connections
if init == True:
print(self.server)
self.connections[self.server].ExecNoQuery(
" INSERT INTO {database} (ProcesStartDate,ProcessStatus) \
SELECT '{date}', 'initialized' "
.format(date=pd.Timestamp(date).date(),
database=self.database))
def __init__(self, config):
self.config = config
self.destinationserver = config['DestinationServer']
self.destinationdb = config['DestinationDB']
self.originserver = config['OriginServer']
self.origindb = config['OriginDB']
self.linkedname = config['LinkedName']
self.schema = config['Schema']
self.dbs = Setup.initDBs()
self.timevar = 'Activity'
self.timevar2 = ''
self.switch = '='
try:
self.timevar = config['TimeField']
except:
pass
try:
self.switch = config['Switch']
except:
pass
try:
self.timevar2 = config['TimeField2']
except:
pass
if __name__ == '__main__':
#time
##first date, last date, version and which websites
##to do: systemint FROM table
##to do: change the Datascientist.dbo.DS_RetentionlistActiveWebsites from [jG} to [190]
parser = argparse.ArgumentParser(description="python ProcessWrapper.py ")
parser.add_argument('--startdays', nargs='?', type=int)
parser.add_argument('--enddays', nargs='?', type=int)
parser.add_argument('--sitesetting', nargs='?', type=str)
parser.add_argument('--systemint', nargs='?', type=int)
##parse arguments
args = Setup.preprocessnew(parser.parse_args())
##load configs and databases
configs = Setup.readConfig(version=args['version'])
dbs = Setup.initDBs()
sites = Setup.setSites(args['systemint'], args['sitesetting'],
args['startdays'], args['enddays'])
date = (datetime.datetime.utcnow() -
datetime.timedelta(days=args['enddays']))
startdate = (datetime.datetime.utcnow() -
datetime.timedelta(days=args['startdays']))
testdate = startdate
#needs implementation and then use the individual wrapper to handle it
Problems = getPreprocessingState(date, date, dbs)
def TransferActivity(Sites, dbs, querytime):
configdata = configs['Database']['TransferRetentionListActivity']
writer = WritetoDB.Writer(configdata)
writer.write(pd.Timestamp(querytime).date())
if __name__ == '__main__':
#time
##first date, last date, version and which websites
parser = argparse.ArgumentParser(description="python ProcessWrapper.py ")
parser.add_argument('--startdays', nargs='?', type=int)
parser.add_argument('--enddays', nargs='?', type=int)
parser.add_argument('--version', nargs='?', type=str)
parser.add_argument('--sitesetting', nargs='?', type=str)
args = Setup.preprocess(parser.parse_args())
print(args)
date = (datetime.datetime.utcnow() - datetime.timedelta(days=0)).strftime(
"%Y-%m-%d 00:00:00.000") #UTC+0
querytime = (datetime.datetime.utcnow() -
datetime.timedelta(days=args['startdays']))
configs = Setup.readConfig(version=args['version'])
dbs = Setup.initDBs()
Sites = Setup.getSites()
Sites2 = Sites.copy()
if args['sitesetting'] != 'all':
for k in Sites2.itertuples():
connections[server].ExecNoQuery(" EXEC {proc} @startdate='{data}' ".
format(proc = proc, data = pd.Timestamp(date).date()))
process.setStep('BalanceCenterUpdatedTime',pd.Timestamp(date).date(), pd.Timestamp(date).date())
except:
pass
def Cleanup(date,connection, aggregationproblems):
process = RetentionListPreprocessManager(date, init = False)
initializedhistory = process.getAllWithStatus('initialized')
sectored = initializedhistory.loc[ initializedhistory.ProcesStartDate == initializedhistory.BalanceCenterUpdatedTime , :]
for k in sectored.iterrows():
process.setStep('ProcessStatus',pd.Timestamp(k.ProcessStartDate).date(),'completed')
if __name__ =='__main__':
#time
date = (datetime.datetime.utcnow() - datetime.timedelta(days = 1)).strftime("%Y-%m-%d 00:00:00.000") #UTC+0
configs = Setup.readConfig()
dbs = Setup.initDBs()
Problems = getBalanceCenterStatus(date, dbs)
AggregateBalanceCenter(date,dbs,Problems )
Cleanup(date, dbs, Problems)
connections[server].ExecNoQuery(
" EXEC {proc} @startdate='{data}' ".format(
proc=proc, data=pd.Timestamp(date).date()))
process.setStep('MonthlyTop10Commissionable', pd.Timestamp(date),
pd.Timestamp(date).now())
except:
pass
def Cleanup(date, connection, Problems):
process = RetentionListPreprocessManager(date, init=False)
initializedhistory = process.getAllWithStatus('initialized')
'''
for k in sectored.iterrows():
process.setStep('ProcessStatus',pd.Timestamp(k[1].ProcesStartDate).date(),'completed')
'''
if __name__ == '__main__':
version = ''
#time
date = (datetime.datetime.utcnow() - datetime.timedelta(days=1)).strftime(
"%Y-%m-%d 00:00:00.000") #UTC+0
configs = Setup.readConfig(version)
dbs = Setup.initDBs()
Problems = getBalanceCenterStatus(date, dbs)
AggregateBalanceCenter(date, dbs, Problems)
CollectTop10Commissionable(date, dbs, Problems)
Cleanup(date, dbs, Problems)