def toggle_last_request(self):
## let it toggle the last request to a given approval only if the chained request allows it
if self.get_attribute('approval') == 'none':
return
ccdb = database('chained_campaigns')
mcm_cc = ccdb.get(self.get_attribute('member_of_campaign'))
(next_campaign_id, flow_name) = mcm_cc['campaigns'][self.get_attribute('step')]
fdb = database('flows')
mcm_f = flow(fdb.get(flow_name))
# check whether we have to do something even more subtle with the request
if mcm_f.get_attribute('approval') == 'submit' or self.get_attribute('approval') == 'submit':
rdb = database('requests')
next_request = request(rdb.get(self.get_attribute('chain')[self.get_attribute('step')]))
current_r_approval = next_request.get_attribute('approval')
time_out = 0
#self.logger.error('Trying to move %s from %s to submit'% (next_request.get_attribute('prepid'), current_r_approval))
while current_r_approval != 'submit' and time_out <= 10:
time_out += 1
#get it back from db to avoid _red issues
next_request = request(rdb.get(next_request.get_attribute('prepid')))
with locker.lock('{0}-wait-for-approval'.format( next_request.get_attribute('prepid') )):
next_request.approve()
request_saved = rdb.save(next_request.json())
if not request_saved:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Could not save the new request %s while trying to move to submit approval' % (
next_request.get_attribute('prepid')))
current_r_approval = next_request.get_attribute('approval')
pass
return True
def processing():
user_password = get_user_name()
if user_password is None or len(user_password) == 0:
user_password = (123456, 'flesdewe234', 'lqz123456789')
try:
print user_password
if user_password is None or len(user_password) == 0:
print 'user name and pass word is over'
return
request = login_chrome_1688(user_password[1], user_password[2],
int(user_password[0]))
time.sleep(3)
for i in range(0, len(regions)):
region = regions[i]
url_region = url_regions[i]
for all_classi in get_all_classi():
print 'region = %s, all class = %s' % (region, all_classi[0])
try:
f = flow(request, region, all_classi[0], url_region)
f.process_classi()
time.sleep(1)
except Exception as e:
print 'traceback.format_exc():\n%s' % traceback.format_exc(
)
finally:
my_sql = Mysql_db()
sql = "update test.crm_clue_password set status = 0 where id = " + str(
user_password[0])
my_sql.excute(sql)
def _flow_align(flow_data, class_map, series, vector_mult=1, interpolate=False):
flowed = flow.flow(flow_data, full_path=True, vector_mult=vector_mult, interpolate=interpolate)
noy, nox, noc = class_map.shape
nof = len(series)
mapped = torch.zeros((noy, nox, noc, nof), device=class_map.device, dtype=class_map.dtype)
for i in range(nof):
mapped[..., i] = map2d.apply(class_map, flowed[..., series[i]])
return mapped
def process_tcp(self, ts, pkt, src, dst):
"""Process a tcp packet"""
ip = pkt.data
tcp = ip.data
this_flow = flow()
this_flow.src = src
this_flow.dst = dst
if type(tcp) != dpkt.tcp.TCP or ip.len == ip.__hdr_len__ + tcp.__hdr_len__:
# no data or bad data, return
return
this_flow.sport = tcp.sport
this_flow.dport = tcp.dport
self.store_packet(ts, this_flow, tcp)
def process_tcp(self, ts, pkt, src, dst):
"""Process a tcp packet"""
ip = pkt.data
tcp = ip.data
this_flow = flow()
this_flow.src = src
this_flow.dst = dst
if type(
tcp
) != dpkt.tcp.TCP or ip.len == ip.__hdr_len__ + tcp.__hdr_len__:
# no data or bad data, return
return
this_flow.sport = tcp.sport
this_flow.dport = tcp.dport
self.store_packet(ts, this_flow, tcp)
def toggle_last_request(self):
## let it toggle the last request to a given approval only if the chained request allows it
if self.get_attribute('approval') == 'none':
return
ccdb = database('chained_campaigns')
mcm_cc = ccdb.get(self.get_attribute('member_of_campaign'))
(next_campaign_id,
flow_name) = mcm_cc['campaigns'][self.get_attribute('step')]
fdb = database('flows')
mcm_f = flow(fdb.get(flow_name))
# check whether we have to do something even more subtle with the request
if mcm_f.get_attribute('approval') == 'submit' or self.get_attribute(
'approval') == 'submit':
rdb = database('requests')
next_request = request(
rdb.get(
self.get_attribute('chain')[self.get_attribute('step')]))
current_r_approval = next_request.get_attribute('approval')
time_out = 0
#self.logger.error('Trying to move %s from %s to submit'% (next_request.get_attribute('prepid'), current_r_approval))
while current_r_approval != 'submit' and time_out <= 10:
time_out += 1
#get it back from db to avoid _red issues
next_request = request(
rdb.get(next_request.get_attribute('prepid')))
with locker.lock('{0}-wait-for-approval'.format(
next_request.get_attribute('prepid'))):
next_request.approve()
request_saved = rdb.save(next_request.json())
if not request_saved:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Could not save the new request %s while trying to move to submit approval'
% (next_request.get_attribute('prepid')))
current_r_approval = next_request.get_attribute('approval')
pass
return True
def flow_to_next_step(self,
input_dataset='',
block_black_list=None,
block_white_list=None,
check_stats=True,
reserve=False):
if not block_white_list: block_white_list = []
if not block_black_list: block_black_list = []
self.logger.log('Flowing chained_request %s to next step...' %
(self.get_attribute('_id')))
if not self.get_attribute('chain'):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'chained_request %s has got no root' %
(self.get_attribute('_id')))
# check on the approval of the chained request before all
## let it be flowing regardless
#if self.get_attribute('approval') == 'none':
# raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
# 'The approval of the chained request is none, and therefore flow cannot happen')
#this operation requires to access all sorts of objects
rdb = database('requests')
cdb = database('campaigns')
ccdb = database('chained_campaigns')
crdb = database('chained_requests')
fdb = database('flows')
adb = database('actions')
l_type = locator()
current_step = len(self.get_attribute(
'chain')) - 1 if reserve else self.get_attribute('step')
current_id = self.get_attribute('chain')[current_step]
next_step = current_step + 1
if not rdb.document_exists(current_id):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the request %s does not exist' % current_id)
current_request = request(rdb.get(current_id))
current_campaign = campaign(
cdb.get(current_request.get_attribute('member_of_campaign')))
if not ccdb.document_exists(self.get_attribute('member_of_campaign')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the chain request %s is member of %s that does not exist' %
(self.get_attribute('_id'),
self.get_attribute('member_of_campaign')))
mcm_cc = ccdb.get(self.get_attribute('member_of_campaign'))
if next_step >= len(mcm_cc['campaigns']):
if reserve: return False
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'chained_campaign %s does not allow any further flowing.' %
(self.get_attribute('member_of_campaign')))
if not reserve:
## is the current request in the proper approval
allowed_request_approvals = ['submit']
if current_request.get_attribute(
'approval') not in allowed_request_approvals:
raise self.NotApprovedException(
current_request.get_attribute('prepid'),
current_request.get_attribute('approval'),
allowed_request_approvals)
## is the current request in the proper status
allowed_request_statuses = ['submitted', 'done']
if current_request.get_attribute(
'status') not in allowed_request_statuses:
raise self.NotInProperStateException(
current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
original_action_id = self.get_attribute('chain')[0]
original_action_item = self.retrieve_original_action_item(
adb, original_action_id)
## what is the campaign to go to next and with which flow
(next_campaign_id, flow_name) = mcm_cc['campaigns'][next_step]
if not fdb.document_exists(flow_name):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The flow %s does not exist' % flow_name)
mcm_f = flow(fdb.get(flow_name))
if not 'sequences' in mcm_f.get_attribute('request_parameters'):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The flow %s does not contain sequences information.' %
(flow_name))
if not cdb.document_exists(next_campaign_id):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The next campaign %s does not exist' % next_campaign_id)
next_campaign = campaign(cdb.get(next_campaign_id))
if len(next_campaign.get_attribute('sequences')) != len(
mcm_f.get_attribute('request_parameters')['sequences']):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the sequences changes in flow %s are not consistent with the next campaign %s'
% (flow_name, next_campaign_id))
if next_campaign.get_attribute(
'energy') != current_campaign.get_attribute('energy'):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'cannot flow any further. Request {0} has inconsistent energy.'
.format(next_campaign.get_attribute("prepid")))
if not next_campaign.is_release_greater_or_equal_to(
current_campaign.get_attribute('cmssw_release')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute("_id"),
'cannot flow any further. Request {0} has lower release version.'
.format(next_campaign.get_attribute("prepid")))
if next_campaign.get_attribute('type') == 'MCReproc' and (
not 'time_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not time per event is specified'
)
if next_campaign.get_attribute('type') == 'MCReproc' and (
not 'size_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not size per event is specified'
)
## check that it is allowed to flow
allowed_flow_approvals = ['flow', 'submit']
###### cascade of checks
"""
if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The flow (%s) is not in proper approval state (%s)'%(
mcm_f.get_attribute('prepid'),
mcm_f.get_attribute('approval')))
if not reserve and not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The chained request (%s) is not in the proper approval state (%s)'% (
self.get_attribute('_id'),
self.get_attribute('approval')))
"""
if not reserve and not mcm_f.get_attribute(
'approval') in allowed_flow_approvals:
if not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Neither the flow (%s) nor the chained request (%s) approvals allow for flowing'
% (mcm_f.get_attribute('approval'),
self.get_attribute('approval')))
if next_campaign.get_attribute('status') == 'stopped':
raise self.CampaignStoppedException(next_campaign_id)
#what is going to be the required number of events for the next request
#update the stats to its best
if not reserve:
current_request.get_stats()
next_total_events = current_request.get_attribute(
'completed_events')
## get the original expected events and allow a margin of 5% less statistics
statistics_fraction = settings().get_value('statistics_fraction')
current_eff_error = 1. - current_request.get_efficiency_error()
statistics_fraction = min(statistics_fraction, current_eff_error)
completed_events_to_pass = int(
current_request.get_attribute('total_events') *
statistics_fraction)
notify_on_fail = True ## to be tuned according to the specific cases
if current_request.get_attribute('completed_events') <= 0:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The number of events completed is negative or null')
else:
allowed_request_statuses = ['done']
## determine if this is a root -> non-root transition to potentially apply staged number
at_a_transition = (current_campaign.get_attribute('root') != 1
and next_campaign.get_attribute('root')
== 1)
if ('staged' in original_action_item or 'threshold'
in original_action_item) and at_a_transition:
allowed_request_statuses.append('submitted')
##check status
if not current_request.get_attribute(
'status') in allowed_request_statuses:
raise self.NotInProperStateException(
current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
##special check at transition that the statistics is good enough
if at_a_transition:
# at a root -> non-root transition only does the staged/threshold functions !
if 'staged' in original_action_item:
next_total_events = int(original_action_item['staged'])
completed_events_to_pass = next_total_events
if 'threshold' in original_action_item:
next_total_events = int(
current_request.get_attribute('total_events') *
float(original_action_item['threshold'] / 100.))
completed_events_to_pass = next_total_events
if check_stats and (
current_request.get_attribute('completed_events') <
completed_events_to_pass):
if notify_on_fail:
current_request.notify(
'Flowing %s with not enough statistics' %
(current_request.get_attribute('prepid')),
'For the request %s, the completed statistics %s is not enough to fullfill the requirement to the next level : need at least %s in chain %s \n\n Please report to the operation HN or at the next MccM what action should be taken.\n\n %srequests?prepid=%s\n%schained_requests?contains=%s\n%schained_requests?prepid=%s '
% (current_request.get_attribute('prepid'),
current_request.get_attribute('completed_events'),
completed_events_to_pass,
self.get_attribute('prepid'), l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(), self.get_attribute('prepid')),
accumulate=True)
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The number of events completed (%s) is not enough for the requirement (%s)'
% (current_request.get_attribute('completed_events'),
completed_events_to_pass))
## select what is to happened : [create, patch, use]
next_id = None
approach = None
next_request = None
if next_step != len(self.get_attribute('chain')):
#not at the end
next_id = self.get_attribute('chain')[next_step]
if not rdb.document_exists(next_id):
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'The next request (%s) according to the step (%s) does not exist'
% (next_id, next_step))
next_request = request(rdb.get(next_id))
if next_request.get_attribute('status') == 'new':
#most likely a rewind + resub
approach = 'patch'
else:
##this is always the case in chains reserved from existing things: so use the next request
approach = 'use'
#raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
#'This should never happen. (%s) is next according to step (%s), but is not in new status (%s)' % (
#next_id, next_step, next_request.get_attribute('status')))
else:
## look in *other* chained campaigns whether you can suck in an existing request
## look up all chained requests that start from the same root request
## remove <pwg>-chain_ and the -serial number, replacing _ with a .
toMatch = '.'.join(
self.get_attribute('prepid').split('_')[1:][0:next_step +
1]).split('-')[0]
## make sure they get ordered by prepid
related_crs = sorted(crdb.queries(
['root_request==%s' % original_action_id]),
key=lambda cr: cr['prepid'])
vetoed_last = []
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid'] == self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute(
'member_of_campaign'):
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) > next_step:
## one existing request in the very same chained campaign has already something used, make sure it is not going to be used
vetoed_last.append(
mcm_cr.get_attribute('chain')[next_step])
continue
else:
continue
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid'] == self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute(
'member_of_campaign'):
continue
truncated = '.'.join(
existing_cr['prepid'].split('_')[1:][0:next_step +
1]).split('-')[0]
self.logger.error('to match : %s , this one %s' %
(toMatch, truncated))
if truncated == toMatch:
#we found a chained request that starts with all same steps
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) <= next_step:
#found one, but it has not enough content either
continue
if mcm_cr.get_attribute('chain')[next_step] in vetoed_last:
continue
next_id = mcm_cr.get_attribute('chain')[next_step]
break
if next_id:
approach = 'use'
else:
approach = 'create'
if approach == 'create':
from rest_api.RequestPrepId import RequestPrepId
next_id = RequestPrepId().next_prepid(
current_request.get_attribute('pwg'), next_campaign_id)
next_request = request(rdb.get(next_id))
request.transfer(current_request, next_request)
self.request_join(next_request)
elif approach == 'use':
## there exists a request in another chained campaign that can be re-used here.
# take this one. advance and go on
next_request = request(rdb.get(next_id))
if not reserve:
self.set_attribute('step', next_step)
self.set_attribute('last_status',
next_request.get_attribute('status'))
self.update_history({'action': 'flow', 'step': str(next_id)})
self.set_attribute('status', 'processing')
if not self.get_attribute("prepid") in next_request.get_attribute(
"member_of_chain"):
## register the chain to the next request
self.request_join(next_request)
saved = rdb.update(next_request.json())
if not saved:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Unable to save %s with updated member_of_chains' %
next_id)
return True
elif approach == 'patch':
## there exists already a request in the chain (step!=last) and it is usable for the next stage
next_request = request(next_campaign.add_request(rdb.get(next_id)))
## propagate again some of the fields of the previous request.
request.transfer(current_request, next_request)
else:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Unrecognized approach %s' % approach)
#current_request -> next_request
#current_campaign -> next_campaign
##determine whether we have an input dataset for the next request
if len(current_request.get_attribute('reqmgr_name')):
last_wma = current_request.get_attribute('reqmgr_name')[-1]
if 'content' in last_wma and 'pdmv_dataset_name' in last_wma[
'content']:
input_dataset = last_wma['content']['pdmv_dataset_name']
else:
statsDB = database('stats',
url='http://cms-pdmv-stats.cern.ch:5984/')
if statsDB.document_exists(last_wma['name']):
latestStatus = statsDB.get(last_wma['name'])
input_dataset = latestStatus['pdmv_dataset_name']
if input_dataset:
next_request.set_attribute('input_dataset', input_dataset)
## set blocks restriction if any
if block_black_list:
next_request.set_attribute('block_black_list', block_black_list)
if block_white_list:
next_request.set_attribute('block_white_list', block_white_list)
## register the flow to the request
next_request.set_attribute('flown_with', flow_name)
##assemble the campaign+flow => request
request.put_together(next_campaign, mcm_f, next_request)
if not reserve:
#already taking stage and threshold into account
next_request.set_attribute('total_events', next_total_events)
next_request.update_history({
'action': 'flow',
'step': self.get_attribute('prepid')
})
request_saved = rdb.save(next_request.json())
if not request_saved:
raise self.ChainedRequestCannotFlowException(
self.get_attribute('_id'),
'Could not save the new request %s' %
(next_request.get_attribute('prepid')))
## inspect priority
self.set_priority(original_action_item['block_number'])
if not reserve:
# sync last status
self.set_attribute('last_status',
next_request.get_attribute('status'))
# we can only be processing at this point
self.set_attribute('status', 'processing')
# set to next step
self.set_attribute('step', next_step)
self.update_history({
'action': 'flow',
'step': next_request.get_attribute('prepid')
})
if not reserve:
notification_subject = 'Flow for request %s in %s' % (
current_request.get_attribute('prepid'), next_campaign_id)
notification_text = 'The request %s has been flown within:\n \t %s \n into campaign:\n \t %s \n using:\n \t %s \n creating the new request:\n \t %s \n as part of:\n \t %s \n and from the produced dataset:\n %s \n\n%srequests?prepid=%s \n%srequests?prepid=%s \n' % (
current_request.get_attribute('prepid'),
self.get_attribute('member_of_campaign'), next_campaign_id,
flow_name, next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_request.get_attribute('input_dataset'), l_type.baseurl(),
current_request.get_attribute('prepid'), l_type.baseurl(),
next_request.get_attribute('prepid'))
current_request.notify(notification_subject,
notification_text,
accumulate=True)
else:
notification_subject = 'Reservation of request {0}'.format(
next_request.get_attribute('prepid'))
notification_text = 'The request {0} of campaign \n\t{2}\nhas been reserved as part of \n\t{1}\nas the next step for {4}\n\n{3}requests?prepid={4}\n{5}requests?prepid={6}\n'.format(
next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_campaign_id,
l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(),
next_request.get_attribute('prepid'),
)
next_request.notify(notification_subject,
notification_text,
accumulate=True)
return True
def _flow_cur_map(flow_data, class_map, vector_mult=1, interpolate=False):
flowed = flow.flow(flow_data, vector_mult=vector_mult, interpolate=interpolate)
flowed_map = map2d.apply(class_map, flowed).type(torch.float)
return flowed_map
def _flow_end_map(flow_data, class_map, vector_mult=1, interpolate=False, use_homography=False):
flowed = flow.flow(flow_data, vector_mult=vector_mult, interpolate=interpolate)
return _end_map(flowed, class_map, use_homography=use_homography)
def _flow_full(flow_data, vector_mult, interpolate=False):
return flow.flow(flow_data, full_path=True, vector_mult=vector_mult, interpolate=interpolate)
def flow_to_next_step(self, input_dataset='', block_black_list=None, block_white_list=None, check_stats=True, reserve=False):
if not block_white_list: block_white_list = []
if not block_black_list: block_black_list = []
self.logger.log('Flowing chained_request %s to next step...' % (self.get_attribute('_id')))
if not self.get_attribute('chain'):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'chained_request %s has got no root' % (
self.get_attribute('_id')))
# check on the approval of the chained request before all
## let it be flowing regardless
#if self.get_attribute('approval') == 'none':
# raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
# 'The approval of the chained request is none, and therefore flow cannot happen')
#this operation requires to access all sorts of objects
rdb = database('requests')
cdb = database('campaigns')
ccdb = database('chained_campaigns')
crdb = database('chained_requests')
fdb = database('flows')
adb = database('actions')
l_type=locator()
current_step = len(self.get_attribute('chain'))-1 if reserve else self.get_attribute('step')
current_id = self.get_attribute('chain')[current_step]
next_step = current_step + 1
if not rdb.document_exists(current_id):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the request %s does not exist' % current_id)
current_request = request(rdb.get(current_id))
current_campaign = campaign(cdb.get(current_request.get_attribute('member_of_campaign')))
if not ccdb.document_exists(self.get_attribute('member_of_campaign')):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the chain request %s is member of %s that does not exist' % (
self.get_attribute('_id'),
self.get_attribute('member_of_campaign')))
mcm_cc = ccdb.get(self.get_attribute('member_of_campaign'))
if next_step >= len(mcm_cc['campaigns']):
if reserve: return False
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'chained_campaign %s does not allow any further flowing.' % (
self.get_attribute('member_of_campaign')))
if not reserve:
## is the current request in the proper approval
allowed_request_approvals = ['submit']
if current_request.get_attribute('approval') not in allowed_request_approvals:
raise self.NotApprovedException(current_request.get_attribute('prepid'),
current_request.get_attribute('approval'), allowed_request_approvals)
## is the current request in the proper status
allowed_request_statuses = ['submitted', 'done']
if current_request.get_attribute('status') not in allowed_request_statuses:
raise self.NotInProperStateException(current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
original_action_id = self.get_attribute('chain')[0]
original_action_item = self.retrieve_original_action_item(adb, original_action_id)
## what is the campaign to go to next and with which flow
(next_campaign_id, flow_name) = mcm_cc['campaigns'][next_step]
if not fdb.document_exists(flow_name):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The flow %s does not exist' % flow_name )
mcm_f = flow(fdb.get(flow_name))
if not 'sequences' in mcm_f.get_attribute('request_parameters'):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The flow %s does not contain sequences information.' % (
flow_name))
if not cdb.document_exists(next_campaign_id):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The next campaign %s does not exist' % next_campaign_id)
next_campaign = campaign(cdb.get(next_campaign_id))
if len(next_campaign.get_attribute('sequences')) != len(mcm_f.get_attribute('request_parameters')['sequences']):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the sequences changes in flow %s are not consistent with the next campaign %s' % (
flow_name, next_campaign_id))
if next_campaign.get_attribute('energy') != current_campaign.get_attribute('energy'):
raise self.EnergyInconsistentException(next_campaign.get_attribute('prepid'))
if next_campaign.get_attribute('type') == 'MCReproc' and (not 'time_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not time per event is specified')
if next_campaign.get_attribute('type') == 'MCReproc' and (not 'size_event' in mcm_f.get_attribute('request_parameters')):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'the flow is getting into a MCReproc campaign but not size per event is specified')
## check that it is allowed to flow
allowed_flow_approvals = ['flow', 'submit']
###### cascade of checks
"""
if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The flow (%s) is not in proper approval state (%s)'%(
mcm_f.get_attribute('prepid'),
mcm_f.get_attribute('approval')))
if not reserve and not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The chained request (%s) is not in the proper approval state (%s)'% (
self.get_attribute('_id'),
self.get_attribute('approval')))
"""
if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals:
if not self.get_attribute('approval') in allowed_flow_approvals:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Neither the flow (%s) nor the chained request (%s) approvals allow for flowing' % (
mcm_f.get_attribute('approval'),
self.get_attribute('approval')))
if next_campaign.get_attribute('status') == 'stopped':
raise self.CampaignStoppedException(next_campaign_id)
#what is going to be the required number of events for the next request
#update the stats to its best
if not reserve:
current_request.get_stats()
next_total_events=current_request.get_attribute('completed_events')
notify_on_fail=True ## to be tuned according to the specific cases
if current_request.get_attribute('completed_events') <= 0:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The number of events completed is negative or null')
else:
allowed_request_statuses = ['done']
## determine if this is a root -> non-root transition to potentially apply staged number
at_a_transition=(current_campaign.get_attribute('root') != 1 and next_campaign.get_attribute('root') == 1)
if ('staged' in original_action_item or 'threshold' in original_action_item) and at_a_transition:
allowed_request_statuses.append('submitted')
##check status
if not current_request.get_attribute('status') in allowed_request_statuses:
raise self.NotInProperStateException(current_request.get_attribute('prepid'),
current_request.get_attribute('status'),
allowed_request_statuses)
##special check at transition that the statistics is good enough
if at_a_transition:
# at a root -> non-root transition only does the staged/threshold functions !
if 'staged' in original_action_item:
next_total_events = int(original_action_item['staged'])
if 'threshold' in original_action_item:
next_total_events = int(current_request.get_attribute('total_events') * float(original_action_item['threshold'] / 100.))
completed_events_to_pass = next_total_events
else:
## get the original expected events and allow a margin of 5% less statistics
completed_events_to_pass = int(current_request.get_attribute('total_events') * 0.95)
if check_stats and (current_request.get_attribute('completed_events') < completed_events_to_pass):
if notify_on_fail:
current_request.notify('Flowing for %s: not enough statistics'%( current_request.get_attribute('prepid')),
'For this request, the completed statistics %s is not enough to fullfill the requirement to the next level : need at least %s \n\n %srequests?prepid=%s'%( current_request.get_attribute('completed_events'),
completed_events_to_pass,
l_type.baseurl(),
current_request.get_attribute('prepid')))
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The number of events completed (%s) is not enough for the requirement (%s)'%(current_request.get_attribute('completed_events'), completed_events_to_pass))
## select what is to happened : [create, patch, use]
next_id = None
approach = None
next_request = None
if next_step != len(self.get_attribute('chain')):
#not at the end
next_id = self.get_attribute('chain')[next_step]
if not rdb.document_exists(next_id):
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'The next request (%s) according to the step (%s) does not exist' % (
next_id, next_step))
next_request = request(rdb.get(next_id))
if next_request.get_attribute('status') == 'new':
#most likely a rewind + resub
approach = 'patch'
else:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'This should never happen. (%s) is next according to step (%s), but is not in new status (%s)' % (
next_id, next_step, next_request.get_attribute('status')))
else:
## look in *other* chained campaigns whether you can suck in an existing request
## look up all chained requests that start from the same root request
## remove <pwg>-chain_ and the -serial number, replacing _ with a .
toMatch = '.'.join(self.get_attribute('prepid').split('_')[1:][0:next_step + 1]).split('-')[0]
## make sure they get ordered by prepid
related_crs = sorted(crdb.queries(['root_request==%s' % original_action_id]), key=lambda cr : cr['prepid'])
vetoed_last=[]
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid']==self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute('member_of_campaign'):
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) > next_step:
## one existing request in the very same chained campaign has already something used, make sure it is not going to be used
vetoed_last.append( mcm_cr.get_attribute('chain')[next_step])
continue
else:
continue
for existing_cr in related_crs:
## exclude itself
if existing_cr['prepid']==self.get_attribute('prepid'):
continue
## prevent from using a request from within the same exact chained_campaigns
if existing_cr['member_of_campaign'] == self.get_attribute('member_of_campaign'):
continue
truncated = '.'.join(existing_cr['prepid'].split('_')[1:][0:next_step + 1]).split('-')[0]
self.logger.error('to match : %s , this one %s' % ( toMatch, truncated ))
if truncated == toMatch:
#we found a chained request that starts with all same steps
mcm_cr = chained_request(crdb.get(existing_cr['prepid']))
if len(mcm_cr.get_attribute('chain')) <= next_step:
#found one, but it has not enough content either
continue
if mcm_cr.get_attribute('chain')[next_step] in vetoed_last:
continue
next_id = mcm_cr.get_attribute('chain')[next_step]
break
if next_id:
approach = 'use'
else:
approach = 'create'
if approach == 'create':
from rest_api.RequestPrepId import RequestPrepId
next_id = RequestPrepId().next_prepid(current_request.get_attribute('pwg'), next_campaign_id)
next_request = request(rdb.get(next_id))
request.transfer( current_request, next_request)
self.request_join(next_request)
elif approach == 'use':
## there exists a request in another chained campaign that can be re-used here.
# take this one. advance and go on
next_request = request(rdb.get(next_id))
if not reserve:
self.set_attribute('step', next_step)
self.set_attribute('last_status', next_request.get_attribute('status'))
self.update_history({'action': 'flow', 'step': str(next_step)})
self.set_attribute('status', 'processing')
if not self.get_attribute("prepid") in next_request.get_attribute("member_of_chain"):
## register the chain to the next request
self.request_join(next_request)
saved = rdb.update(next_request.json())
if not saved:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Unable to save %s with updated member_of_chains' % next_id)
return True
elif approach == 'patch':
## there exists already a request in the chain (step!=last) and it is usable for the next stage
next_request = request( next_campaign.add_request( rdb.get(next_id)))
### shouldn't this be added ?
#transfer( current_request, next_request)
else:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Unrecognized approach %s' % approach )
#current_request -> next_request
#current_campaign -> next_campaign
##determine whether we have an input dataset for the next request
if len(current_request.get_attribute('reqmgr_name')):
last_wma = current_request.get_attribute('reqmgr_name')[-1]
if 'content' in last_wma and 'pdmv_dataset_name' in last_wma['content']:
input_dataset = last_wma['content']['pdmv_dataset_name']
else:
statsDB = database('stats', url='http://cms-pdmv-stats.cern.ch:5984/')
if statsDB.document_exists(last_wma['name']):
latestStatus = statsDB.get(last_wma['name'])
input_dataset = latestStatus['pdmv_dataset_name']
if input_dataset:
next_request.set_attribute('input_filename', input_dataset)
## set blocks restriction if any
if block_black_list:
next_request.set_attribute('block_black_list', block_black_list)
if block_white_list:
next_request.set_attribute('block_white_list', block_white_list)
## register the flow to the request
next_request.set_attribute('flown_with', flow_name)
##assemble the campaign+flow => request
request.put_together(next_campaign, mcm_f, next_request)
if not reserve:
#already taking stage and threshold into account
next_request.set_attribute('total_events', next_total_events)
next_request.update_history({'action': 'flow', 'step': self.get_attribute('prepid')})
request_saved = rdb.save(next_request.json())
if not request_saved:
raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'),
'Could not save the new request %s' % (
next_request.get_attribute('prepid')))
## inspect priority
self.set_priority(original_action_item['block_number'])
if not reserve:
# sync last status
self.set_attribute('last_status', next_request.get_attribute('status'))
# we can only be processing at this point
self.set_attribute('status', 'processing')
# set to next step
self.set_attribute('step', next_step)
if not reserve:
notification_subject = 'Flow for request %s in %s' % (current_request.get_attribute('prepid'), next_campaign_id)
notification_text = 'The request %s has been flown within:\n \t %s \n into campaign:\n \t %s \n using:\n \t %s \n creating the new request:\n \t %s \n as part of:\n \t %s \n and from the produced dataset:\n %s \n\n%srequests?prepid=%s \n%srequests?prepid=%s \n' % (
current_request.get_attribute('prepid'),
self.get_attribute('member_of_campaign'),
next_campaign_id,
flow_name,
next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_request.get_attribute('input_filename'),
l_type.baseurl(),
current_request.get_attribute('prepid'),
l_type.baseurl(),
next_request.get_attribute('prepid')
)
current_request.notify(notification_subject, notification_text)
else:
notification_subject = 'Reservation of request {0}'.format(next_request.get_attribute('prepid'))
notification_text = 'The request {0} of campaign \n\t{2}\nhas been reserved as part of \n\t{1}\nas the next step for {4}\n\n{3}requests?prepid={4}\n{5}requests?prepid={6}\n'.format(
next_request.get_attribute('prepid'),
self.get_attribute('prepid'),
next_campaign_id,
l_type.baseurl(), current_request.get_attribute('prepid'),
l_type.baseurl(), next_request.get_attribute('prepid'),
)
next_request.notify(notification_subject, notification_text)
return True
def opt_pterminal(x0, ic1, ic2, dt):
"""
1.) flow #1, #2 based on x0
2.) run prussing_conway
3.) run shooting method w/ variational eqns
4.) generate delta-vs
5.) generate primer history
6.) extract slopes
7.) compute cost function
"""
"""
FUTURE WORK
1.) add in intelligent step sizing for flow3 = shoot()
"""
# import statements
from flow import P2B as flow
from lambert import prussing_conway
from shootingmodule import firstorder as shoot
from numpy.linalg import norm
from numpy import array, zeros, eye
print ("Starting an iteration of opt_pterminal!")
print ("x0 is: %e, %e " % (x0[0], x0[1]))
# function parameters
""" FW 1.) """
step = 1e-2
tol = 1e-3
# extract mu
mu = ic1[4]
# flow #1, #2 based on x0[0]
tspan = [0, x0[0]]
ministep = abs(x0[0] / 2e2)
print ("Midpoint-A of an iteration of opt_pterminal!")
print ("tspan is: %e, %e " % (tspan[0], tspan[1]))
print ("ministep is: %e " % ministep)
flow1 = flow(ic1, tspan, ministep, trim_output=False, print_status=True, solver="rk4p")
print ("Midpoint-B of an iteration of opt_pterminal!")
flow2 = flow(ic2, tspan, ministep, trim_output=False)
print ("Midpoint-C of an iteration of opt_pterminal!")
# update ic1 and ic2
ic1 = array(flow1["y"][-1])
ic2 = array(flow2["y"][-1])
# update of flow time based on x0
tspan = [0, dt - x0[0] + x0[1]]
# flow #2 based on x0[1]
flow2 = flow(ic2, tspan, tstep=step, trim_output=False)
# target position
target = array(flow2["y"][-1][0:4])
# solve for lambert arc
print ("Midpoint-0 of an iteration of opt_pterminal!")
arc = prussing_conway(ic1, target, mu, TransferTime=abs(tspan[1]))
# setup initial conditions for flow of spacecraft with
# variational equations
ic = zeros(21)
ic[0:2] = ic1[0:2]
ic[2:4] = arc["v1"]
ic[4] = mu
ic[5:] = eye(4).reshape(16)
# flow spacecraft using lambert arc
print ("Midpoint-1 of an iteration of opt_pterminal!")
flow3 = flow(ic, tspan, tstep=step, eom="P2BP_varEqns", trim_output=False)
print ("Midpoint-2 of an iteration of opt_pterminal!")
# position error
poserror = flow3["y"][-1][0:2] - flow2["y"][-1][0:2]
# apply shooting method to reduce error
if norm(poserror) > tol:
flow3 = shoot(
ic, target[0:2], tspan, eom="P2BP_varEqns", tstep=step, damping=1 / 3.0, tol=tol, trim_output=False
)
poserror = flow3["y"][-1][0:2] - flow2["y"][-1][0:2]
print ("Midpoint-3 of an iteration of opt_pterminal!")
# delta-vs
dv1 = array([flow3["y"][0][2], flow3["y"][0][3]]) - array([ic1[2], ic1[3]])
dv2 = -array([flow3["y"][-1][2], flow3["y"][-1][3]]) + array([flow2["y"][-1][2], flow2["y"][-1][3]])
# generate primer vector history
p = phistory(dv1, dv2, flow3)
# extract the slopes of the primer vector
dp0 = pslope(flow3, p, 1, 0)
dp1 = pslope(flow3, p, -1, -2)
# scalar cost function
J = dp0 ** 2 + dp1 ** 2
print ("Finished an iteration of opt_pterminal!")
return J, flow1, flow2, flow3
def firstorder(ic, target, tspan, eom, \
solver = 'rk4', \
tstep = 0, \
damping = 0.1, \
iLimit = 25, \
tol = 1E-3, \
TU = 1, \
guess = None, \
print_status = False, \
trim_output = True):
# import statements
from numpy.linalg import norm, solve
from numpy import zeros, eye, concatenate
from primervectormodule import extractSTM
# if eom argument is not an eom model with variational equations,
#+ then turn on the appropriate model
if eom == 'P2BP': eom = 'P2BP_varEqns'
if eom == 'S2BP': eom = 'S2BP_varEqns'
# determine dimension of problem (i.e. planar or spatial)
#+ and initialize the state vector and residual vector
if len(target) == 2:
from flow import P2B as flow
# initialize a state vector
if len(ic) == 21:
s = ic
if len(ic) == 5:
s = zeros(21)
s[0:5] = ic
# tack identity matrix on end for STM
s[5:] = eye(4).reshape(16,1)
# residual vector
res = s[0:2]
# make a copy of the velocity for computing/printing delta-v
v0 = s[2:4].copy()
elif len(target) == 3:
from flow import S2B as flow
# initialize a state vector
if len(ic) == 43:
s = ic
if len(ic) == 7:
s = zeros(43)
s[0:7] = ic
# tack identity matrix on end for STM
s[7:] = eye(6).reshape(36,1)
# residual vector
res = s[0:3]
# make a copy of the velocity for computing/printing delta-v
v0 = s[3:6].copy()
# calculate an appropriate tstep if not defined
if tstep == 0: tstep = abs(tspan[1] - tspan[0])/5E1
# if an initial guess for perturbation is given then use it,
#+ otherwise perturb the shooting method in the direction of current
#+ velocity, so as to generally seek a lower detla-v solution
if len(target) == 2:
if guess != None: s[2:4] += guess
else: s[2:4] += 0.1*s[2:4]
elif len(target) == 3:
if guess != None: s[3:6] += guess
else: s[3:6] += 0.1*s[3:6]
# set counter
i = 0
# iterate with a first-order shooting method
while ((norm(res) > tol) and (i < iLimit + 1)) or (i == 0):
# solve for 'flow'
solution = flow(s, tspan, tstep, solver, eom, TU, \
trim_output)
# extract the state transition matrix
# and relevant blocks for delta-v update
stm, mblk, nblk = extractSTM(solution, -1)
# solve for delta-v update
dv = solve(nblk, res)
#+ update residual
if len(dv) == 2: res = (solution['y'][-1][0:2] - target)
elif len(dv) == 3: res = (solution['y'][-1][0:3] - target)
# print iterative updates if desired
if print_status and len(dv) == 2:
print ('Iter: %4i Res: %8e ' \
'dv1x: %8.5f dv1y: %8.5f ' \
%(i, norm(res), (s[2] - v0[0]), (s[3] - v0[1])))
if print_status and len(dv) == 3:
print ('Iter: %4i Res: %8e ' \
'dv1x: %8.5f dv1y: %8.5f dv1z: %8.5f ' \
%(i, norm(res), (s[3] - v0[0]), \
(s[4] - v0[1]), (s[5] - v0[2])))
# update initial velocity vector
if len(dv) == 2: s[2:4] = s[2:4] - damping*dv
elif len(dv) == 3: s[3:6] = s[3:6] - damping*dv
# update counter
i += 1
return solution
#--------------------------------------------------------------#
#---------------------- Shooting Module -----------------------#
#--------------------------------------------------------------#
import matplotlib.image as mpimg
import numpy as np
from matplotlib.patches import Rectangle
handle = simulator.simulator("/home/boka/arp/david/")
selection = handle.region()
imagefile = handle.frame()
print("prvo")
if not imagefile:
sys.exit(0)
#image = cv2.imread(imagefile, cv2.IMREAD_GRAYSCALE)
image = cv2.imread(imagefile, cv2.IMREAD_COLOR)
print(imagefile)
tracker = NCCTracker(image, selection)
tracker_flow = flow.flow(image, selection)
tracker_OT = ORF.flow(image, selection)
print("do tukej")
plt.ion()
plt.figure()
while True:
imagefile = handle.frame()
if not imagefile:
break
image = cv2.imread(imagefile, cv2.IMREAD_COLOR)
region = tracker.track(image)
regionOrg = tracker_OT.track(image)
plt.clf()
a = plt.imshow(image)
#tracker_flow.set_region(tracker.position)
def main(argv):
# Check command line parameters
if len(sys.argv) != 3:
print "python networksimulation.py (0,1,2) (0,1)"
print "1st arg: con_ctrl: 0 - none; 1 - TCP Reno; 2 - FAST"
print "2nd arg: verbose: 0 - no; 1 - yes"
print('\n')
sys.exit()
# Store command line parameters
con_ctrl = int(sys.argv[1])
verbose = int(sys.argv[2])
print('\n')
# Give specifications for links file and flows file and then save
# the filename in variables
print("Enter the file of links.")
print("Each line is one link in the following format: ")
print("End1 End2 Rate Delay Capacity\n")
print("Ends=host/source/destination/router ID (ex. H1,S2,T3,R4)")
print("Rate(Mbps), Delay(ms), Capacity(KB)")
linkFile = input('Enter link file: ')
print('\n')
print("Enter the file of flows.")
print("Each line is one flow in the following format: ")
print("Source Dest DataAmount StartTime\n")
print("Source/Dest = ID's (ex. H1,S2,T3,R4)")
print("DataAmount(MB), StartTime(s)")
flowFile = input('Enter flow file: ')
print('\n')
# Variables to store the different network objects
routers = {}
linksLst = []
flowsLst = []
# Try to open the files. If fail, throw an error.
try:
links = open(linkFile, 'r')
flows = open(flowFile, 'r')
except IOError:
print("Link and Flow files do not exist in this directory.")
sys.exit()
# First, make all the links
i = 0
for line in links:
# Split the line into a list, which must have 5 values
info = line.split(' ')
if(len(info) != 5):
print('Each link line must have 5 values')
sys.exit()
# Try converting the rate, delay and capacity. If fail, throw error
try:
rate, delay, capacity = float(info[2]), float(info[3]), float(info[4])
except ValueError:
print("The last 3 values must be numbers")
sys.exit()
# Either make a new entry in the routers table, or grab an entry
# First get the end1 of the link
if(info[0] not in routers.keys()):
router1 = router(info[0])
routers[info[0]] = router1
else:
router1 = routers[info[0]]
# Next get end2
if(info[1] not in routers.keys()):
router2 = router(info[1])
routers[info[1]] = router2
else:
router2 = routers[info[1]]
# Build a link with all the information and add it to the list of links.
temp = link(i, router1, router2, rate, delay, capacity)
linksLst.append(temp)
i += 1
# Now, make all the flows.
for line in flows:
# Split the line into a list, which must have 4 values
info = line.split(' ')
if(len(info) != 4):
print('Each flow line must have 4 values')
sys.exit()
# Try converting the amount and start time. If fail, throw error.
try:
amount, start = int(info[2]), float(info[3])
except ValueError:
print("The last 2 values must be numbers")
sys.exit()
# Try getting the source and dest from the dictionary. If they
# don't exist, it means no link was connected to them, so we
# can't have a flow.
try:
source = routers[info[0]]
destination = routers[info[1]]
except KeyError:
print("Network not connected")
sys.exit()
# Build a flow with all the information and add it to the list of flows.
temp = flow(source, destination, amount, start, con_ctrl)
flowsLst.append(temp)
# Make a list of all the routers using the routers dictionary.
routersLst = []
for key in routers:
routersLst.append(routers[key])
# intitialize an event queue
the_event_queue = event_queue(verbose)
# Statically initialize all the routing tables
globalTable = sP.fillTable(linksLst, flowsLst)
for fl in flowsLst:
fl.src.updateStatic(globalTable)
for rt in routersLst:
rt.updateStatic(globalTable)
# Start the simulation
run_simulation(the_event_queue, flowsLst, linksLst, routersLst, con_ctrl)
