def toggle_last_request(self): ## let it toggle the last request to a given approval only if the chained request allows it if self.get_attribute('approval') == 'none': return ccdb = database('chained_campaigns') mcm_cc = ccdb.get(self.get_attribute('member_of_campaign')) (next_campaign_id, flow_name) = mcm_cc['campaigns'][self.get_attribute('step')] fdb = database('flows') mcm_f = flow(fdb.get(flow_name)) # check whether we have to do something even more subtle with the request if mcm_f.get_attribute('approval') == 'submit' or self.get_attribute('approval') == 'submit': rdb = database('requests') next_request = request(rdb.get(self.get_attribute('chain')[self.get_attribute('step')])) current_r_approval = next_request.get_attribute('approval') time_out = 0 #self.logger.error('Trying to move %s from %s to submit'% (next_request.get_attribute('prepid'), current_r_approval)) while current_r_approval != 'submit' and time_out <= 10: time_out += 1 #get it back from db to avoid _red issues next_request = request(rdb.get(next_request.get_attribute('prepid'))) with locker.lock('{0}-wait-for-approval'.format( next_request.get_attribute('prepid') )): next_request.approve() request_saved = rdb.save(next_request.json()) if not request_saved: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'Could not save the new request %s while trying to move to submit approval' % ( next_request.get_attribute('prepid'))) current_r_approval = next_request.get_attribute('approval') pass return True
def processing(): user_password = get_user_name() if user_password is None or len(user_password) == 0: user_password = (123456, 'flesdewe234', 'lqz123456789') try: print user_password if user_password is None or len(user_password) == 0: print 'user name and pass word is over' return request = login_chrome_1688(user_password[1], user_password[2], int(user_password[0])) time.sleep(3) for i in range(0, len(regions)): region = regions[i] url_region = url_regions[i] for all_classi in get_all_classi(): print 'region = %s, all class = %s' % (region, all_classi[0]) try: f = flow(request, region, all_classi[0], url_region) f.process_classi() time.sleep(1) except Exception as e: print 'traceback.format_exc():\n%s' % traceback.format_exc( ) finally: my_sql = Mysql_db() sql = "update test.crm_clue_password set status = 0 where id = " + str( user_password[0]) my_sql.excute(sql)
def _flow_align(flow_data, class_map, series, vector_mult=1, interpolate=False): flowed = flow.flow(flow_data, full_path=True, vector_mult=vector_mult, interpolate=interpolate) noy, nox, noc = class_map.shape nof = len(series) mapped = torch.zeros((noy, nox, noc, nof), device=class_map.device, dtype=class_map.dtype) for i in range(nof): mapped[..., i] = map2d.apply(class_map, flowed[..., series[i]]) return mapped
def process_tcp(self, ts, pkt, src, dst): """Process a tcp packet""" ip = pkt.data tcp = ip.data this_flow = flow() this_flow.src = src this_flow.dst = dst if type(tcp) != dpkt.tcp.TCP or ip.len == ip.__hdr_len__ + tcp.__hdr_len__: # no data or bad data, return return this_flow.sport = tcp.sport this_flow.dport = tcp.dport self.store_packet(ts, this_flow, tcp)
def process_tcp(self, ts, pkt, src, dst): """Process a tcp packet""" ip = pkt.data tcp = ip.data this_flow = flow() this_flow.src = src this_flow.dst = dst if type( tcp ) != dpkt.tcp.TCP or ip.len == ip.__hdr_len__ + tcp.__hdr_len__: # no data or bad data, return return this_flow.sport = tcp.sport this_flow.dport = tcp.dport self.store_packet(ts, this_flow, tcp)
def toggle_last_request(self): ## let it toggle the last request to a given approval only if the chained request allows it if self.get_attribute('approval') == 'none': return ccdb = database('chained_campaigns') mcm_cc = ccdb.get(self.get_attribute('member_of_campaign')) (next_campaign_id, flow_name) = mcm_cc['campaigns'][self.get_attribute('step')] fdb = database('flows') mcm_f = flow(fdb.get(flow_name)) # check whether we have to do something even more subtle with the request if mcm_f.get_attribute('approval') == 'submit' or self.get_attribute( 'approval') == 'submit': rdb = database('requests') next_request = request( rdb.get( self.get_attribute('chain')[self.get_attribute('step')])) current_r_approval = next_request.get_attribute('approval') time_out = 0 #self.logger.error('Trying to move %s from %s to submit'% (next_request.get_attribute('prepid'), current_r_approval)) while current_r_approval != 'submit' and time_out <= 10: time_out += 1 #get it back from db to avoid _red issues next_request = request( rdb.get(next_request.get_attribute('prepid'))) with locker.lock('{0}-wait-for-approval'.format( next_request.get_attribute('prepid'))): next_request.approve() request_saved = rdb.save(next_request.json()) if not request_saved: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Could not save the new request %s while trying to move to submit approval' % (next_request.get_attribute('prepid'))) current_r_approval = next_request.get_attribute('approval') pass return True
def flow_to_next_step(self, input_dataset='', block_black_list=None, block_white_list=None, check_stats=True, reserve=False): if not block_white_list: block_white_list = [] if not block_black_list: block_black_list = [] self.logger.log('Flowing chained_request %s to next step...' % (self.get_attribute('_id'))) if not self.get_attribute('chain'): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'chained_request %s has got no root' % (self.get_attribute('_id'))) # check on the approval of the chained request before all ## let it be flowing regardless #if self.get_attribute('approval') == 'none': # raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), # 'The approval of the chained request is none, and therefore flow cannot happen') #this operation requires to access all sorts of objects rdb = database('requests') cdb = database('campaigns') ccdb = database('chained_campaigns') crdb = database('chained_requests') fdb = database('flows') adb = database('actions') l_type = locator() current_step = len(self.get_attribute( 'chain')) - 1 if reserve else self.get_attribute('step') current_id = self.get_attribute('chain')[current_step] next_step = current_step + 1 if not rdb.document_exists(current_id): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the request %s does not exist' % current_id) current_request = request(rdb.get(current_id)) current_campaign = campaign( cdb.get(current_request.get_attribute('member_of_campaign'))) if not ccdb.document_exists(self.get_attribute('member_of_campaign')): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the chain request %s is member of %s that does not exist' % (self.get_attribute('_id'), self.get_attribute('member_of_campaign'))) mcm_cc = ccdb.get(self.get_attribute('member_of_campaign')) if next_step >= len(mcm_cc['campaigns']): if reserve: return False raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'chained_campaign %s does not allow any further flowing.' % (self.get_attribute('member_of_campaign'))) if not reserve: ## is the current request in the proper approval allowed_request_approvals = ['submit'] if current_request.get_attribute( 'approval') not in allowed_request_approvals: raise self.NotApprovedException( current_request.get_attribute('prepid'), current_request.get_attribute('approval'), allowed_request_approvals) ## is the current request in the proper status allowed_request_statuses = ['submitted', 'done'] if current_request.get_attribute( 'status') not in allowed_request_statuses: raise self.NotInProperStateException( current_request.get_attribute('prepid'), current_request.get_attribute('status'), allowed_request_statuses) original_action_id = self.get_attribute('chain')[0] original_action_item = self.retrieve_original_action_item( adb, original_action_id) ## what is the campaign to go to next and with which flow (next_campaign_id, flow_name) = mcm_cc['campaigns'][next_step] if not fdb.document_exists(flow_name): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The flow %s does not exist' % flow_name) mcm_f = flow(fdb.get(flow_name)) if not 'sequences' in mcm_f.get_attribute('request_parameters'): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The flow %s does not contain sequences information.' % (flow_name)) if not cdb.document_exists(next_campaign_id): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The next campaign %s does not exist' % next_campaign_id) next_campaign = campaign(cdb.get(next_campaign_id)) if len(next_campaign.get_attribute('sequences')) != len( mcm_f.get_attribute('request_parameters')['sequences']): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the sequences changes in flow %s are not consistent with the next campaign %s' % (flow_name, next_campaign_id)) if next_campaign.get_attribute( 'energy') != current_campaign.get_attribute('energy'): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'cannot flow any further. Request {0} has inconsistent energy.' .format(next_campaign.get_attribute("prepid"))) if not next_campaign.is_release_greater_or_equal_to( current_campaign.get_attribute('cmssw_release')): raise self.ChainedRequestCannotFlowException( self.get_attribute("_id"), 'cannot flow any further. Request {0} has lower release version.' .format(next_campaign.get_attribute("prepid"))) if next_campaign.get_attribute('type') == 'MCReproc' and ( not 'time_event' in mcm_f.get_attribute('request_parameters')): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the flow is getting into a MCReproc campaign but not time per event is specified' ) if next_campaign.get_attribute('type') == 'MCReproc' and ( not 'size_event' in mcm_f.get_attribute('request_parameters')): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'the flow is getting into a MCReproc campaign but not size per event is specified' ) ## check that it is allowed to flow allowed_flow_approvals = ['flow', 'submit'] ###### cascade of checks """ if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The flow (%s) is not in proper approval state (%s)'%( mcm_f.get_attribute('prepid'), mcm_f.get_attribute('approval'))) if not reserve and not self.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The chained request (%s) is not in the proper approval state (%s)'% ( self.get_attribute('_id'), self.get_attribute('approval'))) """ if not reserve and not mcm_f.get_attribute( 'approval') in allowed_flow_approvals: if not self.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Neither the flow (%s) nor the chained request (%s) approvals allow for flowing' % (mcm_f.get_attribute('approval'), self.get_attribute('approval'))) if next_campaign.get_attribute('status') == 'stopped': raise self.CampaignStoppedException(next_campaign_id) #what is going to be the required number of events for the next request #update the stats to its best if not reserve: current_request.get_stats() next_total_events = current_request.get_attribute( 'completed_events') ## get the original expected events and allow a margin of 5% less statistics statistics_fraction = settings().get_value('statistics_fraction') current_eff_error = 1. - current_request.get_efficiency_error() statistics_fraction = min(statistics_fraction, current_eff_error) completed_events_to_pass = int( current_request.get_attribute('total_events') * statistics_fraction) notify_on_fail = True ## to be tuned according to the specific cases if current_request.get_attribute('completed_events') <= 0: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The number of events completed is negative or null') else: allowed_request_statuses = ['done'] ## determine if this is a root -> non-root transition to potentially apply staged number at_a_transition = (current_campaign.get_attribute('root') != 1 and next_campaign.get_attribute('root') == 1) if ('staged' in original_action_item or 'threshold' in original_action_item) and at_a_transition: allowed_request_statuses.append('submitted') ##check status if not current_request.get_attribute( 'status') in allowed_request_statuses: raise self.NotInProperStateException( current_request.get_attribute('prepid'), current_request.get_attribute('status'), allowed_request_statuses) ##special check at transition that the statistics is good enough if at_a_transition: # at a root -> non-root transition only does the staged/threshold functions ! if 'staged' in original_action_item: next_total_events = int(original_action_item['staged']) completed_events_to_pass = next_total_events if 'threshold' in original_action_item: next_total_events = int( current_request.get_attribute('total_events') * float(original_action_item['threshold'] / 100.)) completed_events_to_pass = next_total_events if check_stats and ( current_request.get_attribute('completed_events') < completed_events_to_pass): if notify_on_fail: current_request.notify( 'Flowing %s with not enough statistics' % (current_request.get_attribute('prepid')), 'For the request %s, the completed statistics %s is not enough to fullfill the requirement to the next level : need at least %s in chain %s \n\n Please report to the operation HN or at the next MccM what action should be taken.\n\n %srequests?prepid=%s\n%schained_requests?contains=%s\n%schained_requests?prepid=%s ' % (current_request.get_attribute('prepid'), current_request.get_attribute('completed_events'), completed_events_to_pass, self.get_attribute('prepid'), l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), self.get_attribute('prepid')), accumulate=True) raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The number of events completed (%s) is not enough for the requirement (%s)' % (current_request.get_attribute('completed_events'), completed_events_to_pass)) ## select what is to happened : [create, patch, use] next_id = None approach = None next_request = None if next_step != len(self.get_attribute('chain')): #not at the end next_id = self.get_attribute('chain')[next_step] if not rdb.document_exists(next_id): raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'The next request (%s) according to the step (%s) does not exist' % (next_id, next_step)) next_request = request(rdb.get(next_id)) if next_request.get_attribute('status') == 'new': #most likely a rewind + resub approach = 'patch' else: ##this is always the case in chains reserved from existing things: so use the next request approach = 'use' #raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), #'This should never happen. (%s) is next according to step (%s), but is not in new status (%s)' % ( #next_id, next_step, next_request.get_attribute('status'))) else: ## look in *other* chained campaigns whether you can suck in an existing request ## look up all chained requests that start from the same root request ## remove <pwg>-chain_ and the -serial number, replacing _ with a . toMatch = '.'.join( self.get_attribute('prepid').split('_')[1:][0:next_step + 1]).split('-')[0] ## make sure they get ordered by prepid related_crs = sorted(crdb.queries( ['root_request==%s' % original_action_id]), key=lambda cr: cr['prepid']) vetoed_last = [] for existing_cr in related_crs: ## exclude itself if existing_cr['prepid'] == self.get_attribute('prepid'): continue ## prevent from using a request from within the same exact chained_campaigns if existing_cr['member_of_campaign'] == self.get_attribute( 'member_of_campaign'): mcm_cr = chained_request(crdb.get(existing_cr['prepid'])) if len(mcm_cr.get_attribute('chain')) > next_step: ## one existing request in the very same chained campaign has already something used, make sure it is not going to be used vetoed_last.append( mcm_cr.get_attribute('chain')[next_step]) continue else: continue for existing_cr in related_crs: ## exclude itself if existing_cr['prepid'] == self.get_attribute('prepid'): continue ## prevent from using a request from within the same exact chained_campaigns if existing_cr['member_of_campaign'] == self.get_attribute( 'member_of_campaign'): continue truncated = '.'.join( existing_cr['prepid'].split('_')[1:][0:next_step + 1]).split('-')[0] self.logger.error('to match : %s , this one %s' % (toMatch, truncated)) if truncated == toMatch: #we found a chained request that starts with all same steps mcm_cr = chained_request(crdb.get(existing_cr['prepid'])) if len(mcm_cr.get_attribute('chain')) <= next_step: #found one, but it has not enough content either continue if mcm_cr.get_attribute('chain')[next_step] in vetoed_last: continue next_id = mcm_cr.get_attribute('chain')[next_step] break if next_id: approach = 'use' else: approach = 'create' if approach == 'create': from rest_api.RequestPrepId import RequestPrepId next_id = RequestPrepId().next_prepid( current_request.get_attribute('pwg'), next_campaign_id) next_request = request(rdb.get(next_id)) request.transfer(current_request, next_request) self.request_join(next_request) elif approach == 'use': ## there exists a request in another chained campaign that can be re-used here. # take this one. advance and go on next_request = request(rdb.get(next_id)) if not reserve: self.set_attribute('step', next_step) self.set_attribute('last_status', next_request.get_attribute('status')) self.update_history({'action': 'flow', 'step': str(next_id)}) self.set_attribute('status', 'processing') if not self.get_attribute("prepid") in next_request.get_attribute( "member_of_chain"): ## register the chain to the next request self.request_join(next_request) saved = rdb.update(next_request.json()) if not saved: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Unable to save %s with updated member_of_chains' % next_id) return True elif approach == 'patch': ## there exists already a request in the chain (step!=last) and it is usable for the next stage next_request = request(next_campaign.add_request(rdb.get(next_id))) ## propagate again some of the fields of the previous request. request.transfer(current_request, next_request) else: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Unrecognized approach %s' % approach) #current_request -> next_request #current_campaign -> next_campaign ##determine whether we have an input dataset for the next request if len(current_request.get_attribute('reqmgr_name')): last_wma = current_request.get_attribute('reqmgr_name')[-1] if 'content' in last_wma and 'pdmv_dataset_name' in last_wma[ 'content']: input_dataset = last_wma['content']['pdmv_dataset_name'] else: statsDB = database('stats', url='http://cms-pdmv-stats.cern.ch:5984/') if statsDB.document_exists(last_wma['name']): latestStatus = statsDB.get(last_wma['name']) input_dataset = latestStatus['pdmv_dataset_name'] if input_dataset: next_request.set_attribute('input_dataset', input_dataset) ## set blocks restriction if any if block_black_list: next_request.set_attribute('block_black_list', block_black_list) if block_white_list: next_request.set_attribute('block_white_list', block_white_list) ## register the flow to the request next_request.set_attribute('flown_with', flow_name) ##assemble the campaign+flow => request request.put_together(next_campaign, mcm_f, next_request) if not reserve: #already taking stage and threshold into account next_request.set_attribute('total_events', next_total_events) next_request.update_history({ 'action': 'flow', 'step': self.get_attribute('prepid') }) request_saved = rdb.save(next_request.json()) if not request_saved: raise self.ChainedRequestCannotFlowException( self.get_attribute('_id'), 'Could not save the new request %s' % (next_request.get_attribute('prepid'))) ## inspect priority self.set_priority(original_action_item['block_number']) if not reserve: # sync last status self.set_attribute('last_status', next_request.get_attribute('status')) # we can only be processing at this point self.set_attribute('status', 'processing') # set to next step self.set_attribute('step', next_step) self.update_history({ 'action': 'flow', 'step': next_request.get_attribute('prepid') }) if not reserve: notification_subject = 'Flow for request %s in %s' % ( current_request.get_attribute('prepid'), next_campaign_id) notification_text = 'The request %s has been flown within:\n \t %s \n into campaign:\n \t %s \n using:\n \t %s \n creating the new request:\n \t %s \n as part of:\n \t %s \n and from the produced dataset:\n %s \n\n%srequests?prepid=%s \n%srequests?prepid=%s \n' % ( current_request.get_attribute('prepid'), self.get_attribute('member_of_campaign'), next_campaign_id, flow_name, next_request.get_attribute('prepid'), self.get_attribute('prepid'), next_request.get_attribute('input_dataset'), l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), next_request.get_attribute('prepid')) current_request.notify(notification_subject, notification_text, accumulate=True) else: notification_subject = 'Reservation of request {0}'.format( next_request.get_attribute('prepid')) notification_text = 'The request {0} of campaign \n\t{2}\nhas been reserved as part of \n\t{1}\nas the next step for {4}\n\n{3}requests?prepid={4}\n{5}requests?prepid={6}\n'.format( next_request.get_attribute('prepid'), self.get_attribute('prepid'), next_campaign_id, l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), next_request.get_attribute('prepid'), ) next_request.notify(notification_subject, notification_text, accumulate=True) return True
def _flow_cur_map(flow_data, class_map, vector_mult=1, interpolate=False): flowed = flow.flow(flow_data, vector_mult=vector_mult, interpolate=interpolate) flowed_map = map2d.apply(class_map, flowed).type(torch.float) return flowed_map
def _flow_end_map(flow_data, class_map, vector_mult=1, interpolate=False, use_homography=False): flowed = flow.flow(flow_data, vector_mult=vector_mult, interpolate=interpolate) return _end_map(flowed, class_map, use_homography=use_homography)
def _flow_full(flow_data, vector_mult, interpolate=False): return flow.flow(flow_data, full_path=True, vector_mult=vector_mult, interpolate=interpolate)
def flow_to_next_step(self, input_dataset='', block_black_list=None, block_white_list=None, check_stats=True, reserve=False): if not block_white_list: block_white_list = [] if not block_black_list: block_black_list = [] self.logger.log('Flowing chained_request %s to next step...' % (self.get_attribute('_id'))) if not self.get_attribute('chain'): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'chained_request %s has got no root' % ( self.get_attribute('_id'))) # check on the approval of the chained request before all ## let it be flowing regardless #if self.get_attribute('approval') == 'none': # raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), # 'The approval of the chained request is none, and therefore flow cannot happen') #this operation requires to access all sorts of objects rdb = database('requests') cdb = database('campaigns') ccdb = database('chained_campaigns') crdb = database('chained_requests') fdb = database('flows') adb = database('actions') l_type=locator() current_step = len(self.get_attribute('chain'))-1 if reserve else self.get_attribute('step') current_id = self.get_attribute('chain')[current_step] next_step = current_step + 1 if not rdb.document_exists(current_id): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'the request %s does not exist' % current_id) current_request = request(rdb.get(current_id)) current_campaign = campaign(cdb.get(current_request.get_attribute('member_of_campaign'))) if not ccdb.document_exists(self.get_attribute('member_of_campaign')): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'the chain request %s is member of %s that does not exist' % ( self.get_attribute('_id'), self.get_attribute('member_of_campaign'))) mcm_cc = ccdb.get(self.get_attribute('member_of_campaign')) if next_step >= len(mcm_cc['campaigns']): if reserve: return False raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'chained_campaign %s does not allow any further flowing.' % ( self.get_attribute('member_of_campaign'))) if not reserve: ## is the current request in the proper approval allowed_request_approvals = ['submit'] if current_request.get_attribute('approval') not in allowed_request_approvals: raise self.NotApprovedException(current_request.get_attribute('prepid'), current_request.get_attribute('approval'), allowed_request_approvals) ## is the current request in the proper status allowed_request_statuses = ['submitted', 'done'] if current_request.get_attribute('status') not in allowed_request_statuses: raise self.NotInProperStateException(current_request.get_attribute('prepid'), current_request.get_attribute('status'), allowed_request_statuses) original_action_id = self.get_attribute('chain')[0] original_action_item = self.retrieve_original_action_item(adb, original_action_id) ## what is the campaign to go to next and with which flow (next_campaign_id, flow_name) = mcm_cc['campaigns'][next_step] if not fdb.document_exists(flow_name): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The flow %s does not exist' % flow_name ) mcm_f = flow(fdb.get(flow_name)) if not 'sequences' in mcm_f.get_attribute('request_parameters'): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The flow %s does not contain sequences information.' % ( flow_name)) if not cdb.document_exists(next_campaign_id): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The next campaign %s does not exist' % next_campaign_id) next_campaign = campaign(cdb.get(next_campaign_id)) if len(next_campaign.get_attribute('sequences')) != len(mcm_f.get_attribute('request_parameters')['sequences']): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'the sequences changes in flow %s are not consistent with the next campaign %s' % ( flow_name, next_campaign_id)) if next_campaign.get_attribute('energy') != current_campaign.get_attribute('energy'): raise self.EnergyInconsistentException(next_campaign.get_attribute('prepid')) if next_campaign.get_attribute('type') == 'MCReproc' and (not 'time_event' in mcm_f.get_attribute('request_parameters')): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'the flow is getting into a MCReproc campaign but not time per event is specified') if next_campaign.get_attribute('type') == 'MCReproc' and (not 'size_event' in mcm_f.get_attribute('request_parameters')): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'the flow is getting into a MCReproc campaign but not size per event is specified') ## check that it is allowed to flow allowed_flow_approvals = ['flow', 'submit'] ###### cascade of checks """ if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The flow (%s) is not in proper approval state (%s)'%( mcm_f.get_attribute('prepid'), mcm_f.get_attribute('approval'))) if not reserve and not self.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The chained request (%s) is not in the proper approval state (%s)'% ( self.get_attribute('_id'), self.get_attribute('approval'))) """ if not reserve and not mcm_f.get_attribute('approval') in allowed_flow_approvals: if not self.get_attribute('approval') in allowed_flow_approvals: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'Neither the flow (%s) nor the chained request (%s) approvals allow for flowing' % ( mcm_f.get_attribute('approval'), self.get_attribute('approval'))) if next_campaign.get_attribute('status') == 'stopped': raise self.CampaignStoppedException(next_campaign_id) #what is going to be the required number of events for the next request #update the stats to its best if not reserve: current_request.get_stats() next_total_events=current_request.get_attribute('completed_events') notify_on_fail=True ## to be tuned according to the specific cases if current_request.get_attribute('completed_events') <= 0: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The number of events completed is negative or null') else: allowed_request_statuses = ['done'] ## determine if this is a root -> non-root transition to potentially apply staged number at_a_transition=(current_campaign.get_attribute('root') != 1 and next_campaign.get_attribute('root') == 1) if ('staged' in original_action_item or 'threshold' in original_action_item) and at_a_transition: allowed_request_statuses.append('submitted') ##check status if not current_request.get_attribute('status') in allowed_request_statuses: raise self.NotInProperStateException(current_request.get_attribute('prepid'), current_request.get_attribute('status'), allowed_request_statuses) ##special check at transition that the statistics is good enough if at_a_transition: # at a root -> non-root transition only does the staged/threshold functions ! if 'staged' in original_action_item: next_total_events = int(original_action_item['staged']) if 'threshold' in original_action_item: next_total_events = int(current_request.get_attribute('total_events') * float(original_action_item['threshold'] / 100.)) completed_events_to_pass = next_total_events else: ## get the original expected events and allow a margin of 5% less statistics completed_events_to_pass = int(current_request.get_attribute('total_events') * 0.95) if check_stats and (current_request.get_attribute('completed_events') < completed_events_to_pass): if notify_on_fail: current_request.notify('Flowing for %s: not enough statistics'%( current_request.get_attribute('prepid')), 'For this request, the completed statistics %s is not enough to fullfill the requirement to the next level : need at least %s \n\n %srequests?prepid=%s'%( current_request.get_attribute('completed_events'), completed_events_to_pass, l_type.baseurl(), current_request.get_attribute('prepid'))) raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The number of events completed (%s) is not enough for the requirement (%s)'%(current_request.get_attribute('completed_events'), completed_events_to_pass)) ## select what is to happened : [create, patch, use] next_id = None approach = None next_request = None if next_step != len(self.get_attribute('chain')): #not at the end next_id = self.get_attribute('chain')[next_step] if not rdb.document_exists(next_id): raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'The next request (%s) according to the step (%s) does not exist' % ( next_id, next_step)) next_request = request(rdb.get(next_id)) if next_request.get_attribute('status') == 'new': #most likely a rewind + resub approach = 'patch' else: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'This should never happen. (%s) is next according to step (%s), but is not in new status (%s)' % ( next_id, next_step, next_request.get_attribute('status'))) else: ## look in *other* chained campaigns whether you can suck in an existing request ## look up all chained requests that start from the same root request ## remove <pwg>-chain_ and the -serial number, replacing _ with a . toMatch = '.'.join(self.get_attribute('prepid').split('_')[1:][0:next_step + 1]).split('-')[0] ## make sure they get ordered by prepid related_crs = sorted(crdb.queries(['root_request==%s' % original_action_id]), key=lambda cr : cr['prepid']) vetoed_last=[] for existing_cr in related_crs: ## exclude itself if existing_cr['prepid']==self.get_attribute('prepid'): continue ## prevent from using a request from within the same exact chained_campaigns if existing_cr['member_of_campaign'] == self.get_attribute('member_of_campaign'): mcm_cr = chained_request(crdb.get(existing_cr['prepid'])) if len(mcm_cr.get_attribute('chain')) > next_step: ## one existing request in the very same chained campaign has already something used, make sure it is not going to be used vetoed_last.append( mcm_cr.get_attribute('chain')[next_step]) continue else: continue for existing_cr in related_crs: ## exclude itself if existing_cr['prepid']==self.get_attribute('prepid'): continue ## prevent from using a request from within the same exact chained_campaigns if existing_cr['member_of_campaign'] == self.get_attribute('member_of_campaign'): continue truncated = '.'.join(existing_cr['prepid'].split('_')[1:][0:next_step + 1]).split('-')[0] self.logger.error('to match : %s , this one %s' % ( toMatch, truncated )) if truncated == toMatch: #we found a chained request that starts with all same steps mcm_cr = chained_request(crdb.get(existing_cr['prepid'])) if len(mcm_cr.get_attribute('chain')) <= next_step: #found one, but it has not enough content either continue if mcm_cr.get_attribute('chain')[next_step] in vetoed_last: continue next_id = mcm_cr.get_attribute('chain')[next_step] break if next_id: approach = 'use' else: approach = 'create' if approach == 'create': from rest_api.RequestPrepId import RequestPrepId next_id = RequestPrepId().next_prepid(current_request.get_attribute('pwg'), next_campaign_id) next_request = request(rdb.get(next_id)) request.transfer( current_request, next_request) self.request_join(next_request) elif approach == 'use': ## there exists a request in another chained campaign that can be re-used here. # take this one. advance and go on next_request = request(rdb.get(next_id)) if not reserve: self.set_attribute('step', next_step) self.set_attribute('last_status', next_request.get_attribute('status')) self.update_history({'action': 'flow', 'step': str(next_step)}) self.set_attribute('status', 'processing') if not self.get_attribute("prepid") in next_request.get_attribute("member_of_chain"): ## register the chain to the next request self.request_join(next_request) saved = rdb.update(next_request.json()) if not saved: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'Unable to save %s with updated member_of_chains' % next_id) return True elif approach == 'patch': ## there exists already a request in the chain (step!=last) and it is usable for the next stage next_request = request( next_campaign.add_request( rdb.get(next_id))) ### shouldn't this be added ? #transfer( current_request, next_request) else: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'Unrecognized approach %s' % approach ) #current_request -> next_request #current_campaign -> next_campaign ##determine whether we have an input dataset for the next request if len(current_request.get_attribute('reqmgr_name')): last_wma = current_request.get_attribute('reqmgr_name')[-1] if 'content' in last_wma and 'pdmv_dataset_name' in last_wma['content']: input_dataset = last_wma['content']['pdmv_dataset_name'] else: statsDB = database('stats', url='http://cms-pdmv-stats.cern.ch:5984/') if statsDB.document_exists(last_wma['name']): latestStatus = statsDB.get(last_wma['name']) input_dataset = latestStatus['pdmv_dataset_name'] if input_dataset: next_request.set_attribute('input_filename', input_dataset) ## set blocks restriction if any if block_black_list: next_request.set_attribute('block_black_list', block_black_list) if block_white_list: next_request.set_attribute('block_white_list', block_white_list) ## register the flow to the request next_request.set_attribute('flown_with', flow_name) ##assemble the campaign+flow => request request.put_together(next_campaign, mcm_f, next_request) if not reserve: #already taking stage and threshold into account next_request.set_attribute('total_events', next_total_events) next_request.update_history({'action': 'flow', 'step': self.get_attribute('prepid')}) request_saved = rdb.save(next_request.json()) if not request_saved: raise self.ChainedRequestCannotFlowException(self.get_attribute('_id'), 'Could not save the new request %s' % ( next_request.get_attribute('prepid'))) ## inspect priority self.set_priority(original_action_item['block_number']) if not reserve: # sync last status self.set_attribute('last_status', next_request.get_attribute('status')) # we can only be processing at this point self.set_attribute('status', 'processing') # set to next step self.set_attribute('step', next_step) if not reserve: notification_subject = 'Flow for request %s in %s' % (current_request.get_attribute('prepid'), next_campaign_id) notification_text = 'The request %s has been flown within:\n \t %s \n into campaign:\n \t %s \n using:\n \t %s \n creating the new request:\n \t %s \n as part of:\n \t %s \n and from the produced dataset:\n %s \n\n%srequests?prepid=%s \n%srequests?prepid=%s \n' % ( current_request.get_attribute('prepid'), self.get_attribute('member_of_campaign'), next_campaign_id, flow_name, next_request.get_attribute('prepid'), self.get_attribute('prepid'), next_request.get_attribute('input_filename'), l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), next_request.get_attribute('prepid') ) current_request.notify(notification_subject, notification_text) else: notification_subject = 'Reservation of request {0}'.format(next_request.get_attribute('prepid')) notification_text = 'The request {0} of campaign \n\t{2}\nhas been reserved as part of \n\t{1}\nas the next step for {4}\n\n{3}requests?prepid={4}\n{5}requests?prepid={6}\n'.format( next_request.get_attribute('prepid'), self.get_attribute('prepid'), next_campaign_id, l_type.baseurl(), current_request.get_attribute('prepid'), l_type.baseurl(), next_request.get_attribute('prepid'), ) next_request.notify(notification_subject, notification_text) return True
def opt_pterminal(x0, ic1, ic2, dt): """ 1.) flow #1, #2 based on x0 2.) run prussing_conway 3.) run shooting method w/ variational eqns 4.) generate delta-vs 5.) generate primer history 6.) extract slopes 7.) compute cost function """ """ FUTURE WORK 1.) add in intelligent step sizing for flow3 = shoot() """ # import statements from flow import P2B as flow from lambert import prussing_conway from shootingmodule import firstorder as shoot from numpy.linalg import norm from numpy import array, zeros, eye print ("Starting an iteration of opt_pterminal!") print ("x0 is: %e, %e " % (x0[0], x0[1])) # function parameters """ FW 1.) """ step = 1e-2 tol = 1e-3 # extract mu mu = ic1[4] # flow #1, #2 based on x0[0] tspan = [0, x0[0]] ministep = abs(x0[0] / 2e2) print ("Midpoint-A of an iteration of opt_pterminal!") print ("tspan is: %e, %e " % (tspan[0], tspan[1])) print ("ministep is: %e " % ministep) flow1 = flow(ic1, tspan, ministep, trim_output=False, print_status=True, solver="rk4p") print ("Midpoint-B of an iteration of opt_pterminal!") flow2 = flow(ic2, tspan, ministep, trim_output=False) print ("Midpoint-C of an iteration of opt_pterminal!") # update ic1 and ic2 ic1 = array(flow1["y"][-1]) ic2 = array(flow2["y"][-1]) # update of flow time based on x0 tspan = [0, dt - x0[0] + x0[1]] # flow #2 based on x0[1] flow2 = flow(ic2, tspan, tstep=step, trim_output=False) # target position target = array(flow2["y"][-1][0:4]) # solve for lambert arc print ("Midpoint-0 of an iteration of opt_pterminal!") arc = prussing_conway(ic1, target, mu, TransferTime=abs(tspan[1])) # setup initial conditions for flow of spacecraft with # variational equations ic = zeros(21) ic[0:2] = ic1[0:2] ic[2:4] = arc["v1"] ic[4] = mu ic[5:] = eye(4).reshape(16) # flow spacecraft using lambert arc print ("Midpoint-1 of an iteration of opt_pterminal!") flow3 = flow(ic, tspan, tstep=step, eom="P2BP_varEqns", trim_output=False) print ("Midpoint-2 of an iteration of opt_pterminal!") # position error poserror = flow3["y"][-1][0:2] - flow2["y"][-1][0:2] # apply shooting method to reduce error if norm(poserror) > tol: flow3 = shoot( ic, target[0:2], tspan, eom="P2BP_varEqns", tstep=step, damping=1 / 3.0, tol=tol, trim_output=False ) poserror = flow3["y"][-1][0:2] - flow2["y"][-1][0:2] print ("Midpoint-3 of an iteration of opt_pterminal!") # delta-vs dv1 = array([flow3["y"][0][2], flow3["y"][0][3]]) - array([ic1[2], ic1[3]]) dv2 = -array([flow3["y"][-1][2], flow3["y"][-1][3]]) + array([flow2["y"][-1][2], flow2["y"][-1][3]]) # generate primer vector history p = phistory(dv1, dv2, flow3) # extract the slopes of the primer vector dp0 = pslope(flow3, p, 1, 0) dp1 = pslope(flow3, p, -1, -2) # scalar cost function J = dp0 ** 2 + dp1 ** 2 print ("Finished an iteration of opt_pterminal!") return J, flow1, flow2, flow3
def firstorder(ic, target, tspan, eom, \ solver = 'rk4', \ tstep = 0, \ damping = 0.1, \ iLimit = 25, \ tol = 1E-3, \ TU = 1, \ guess = None, \ print_status = False, \ trim_output = True): # import statements from numpy.linalg import norm, solve from numpy import zeros, eye, concatenate from primervectormodule import extractSTM # if eom argument is not an eom model with variational equations, #+ then turn on the appropriate model if eom == 'P2BP': eom = 'P2BP_varEqns' if eom == 'S2BP': eom = 'S2BP_varEqns' # determine dimension of problem (i.e. planar or spatial) #+ and initialize the state vector and residual vector if len(target) == 2: from flow import P2B as flow # initialize a state vector if len(ic) == 21: s = ic if len(ic) == 5: s = zeros(21) s[0:5] = ic # tack identity matrix on end for STM s[5:] = eye(4).reshape(16,1) # residual vector res = s[0:2] # make a copy of the velocity for computing/printing delta-v v0 = s[2:4].copy() elif len(target) == 3: from flow import S2B as flow # initialize a state vector if len(ic) == 43: s = ic if len(ic) == 7: s = zeros(43) s[0:7] = ic # tack identity matrix on end for STM s[7:] = eye(6).reshape(36,1) # residual vector res = s[0:3] # make a copy of the velocity for computing/printing delta-v v0 = s[3:6].copy() # calculate an appropriate tstep if not defined if tstep == 0: tstep = abs(tspan[1] - tspan[0])/5E1 # if an initial guess for perturbation is given then use it, #+ otherwise perturb the shooting method in the direction of current #+ velocity, so as to generally seek a lower detla-v solution if len(target) == 2: if guess != None: s[2:4] += guess else: s[2:4] += 0.1*s[2:4] elif len(target) == 3: if guess != None: s[3:6] += guess else: s[3:6] += 0.1*s[3:6] # set counter i = 0 # iterate with a first-order shooting method while ((norm(res) > tol) and (i < iLimit + 1)) or (i == 0): # solve for 'flow' solution = flow(s, tspan, tstep, solver, eom, TU, \ trim_output) # extract the state transition matrix # and relevant blocks for delta-v update stm, mblk, nblk = extractSTM(solution, -1) # solve for delta-v update dv = solve(nblk, res) #+ update residual if len(dv) == 2: res = (solution['y'][-1][0:2] - target) elif len(dv) == 3: res = (solution['y'][-1][0:3] - target) # print iterative updates if desired if print_status and len(dv) == 2: print ('Iter: %4i Res: %8e ' \ 'dv1x: %8.5f dv1y: %8.5f ' \ %(i, norm(res), (s[2] - v0[0]), (s[3] - v0[1]))) if print_status and len(dv) == 3: print ('Iter: %4i Res: %8e ' \ 'dv1x: %8.5f dv1y: %8.5f dv1z: %8.5f ' \ %(i, norm(res), (s[3] - v0[0]), \ (s[4] - v0[1]), (s[5] - v0[2]))) # update initial velocity vector if len(dv) == 2: s[2:4] = s[2:4] - damping*dv elif len(dv) == 3: s[3:6] = s[3:6] - damping*dv # update counter i += 1 return solution #--------------------------------------------------------------# #---------------------- Shooting Module -----------------------# #--------------------------------------------------------------#
import matplotlib.image as mpimg import numpy as np from matplotlib.patches import Rectangle handle = simulator.simulator("/home/boka/arp/david/") selection = handle.region() imagefile = handle.frame() print("prvo") if not imagefile: sys.exit(0) #image = cv2.imread(imagefile, cv2.IMREAD_GRAYSCALE) image = cv2.imread(imagefile, cv2.IMREAD_COLOR) print(imagefile) tracker = NCCTracker(image, selection) tracker_flow = flow.flow(image, selection) tracker_OT = ORF.flow(image, selection) print("do tukej") plt.ion() plt.figure() while True: imagefile = handle.frame() if not imagefile: break image = cv2.imread(imagefile, cv2.IMREAD_COLOR) region = tracker.track(image) regionOrg = tracker_OT.track(image) plt.clf() a = plt.imshow(image) #tracker_flow.set_region(tracker.position)
def main(argv): # Check command line parameters if len(sys.argv) != 3: print "python networksimulation.py (0,1,2) (0,1)" print "1st arg: con_ctrl: 0 - none; 1 - TCP Reno; 2 - FAST" print "2nd arg: verbose: 0 - no; 1 - yes" print('\n') sys.exit() # Store command line parameters con_ctrl = int(sys.argv[1]) verbose = int(sys.argv[2]) print('\n') # Give specifications for links file and flows file and then save # the filename in variables print("Enter the file of links.") print("Each line is one link in the following format: ") print("End1 End2 Rate Delay Capacity\n") print("Ends=host/source/destination/router ID (ex. H1,S2,T3,R4)") print("Rate(Mbps), Delay(ms), Capacity(KB)") linkFile = input('Enter link file: ') print('\n') print("Enter the file of flows.") print("Each line is one flow in the following format: ") print("Source Dest DataAmount StartTime\n") print("Source/Dest = ID's (ex. H1,S2,T3,R4)") print("DataAmount(MB), StartTime(s)") flowFile = input('Enter flow file: ') print('\n') # Variables to store the different network objects routers = {} linksLst = [] flowsLst = [] # Try to open the files. If fail, throw an error. try: links = open(linkFile, 'r') flows = open(flowFile, 'r') except IOError: print("Link and Flow files do not exist in this directory.") sys.exit() # First, make all the links i = 0 for line in links: # Split the line into a list, which must have 5 values info = line.split(' ') if(len(info) != 5): print('Each link line must have 5 values') sys.exit() # Try converting the rate, delay and capacity. If fail, throw error try: rate, delay, capacity = float(info[2]), float(info[3]), float(info[4]) except ValueError: print("The last 3 values must be numbers") sys.exit() # Either make a new entry in the routers table, or grab an entry # First get the end1 of the link if(info[0] not in routers.keys()): router1 = router(info[0]) routers[info[0]] = router1 else: router1 = routers[info[0]] # Next get end2 if(info[1] not in routers.keys()): router2 = router(info[1]) routers[info[1]] = router2 else: router2 = routers[info[1]] # Build a link with all the information and add it to the list of links. temp = link(i, router1, router2, rate, delay, capacity) linksLst.append(temp) i += 1 # Now, make all the flows. for line in flows: # Split the line into a list, which must have 4 values info = line.split(' ') if(len(info) != 4): print('Each flow line must have 4 values') sys.exit() # Try converting the amount and start time. If fail, throw error. try: amount, start = int(info[2]), float(info[3]) except ValueError: print("The last 2 values must be numbers") sys.exit() # Try getting the source and dest from the dictionary. If they # don't exist, it means no link was connected to them, so we # can't have a flow. try: source = routers[info[0]] destination = routers[info[1]] except KeyError: print("Network not connected") sys.exit() # Build a flow with all the information and add it to the list of flows. temp = flow(source, destination, amount, start, con_ctrl) flowsLst.append(temp) # Make a list of all the routers using the routers dictionary. routersLst = [] for key in routers: routersLst.append(routers[key]) # intitialize an event queue the_event_queue = event_queue(verbose) # Statically initialize all the routing tables globalTable = sP.fillTable(linksLst, flowsLst) for fl in flowsLst: fl.src.updateStatic(globalTable) for rt in routersLst: rt.updateStatic(globalTable) # Start the simulation run_simulation(the_event_queue, flowsLst, linksLst, routersLst, con_ctrl)