def enhance_symbol_flags(self):
is_float_function_pattern = re.compile(
r"^__aeabi_(f.*|.*2f)|__addsf3$")
def is_float_function_name(n):
return is_float_function_pattern.match(n)
float_functions = [
f for f in self.all_functions() if is_float_function_name(f[NAME])
]
for f in self.all_functions():
callees = f[CALLEES]
f["calls_float_function"] = any(
[ff in callees for ff in float_functions])
for file in self.all_files():
file["calls_float_function"] = any(
[f["calls_float_function"] for f in file[FUNCTIONS]])
def folder_calls_float_function(folder):
result = any([f["calls_float_function"] for f in folder[FILES]])
for sub_folder in folder[SUB_FOLDERS]:
if folder_calls_float_function(sub_folder):
result = True
folder["calls_float_function"] = result
return result
for folder in self.root_folders():
folder_calls_float_function(folder)
def folder_calls_float_function(folder):
result = any([f["calls_float_function"] for f in folder[FILES]])
for sub_folder in folder[SUB_FOLDERS]:
if folder_calls_float_function(sub_folder):
result = True
folder["calls_float_function"] = result
return result
def parse_incoming_email(self, email_pickle):
self.log.write("Parsing incoming")
email = pickle.loads(email_pickle)
if not any(eml.endswith("@armadillo.xvm.mit.edu")
for eml in email['rcpttos']):
# ignore emails not sent to a user on this server.
return
text = email["data"]
try:
self.log.write("subject: %s" % email["subject"])
except Exception as e:
pass
self.log.flush()
processed = self.preprocessor.parse(text)
#add known exception subjects lines. Like
exceptions = ["Reuse-ask Summary"]
if processed["subject"] in exceptions:
pass
else:
parsed = self.parser.parse(processed)
self.handle_parsed_email(parsed)
def folder_calls_float_function(folder):
result = any([f["calls_float_function"] for f in folder[FILES]])
for sub_folder in folder[SUB_FOLDERS]:
if folder_calls_float_function(sub_folder):
result = True
folder["calls_float_function"] = result
return result
def checkForOutputFailure(self, output, workingDir):
"""
This method is called by the RAVEN code at the end of each run if the return code is == 0.
This method needs to be implemented by the codes that, if the run fails, return a return code that is 0
This can happen in those codes that record the failure of the job (e.g. not converged, etc.) as normal termination (returncode == 0)
This method can be used, for example, to parse the outputfile looking for a special keyword that testifies that a particular job got failed
(e.g. in RELAP5 would be the keyword "********")
@ In, output, string, the Output name root
@ In, workingDir, string, current working dir
@ Out, failure, bool, True if the job is failed, False otherwise
"""
failure = True
goodWord = [
"Transient terminated by end of time step cards",
"Transient terminated by trip"
]
try:
outputToRead = open(os.path.join(workingDir, output + '.o'), "r")
except:
return failure
readLines = outputToRead.readlines()
for goodMsg in goodWord:
if any(goodMsg in x for x in readLines[-20:]):
failure = False
return failure
def get_reverted_revs(revs):
reverted_revs = []
diff_sizes = get_diff_sizes(revs)
for rev_num, (editor, size) in enumerate(diff_sizes):
rev_range = [x[1] for x in diff_sizes[rev_num:rev_num+REVERT_LOOKAHEAD+1]]
if any([ r != 0 for r in rev_range ]) and sum(rev_range) == REVERT_THRESHOLD:
reverted_revs.append(rev_num)
return reverted_revs
def zip_(*args):
if __builtin__.any(utils.isvar(a) for a in args):
raise TypeError(
'Called zip() on Tensor but Tensors '
'do not support iteration. Maybe try escaping '
'the tensor?')
else:
return zip(*args)
def process_request(self, req):
"""
Implements IRequestHandler.process_request
Build a dict of search criteria from the user and request results from
the active AdvancedSearchBackend.
"""
req.perm.assert_permission('SEARCH_VIEW')
try:
per_page = int(req.args.getfirst('per_page',
self.DEFAULT_PER_PAGE))
except ValueError:
self.log.warn('Could not set per_page to %s' %
req.args.getfirst('per_page'))
per_page = self.DEFAULT_PER_PAGE
try:
page = int(req.args.getfirst('page', 1))
except ValueError:
page = 1
sort_order = req.args.getfirst('sort_order', 'relevance')
data = {
'source': self._get_filter_dicts(req.args),
'author': [auth for auth in req.args.getlist('author') if auth],
'date_start': req.args.getfirst('date_start'),
'date_end': req.args.getfirst('date_end'),
'q': req.args.get('q'),
'start_points': StartPoints.parse_args(req.args, self.providers),
'per_page': per_page,
'sort_order': sort_order,
'ticket_statuses': self._get_ticket_statuses(req.args),
}
# Initial page request
if not any(
(data['q'], data['author'], data['date_start'], data['date_end'])):
return self._send_response(req, data)
# Look for quickjump
quickjump = self._get_quickjump(req, data['q'])
if quickjump:
req.redirect(quickjump)
# perform query using backend if q is set
result_map = {}
total_count = 0
for provider in self.providers:
result_count, result_list = 0, []
try:
result_count, result_list = provider.query_backend(data)
except SearchBackendException, e:
add_warning(req, _('SearchBackendException: %s' % e))
total_count += result_count
result_map[provider.get_name()] = result_list
def retrieve_character_real_name(sub_wikia, movie_character_name):
"""Retrieve the complete name of the movie character.
The character's names in the movie script are often incomplete or
shortened (for example, "Samwise Gamgee" appears as "Sam" in "The Lord
of the Rings") so this function uses the movie's sub-wikia to detect
the character's real name.
The script will query the wikia search for the short name (for example,
"Sam") and it will accept anything that shows up as the first result as
the correct real name.
It is likely that the movie script contains generic characters which do
not properly exist in the wikia. For example, in "The Lord of the Rings:
The Fellowship of the Ring":
- Character name: "ORC OVERSEER"
- Wikia search result: "List of unnamed original characters of the
books and films"
In cases in which the wikia search result returns "List of", the function
will return None as the character's real name.
Sometimes the search result will return something like this:
- Character name: "HALDIR"
- Wikia search result: "Haldir (disambiguation)"
This might happen when there are multiple characters with the same name in
the movie lore. The function will assume that the less important characters
with the same name are irrelevant and it will automatically strip the
substring "(disambiguation)".
Args:
sub_wikia (String): The sub-wikia to be queried
movie_character_name (String): The character's
Returns:
String: The character's real name
"""
logger = logging.getLogger(__name__)
black_list = ['List of']
real_name = wikia.search(sub_wikia, movie_character_name)[0]
logger.info('Resolved ' + movie_character_name + ' to ' + real_name)
if __builtin__.any(x in real_name for x in black_list):
logger.info('Rejecting possible invalid character.' + ' Name: ' +
movie_character_name + ' ; Real name: ' + real_name)
real_name = None
else:
# Removing any "(disambiguation)" sub-strings
real_name = real_name.split('(')[0].strip(' ')
# Remove any special accents from the string
real_name = unidecode.unidecode(unicode(real_name))
return real_name
def checkForOutputFailure(self, output, workingDir):
"""
This method is called by the RAVEN code at the end of each run if the return code is == 0.
This method needs to be implemented by the codes that, if the run fails, return a return code that is 0
This can happen in those codes that record the failure of the job (e.g. not converged, etc.) as normal termination (returncode == 0)
This method can be used, for example, to parse the outputfile looking for a special keyword that testifies that a particular job got failed
(e.g. in RAVEN would be the keyword "raise")
@ In, output, string, the Output name root
@ In, workingDir, string, current working dir
@ Out, failure, bool, True if the job is failed, False otherwise
"""
failure = False
try:
outputToRead = open(os.path.join(workingDir, output), "r")
except IOError:
failure = True
print(self.printTag + ' ERROR: The RAVEN SLAVE log file "' +
str(os.path.join(workingDir, output)) + '" does not exist!')
if not failure:
readLines = outputToRead.readlines()
if not any("Run complete" in x for x in readLines[-20:]):
failure = True
del readLines
if not failure:
for filename in self.linkedDataObjectOutStreamsNames:
outStreamFile = os.path.join(workingDir, self.innerWorkingDir,
filename + ".csv")
try:
fileObj = open(outStreamFile, "r")
except IOError:
print(self.printTag +
' ERROR: The RAVEN SLAVE output file "' +
str(outStreamFile) + '" does not exist!')
failure = True
if not failure:
readLines = fileObj.readlines()
if any("nan" in x.lower() for x in readLines):
failure = True
print(self.printTag +
' ERROR: Found nan in RAVEN SLAVE output "' +
str(outStreamFile) + '!')
break
del readLines
return failure
def process_request(self, req):
"""
Implements IRequestHandler.process_request
Build a dict of search criteria from the user and request results from
the active AdvancedSearchBackend.
"""
req.perm.assert_permission('SEARCH_VIEW')
try:
per_page = int(req.args.getfirst('per_page',
self.DEFAULT_PER_PAGE))
except ValueError:
self.log.warn('Could not set per_page to %s' %
req.args.getfirst('per_page'))
per_page = self.DEFAULT_PER_PAGE
try:
page = int(req.args.getfirst('page', 1))
except ValueError:
page = 1
sort_order = req.args.getfirst('sort_order', 'relevance')
data = {
'source': self._get_filter_dicts(req.args),
'author': [auth for auth in req.args.getlist('author') if auth],
'date_start': req.args.getfirst('date_start'),
'date_end': req.args.getfirst('date_end'),
'q': req.args.get('q'),
'start_points': StartPoints.parse_args(req.args, self.providers),
'per_page': per_page,
'sort_order': sort_order,
'ticket_statuses': self._get_ticket_statuses(req.args),
}
# Initial page request
if not any((data['q'], data['author'], data['date_start'], data['date_end'])):
return self._send_response(req, data)
# Look for quickjump
quickjump = self._get_quickjump(req, data['q'])
if quickjump:
req.redirect(quickjump)
# perform query using backend if q is set
result_map = {}
total_count = 0
for provider in self.providers:
result_count, result_list = 0, []
try:
result_count, result_list = provider.query_backend(data)
except SearchBackendException, e:
add_warning(req, _('SearchBackendException: %s' % e))
total_count += result_count
result_map[provider.get_name()] = result_list
def select_necessary(covers):
"""Select winners covered by only one component; remove from covers.
Return a pair of (covers, necessary)."""
counts = Counter(w for r in covers for w in covers[r])
necessary = {r for r in covers if any(counts[w] == 1 for w in covers[r])}
if necessary:
covered = {w for r in necessary for w in covers[r]}
covers = {r: covers[r] - covered for r in covers if r not in necessary}
return covers, OR(necessary)
else:
return covers, None
def any(sequence):
"""
Returns True if any element of sequence is True. It is equivalent
to calling reduce(op_or, sequence, False).
>>> any([True, False, False])
True
>>> any([])
False
"""
return __builtin__.any(sequence)
def any(sequence):
"""
Returns True if any element of sequence is True. It is equivalent
to calling reduce(op_or, sequence, False).
>>> any([True, False, False])
True
>>> any([])
False
"""
return __builtin__.any(sequence)
def upstreamEstuary(llid, rm, estuary_stations):
stations_lyr = "stations_2010_lyr"
st_query = '"ESTUARY" = 1'
arcpy.MakeFeatureLayer_management(estuary_stations, stations_lyr, st_query)
est = pd.DataFrame({'llid': [], 'rm':[], 'estuary':[]})
with arcpy.da.SearchCursor(stations_lyr, ['LLID', 'RIVER_MILE', 'ESTUARY']) as cursor:
for row in cursor:
if row[0] == llid:
est = est.append({'llid': str(row[0]), 'rm':row[1], 'estuary':row[2]}, ignore_index=True)
est = est[est['rm']>= rm]
if any(x==1 for x in est['estuary']):
return('Estuary')
else:
return('Needs Further Review')
def enhance_symbol_flags(self):
is_float_function_pattern = re.compile(r"^__aeabi_(f.*|.*2f)|__addsf3$")
def is_float_function_name(n):
return is_float_function_pattern.match(n)
float_functions = [f for f in self.all_functions() if is_float_function_name(f[NAME])]
for f in self.all_functions():
callees = f[CALLEES]
f["calls_float_function"] = any([ff in callees for ff in float_functions])
for file in self.all_files():
file["calls_float_function"] = any([f["calls_float_function"] for f in file[FUNCTIONS]])
def folder_calls_float_function(folder):
result = any([f["calls_float_function"] for f in folder[FILES]])
for sub_folder in folder[SUB_FOLDERS]:
if folder_calls_float_function(sub_folder):
result = True
folder["calls_float_function"] = result
return result
for folder in self.root_folders():
folder_calls_float_function(folder)
def eliminate_dominated(covers):
"""Given a dict of {regex: {winner...}}, make a new dict with only the regexes
that are not dominated by any others. A regex r is dominated by r2 if r2 covers
a superset of the matches covered by r, and r2 is shorter."""
newcovers = {}
def signature(r):
return (-len(covers[r]), len(r))
for r in sorted(covers, key=signature):
if not covers[r]: break # All remaining r must not cover anything
# r goes in newcache if it is not dominated by any other regex
if not any(covers[r2] >= covers[r] and len(r2) <= len(r)
for r2 in newcovers):
newcovers[r] = covers[r]
return newcovers
def multiStreamTest(self, cx=False):
self.comp.zeroCenter=False;
self.comp.zeroMean=False;
self.comp.correlationSize=50
self.comp.inputOverlap=10
self.comp.numAverages=0
sampleRate=10e3
outFrameA, mainInput = self.datatest(cx, sampleRate, streamID='stream_a')
output = self.main(mainInput, cx, sampleRate,streamID = 'stream_a')
outFrameB = output[0]
assert len(outFrameB)==len(outFrameA)
self.assertTrue(__builtin__.any([abs(x-y)>.1 for x, y in zip(outFrameA, outFrameB)]))
output = self.main(mainInput, cx, sampleRate, streamID = 'stream_b')
outFrameC = output[0]
assert len(outFrameC)==len(outFrameA)
self.assertTrue(all([abs(x-y)<.1 for x, y in zip(outFrameC, outFrameA)]))
def _get_ticket_statuses(self, req_args):
"""Create map of ticket statuses."""
status_values = _get_config_values(self.config, 'ticket_status')
statuses = []
# Default to new/assigned/reopened
defaults = set(_get_config_values(self.config, 'ticket_status_enable'))
if any((req_args.get('status_%s' % s) for s in status_values)):
defaults = set()
for status in status_values:
field_name = 'status_%s' % status
statuses.append({
'name': status,
'active': req_args.get(field_name) or (status in defaults),
'field_name': field_name,
})
return statuses
def _get_ticket_statuses(self, req_args):
"""Create map of ticket statuses."""
status_values = ('new', 'assigned', 'closed')
statuses = []
# Default to new/assigned
defaults = set(('new', 'assigned'))
if any((req_args.get('status_%s' % s) for s in status_values)):
defaults = set()
for status in status_values:
field_name = 'status_%s' % status
statuses.append({
'name': status,
'active': req_args.get(field_name) or (status in defaults),
'field_name': field_name,
})
return statuses
def user_location(address):
'''
This function uses the pygeocoder.Geocoder module to reformat an user input into the suggested format by Google
The format is as follow: House_Number Street_Address, City, State Zip_Code, Country
Args:
address: string, user input address
Returns:
address_state.formatted_address: string, reformatted address
'''
# if original string does not contain variation of NY, add in the string to make sure the addressed searched for is definitely in NYC.
state_city = set(['new york','New York','ny','NY','Ny','New york','new York'])
if __builtin__.any(keyword in address for keyword in state_city):
address_state = Geocoder.geocode(address)
else:
address_state = Geocoder.geocode(address + "New York")
return address_state.formatted_address
def compare(mobile_path, desktop_path):
"""Returns the set intersection and outliers for all wpr resources
:param mobile_path: str path to wpr-mobile directory
:param desktop_path: str path to wpr-desktop directory
"""
mobile_files = glob(path.join(mobile_path,'*.inter'))
desktop_files = glob(path.join(desktop_path, '*.inter'))
mobile_files.sort()
desktop_files.sort()
assert len(mobile_files) == len(desktop_files), 'Directories must have the '
'same number of files'
for mobile_file in mobile_files:
file_name = mobile_file.split('/')[-1]
if any(file_name in x for x in desktop_files):
desktop_file = path.join(desktop_path, file_name)
mobile_dict = pickle.load(open(mobile_file, 'rb'))['data']
mobile_requests = set(mobile_dict[0])
mobile_responses = set(mobile_dict[1])
desktop_dict = pickle.load(open(desktop_file, 'rb'))['data']
desktop_requests = set(desktop_dict[0])
desktop_responses = set(desktop_dict[1])
mobile_intersection, mobile_difference = \
compare_wpr_resources(mobile_requests, desktop_requests)
desktop_intersection, desktop_difference = \
compare_wpr_resources(mobile_responses, desktop_responses)
if mobile_difference:
print '\n'
print 'Found difference in mobile: {0}'.format(mobile_file)
print mobile_difference
if desktop_difference:
print '\n'
print 'Found difference in desktop: {0}'.format(desktop_file)
print desktop_difference
else:
print 'File not found in desktop: {0}'.format(file_name)
def _get_ticket_statuses(self, req_args):
"""Create map of ticket statuses."""
status_values = self.config.get('advanced_search_plugin', 'ticket_status_filters', '').split(',')
statuses = []
# Default to new/assigned
defaults = self.config.get('advanced_search_plugin', 'status_filter_defaults', '').split(',')
if any((req_args.get('status_%s' % s) for s in status_values)):
defaults = set()
for status in status_values:
field_name = 'status_%s' % status
statuses.append({
'name': status,
'active': req_args.get(field_name) or (status in defaults),
'field_name': field_name,
})
return statuses
def _get_ticket_statuses(self, req_args):
"""Create map of ticket statuses."""
status_values = _get_config_values(self.config, 'ticket_status')
statuses = []
# Default to new/assigned/reopened
defaults = set(_get_config_values(self.config, 'ticket_status_enable'))
if any((req_args.get('status_%s' % s) for s in status_values)):
defaults = set()
for status in status_values:
field_name = 'status_%s' % status
statuses.append({
'name':
status,
'active':
req_args.get(field_name) or (status in defaults),
'field_name':
field_name,
})
return statuses
def eosTest(self, cx=False):
self.comp.zeroCenter=False;
self.comp.zeroMean=False;
self.comp.correlationSize=50
self.comp.inputOverlap=10
self.comp.numAverages=0
sampleRate=10e3
streamID = 'testStream'
outFrameA, mainInput = self.datatest(cx, sampleRate,streamID=streamID)
output = self.main(mainInput, cx, sampleRate,eos=True, streamID=streamID)
outFrameB = output[0]
assert len(outFrameB)==len(outFrameA)
self.assertTrue(__builtin__.any([abs(x-y)>.1 for x, y in zip(outFrameA, outFrameB)]))
#work arround a bug here to force an SRI push by the src
self.src._sri=None
output = self.main(mainInput, cx, sampleRate, streamID=streamID)
outFrameC = output[0]
assert len(outFrameC)==len(outFrameA)
self.assertTrue(all([abs(x-y)<.1 for x, y in zip(outFrameC, outFrameA)]))
#[x for x in set(''.join(list(clean.loc[pd.isnull(clean.official) == False].official))) if re.compile('[a-z]', re.I).search(x) == None]
#
#clean.loc[clean.official.str.contains('lonnie', regex = True)]
name_str = ' (?!(jr|sr|ii))(?!(van|von) )(?!(mc|st) )(?!de la )'
for i in range(2):
multiple = set(clean_df.loc[clean_df.official.apply(lambda x: len(re.findall(name_str, x)) > 2)].official)
single = set(clean_df.loc[clean_df.official.apply(lambda x: len(x) >= 6)
& clean_df.official.apply(lambda x: len(re.findall(name_str, x)) in [1,2])
& clean_df.official.apply(lambda x: re.compile('^[a-z] [a-z\-]+$').search(x) == None)].official)
for string in multiple:
matches = [x[0] for x in process.extractBests(string, single, scorer = fuzz.partial_ratio, score_cutoff = 100)]
matches = [x for x in matches if not any(x in y for y in set(matches) - set([x]))]
if len(matches) > 0:
new_string = string
new_names = []
for name in matches:
string_start = string.find(name)
string_end = string.find(name) + len(name)
if 0 in [string_start, string_end]:
new_string = re.sub(name,'',new_string).strip()
else:
new_string = re.sub(name,'&&',new_string).strip()
new_names = matches + ([x.strip() for x in new_string.split('&&') if x.strip() != ''] if new_string != '' else [])
def getGoals (rYear):
rounds = {'r2l1' : ["R16 L1", 8] , 'r2l2' : ["R16 L2", 8],
's0l1' : ["Quarters L1", 4], 's0l2' : ["Quarters L2", 4],
't0l1' : ["Semis L1", 2], 't0l2' : ["Semis L2", 4],
'u0' : ["Finals", 1]}
goalArray = []
temp = rYear-2000
year = str(temp)+str(temp+1)
goalTimeArray = {}
rx = {}
teamNo = 0
htSquad = []
atSquad = []
homeTeam = ""
awayTeam = ""
hScore = 0
aScore = 0
for r in rounds.keys():
#print "Parsing Round: " + rounds[r][0]
url = "http://www.soccerassociation.com/CL/%s/%s.htm" % (year, r)
#get request that ish
while True:
try:
toScrape = requests.get(url)
break
except ConnectionError:
print "Connection Error. Will try again"
#soup it too
soup = BeautifulSoup(toScrape.content)
#this will give us the results section
results = soup.findAll('table', { "cellspacing" : "1", "cellpadding" : "1"})[0]
#this gives us the
teamsAndPlayers = results.findAll('td')
#get all the teams playing
#all the data we need is in 'td' tags but they aren't ordered by match
rows = results.findAll('td')
acount = 0
hcount = 0
awayGoal = False
homeGoal = False
#now we have each match to iterate through
#if a 'td' tag with attributes align center and colspan is found, the a tags within have the team name
for row in rows:
if row.has_attr('align') and row.has_attr('colspan'):
if row['align'] == 'center' and row['colspan'] == '2':
#if we're here, we're looking at a new match (or the first one)
#so now we make the goal objects and add to goalArray
if teamNo%2==0:
while len(goalTimeArray) > 0:
#this will give us the min key in the array
#this is also the first goal
gt = 120
for key in goalTimeArray.keys():
#print goalTimeArray[key]
if key <= gt:
gt = key
#now we have the smallest remaining value in the array at key = gt
minScorer = goalTimeArray.pop(gt)
rnd = rx.pop(gt)
#if (homeTeam == 'Real Madrid'):
# print goalTimeArray[19]
if any(minScorer in names for names in htSquad):
print "Scorer: " + minScorer + " at minute " + str(gt)
#for final, assume both teams are away
if rounds[r][0] == 'Finals':
newGoal = Goal(minScorer, gt, False, hScore, aScore, rnd)
else:
newGoal = Goal(minScorer, gt, True, hScore, aScore, rnd)
goalArray.append(newGoal)
hScore+=1
elif any(minScorer in names for names in atSquad):
print "Scorer: " + minScorer + " at minute " + str(gt)
newGoal = Goal(minScorer, gt, False, aScore, hScore, rnd)
goalArray.append(newGoal)
aScore+=1
#if teamNo%2 is 0 then home team, else away team. increment every time team is found
#update homeTeam & awayTeam and get their squads
#also set the scores to 0-0
hScore = 0
aScore = 0
#goalTimeArray = {}
teamName = row.find('a')
strTeamName = str(teamName)
strTeamName = re.sub('<[^<]+?>', '', strTeamName)
if (teamNo%2==0):
homeTeam = strTeamName
htSquad = getRoster(homeTeam, rYear)
else:
awayTeam = strTeamName
atSquad = getRoster(awayTeam, rYear)
teamNo+=1
#if 'td' tag has attributes align right, then its a goal - this contains the goal time
elif row.has_attr('align') and row['align'] == 'right':
#for the name, look at the td tag immediately after the goal time thing
#the a tags within that have the name of the team
#check if player name is in htSquad or atSquad and increment scores accordingly
#then create new instance of Goal class and add to array
goalTime = str(row)
goalTime = re.sub('<[^<]+?>', '', goalTime)
goalTime = goalTime.replace('(p)', '')
#we dont want own goals to count for anything
if ('(og)' not in goalTime):
gTime = int(goalTime)
elif (len(row.findAll('a')) > 0) and not row.has_attr('bgcolor'):
scorer = row.find('a')
scorer = str(scorer)
scorer = re.sub('<[^<]+?>', '', scorer)
if any(scorer in names for names in atSquad):
#make the goal object now - note that it requires the score BEFORE the goal is scored
#if this is the finals, assume Away
if acount%2==0:
goalTimeArray[gTime] = scorer
rx[gTime] = rounds[r][0]
#print "Scorer: " + scorer + " at minute " + str(gTime)
#add time of the goal and the scorer to the goaltimearray
#we need this because goals are not ordered by time on the website we're scraping from
awayGoal = True
homeGoal = False
elif any(scorer in names for names in htSquad):
if hcount%2==0:
#print "Scorer: " + scorer + " at minute " + str(gTime)
goalTimeArray[gTime] = scorer
rx[gTime] = rounds[r][0]
#print goalTimeArray[scorer]
awayGoal = False
homeGoal = True
#all of this shpiel is because scorers get counted twice for some odd reason
if awayGoal:
acount+=1
elif homeGoal:
hcount+=1
return goalArray
def getGoals(rYear):
rounds = {
'r2l1': ["R16 L1", 8],
'r2l2': ["R16 L2", 8],
's0l1': ["Quarters L1", 4],
's0l2': ["Quarters L2", 4],
't0l1': ["Semis L1", 2],
't0l2': ["Semis L2", 4],
'u0': ["Finals", 1]
}
goalArray = []
temp = rYear - 2000
year = str(temp) + str(temp + 1)
goalTimeArray = {}
rx = {}
teamNo = 0
htSquad = []
atSquad = []
homeTeam = ""
awayTeam = ""
hScore = 0
aScore = 0
for r in rounds.keys():
#print "Parsing Round: " + rounds[r][0]
url = "http://www.soccerassociation.com/CL/%s/%s.htm" % (year, r)
#get request that ish
while True:
try:
toScrape = requests.get(url)
break
except ConnectionError:
print "Connection Error. Will try again"
#soup it too
soup = BeautifulSoup(toScrape.content)
#this will give us the results section
results = soup.findAll('table', {
"cellspacing": "1",
"cellpadding": "1"
})[0]
#this gives us the
teamsAndPlayers = results.findAll('td')
#get all the teams playing
#all the data we need is in 'td' tags but they aren't ordered by match
rows = results.findAll('td')
acount = 0
hcount = 0
awayGoal = False
homeGoal = False
#now we have each match to iterate through
#if a 'td' tag with attributes align center and colspan is found, the a tags within have the team name
for row in rows:
if row.has_attr('align') and row.has_attr('colspan'):
if row['align'] == 'center' and row['colspan'] == '2':
#if we're here, we're looking at a new match (or the first one)
#so now we make the goal objects and add to goalArray
if teamNo % 2 == 0:
while len(goalTimeArray) > 0:
#this will give us the min key in the array
#this is also the first goal
gt = 120
for key in goalTimeArray.keys():
#print goalTimeArray[key]
if key <= gt:
gt = key
#now we have the smallest remaining value in the array at key = gt
minScorer = goalTimeArray.pop(gt)
rnd = rx.pop(gt)
#if (homeTeam == 'Real Madrid'):
# print goalTimeArray[19]
if any(minScorer in names for names in htSquad):
print "Scorer: " + minScorer + " at minute " + str(
gt)
#for final, assume both teams are away
if rounds[r][0] == 'Finals':
newGoal = Goal(minScorer, gt, False,
hScore, aScore, rnd)
else:
newGoal = Goal(minScorer, gt, True, hScore,
aScore, rnd)
goalArray.append(newGoal)
hScore += 1
elif any(minScorer in names for names in atSquad):
print "Scorer: " + minScorer + " at minute " + str(
gt)
newGoal = Goal(minScorer, gt, False, aScore,
hScore, rnd)
goalArray.append(newGoal)
aScore += 1
#if teamNo%2 is 0 then home team, else away team. increment every time team is found
#update homeTeam & awayTeam and get their squads
#also set the scores to 0-0
hScore = 0
aScore = 0
#goalTimeArray = {}
teamName = row.find('a')
strTeamName = str(teamName)
strTeamName = re.sub('<[^<]+?>', '', strTeamName)
if (teamNo % 2 == 0):
homeTeam = strTeamName
htSquad = getRoster(homeTeam, rYear)
else:
awayTeam = strTeamName
atSquad = getRoster(awayTeam, rYear)
teamNo += 1
#if 'td' tag has attributes align right, then its a goal - this contains the goal time
elif row.has_attr('align') and row['align'] == 'right':
#for the name, look at the td tag immediately after the goal time thing
#the a tags within that have the name of the team
#check if player name is in htSquad or atSquad and increment scores accordingly
#then create new instance of Goal class and add to array
goalTime = str(row)
goalTime = re.sub('<[^<]+?>', '', goalTime)
goalTime = goalTime.replace('(p)', '')
#we dont want own goals to count for anything
if ('(og)' not in goalTime):
gTime = int(goalTime)
elif (len(row.findAll('a')) > 0) and not row.has_attr('bgcolor'):
scorer = row.find('a')
scorer = str(scorer)
scorer = re.sub('<[^<]+?>', '', scorer)
if any(scorer in names for names in atSquad):
#make the goal object now - note that it requires the score BEFORE the goal is scored
#if this is the finals, assume Away
if acount % 2 == 0:
goalTimeArray[gTime] = scorer
rx[gTime] = rounds[r][0]
#print "Scorer: " + scorer + " at minute " + str(gTime)
#add time of the goal and the scorer to the goaltimearray
#we need this because goals are not ordered by time on the website we're scraping from
awayGoal = True
homeGoal = False
elif any(scorer in names for names in htSquad):
if hcount % 2 == 0:
#print "Scorer: " + scorer + " at minute " + str(gTime)
goalTimeArray[gTime] = scorer
rx[gTime] = rounds[r][0]
#print goalTimeArray[scorer]
awayGoal = False
homeGoal = True
#all of this shpiel is because scorers get counted twice for some odd reason
if awayGoal:
acount += 1
elif homeGoal:
hcount += 1
return goalArray
def any(f, x):
return __builtin__.any(__builtin__.map(f, x))
def any(iterable, pred):
"Returns True if ANY element in the given iterable is True for the given pred function"
return builtins.any(pred(x) for x in iterable)
def detectInlineValidation(validation_module):
validation_module_path = get_source_file_path(validation_module)
path = operator.itemgetter(1)
return any(path(frame) == validation_module_path
for frame in inspect.stack())
def any(iterable):
warnings.warn("django.utils.itercompat.any is deprecated; use the native version instead",
DeprecationWarning)
return __builtin__.any(iterable)
def createNewInput(self,currentInputFiles,oriInputFiles,samplerType,**Kwargs):
"""
this generates a new input file depending on which sampler has been chosen
@ In, currentInputFiles, list, list of current input files (input files from last this method call)
@ In, oriInputFiles, list, list of the original input files
@ In, samplerType, string, Sampler type (e.g. MonteCarlo, Adaptive, etc. see manual Samplers section)
@ In, Kwargs, dictionary, kwarded dictionary of parameters. In this dictionary there is another dictionary called "SampledVars"
where RAVEN stores the variables that got sampled (e.g. Kwargs['SampledVars'] => {'var1':10,'var2':40})
@ Out, newInputFiles, list, list of newer input files, list of the new input files (modified and not)
"""
import RAVENparser
if 'dynamiceventtree' in str(samplerType).strip().lower():
raise IOError(self.printTag+' ERROR: DynamicEventTree-based sampling not supported!')
index = self.__findInputFile(currentInputFiles)
parser = RAVENparser.RAVENparser(currentInputFiles[index].getAbsFile())
# get the OutStreams names
self.outStreamsNamesAndType = parser.returnOutstreamsNamesAnType()
# check if the linked DataObjects are among the Outstreams
pointSetNumber, historySetNumber = 0, 0
for outstream, dataObj in self.outStreamsNamesAndType.items():
if outstream in self.linkedDataObjectOutStreamsNames:
if dataObj[1].strip() == 'PointSet':
pointSetNumber+=1
else:
historySetNumber+=1
if pointSetNumber > 1 or historySetNumber > 1:
raise IOError(self.printTag+' ERROR: Only one OutStream for PointSet and/or one for HistorySet can be linked as output export!')
if pointSetNumber == 0 and historySetNumber == 0:
raise IOError(self.printTag+' ERROR: No one of the OutStreams linked to this interface have been found in the SLAVE RAVEN!'
+' Expected: "'+' '.join(self.linkedDataObjectOutStreamsNames)+'" but found "'
+' '.join(self.outStreamsNamesAndType.keys())+'"!')
# get variable groups
varGroupNames = parser.returnVarGroups()
if len(varGroupNames) > 0:
# check if they are not present in the linked outstreams
for outstream in self.linkedDataObjectOutStreamsNames:
inputNode = self.outStreamsNamesAndType[outstream][2].find("Input")
outputNode = self.outStreamsNamesAndType[outstream][2].find("Output")
inputVariables = inputNode.text.split(",") if inputNode is not None else []
outputVariables = outputNode.text.split(",") if outputNode is not None else []
if any (varGroupName in inputVariables+outputVariables for varGroupName in varGroupNames):
raise IOError(self.printTag+' ERROR: The VariableGroup system is not supported in the current ' +
'implementation of the interface for the DataObjects specified in the '+
'<outputExportOutStreams> XML node!')
# get inner working dir
self.innerWorkingDir = parser.workingDir
# get sampled variables
modifDict = Kwargs['SampledVars']
# apply conversion scripts
for source,convDict in self.conversionDict.items():
module = utils.importFromPath(source)
varVals = dict((var,np.asarray(modifDict[var])) for var in convDict['variables'])
# modify vector+ variables that need to be flattened
if convDict['noScalar']:
# call conversion
newVars = module.convertNotScalarSampledVariables(varVals)
# check type
if type(newVars).__name__ != 'dict':
raise IOError(self.printTag+' ERROR: convertNotScalarSampledVariables in "{}" must return a dictionary!'.format(source))
# apply new and/or updated values
modifDict.update(newVars)
# modify scalar variables
if convDict['scalar']:
# call conversion, value changes happen in-place
module.manipulateScalarSampledVariables(modifDict)
# we work on batchSizes here
newBatchSize = Kwargs['NumMPI']
internalParallel = Kwargs.get('internalParallel',False)
if int(Kwargs['numberNodes']) > 0:
# we are in a distributed memory machine => we allocate a node file
nodeFileToUse = os.path.join(Kwargs['BASE_WORKING_DIR'],"node_" +str(Kwargs['INDEX']))
if os.path.exists(nodeFileToUse):
modifDict['RunInfo|mode' ] = 'mpi'
modifDict['RunInfo|mode|nodefile' ] = nodeFileToUse
else:
raise IOError(self.printTag+' ERROR: The nodefile "'+str(nodeFileToUse)+'" does not exist!')
if internalParallel or newBatchSize > 1:
# either we have an internal parallel or NumMPI > 1
modifDict['RunInfo|batchSize' ] = newBatchSize
#modifDict['RunInfo|internalParallel'] = internalParallel
#make tree
modifiedRoot = parser.modifyOrAdd(modifDict,save=True,allowAdd = True)
#make input
parser.printInput(modifiedRoot,currentInputFiles[index].getAbsFile())
# copy slave files
parser.copySlaveFiles(currentInputFiles[index].getPath())
return currentInputFiles
""" Compatibility functions for Python 1.5 to 2.5. any() function ============== any() returns True if at least one items is True, or False otherwise. >>> any([False, True]) True >>> any([True, True]) True >>> any([False, False]) False all() function ============== all() returns True if all items are True, or False otherwise. This function is just apply binary and operator (&) on all values. >>> all([True, True]) True >>> all([False, True]) False >>> all([False, False]) False """
""" Compatibility functions for Python 2.4. any() function ============== any() returns True if at least one items is True, or False otherwise. >>> any([False, True]) True >>> any([True, True]) True >>> any([False, False]) False all() function ============== all() returns True if all items are True, or False otherwise. This function is just apply binary and operator (&) on all values. >>> all([True, True]) True >>> all([False, True]) False >>> all([False, False]) False """
def any(iterable):
warnings.warn(
"django.utils.itercompat.any is deprecated; use the native version instead",
PendingDeprecationWarning)
return __builtin__.any(iterable)
def exists(sequence, function):
return __builtin__.any(function(x) for x in sequence)
def any(f, x):
return __builtin__.any(__builtin__.map(f, x))
r = requests.put("{}/{}/_mapping/search".format(request_url,
es_kibana_index),
headers=headers,
data=data)
log_message(r.text)
try:
data_paths = literal_eval(os.getenv('SHARED_DATA_PATHS'))
except ValueError:
log_message("Could not parse script paths from the environment",
"critical")
exit(1)
# We sit and poll to see if elastic search has started, if not, wait 30 seconds and try again
while True:
log_message("Elastic search has not started, waiting.")
time.sleep(30)
res = subprocess.Popen(["/usr/bin/sv", "status", "elasticsearch"],
stdout=subprocess.PIPE)
if any(es_running in item for item in res.stdout.readlines()):
log_message("Elastic search is running")
log_message("Waiting {} seconds for indices to be created".format(
sleep_timeout))
time.sleep(sleep_timeout)
create_default_indices()
find_and_install_scripts()
logfile.close()
exit(0)
def any(xs, predicate=lambda x: True):
return __builtin__.any(x for x in xs if predicate(x))
print x
order = ""
rollback = ""
ins_exe = []
locktable = []
point = [0] * tno
tstate = ["0"] * (tno)
while True:
for p in range(tno):
count = 0
while count < 3:
if (tstate[p] == 'committed'):
break
if x[p][point[p]][0] == 'r' and any(
x[p][point[p]][1] in c
for c in locktable) and not (str(p) + x[p][point[p]][1]
in locktable):
m = [m for m in locktable if x[p][point[p]][1] in m]
rmlock = []
if p > int(m[0][0]):
for i in locktable:
if str(p) != i[0]:
rmlock.append(i)
ins_exe.append("rollback" + str(p))
point[p] = 0
locktable = rmlock
rollback = rollback + str(p)
break
if x[p][point[p]][0] == 'r' and any(
x[p][point[p]][1] in c
def any (self, constraint, *constraints):
for extra in constraints:
constraint = constraint & extra
return Filter(lambda x: __builtin__.any(constraint(inst) for inst in \
x.field_get(self.name)))
def detectInlineValidation(validation_module):
validation_module_path = get_source_file_path(validation_module)
path = operator.itemgetter(1)
return any(path(frame) == validation_module_path
for frame in inspect.stack())
