def main():
while True:
sleep(0.01)
command_handler(r, meassys)
status_setter(meassys, r)
setting_handler(r)
if next(GENS['local_pub']):
dttm = get_utc_time()
data = meassys.data
if data is None:
print(dttm, 'No data.')
continue
data, status, errors = data
data = {
'dttm': dttm,
'data': data,
'status': status,
'errors': errors
}
print(dttm, data['data']['Status'], data['status'])
json_data = json.dumps(data)
push_to_redis(r, 'data', json_data)
if next(GENS['local_record']):
save_data(data)
if meassys.status != 'Idle':
if is_auto_zero():
meassys.cal_gas_zero_start()
else:
meassys.cal_gas_zero_stop()
def fetch_groups(gl):
'''
Download all the groups on the https://gitlab.gnome.org.
'''
start = time.time()
print('Fetching groups.')
# donot include the `Archive` group
blacklist = [4001]
groups = json.loads(
requests.get('https://gitlab.gnome.org/api/v4/groups',
params={
'per_page': 100
}).text)
save_data(groups, 'groups.json')
print(f'Downloaded and saved {len(groups)} groups.')
# create a list of group_ids for downloading the projects in the each group
group_ids = []
for group in groups:
if group['id'] not in blacklist:
group_ids.append(group['id'])
finish = time.time()
print(f'Took {round(finish-start, 2)} seconds.')
return group_ids
def fetch_projects(gl, group_ids):
'''
Download all the projects on the https://gitlab.gnome.org.
'''
start = time.time()
print('Fetching projects.')
# get the all the projects in each group
projects = []
for group_id in group_ids:
group = gl.groups.get(id=group_id, lazy=True)
group_projects = group.projects.list(all=True)
projects += group_projects
projects = [project.attributes for project in projects]
save_data(projects, 'projects.json')
print(f'Downloaded and saved {len(projects)} projects.')
# create a list of project_ids for downloading the issues, merge_requests, commits in the each project
project_ids = []
for project in projects:
project_ids.append(project['id'])
finish = time.time()
print(f'Took {round(finish-start, 2)} seconds.')
return project_ids
def clean_fixtures_data():
"""
Cleans the fixtures data and filters it.
"""
# load the fixtures data
all_fixtures = load_data("fixtures.json", "data/original")
# define headers that we need to keep
headers = headers = [
"event", "finished", "team_a", "team_a_difficulty", "team_h",
"team_h_difficulty"
]
# list to store the filtered information
filtered_fixtures = []
# iterate over all the teams and remove unwanted information
for fixture in all_fixtures:
if fixture["event"] is not None:
# remove unwanted keys from the fixture's data
fixture = {header: fixture[header] for header in headers}
filtered_fixtures.append(fixture)
# only retain the fixtures that are yet to take place
filtered_fixtures = [
fixture for fixture in filtered_fixtures
if fixture['event'] >= next_event
]
# save the data in a JSON file
save_data(filtered_fixtures, "filtered_fixtures.json", "data")
def get_weather_data(apikey, locs, cols, start_date, end_date, offset):
""" Retrieves daily historical weather data for the specified locations
using the Dark Sky API. Output is saved as a CSV in the 'data' folder.
Args:
api_key (str): Dark Sky API key.
locs (str): Geocoded locations file name (with extension).
cols (str): File name contain custom column names.
start_date (datetime.datetime): Start date for historical data range.
end_date (datetime.datetime): End date for historical data range.
offset (int): Step size for iterator (number of days).
"""
locs_path = get_datafile(locs)
locs = pd.read_csv(locs_path)
# get columns list
columns = get_datafile(cols)
with open(columns) as f:
cols = [line.strip() for line in f]
# extract data for each location for date range b/w start and end date
tbl = []
for index, row in locs.iterrows():
for single_date in daterange(start_date, end_date, offset):
forecast = forecastio.load_forecast(apikey,
row['Lat'],
row['Lng'],
time=single_date,
units='si')
h = forecast.daily()
tz = forecast.json['timezone']
d = h.data
for p in d:
# get date info
utc = p.d['time']
dts = dt.datetime.utcfromtimestamp(utc)
isodate = dt.datetime.utcfromtimestamp(utc).isoformat()
date_info = [tz, isodate, dts.year, dts.month, dts.day]
# get location info
loc, lat, lng = row['Location'], row['Lat'], row['Lng']
elevation = row['Elevation']
loc_info = [loc, lat, lng, elevation]
# get weather attributes - need to handle possible KeyErrors
temp_high = p.d.get('temperatureHigh', None)
temp_low = p.d.get('temperatureLow', None)
humidity = p.d.get('humidity', None) * 100
pressure = p.d.get('pressure', None)
attr_info = [temp_high, temp_low, humidity, pressure]
tbl.append(loc_info + date_info + attr_info)
# convert output to data frame
df = pd.DataFrame(tbl)
df.columns = cols
filename = 'historical_data.csv'
save_data(df, filename, sep='|')
def save_twitter_filter_data(self, json_data):
'''Saves the given json data into the filter directory'''
time_folder = int(time.time() / 300) * 300
filelocation = os.path.join(self.conf['save_data_location'],
"statuses/filter", str(time_folder),
str(int(time.time())) + "_" +
uuid.uuid4().hex +
".json")
save_data(json_data, filelocation)
def save_twitter_sample_data(self, json_data):
'''Saves data from the sample API to json files'''
time_folder = int(time.time() / 300) * 300
filelocation = os.path.join(self.conf['save_data_location'],
"statuses/sample", str(time_folder),
str(int(time.time())) + "_" +
md5.md5(json.dumps(json_data)).hexdigest() +
".json")
save_data(json_data, filelocation)
def download_fixtures_data():
"""
Downloads the fixtures data.
"""
# load the fixture specific data from the FPL RestAPI Endpoint
fixtures = json.loads(
requests.get('https://fantasy.premierleague.com/api/fixtures/').text)
# save the data in a JSON file
save_data(fixtures, 'fixtures.json', 'data/original')
def process_awards():
'''
Process the awards for top 10 users.
'''
try:
# awards = load_data('awards.json')
awards = requests.get(
'https://raw.githubusercontent.com/ravgeetdhillon/gnome-hackers/website/artifacts/data/awards.json'
)
awards = json.loads(awards.text)
except:
awards = []
# sort the data for each criteria and save them in their respective json files
criteria = ['days_1', 'days_7', 'days_15', 'days_30']
for key in criteria:
users = load_data('processed_users.json')
users = sorted(users, key=lambda k: k['points'][key],
reverse=True)[:10]
for user in users:
for u in awards:
if user['id'] == u['id']:
break
else:
awards.append({
'id': user['id'],
'awards': {
'gold': 0,
'silver': 0,
'bronze': 0,
'top10': 0,
}
})
for u in awards:
for index, user in enumerate(users, start=1):
if u['id'] == user['id']:
if index == 1:
u['awards']['gold'] += 1
elif index == 2:
u['awards']['silver'] += 1
elif index == 3:
u['awards']['bronze'] += 1
u['awards']['top10'] += 1
break
save_data(awards, 'awards.json')
def download_teams_data():
"""
Downloads the teams data.
"""
# load the team specific data from the FPL RestAPI Endpoint
data = json.loads(
requests.get(
'https://fantasy.premierleague.com/api/bootstrap-static/').text)
teams = data['teams']
# save the data in a JSON file
save_data(teams, 'teams.json', 'data/original')
def download_gameweeks_data():
"""
Downloads the gameweeks data.
"""
# load the fixture specific data from the FPL RestAPI Endpoint
data = json.loads(
requests.get(
'https://fantasy.premierleague.com/api/bootstrap-static/').text)
gameweeks = data['events']
# save the data in a JSON file
save_data(gameweeks, 'gameweeks.json', 'data/original')
def download_users_team_data(user_team_id, recent_gw_id):
"""
Downloads the users team data.
"""
# load the fixture specific data from the FPL RestAPI Endpoint
user_team = json.loads(
requests.get(
f'https://fantasy.premierleague.com/api/entry/{user_team_id}/event/{recent_gw_id}/picks/'
).text)
# save the data in a JSON file
save_data(user_team, 'user_team.json', 'data/original')
def fetch_users(gl):
'''
Download all the users on the https://gitlab.gnome.org.
'''
start = time.time()
print('Fetching users.')
users = gl.users.list(all=True)
users = [user.attributes for user in users]
save_data(users, 'users.json')
print(f'Downloaded and saved {len(users)} users.')
finish = time.time()
print(f'Took {round(finish-start, 2)} seconds.')
def add_awards():
'''
Add the processed awards to the processed users.
'''
awards = load_data('awards.json')
users = load_data('processed_users.json')
for user in users:
for u in awards:
if u['id'] == user['id']:
user['awards'] = u['awards']
break
save_data(users, 'processed_users.json')
return users
def app():
"""
Main function for the app.
"""
# generate the credits
drive_creds = create_crendentials(DRIVE_REFRESH_TOKEN, DRIVE_SCOPES)
youtube_creds = create_crendentials(YOUTUBE_REFRESH_TOKEN, YOUTUBE_SCOPES)
# get a list of files to select which song to upload
files = get_files(creds=drive_creds)
# check if there is any new music to upload
if select_audio_image(files) is not None:
# select an audio file and it's corresponding image
audio, image = select_audio_image(files)
audio_id = audio['id']
audio_name = audio['name']
image_id = image['id']
image_name = image['name']
# download the selected audio and image
download_file(audio_id, audio_name, creds=drive_creds)
download_file(image_id, image_name, creds=drive_creds)
# convert the downloaded audio and image into a video
conversion = convert_to_video(audio_name, image_name)
print(conversion)
# if conversion is successful, upload the video to Youtube
if conversion is not False:
response = upload_video(conversion, creds=youtube_creds)
print(response)
print('New track released.')
# save the audio id in uploaded
uploaded = load_data('uploaded.json')
uploaded.append(audio_id)
save_data(uploaded, 'uploaded.json')
else:
print('No new track to release.')
def main():
logging.info('Iniciando procedimento')
logging.info(
'Checando se o arquivo de dados consolidados por partido existe')
if os.path.exists(constants.PARTIDO_FILE_PATH):
logging.info('Arquivo existe')
else:
logging.info('Arquivo não existe')
data = pd.DataFrame(parser.generate_party_data())
logging.info('Salvando dados')
save_data(df=data, filepath=constants.PARTIDO_FILE_PATH)
logging.info(
'Checando se o arquivo de dados consolidados por candidato existe')
if os.path.exists(constants.CANDIDATO_FILE_PATH):
logging.info('Arquivo existe')
else:
logging.info('Arquivo não existe')
data = pd.DataFrame(parser.generate_candidate_data())
logging.info('Salvando dados')
save_data(df=data, filepath=constants.CANDIDATO_FILE_PATH)
def download_players_data():
"""
Downloads the players data.
"""
# load the player specific data from the FPL RestAPI Endpoint
data = json.loads(
requests.get(
'https://fantasy.premierleague.com/api/bootstrap-static/').text)
players = data['elements']
for player in players:
player_id = player['id']
print(f'Downloading data for player: {player_id}')
data = json.loads(
requests.get(
f'https://fantasy.premierleague.com/api/element-summary/{player_id}/'
).text)
player['history'] = data['history']
# save the data in a JSON file
save_data(players, 'players.json', 'data/original')
def clean_teams_data():
"""
Cleans the data for the teams.
"""
# load the teams data
all_teams = load_data("teams.json", "data/original")
# define headers that we need to keep
headers = ["id", "name"]
# list to store the filtered information
filtered_teams = []
# iterate over all the teams and remove unwanted information
for team in all_teams:
# remove unwanted keys from the team's data
team = {header: team[header] for header in headers}
filtered_teams.append(team)
# save the data in a JSON file
save_data(filtered_teams, "filtered_teams.json", "data")
def main():
'''
Main function for the process.py.
'''
# initialize the users array to store the data about the users contributing to the GNOME
users = []
# load the commits, merge requests and issues
commits = load_data('commits.json')
merge_requests = load_data('merge_requests.json')
issues = load_data('issues.json')
all_users = load_data('users.json')
# process the commits, merge requests and issues and generate points for the users
users = process_issues(users, issues)
users = process_merge_requests(users, merge_requests)
users = process_commits(users, commits)
users = process_users(users, all_users)
# download the avatar image from each user
fetch_images(users)
save_data(users, 'processed_users.json')
users = []
# load the commits, merge requests and issues
commits = load_data('commits.json')
merge_requests = load_data('merge_requests.json')
issues = load_data('issues.json')
all_users = load_data('users.json')
# process the commits, merge requests and issues and generate points for the users
users = process_issues(users, issues)
users = process_merge_requests(users, merge_requests)
users = process_commits(users, commits)
users = process_users(users, all_users)
# download the avatar image from each user
fetch_images(users)
save_data(users, 'processed_users.json')
if __name__ == '__main__':
main()
process_awards()
users = add_awards()
# sort the data for each criteria and save them in their respective json files
criteria = ['days_1', 'days_7', 'days_15', 'days_30']
for key in criteria:
users = sorted(users, key=lambda k: k['points'][key], reverse=True)
save_data(users, f'sorted_users_acc_{key}.json')
def fetch_projects_data(gl, project_ids):
'''
Download all the merge requests, issues and commits for each project on the https://gitlab.gnome.org.
'''
start = time.time()
print('Fetching merge requests, issues and commits.')
merge_requests = []
issues = []
commits = []
for index, project_id in enumerate(project_ids):
print(index, end=', ')
project = gl.projects.get(id=project_id, lazy=True)
since = get_date_30_days_now()
try:
project_merge_requests = project.mergerequests.list(
all=True,
query_parameters={
'state': 'all',
'created_after': since
})
merge_requests += project_merge_requests
except Exception as e:
print(f'{e}. Raised while getting merge requests.')
try:
project_issues = project.issues.list(
all=True, query_parameters={'created_after': since})
issues += project_issues
except Exception as e:
print(f'{e}. Raised while getting issues.')
try:
project_commits = project.commits.list(
all=True, query_parameters={'since': since})
except Exception as e:
print(f'{e}. Raised while getting commits.')
for commit in project_commits:
commit = commit.attributes
commit = gl.projects.get(id=commit['project_id'],
lazy=True).commits.get(id=commit['id'])
commits.append(commit)
merge_requests = [
merge_request.attributes for merge_request in merge_requests
]
issues = [issue.attributes for issue in issues]
commits = [commit.attributes for commit in commits]
save_data(merge_requests, 'merge_requests.json')
print(f'Downloaded and saved {len(merge_requests)} merge requests.')
save_data(issues, 'issues.json')
print(f'Downloaded and saved {len(issues)} issues.')
save_data(commits, 'commits.json')
print(f'Downloaded and saved {len(commits)} commits.')
finish = time.time()
print(f'Took {round(finish-start, 2)} seconds.')
geodata = 'geocoded_locations.txt'
obs = int(sys.argv[1])
# throw error if number of obs is 0 or less
if obs < 1:
raise ValueError('Number of observations cannot be less than 1')
# instantiate new WeatherGenerator object
weatherGenerator = WeatherGenerator(obs, start_date, end_date, histdata,
geodata)
# start timing
start_time = timeit.default_timer()
# generate random weather observations
weatherGenerator.initialize_output()
weatherGenerator.generate_position_data()
weatherGenerator.generate_time_data()
weatherGenerator.generate_weather_data()
weatherGenerator.generate_condition_data()
weatherGenerator.order_output()
output = weatherGenerator.return_output()
# save data frame to CSV in 'output' directory
save_data(output, filename='generated_weather_data.csv', subdir='output')
# stop timing and print elapsed time
elapsed = str("{:.4f}".format(timeit.default_timer() - start_time))
print('Generated ' + str(obs) + ' random weather observations in ' +
elapsed + ' seconds')
def save_trend_data(self, json_data, woeid):
'''Saves Twitter Trend data to a json file'''
filelocation = os.path.join(self.conf['save_data_location'],
"trends/place/weoid_" + str(woeid) + "_" +
str(int(time.time())) + ".json")
save_data(json_data, filelocation)
for ckey, val in comb.items():
keys = ckey.split('.') # get list from dotted string
replace(params, keys, val)
# save parameters in the data_folder
if not os.path.exists(opts.data_folder):
os.makedirs(opts.data_folder)
shutil.copy('./' + opts.param_file,
opts.data_folder + '/' + str(comb) + '_' + opts.param_file)
if not opts.analysis:
Populations = h.build_network(sim, params)
h.record_data(params, Populations)
h.perform_injections(params, Populations)
h.run_simulation(sim, params)
h.save_data(Populations, opts.data_folder, str(comb))
sim.end()
#if opts.analysis:
scores = h.analyse(params, opts.data_folder, str(comb), opts.remove)
# map storage of scores
if search and opts.map:
if i == 0:
with open(opts.data_folder + '/map.csv', "w",
newline="") as csvfile:
fh = csv.writer(csvfile)
fh.writerow([
'#' + testParams[1] + ':[' +
",".join(map(str, search[testParams[1]])) + "]"
])
def clean_players_data():
"""
Cleans the data for all the players.
"""
# load the players data
all_players = load_data("players.json", "data/original")
# define headers that we need to keep
headers = [
"first_name", "second_name", "minutes", "total_points",
"points_per_game", "team", "element_type", "now_cost", "status", "id",
"history"
]
# list to store the filtered information
filtered_players = []
# only keep the required headers and remove the unwanted information
for player in all_players:
# remove unwanted keys from the player's data
player = {header: player[header] for header in headers}
# add player's full name
player_name = f"{player['first_name']} {player['second_name']}"
player["full_name"] = player_name.lower()
# convert floats casted into a string to floats
player["points_per_game"] = float(player["points_per_game"])
# divide stats according to the season
stats_headers = [
"minutes", "total_points", "points_per_game", "now_cost", "history"
]
for season in all_seasons:
player_season_stats = {"season": season}
for header in stats_headers:
player_season_stats[header] = player[header]
del player[header]
player["seasons"] = [player_season_stats]
# calculate the net points only, remove the playing points
for season in player["seasons"]:
player_gw_history = []
for count, gw in enumerate(season["history"][::-1]):
if count < 5:
if gw["minutes"] >= 60:
net_points = gw["total_points"] - 2
elif 0 < gw["minutes"] < 60:
net_points = gw["total_points"] - 1
else:
net_points = gw["total_points"]
player_gw_history.append(net_points)
season["gw_history"] = player_gw_history
del season["history"]
filtered_players.append(player)
# only retain the players who have played atleast one minute in the season
filtered_players = [
player for player in filtered_players
if (player["seasons"][0]["minutes"] > 0
and player["seasons"][0]["total_points"] > 0
and len(player["seasons"][0]["gw_history"]) != 0)
]
# save the data in a JSON file
save_data(filtered_players, "filtered_players.json", "data")
def simulate_mp(params):
N, grid_size, exp, xmin, i = params
model = ContactModel(N, grid_size, grid_size, exp, STEPS, xmin, seed=8)
model.run(STEPS)
save_data(model, N, i, exp, grid_size, xmin)
fer.append(1 - 0.1 * fixture["team_h_difficulty"])
if len(fer) == 5:
break
team["fer"] = fer
team["fer_points"] = np.mean(fer) * (1 - np.var(fer))
avg_fer_points += team["fer_points"]
avg_fer_points /= len(teams)
for team in teams:
team["fer_points"] = round(team["fer_points"] / avg_fer_points, 3)
if team["fer_points"] > max_fer_points:
max_fer_points = team["fer_points"]
teams = sorted(teams, key=lambda k: k["fer_points"], reverse=True)
save_data(teams, "teams_cleaned.json", "data")
# generate the desired data for each player for each season.
max_consistency = {}
for season in all_seasons:
max_consistency[season] = 0
for player in players:
# get player's position
player["position"] = positions[player["element_type"]]
player["value_points"] = 0
# get player's team and next fixture difficulty rating
for team in teams:
os.makedirs(opts.data_folder+str(run))
shutil.copy('./'+opts.param_file, opts.data_folder+ str(run)+'/'+opts.param_file+'_'+str(comb)+'.py')
if not opts.analysis:
already_computed = 0
for pop in params['Populations'].keys():
if os.path.exists(opts.data_folder + str(run) +'/'+pop+str(comb)+'.pkl'):
already_computed = already_computed + 1
if already_computed > 0:
print "already computed"
else:
Populations = h.build_network(sim,params)
h.record_data(params, Populations)
h.perform_injections(params, Populations)
h.run_simulation(sim,params)
h.save_data(Populations, opts.data_folder + str(run), str(comb))
sim.end()
else :
if search:
already_computed = 0
for pop in params['Populations'].keys():
if os.path.exists(opts.data_folder + str(run) +'/'+pop+str(comb)+'.png'):
already_computed = already_computed + 1
if already_computed > len(params['Populations']) - 1:
print "already analysed"
else:
ratio,fqcy,psd,freq, fqcy_ratio = h.analyse(params, opts.data_folder + str(run), str(comb), removeDataFile)
print "ratio",ratio,"fqcy",fqcy,"psd",psd,"freq",freq
gen = (pop for pop in params['Populations'].keys() if pop != 'ext')
for pop in gen:
def run_simulation(run):
info = {}
# run combinations
for i,comb in enumerate(combinations):
print "param combination",i, "trial",run
print "current set:",comb
# replacement
for ckey,val in comb.iteritems():
keys = ckey.split('.') # get list from dotted string
replace(params,keys,val)
# save parameters in the data_folder
if not os.path.exists(opts.data_folder+str(run)):
os.makedirs(opts.data_folder+str(run))
shutil.copy('./'+opts.param_file, opts.data_folder + str(run)+'/'+opts.param_file+'_'+str(comb)+'.py')
if not opts.analysis: #run simulation
# run simulation if it hasn't already been ran for these parameters
already_computed = 0
for pop in params['Populations'].keys():
if os.path.exists(opts.data_folder + str(run) +'/'+pop+str(comb)+'.pkl'):
already_computed = already_computed + 1
if already_computed > 0:
print "already computed"
else:
Populations = h.build_network(sim,params)
h.record_data(params, Populations)
h.perform_injections(params, Populations)
print "Running Network"
timer = Timer()
timer.reset()
inject_spikes = partial(inject_spikes_pop,Populations = Populations)
sim.run(params['run_time'], [inject_spikes])
simCPUtime = timer.elapsedTime()
print "Simulation Time: %s" % str(simCPUtime)
h.save_data(Populations, opts.data_folder + str(run), str(comb))
sim.end()
else : #do analysis
if search: #then compute the csv needed for the search maps
already_analysed = 0 #analyse only those that haven't already been analysed
for pop in params['Populations'].keys():
if os.path.exists(opts.data_folder + str(run) +'/'+pop+str(comb)+'.png'):
already_analysed = already_analysed + 1
if already_analysed >= len(params['Populations'])-1 :
print "already analysed"
else:
ratio,fqcy,psd,freq, fqcy_ratio = h.analyse(params, opts.data_folder + str(run), str(comb), opts.remove)
#save computed values in csv files
gen = (pop for pop in params['Populations'].keys() if pop != 'ext')
for pop in gen:
if i == 0:
with open(opts.data_folder+ str(run)+'/map-'+pop+'.csv', 'wb') as csvfile:
mywriter = csv.writer(csvfile)
mywriter.writerow( ['#'+str(testParams[1])+ ':' +str(search[testParams[1]]) ] )
mywriter.writerow( ['#'+str(testParams[0])+ ':' +str(search[testParams[0]]) ] )
with open(opts.data_folder+ str(run)+'/psdmap-'+pop+'.csv', 'wb') as csvfile:
mywriter = csv.writer(csvfile)
mywriter.writerow( ['#'+str(testParams[1])+ ':' +str(search[testParams[1]]) ] )
mywriter.writerow( ['#'+str(testParams[0])+ ':' +str(search[testParams[0]]) ] )
if pop in freq:
mywriter.writerow(freq[pop])
info[pop] = []
if pop in ratio and pop in fqcy:
info[pop].append([ratio[pop],fqcy[pop],fqcy_ratio[pop]])
if (i+1)%len(search[testParams[1]]) == 0:
with open(opts.data_folder+str(run)+'/map-'+pop+'.csv', 'a') as csvfile:
mywriter = csv.writer(csvfile)
mywriter.writerow(info[pop])
info[pop] = []
if pop in psd:
with open(opts.data_folder+str(run)+'/psdmap-'+pop+'.csv', 'a') as csvfile:
mywriter = csv.writer(csvfile)
mywriter.writerow(psd[pop])
else:
h.analyse(params, opts.data_folder+str(run), str(comb), opts.remove)