def test_pagination(self, params, get_response):
if params['page'] > 0:
with allure.step(f'Получить списки названий регионов '):
_first_page_items = get_data(page=1,
page_size=15).json().get('items')
first_regions_names = set(
[region.name for region in get_regions(_first_page_items)])
other_page_items = get_response.json().get('items')
other_regions_names = set(
[region.name for region in get_regions(other_page_items)])
with allure.step(
f'Список регионов первой и {params["page"]} страницы отличны'
):
assert not first_regions_names == other_regions_names
else:
with allure.step('Запрос не обработан сервером'):
assert get_response.status_code == 500
def test_default_page_number(self, get_regions_list_class, page=1):
with allure.step('Получить списки названий регионов'):
default_regions_names = set(
[region.name for region in get_regions_list_class])
_response_page_items = get_data(page=page).json().get('items')
response_regions_names = set(
[region.name for region in get_regions(_response_page_items)])
with allure.step(f'Номер страницы соответствует {page}'):
assert default_regions_names == response_regions_names
def _updateRegions():
global locations
log.info("started updating regions")
accounts = Accounty.select()
for account in accounts:
log.info(account.api_key)
regions = get_regions(account.api_key)
log.info(str(regions))
locations[account.id] = regions
log.info("finished updating regions")
def get_region_plot(graph, data, layout, columns_to_color, node_elements,
colorscale):
'''Function to generate a figure of the mapper graph colored by identified
regions
Parameters
----------
graph : igraph object
Mapper graph
data : ndarray (n_samples x n_dim)
Data used for mapper
layout : igraph.layout.Layout
Layout of graph
columns_to_color : list
List of columns to color by
node_elements : tuple
Tuple of arrays where array at positin x contains the data points for
node x
colorscale : list
List of colors to use for each region
Returns
-------
fig : igraph object
'''
regions = utils.get_regions()
# set node color:
# 1. assign to each node of a region its color (zip())
# 2. convert zip elements to list (map())
# 3. flatten list (itertools.chain())
# 4. sort values by keys
# 5. convert to ordered dictionary
# 6. extract values and convert to list
node_color = list(
collections.OrderedDict(
sorted(itertools.chain(
*map(list,
[zip(regions[region],
itertools.repeat(colorscale[region]))
for region in range(len(regions))])))).values())
# set plotly arguments:
# 1. set uniform node size
# 2. hide scale of marker color
plotly_kwargs = {
'node_trace_marker_size': [1] * len(node_elements),
'node_trace_marker_showscale': False}
return visualization.create_network_2d(graph, data, layout, node_color,
columns_to_color=columns_to_color,
plotly_kwargs=plotly_kwargs)
CHAINS = [PREROUTING, OUTPUT, SNAT]
logger = logging.getLogger('repair-floating-ips')
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('/var/log/dair/repair-floating-ips.log')
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)
region_rc_filename = sys.argv[1]
region = utils.get_regions(region_rc_filename)[0]
url_parsed = urlparse(region.url)
novaRegion=boto.ec2.regioninfo.RegionInfo(name = "nova", endpoint = url_parsed.hostname)
conn = boto.connect_ec2(aws_access_key_id=region.access_key,
aws_secret_access_key=region.secret_access_key,
is_secure=True,
region=novaRegion,
port=url_parsed.port,
path=url_parsed.path)
addresses = conn.get_all_addresses()
used_floating_ips = set()
for address in addresses:
if not address.instance_id.startswith('None'):
def get_regions_list_class(get_response_class):
yield get_regions(get_response_class.json().get('items'))
def get_regions_list(get_response):
yield get_regions(get_response.json().get('items'))
path = os.path.join(os.path.dirname(__file__),'templates/version3/includes/guardian-data.html')
self.response.out.write(template.render(path, global_mashup))
else:
memcacheKey = str(destination)+":"+str(price)+":"+str(startDate.date().isoformat())+":"+str(endDate.date().isoformat())
memcachedHotels = memcache.get(key=memcacheKey, namespace='lastminute')
logging.info("Looking up MEMCACHE for : "+memcacheKey)
logging.info(memcachedHotels)
if memcachedHotels is not None:
global_mashup['hotels'] = memcachedHotels
path = os.path.join(os.path.dirname(__file__),'templates/version3/includes/'+info_type+'.html')
self.response.out.write(template.render(path, global_mashup))
logging.info("AjaxAPIHandler_v3() : Retrieving Hotels from MEMCACHE for destination "+str(destination))
else:
regions = utils.get_regions()
if destination in regions:
hotelsData = datastore.get_hotels_by_region(destination, price)
logging.info(hotelsData)
if hotelsData is None:
logging.info("No results for "+str(destination))
path = os.path.join(os.path.dirname(__file__),'templates/version3/includes/no-results.html')
self.response.out.write(template.render(path, global_mashup))
return
logging.info("Setting hotels to MEMCACHE with key : "+memcacheKey)
memcache.set(key=memcacheKey, value=hotelsData, time=6000, namespace='lastminute')
global_mashup['region'] = destination
else:
# [ST]TODO: Reinstate arguments: rating
hotelsData = datastore.get_hotels(destination, price, startDate, endDate, None)
import os
import csv
from utils import paths, clean, get_regions
regions, rev_regions = get_regions()
os.system("unzip data/datagov.zip -d data")
for category, category_paths in paths.iteritems():
for path, sti in category_paths:
path = "data/datagov/" + category + "/" + path
data = []
with open(path, "rb") as csvfile:
reader = csv.reader(csvfile, delimiter=",", quotechar="\"")
for row in reader:
data.append([clean(field) for field in row])
with open(path, "wb") as csvfile:
writer = csv.writer(csvfile,
delimiter=",",
quotechar="\"",
quoting=csv.QUOTE_NONNUMERIC)
for row in data:
writer.writerow(row)
def main():
# Loads and manages the input arguments
args = arg_parser()
# Config file
if args.config_file is not None:
config_file = args.config_file
else:
config_file = "config.ini"
if args.start_day is None or args.end_day is None:
# Setup default temporal period in which to work
# TODO: Print on logger we are using default dates
start = utils.get_yesterday_midnight_datetime()
end = utils.get_today_midnight_datetime()
else:
# Setup user temporal period in which to work
# TODO: Print on logger we are using user dates
start = utils.get_datetime_from_args(args.start_day)
end = utils.get_datetime_from_args(args.end_day)
start_timestamp = utils.get_timestamp(start)
end_timestamp = utils.get_timestamp(end)
# Read config file
if not os.path.isfile(config_file):
print("Configuration file not found: {}").format(config_file)
sys.exit(-1)
try:
config = ConfigParser()
config.optionxform = str
config.read(config_file)
except Exception as e:
print("Problem parsing config file: {}").format(e)
sys.exit(-1)
# Read main config file
mainconfig_file = config.get("mainconfig", "path")
if not os.path.isfile(mainconfig_file):
print("Main configuration file not found: {}").format(mainconfig_file)
sys.exit(-1)
try:
main_config = ConfigParser()
# Preserve case when reading configfile
main_config.optionxform = str
main_config.read(mainconfig_file)
except Exception as e:
print("Problem parsing main config file: {}").format(e)
sys.exit(-1)
# Get excluded regions, if any
excluded_regions = json.loads(config.get("regionexclude","regions"))
# Get included services, if any
included_services = json.loads(config.get("servicesinclude","services"))
# Setup monasca collector
CONF_M_SECTION = 'monasca'
keystone_endpoint = config.get('keystone','uri')
monasca_endpoint = config.get('monasca','uri')
user = config.get('profile','user')
password = config.get('profile','password')
collector = CollectorMonasca(user, password, monasca_endpoint, keystone_endpoint)
# Setup mysql persister
persister = PersisterMysql(config, start, end)
# Get regions to work on
if args.region_id is None:
regions = utils.get_regions(main_config)
else:
regions = [args.region_id]
# Loop each region
for region in regions:
# Skip excluded regions
if region in excluded_regions:
continue
# Retrieve sanity checks aggregation
day_agg = model.Aggregation('d', 86400, 'avg')
sanities_data = collector.get_sanities_avg(region,day_agg.period, start_timestamp, end_timestamp)
# Adapt and persist daily average of sanity checks aggregation into hourly base
sanities = []
for sanity_data in sanities_data:
sanity = model_adapter.from_monasca_sanity_to_sanity(sanity_data, day_agg)
sanities.append(sanity)
persister.persist_sanity(sanities)
# Retrieve processes aggregation
hour_agg = model.Aggregation('h', 3600, 'avg')
services_processes = collector.get_services_processes_avg(
region, hour_agg.period, start_timestamp, end_timestamp, included_services)
# Calculate and map processes aggregation
processes = []
for service in services_processes:
for process_name in services_processes[service].keys():
process_values = services_processes[service][process_name]
process = model_adapter.from_monasca_process_to_process(process_values, hour_agg)
processes.append(process)
# Adapt and persist processes aggregation
persister.persist_process(processes)
# Calculate and persist host_service daily aggregation
persister.persist_host_service_daily_avg(start, end)
# Calculate and persist host_service daily aggregation
if start.month != end.month:
s_month, e_month = utils.get_range_for_daily_agg(start, end)
persister.persist_host_service_monthly_avg(s_month, e_month)
def get_county_plot_by_region(data, colorscale, node_elements, fips):
'''Function to create figure of US with counties colored by region they
belong to
Parameters
----------
data : ndarray (n_samples x n_dim)
Data used for mapper
colorscale : list
List with colors to color counties by
node_elements : tuple
Tuple of arrays where array at positin x contains the data points for
node x
fips : list
List of Federal Information Processing Standard (FIPS) county codes
Returns
-------
fig: plotly figure object
'''
# convert colorscale from hex format to rgb
colorscale = dict(zip(map(str, range(len(colorscale))),
utils.hex2rgb(colorscale.values())))
# define color of counties belonging to two regions as their mean rgb value
colorscale['1-3'] = utils.mean_rgb([colorscale['1'],
colorscale['3']])
colorscale['2-3'] = utils.mean_rgb([colorscale['2'],
colorscale['3']])
colorscale['3-4'] = utils.mean_rgb([colorscale['3'],
colorscale['4']])
colorscale['4-5'] = utils.mean_rgb([colorscale['4'],
colorscale['5']])
elements_per_region = utils.get_data_per_region(utils.get_regions(),
node_elements)
# assign each county its color
county_color = np.zeros(data.shape[0], dtype='int')
# region 0
county_color[list(elements_per_region[0])] = 0
# region 1 and not 3
county_color[list(elements_per_region[1]
.difference(elements_per_region[3]))] = 1
# region 2 and not 3
county_color[list(elements_per_region[2]
.difference(elements_per_region[3]))] = 2
# region 3 and neither 1 nor 2
county_color[list(elements_per_region[3]
.difference(elements_per_region[1])
.difference(elements_per_region[2]))] = 3
# region 4 but not 3
county_color[list(elements_per_region[4]
.difference(elements_per_region[3]))] = 4
# region 5 but not 4
county_color[list(elements_per_region[5]
.difference(elements_per_region[4]))] = 5
# region 1 and 3
county_color[list(elements_per_region[1]
.intersection(elements_per_region[3]))] = 6
# region 2 and 3
county_color[list(elements_per_region[2]
.intersection(elements_per_region[3]))] = 7
# region 3 and 4
county_color[list(elements_per_region[3]
.intersection(elements_per_region[4]))] = 8
# region 4 and 5
county_color[list(elements_per_region[4]
.intersection(elements_per_region[5]))] = 9
county_color = county_color.tolist()
return get_county_plot(
fips=fips, values=county_color,
colorscale=[f'rgb{rgb}' for rgb in list(colorscale.values())])
def main():
# or call logging.basicConfig()
logging.info('Starting logger for monitoringHisto')
LOG = logging.getLogger(__name__)
# Loads and manages the input arguments
args = arg_parser()
# Config file
if args.config_file is not None:
config_file = args.config_file
else:
config_file = "config.ini"
if args.start_day is None or args.end_day is None:
# Setup default temporal period in which to work
# TODO: Print on logger we are using default dates
start = utils.get_yesterday_midnight_datetime()
end = utils.get_today_midnight_datetime()
else:
# Setup user temporal period in which to work
# TODO: Print on logger we are using user dates
start = utils.get_datetime_from_args(args.start_day)
end = utils.get_datetime_from_args(args.end_day)
start_timestamp = utils.get_timestamp(start)
end_timestamp = utils.get_timestamp(end)
# Read config file
if not os.path.isfile(config_file):
print("Configuration file not found: {}").format(config_file)
sys.exit(-1)
try:
config = ConfigParser()
config.optionxform = str
config.read(config_file)
except Exception as e:
print("Problem parsing config file: {}").format(e)
sys.exit(-1)
# Read main config file
mainconfig_file = config.get("mainconfig", "path")
if not os.path.isfile(mainconfig_file):
print("Main configuration file not found: {}").format(mainconfig_file)
sys.exit(-1)
try:
main_config = ConfigParser()
# Preserve case when reading configfile
main_config.optionxform = str
main_config.read(mainconfig_file)
except Exception as e:
print("Problem parsing main config file: {}").format(e)
sys.exit(-1)
# Set logger level, if any
l_level = config.get("logger", "log_level")
if l_level:
LOG.setLevel(l_level)
LOG.debug("Prova")
# Get excluded regions, if any
excluded_regions = json.loads(config.get("regionexclude", "regions"))
# Get included services, if any
included_services = json.loads(config.get("servicesinclude", "services"))
# Setup monasca collector
CONF_M_SECTION = 'monasca'
keystone_endpoint = config.get('keystone', 'uri')
monasca_endpoint = config.get('monasca', 'uri')
user = config.get('profile', 'user')
password = config.get('profile', 'password')
collector = CollectorMonasca(
user, password, monasca_endpoint, keystone_endpoint)
# Setup mysql persister
persister = PersisterMysql(config, start, end)
# Get regions to work on
if args.region_id is None:
regions = utils.get_regions(main_config)
else:
regions = [args.region_id]
# Loop each region
for region in regions:
# Skip excluded regions
if region in excluded_regions:
continue
# Retrieve sanity checks aggregation
day_agg = model.Aggregation('d', 86400, 'avg')
sanities_data = collector.get_sanities_avg(
region, day_agg.period, start_timestamp, end_timestamp)
if sanities_data is not None:
# Adapt and persist daily average of sanity checks aggregation into
# hourly base
sanities = []
for sanity_data in sanities_data:
sanity = model_adapter.from_monasca_sanity_to_sanity(
sanity_data, day_agg)
sanities.append(sanity)
persister.persist_sanity(sanities)
else:
# No sanities data available on the specified range, rise a warning
LOG.warning("No sanities data available on the specified range "
"for region: %s", region)
# Retrieve processes aggregation
hour_agg = model.Aggregation('h', 3600, 'avg')
services_processes = collector.get_services_processes_avg(
region, hour_agg.period, start_timestamp,
end_timestamp, included_services)
# Calculate and map processes aggregation
processes = []
for service in services_processes:
if services_processes[service] is None:
LOG.warning("No services process data available on the "
"specified range for region: %s and service: %s",
region, service)
continue
for process_name in services_processes[service].keys():
process_values = services_processes[service][process_name]
process = model_adapter.from_monasca_process_to_process(
process_values, hour_agg)
processes.append(process)
# Adapt and persist processes aggregation
persister.persist_process(processes)
# Calculate and persist host_service daily aggregation
persister.persist_host_service_daily_avg(start, end)
# Calculate and persist host_service monthly aggregation
if start.month != end.month:
s_month, e_month = utils.get_range_for_daily_agg(start, end)
persister.persist_host_service_monthly_avg(s_month, e_month)
from utils import read_data, get_data, get_regions
from charts import get_confirmed_trend, get_recovered_trend, get_death_trend
from update import update_data
from constants import DATA_SOURCE, CODE_SOURCE
external_stylesheets = ['https://codepen.io/chriddyp/pen/bWLwgP.css']
covidtrends_flask = flask.Flask(__name__)
app = dash.Dash(__name__,
server=covidtrends_flask,
external_stylesheets=external_stylesheets)
# Data processing
update_data()
confirmed_df, recovered_df, death_df = read_data()
regions = get_regions(confirmed_df)
# data_text = f'Data Source: <a href="{DATA_SOURCE}">Johns Hopkins CSEE</a>'
# code_text = f'Code Source: <a href="{CODE_SOURCE}">Github</a>'
footer_text_md = dcc.Markdown(f"""
Data Source: [Johns Hopkins CSEE]({DATA_SOURCE})
Code Source: [Github]({CODE_SOURCE})
""")
dropdown_layout = html.Div([
html.Div([
html.B('Country'),
dcc.Dropdown(id='countries-dd',
options=[{
'label': k,
logger = logging.getLogger('repair-floating-ips')
logger.setLevel(logging.DEBUG)
fh = logging.FileHandler('/var/log/dair/repair-floating-ips.log')
fh.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
ch.setFormatter(formatter)
logger.addHandler(fh)
logger.addHandler(ch)
region_rc_filename = sys.argv[1]
region = utils.get_regions(region_rc_filename)[0]
url_parsed = urlparse(region.url)
novaRegion = boto.ec2.regioninfo.RegionInfo(name="nova",
endpoint=url_parsed.hostname)
conn = boto.connect_ec2(aws_access_key_id=region.access_key,
aws_secret_access_key=region.secret_access_key,
is_secure=True,
region=novaRegion,
port=url_parsed.port,
path=url_parsed.path)
addresses = conn.get_all_addresses()
used_floating_ips = set()
for address in addresses:
def get_region_plot(pipe, data, layout, node_elements, colorscale):
'''Function to generate a figure of the mapper graph colored by identified
regions
Parameters
----------
pipe : MapperPipeline
The Mapper pipeline to compute the mapper-graph
data : ndarray (n_samples x n_dim)
Data used for mapper
layout : igraph.layout.Layout
Layout of graph
node_elements : tuple
Tuple of arrays where array at positin x contains the data points for
node x
colorscale : list
List of colors to use for each region
Returns
-------
fig : igraph object
'''
regions = utils.get_regions()
# set node color:
# 1. assign to each node of a region its color (zip())
# 2. convert zip elements to list (map())
# 3. flatten list (itertools.chain())
# 4. sort values by keys
# 5. convert to ordered dictionary
# 6. extract values and convert to list
node_color = np.array(
list(
collections.OrderedDict(
sorted(
itertools.chain(*map(list, [
zip(regions[region],
itertools.repeat(colorscale[region]))
for region in range(len(regions))
])))).values()))
# set plotly arguments:
# 1. set uniform node size
# 2. hide scale of marker color
plotly_kwargs = {
'node_trace_marker_size': [1] * len(node_elements),
'node_trace_marker_showscale': False,
'node_trace_hoverlabel': node_color,
'node_trace_marker_color': node_color
}
fig = plot_static_mapper_graph(pipe,
data,
layout,
layout_dim=2,
color_by_columns_dropdown=False,
plotly_kwargs=plotly_kwargs)
# update colors to fig
fig._data[1]['marker'][
'color'] = node_color # hack around with the new api
return fig
