def sites(request):
schema_version = request.GET.get('schema_version')
if schema_version is None:
schema_version = SiteData.get_latest_schema()
else:
schema_version = float(schema_version)
supported_vos = request.GET.get('supported_vos')
if supported_vos is None:
supported_vos = None
else:
supported_vos = supported_vos.split(',')
filters = {
'name': request.GET.get('name'),
'tier': request.GET.get('tier'),
'sitevo__vo__name__in': supported_vos,
'country': request.GET.get('country'),
'country_code': request.GET.get('country_code'),
'federation': request.GET.get('federation')
}
trimmed_filters = {}
for key, val in filters.items():
if val is not None:
trimmed_filters[key] = val
filtered_sites = SiteService.get_active_sites()
filtered_sites = filtered_sites.complex_filter(trimmed_filters)
output = SiteService.generate_site_data(filtered_sites, schema_version)
return HttpResponse(output)
def site(request, site_id):
schema_version = request.GET.get('schema_version')
if schema_version is None:
schema_version = SiteData.get_latest_schema()
else:
schema_version = float(schema_version)
try:
the_site = SiteService.get_by_id(site_id)
except Site.DoesNotExist:
raise Http404("Site does not exist!")
output = SiteService.generate_site_data(the_site, schema_version)
return HttpResponse(output)
def process_rebus_sites(rebus_sites_file):
with open(rebus_sites_file, 'r') as file:
sites = json.load(file)
for site in sites:
site_name = site["Site"]
try:
site_obj = SiteService.get(site_name)
except:
logger.error(
"{site_name} not found in database. Cannot update storage info from REBUS"
.format(site_name=site_name))
continue
site_obj.total_online_storage = site['TotalNearlineSize']
site_obj.total_online_storage = site['TotalOnlineSize']
SiteService.save(site_obj)
def execute(self):
self.status = SequenceStatus.INIT
self.status = SequenceStatus.PARSE_CRIC_FEDERATION
CRICService.process_cric_federations(self.data["cric_federations"])
self.status = SequenceStatus.PARSE_CRIC_SITES
CRICService.process_cric_sites(self.data["cric_sites"])
self.status = SequenceStatus.PARSE_REBUS_SITES
RebusService.process_rebus_sites(self.data["rebus_sites"])
self.status = SequenceStatus.PROCESS_SITES
SiteService.post_process()
self.status = SequenceStatus.ANALYSE
SiteService.analyse(SiteService.get_all())
self.status = SequenceStatus.END
EventHub.announce_event(
SequenceCompletedEvent(created_by=Sequence.__name__, holder=self))
def get_all():
active_sites = SiteService.get_active_sites()
countries = []
for site in active_sites:
if site.country not in countries:
countries.append(site.country)
return countries
def __init__(self):
Singleton.__init__(self)
self.event_hub = EventHub()
self.google_earth_service = GoogleEarthService()
self.cric_service = CRICService()
self.site_service = SiteService()
self.atlas_service = AtlasService()
self.rebus_service = RebusService()
self.sequence_service = SequenceService()
self.recon_chewbacca = ReconChewbacca()
def process_cric_sites(cric_sites_file):
with open(cric_sites_file) as file:
sites = json.load(file)
for site_name, site in sites.items():
if site_name == "NULL":
continue
# print(site_name)
site = CRICService.cric_sites_location_patch(site_name, site)
site_obj = SiteService.get_or_create(site_name)
site_obj.sources.append('cric_sites')
site_obj.latitude = site['latitude']
site_obj.longitude = site['longitude']
site_obj.hepspec06 = site['capacity']
site_obj.cores = site['cores']
for site_vo in site['sites']:
vo_obj = VOService.get_or_create(site_vo['vo_name'])
site_vo_obj = SiteVOService.get_or_create(
site_vo['name'], site_obj, vo_obj)
site_obj.supported_vos.add(vo_obj)
site_obj.sitevo_set.add(site_vo_obj)
vo_obj.sitevo_set.add(site_vo_obj)
vo_obj.site_set.add(site_obj)
if site['country'] == "NULL":
if "KISTI" in site_name:
site_obj.country = "South Korea"
else:
site_obj.country = get_country(site['latitude'],
site['longitude']).name
else:
site_obj.country = site['country']
if site['country_code'] is not None or len(
site['country_code']) > 0:
site_obj.country_code = site['country_code']
else:
if "KISTI" in site_name:
site_obj.country_code = "KR"
else:
site_obj.country = get_country(
site['latitude'], site['longitude']).alpha_2
SiteService.save(site_obj)
def generate_v1_0(country):
sites = []
country_code = None
for site in SiteService.get_active_sites().filter(country=country):
sites.append(site.name)
country_code = site.country_code
pycountry = get_py_country(country)
if pycountry is not None:
country_code = pycountry.alpha_2
country_dict = {
"country": country,
"country_code": country_code,
"sites": sites
}
return country_dict
def process_cric_federations(cric_federations_file):
with open(cric_federations_file) as file:
federations = json.load(file)
for federation_name, federation in federations.items():
if federation_name == "NULL":
continue
fed_obj = FederationsService.get_or_create(federation_name)
fed_obj.accounting_name = federation['accounting_name']
fed_obj.tier = federation['tier_level']
for vo_name in federation['vos']:
vo = VOService.get_or_create(vo_name)
fed_obj.supported_vos.add(vo)
pledges = federation['pledges']
if federation['rcsites'] is not None:
for site in federation['rcsites']:
site_obj = SiteService.get_or_create(site)
site_obj.federation = fed_obj
site_obj.sources.append('cric-federations')
site_obj.tier = federation['tier_level']
SiteService.save(site_obj)
fed_obj.site_set.add(site_obj)
for year, vos in pledges.items():
for vo, pledge in vos.items():
vo_obj = VOService.get_or_create(vo)
pledge_obj = PledgeService.get_or_create(
fed_obj, vo_obj)
pledge_obj.year = year
if 'CPU' in pledge:
pledge_obj.cpu = pledge['CPU']
if 'Disk' in pledge:
pledge_obj.disk = pledge['Disk']
if 'Tape' in pledge:
pledge_obj.tape = pledge['Tape']
PledgeService.save(pledge_obj)
fed_obj.pledge_set.add(pledge_obj)
FederationsService.save(fed_obj)