def subscribe(self, subscriptions):
"""
Make subscriptions to phedex
subscriptions = [(dataset_name, site_name), ...]
"""
new_subscriptions = dict()
for subscription in subscriptions:
dataset_name = subscription[0]
site_name = subscription[1]
try:
new_subscriptions[site_name].append(dataset_name)
except:
new_subscriptions[site_name] = list()
new_subscriptions[site_name].append(dataset_name)
for site_name, dataset_names in new_subscriptions.items():
data = self.phedex.generate_xml(dataset_names)
comments = (
"This dataset is predicted to become popular and has therefore been automatically replicated by cuadrnt"
)
api = "subscribe"
params = [
("node", site_name),
("data", data),
("level", "dataset"),
("move", "n"),
("custodial", "n"),
("group", "AnalysisOps"),
("request_only", "n"),
("no_mail", "n"),
("comments", comments),
]
json_data = self.phedex.fetch(api=api, params=params, method="post")
# insert into db
group_name = "AnalysisOps"
request_id = 0
request_type = 0
try:
request = json_data["phedex"]
request_id = request["request_created"][0]["id"]
request_created = timestamp_to_datetime(request["request_timestamp"])
except:
self.logger.warning(
"Subscription did not succeed\n\tSite:%s\n\tDatasets: %s", str(site_name), str(dataset_names)
)
continue
for dataset_name in dataset_names:
coll = "dataset_rankings"
date = datetime_day(datetime.datetime.utcnow())
pipeline = list()
match = {"$match": {"name": dataset_name, "date": date}}
pipeline.append(match)
project = {"$project": {"delta_rank": 1, "_id": 0}}
pipeline.append(project)
data = self.storage.get_data(coll=coll, pipeline=pipeline)
dataset_rank = data[0]["delta_rank"]
query = "INSERT INTO Requests(RequestId, RequestType, DatasetId, SiteId, GroupId, Rank, Date) SELECT %s, %s, Datasets.DatasetId, Sites.SiteId, Groups.GroupId, %s, %s FROM Datasets, Sites, Groups WHERE Datasets.DatasetName=%s AND Sites.SiteName=%s AND Groups.GroupName=%s"
values = (request_id, request_type, dataset_rank, request_created, dataset_name, site_name, group_name)
self.mit_db.query(query=query, values=values, cache=False)
def insert_dbs_data(self, dataset_name):
"""
Fetch dbs data about dataset and insert into database
"""
api = 'datasets'
params = {'dataset':dataset_name, 'detail':True, 'dataset_access_type':'*'}
dbs_data = self.dbs.fetch(api=api, params=params)
dataset_data = get_json(dbs_data, 'data')[0]
ds_name = get_json(dataset_data, 'primary_ds_name')
physics_group = get_json(dataset_data, 'physics_group_name')
data_tier = get_json(dataset_data, 'data_tier_name')
creation_date = datetime_day(timestamp_to_datetime(get_json(dataset_data, 'creation_date')))
ds_type = get_json(dataset_data, 'primary_ds_type')
coll = 'dataset_data'
query = {'name':dataset_name}
data = {'$set':{'ds_name':ds_name, 'physics_group':physics_group, 'data_tier':data_tier, 'creation_date':creation_date, 'ds_type':ds_type}}
self.storage.update_data(coll=coll, query=query, data=data, upsert=False)
def subscribe(self, subscriptions):
"""
Make subscriptions to phedex
subscriptions = [(dataset_name, site_name), ...]
"""
new_subscriptions = dict()
for subscription in subscriptions:
dataset_name = subscription[0]
site_name = subscription[1]
try:
new_subscriptions[site_name].append(dataset_name)
except:
new_subscriptions[site_name] = list()
new_subscriptions[site_name].append(dataset_name)
for site_name, dataset_names in new_subscriptions.items():
data = self.phedex.generate_xml(dataset_names)
comments = 'This dataset is predicted to become popular and has therefore been automatically replicated by cuadrnt'
api = 'subscribe'
params = [('node', site_name), ('data', data), ('level','dataset'), ('move', 'n'), ('custodial', 'n'), ('group', 'AnalysisOps'), ('request_only', 'n'), ('no_mail', 'n'), ('comments', comments)]
json_data = self.phedex.fetch(api=api, params=params, method='post')
# insert into db
group_name = 'AnalysisOps'
request_id = 0
request_type = 0
try:
request = json_data['phedex']
request_id = request['request_created'][0]['id']
request_created = timestamp_to_datetime(request['request_timestamp'])
except:
self.logger.warning('Subscription did not succeed\n\tSite:%s\n\tDatasets: %s', str(site_name), str(dataset_names))
continue
for dataset_name in dataset_names:
coll = 'dataset_popularity'
date = datetime_day(datetime.datetime.utcnow())
pipeline = list()
match = {'$match':{'name':dataset_name, 'date':date}}
pipeline.append(match)
project = {'$project':{'delta_popularity':1, '_id':0}}
pipeline.append(project)
data = self.storage.get_data(coll=coll, pipeline=pipeline)
dataset_rank = data[0]['delta_popularity']
query = "INSERT INTO Requests(RequestId, RequestType, DatasetId, SiteId, GroupId, Rank, Date) SELECT %s, %s, Datasets.DatasetId, Sites.SiteId, Groups.GroupId, %s, %s FROM Datasets, Sites, Groups WHERE Datasets.DatasetName=%s AND Sites.SiteName=%s AND Groups.GroupName=%s"
values = (request_id, request_type, dataset_rank, request_created, dataset_name, site_name, group_name)
self.mit_db.query(query=query, values=values, cache=False)
def subscribe(self, subscriptions):
"""
Make subscriptions to phedex
subscriptions = [(dataset_name, site_name), ...]
"""
new_subscriptions = dict()
for subscription in subscriptions:
dataset_name = subscription[0]
site_name = subscription[1]
try:
new_subscriptions[site_name].append(dataset_name)
except:
new_subscriptions[site_name] = list()
new_subscriptions[site_name].append(dataset_name)
for site_name, dataset_names in new_subscriptions.items():
data = self.phedex.generate_xml(dataset_names)
comments = 'This dataset is predicted to become popular and has therefore been automatically replicated by cuadrnt'
api = 'subscribe'
params = [('node', site_name), ('data', data), ('level','dataset'), ('move', 'n'), ('custodial', 'n'), ('group', 'AnalysisOps'), ('request_only', 'n'), ('no_mail', 'n'), ('comments', comments)]
json_data = self.phedex.fetch(api=api, params=params, method='post')
# insert into db
group_name = 'AnalysisOps'
request_id = 0
request_type = 0
try:
request = json_data['phedex']
request_id = request['request_created'][0]['id']
request_created = timestamp_to_datetime(request['request_timestamp'])
except:
self.logger.warning('Subscription did not succeed\n\tSite:%s\n\tDatasets: %s', str(site_name), str(dataset_names))
continue
for dataset_name in dataset_names:
coll = 'dataset_rankings'
date = datetime_day(datetime.datetime.utcnow())
pipeline = list()
match = {'$match':{'name':dataset_name, 'date':date}}
pipeline.append(match)
project = {'$project':{'delta_rank':1, '_id':0}}
pipeline.append(project)
data = self.storage.get_data(coll=coll, pipeline=pipeline)
dataset_rank = data[0]['delta_rank']
query = "INSERT INTO Requests(RequestId, RequestType, DatasetId, SiteId, GroupId, Rank, Date) SELECT %s, %s, Datasets.DatasetId, Sites.SiteId, Groups.GroupId, %s, %s FROM Datasets, Sites, Groups WHERE Datasets.DatasetName=%s AND Sites.SiteName=%s AND Groups.GroupName=%s"
values = (request_id, request_type, dataset_rank, request_created, dataset_name, site_name, group_name)
self.mit_db.query(query=query, values=values, cache=False)
def test_timestamp_to_datetime(self):
"Test timestamp_to_datetime function"
timestamp = 1434989041.102534
expected = datetime.datetime(2015, 6, 22, 16, 4, 1)
result = timestamp_to_datetime(timestamp)
self.assertEqual(result, expected)
def test_timestamp_to_datetime(self):
"Test timestamp_to_datetime function"
timestamp = 1434989041.102534
expected = datetime.datetime(2015, 6, 22, 16, 4, 1)
result = timestamp_to_datetime(timestamp)
self.assertEqual(result, expected)