def main():
# Read the command line arguments
parser = argparse.ArgumentParser(description="Force summarization for a given time period")
parser.add_argument('URL', metavar='url', help="AMQP URL to connect. For example: amqps://username:password@host:port/vhost")
parser.add_argument('from_str', metavar='from', help="Dateutil parsable date beginning of summarization period")
parser.add_argument('to_str', metavar='to', help="Dateutil parsable date end of summarization period")
# Arguments for the AMQP
parser.add_argument('--exchange', dest='exchange', help="Exchange to send summarize request", default="gracc.osg.requests")
parser.add_argument('--routing_key', dest='routing_key', help="Routing key to use for summarize request", default="gracc.osg.requests")
# Arguments to send to the remote summarizer
parser.add_argument('--destination_exchange', dest='destination_exchange', help="Exchange to send summarized records", default="gracc.osg.summary")
parser.add_argument('--destination_key', dest='destination_key', help="Routing key to send summarized records", default="gracc.osg.summary")
parser.add_argument('--type', dest='type', help="Type of summarization to retrieve, summary or transfer_summary. Default: summary", default="summary")
# Parse the arguments
args = parser.parse_args()
# Create the client
client = Client(exchange=args.exchange, routing_key=args.routing_key, url=args.URL)
# Break the summarize period into 7 day increments, so we don't overload anything!
# (no testing went into this decision, 7 days was picked arbitrarily)
from_date = parse(args.from_str)
to_date = parse(args.to_str)
while (from_date < to_date):
tmp_to_date = min(from_date + datetime.timedelta(days=7), to_date)
print "Summarizing %s to %s" % (from_date.isoformat(), tmp_to_date.isoformat())
client.query(from_date, tmp_to_date, args.type, destination_exchange=args.destination_exchange, destination_key=args.destination_key)
# Update the from date
from_date = tmp_to_date
def test_transfer_summary(self):
start_time = datetime(2016, 6, 1)
end_time = start_time + timedelta(days=31)
status = {'num_messages': 0}
def getMessage(msg):
status['num_messages'] += 1
client = Client("gracc.osg.requests", "gracc.osg.requests")
client.query(start_time, end_time, 'transfer_summary', getMessage)
self.assertGreater(status['num_messages'], 1)
def test_transfer_summary(self):
start_time = datetime(2016, 6, 1)
end_time = start_time + timedelta(days=31)
status = {'num_messages': 0}
def getMessage(msg):
status['num_messages'] += 1
client = Client("gracc.osg.requests", "gracc.osg.requests")
client.query(start_time, end_time, 'transfer_summary', getMessage)
self.assertGreater(status['num_messages'], 1)
def test_exchange_raw(self):
# Set the timerange
start_time = datetime(2016, 6, 1)
end_time = start_time + timedelta(days=32)
client = Client("gracc.osg.requests", "gracc.osg.requests")
client.query(start_time,
end_time,
'raw',
destination_exchange="gracc.osg.raw",
destination_key="gracc.osg.raw")
def test_raw(self):
# Set the timerange
start_time = datetime(2016, 6, 1)
end_time = start_time + timedelta(days=32)
status = {'num_messages': 0}
def getMessage(msg):
status['num_messages'] += 1
client = Client("gracc.osg.requests", "gracc.osg.requests")
client.query(start_time, end_time, 'raw', getMessage)
self.assertGreater(status['num_messages'], 1)
def test_client_long_range(self):
# Set the timerange
start_time = datetime(2016, 6, 1)
end_time = start_time + timedelta(days=31)
print start_time
print end_time
status = {'num_messages': 0}
def getMessage(msg):
status['num_messages'] += 1
client = Client("gracc.osg.requests", "gracc.osg.requests")
client.query(start_time, end_time, 'summary', getMessage)
self.assertGreater(status['num_messages'], 1)
def runRules(self, restrict_type=None):
for summary_name in self._config['Summary']:
cur_type = self._config['Summary'][summary_name]
logging.debug("Starting the summary: %s" % summary_name)
# Every 15 minutes, resummarize the last 7 days
client = Client(
exchange=self._config['PeriodicSummarizer']
['request_exchange'],
routing_key=self._config['PeriodicSummarizer']['request_key'],
url=self._config['AMQP']['url'])
# Get today's date, and the date 7 days ago
end_time = datetime.today()
start_time = end_time - timedelta(days=7)
logging.debug("Starting query to remote requster")
client.query(start_time,
end_time,
cur_type['summary_type'],
destination_exchange=cur_type['destination_exchange'],
destination_key=cur_type['destination_key'])
def runRules(self, timeperiod, restrict_type = None):
for summary_name in self._config['Summary']:
cur_type = self._config['Summary'][summary_name]
logging.debug("Starting the summary: %s" % summary_name)
# Every 15 minutes, resummarize the last 7 days
client = Client(
exchange=self._config['PeriodicSummarizer']['request_exchange'],
routing_key=self._config['PeriodicSummarizer']['request_key'],
url=self._config['AMQP']['url'])
# Get today's date, and the date 7 days ago
end_time = datetime.today()
start_time = end_time - timedelta(days=timeperiod)
while (start_time < end_time):
tmp_to_date = min(start_time + timedelta(days=7), end_time)
print "Summarizing %s to %s" % (start_time.isoformat(), tmp_to_date.isoformat())
client.query(start_time, tmp_to_date, cur_type['summary_type'], destination_exchange=cur_type['destination_exchange'], destination_key=cur_type['destination_key'])
# Update the from date
start_time = tmp_to_date
def runRules(self, timeperiod, restrict_type = None):
for summary_name in self._config['Summary']:
cur_type = self._config['Summary'][summary_name]
logging.debug("Starting the summary: %s" % summary_name)
# Every 15 minutes, resummarize the last 7 days
client = Client(
exchange=self._config['PeriodicSummarizer']['request_exchange'],
routing_key=self._config['PeriodicSummarizer']['request_key'],
url=self._config['AMQP']['url'])
# Get today's date, and the date 7 days ago
end_time = datetime.today()
start_time = end_time - timedelta(days=timeperiod)
while (start_time < end_time):
tmp_to_date = min(start_time + timedelta(days=7), end_time)
print("Summarizing %s to %s" % (start_time.isoformat(), tmp_to_date.isoformat()))
client.query(start_time, tmp_to_date, cur_type['summary_type'], destination_exchange=cur_type['destination_exchange'], destination_key=cur_type['destination_key'])
# Update the from date
start_time = tmp_to_date
def main():
# Read the command line arguments
parser = argparse.ArgumentParser(
description="Force summarization for a given time period")
parser.add_argument(
'URL',
metavar='url',
help=
"AMQP URL to connect. For example: amqps://username:password@host:port/vhost"
)
parser.add_argument(
'from_str',
metavar='from',
help="Dateutil parsable date beginning of summarization period")
parser.add_argument(
'to_str',
metavar='to',
help="Dateutil parsable date end of summarization period")
# Arguments for the AMQP
parser.add_argument('--exchange',
dest='exchange',
help="Exchange to send summarize request",
default="gracc.osg.requests")
parser.add_argument('--routing_key',
dest='routing_key',
help="Routing key to use for summarize request",
default="gracc.osg.requests")
# Arguments to send to the remote summarizer
parser.add_argument('--destination_exchange',
dest='destination_exchange',
help="Exchange to send summarized records",
default="gracc.osg.summary")
parser.add_argument('--destination_key',
dest='destination_key',
help="Routing key to send summarized records",
default="gracc.osg.summary")
parser.add_argument(
'--type',
dest='type',
help=
"Type of summarization to retrieve, summary or transfer_summary. Default: summary",
default="summary")
# Parse the arguments
args = parser.parse_args()
# Create the client
client = Client(exchange=args.exchange,
routing_key=args.routing_key,
url=args.URL)
# Break the summarize period into 7 day increments, so we don't overload anything!
# (no testing went into this decision, 7 days was picked arbitrarily)
from_date = parse(args.from_str)
to_date = parse(args.to_str)
while (from_date < to_date):
tmp_to_date = min(from_date + datetime.timedelta(days=7), to_date)
print "Summarizing %s to %s" % (from_date.isoformat(),
tmp_to_date.isoformat())
client.query(from_date,
tmp_to_date,
args.type,
destination_exchange=args.destination_exchange,
destination_key=args.destination_key)
# Update the from date
from_date = tmp_to_date