def get_options():
"""Generates command-line options."""
global OPTIONS, FILENAMES
parser = optparse.OptionParser()
# Standard options
parser.add_option(
"-b",
"--use_base",
action="store_true",
default=False,
dest="use_base",
help="Update the Base index. Can be used with --use_solr.",
)
parser.add_option(
"-s",
"--use_solr",
action="store_true",
default=False,
dest="use_solr",
help="Update the Solr index. Can be used with --use_base.",
)
parser.add_option(
"-t",
"--test_mode",
action="store_true",
default=False,
dest="test_mode",
help="Don't process or upload the data files",
)
# Base options
base_group = parser.add_option_group("Google Base options")
base_group.add_option(
"--base_ftp_user", default=pipeline_keys.BASE_FTP_USER, dest="base_ftp_user", help="GBase username"
)
base_group.add_option(
"--base_ftp_pass", default=pipeline_keys.BASE_FTP_PASS, dest="base_ftp_pass", help="GBase password"
)
base_group.add_option(
"--base_cust_id", default=pipeline_keys.BASE_CUSTOMER_ID, dest="base_cust_id", help="GBase customer ID."
)
# Solr options
solr_group = parser.add_option_group("Solr options")
solr_group.add_option(
"--solr_url",
default=pipeline_keys.SOLR_URLS,
dest="solr_urls",
action="append",
help="URL of a Solr instance to be updated. " + "This option may be used multiple times.",
)
solr_group.add_option("--solr_user", default=pipeline_keys.SOLR_USER, dest="solr_user", help="Solr username.")
solr_group.add_option("--solr_pass", default=pipeline_keys.SOLR_PASS, dest="solr_pass", help="Solr password")
solr_group.add_option("--feed_providername", default=None, dest="feed_providername")
(OPTIONS, FILENAMES) = parser.parse_args()
def get_parser():
"""
Creates and returns Parser object
:return: optparse.OptionParser
"""
parser = OptionParser(
description="Situation Data App Example",
conflict_handler="resolve")
parser.add_option('-?', action='callback', callback=print_help, help=SUPPRESS_HELP)
req_group = OptionGroup(parser, "Required Parameters")
req_group.add_option('-t', '--type', help='Request Type Query - qry, Subscription = sub )',
metavar='type',
dest='REQUEST_TYPE',
default='sub')
req_group.add_option('-d', '--data', help='Data Type (Vehicle, Intersection, Aggregate, Advisory)',
metavar='data',
dest='DATA',
default='veh')
req_group.add_option('-h', '--host', help='Hostname',
metavar='host',
dest='HOST',
default='localhost:8080/ode')
parser.add_option_group(req_group)
group = OptionGroup(parser, "Optional Parameters")
group.add_option('-f', '--file', help='Full Path to a File containing JSON output that will be used to'
'validaate ODE output instead of default JSON files',
metavar='file',
dest='VALIDATION_FILE',
default=None, )
group.add_option('-c', '--config', help='Full path to config file File that can be used to override all Settings.'
'Config file will override all command line parameters',
metavar='config_file',
dest='CONFIG_FILE',
default=None)
parser.add_option_group(group)
return parser
def main():
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option(
'-u',
'--user',
metavar='<email>',
# Look for USER and USERNAME (Windows) environment variables.
default=os.environ.get('USER', os.environ.get('USERNAME')),
help='Filter on user, default=%default')
parser.add_option('-b',
'--begin',
metavar='<date>',
help='Filter issues created after the date (mm/dd/yy)')
parser.add_option('-e',
'--end',
metavar='<date>',
help='Filter issues created before the date (mm/dd/yy)')
quarter_begin, quarter_end = get_quarter_of(datetime.today() -
relativedelta(months=2))
parser.add_option(
'-Q',
'--last_quarter',
action='store_true',
help='Use last quarter\'s dates, i.e. %s to %s' %
(quarter_begin.strftime('%Y-%m-%d'), quarter_end.strftime('%Y-%m-%d')))
parser.add_option('-Y',
'--this_year',
action='store_true',
help='Use this year\'s dates')
parser.add_option('-w',
'--week_of',
metavar='<date>',
help='Show issues for week of the date (mm/dd/yy)')
parser.add_option(
'-W',
'--last_week',
action='count',
help='Show last week\'s issues. Use more times for more weeks.')
parser.add_option(
'-a',
'--auth',
action='store_true',
help='Ask to authenticate for instances with no auth cookie')
parser.add_option('-d',
'--deltas',
action='store_true',
help='Fetch deltas for changes.')
parser.add_option(
'--no-referenced-issues',
action='store_true',
help='Do not fetch issues referenced by owned changes. Useful in '
'combination with --changes-by-issue when you only want to list '
'issues that have also been modified in the same time period.')
parser.add_option(
'--skip-own-issues-without-changes',
action='store_true',
help='Skips listing own issues without changes when showing changes '
'grouped by referenced issue(s). See --changes-by-issue for more '
'details.')
activity_types_group = optparse.OptionGroup(
parser, 'Activity Types',
'By default, all activity will be looked up and '
'printed. If any of these are specified, only '
'those specified will be searched.')
activity_types_group.add_option('-c',
'--changes',
action='store_true',
help='Show changes.')
activity_types_group.add_option('-i',
'--issues',
action='store_true',
help='Show issues.')
activity_types_group.add_option('-r',
'--reviews',
action='store_true',
help='Show reviews.')
activity_types_group.add_option(
'--changes-by-issue',
action='store_true',
help='Show changes grouped by referenced issue(s).')
parser.add_option_group(activity_types_group)
output_format_group = optparse.OptionGroup(
parser, 'Output Format',
'By default, all activity will be printed in the '
'following format: {url} {title}. This can be '
'changed for either all activity types or '
'individually for each activity type. The format '
'is defined as documented for '
'string.format(...). The variables available for '
'all activity types are url, title, author, '
'created and modified. Format options for '
'specific activity types will override the '
'generic format.')
output_format_group.add_option(
'-f',
'--output-format',
metavar='<format>',
default=u'{url} {title}',
help='Specifies the format to use when printing all your activity.')
output_format_group.add_option(
'--output-format-changes',
metavar='<format>',
default=None,
help='Specifies the format to use when printing changes. Supports the '
'additional variable {reviewers}')
output_format_group.add_option(
'--output-format-issues',
metavar='<format>',
default=None,
help='Specifies the format to use when printing issues. Supports the '
'additional variable {owner}.')
output_format_group.add_option(
'--output-format-reviews',
metavar='<format>',
default=None,
help='Specifies the format to use when printing reviews.')
output_format_group.add_option(
'--output-format-heading',
metavar='<format>',
default=u'{heading}:',
help='Specifies the format to use when printing headings.')
output_format_group.add_option(
'--output-format-no-url',
default='{title}',
help='Specifies the format to use when printing activity without url.')
output_format_group.add_option(
'-m',
'--markdown',
action='store_true',
help='Use markdown-friendly output (overrides --output-format '
'and --output-format-heading)')
output_format_group.add_option(
'-j',
'--json',
action='store_true',
help='Output json data (overrides other format options)')
parser.add_option_group(output_format_group)
auth.add_auth_options(parser)
parser.add_option('-v',
'--verbose',
action='store_const',
dest='verbosity',
default=logging.WARN,
const=logging.INFO,
help='Output extra informational messages.')
parser.add_option('-q',
'--quiet',
action='store_const',
dest='verbosity',
const=logging.ERROR,
help='Suppress non-error messages.')
parser.add_option('-M',
'--merged-only',
action='store_true',
dest='merged_only',
default=False,
help='Shows only changes that have been merged.')
parser.add_option(
'-C',
'--completed-issues',
action='store_true',
dest='completed_issues',
default=False,
help='Shows only monorail issues that have completed (Fixed|Verified) '
'by the user.')
parser.add_option(
'-o',
'--output',
metavar='<file>',
help='Where to output the results. By default prints to stdout.')
# Remove description formatting
parser.format_description = (lambda _: parser.description) # pylint: disable=no-member
options, args = parser.parse_args()
options.local_user = os.environ.get('USER')
if args:
parser.error('Args unsupported')
if not options.user:
parser.error('USER/USERNAME is not set, please use -u')
# Retains the original -u option as the email address.
options.email = options.user
options.user = username(options.email)
logging.basicConfig(level=options.verbosity)
# python-keyring provides easy access to the system keyring.
try:
import keyring # pylint: disable=unused-import,unused-variable,F0401
except ImportError:
logging.warning('Consider installing python-keyring')
if not options.begin:
if options.last_quarter:
begin, end = quarter_begin, quarter_end
elif options.this_year:
begin, end = get_year_of(datetime.today())
elif options.week_of:
begin, end = (get_week_of(
datetime.strptime(options.week_of, '%m/%d/%y')))
elif options.last_week:
begin, end = (
get_week_of(datetime.today() -
timedelta(days=1 + 7 * options.last_week)))
else:
begin, end = (get_week_of(datetime.today() - timedelta(days=1)))
else:
begin = dateutil.parser.parse(options.begin)
if options.end:
end = dateutil.parser.parse(options.end)
else:
end = datetime.today()
options.begin, options.end = begin, end
if begin >= end:
# The queries fail in peculiar ways when the begin date is in the future.
# Give a descriptive error message instead.
logging.error(
'Start date (%s) is the same or later than end date (%s)' %
(begin, end))
return 1
if options.markdown:
options.output_format_heading = '### {heading}\n'
options.output_format = ' * [{title}]({url})'
options.output_format_no_url = ' * {title}'
logging.info('Searching for activity by %s', options.user)
logging.info('Using range %s to %s', options.begin, options.end)
my_activity = MyActivity(options)
my_activity.show_progress('Loading data')
if not (options.changes or options.reviews or options.issues
or options.changes_by_issue):
options.changes = True
options.issues = True
options.reviews = True
# First do any required authentication so none of the user interaction has to
# wait for actual work.
if options.changes or options.changes_by_issue:
my_activity.auth_for_changes()
if options.reviews:
my_activity.auth_for_reviews()
logging.info('Looking up activity.....')
try:
if options.changes or options.changes_by_issue:
my_activity.get_changes()
if options.reviews:
my_activity.get_reviews()
if options.issues or options.changes_by_issue:
my_activity.get_issues()
if not options.no_referenced_issues:
my_activity.get_referenced_issues()
except auth.AuthenticationError as e:
logging.error('auth.AuthenticationError: %s', e)
my_activity.show_progress('\n')
my_activity.print_access_errors()
output_file = None
try:
if options.output:
output_file = open(options.output, 'w')
logging.info('Printing output to "%s"', options.output)
sys.stdout = output_file
except (IOError, OSError) as e:
logging.error('Unable to write output: %s', e)
else:
if options.json:
my_activity.dump_json()
else:
if options.changes:
my_activity.print_changes()
if options.reviews:
my_activity.print_reviews()
if options.issues:
my_activity.print_issues()
if options.changes_by_issue:
my_activity.print_changes_by_issue(
options.skip_own_issues_without_changes)
finally:
if output_file:
logging.info('Done printing to file.')
sys.stdout = sys.__stdout__
output_file.close()
return 0
def get_options(argv):
parser = OptionParser(usage=usage, version=version)
parser.description = description
parser.add_option(
"-v",
"--verbosity",
action="count",
default=0,
help="Specify up to three times to increase verbosity, i.e. -v to see warnings, -vv for information messages, or -vvv for debug messages.",
)
filter_group = OptionGroup(parser, "Filter Options", "Options to filter the revisions included in the output.")
filter_group.add_option(
"-a",
"--author",
dest="author_filter",
metavar="AUTHOR",
help="Filter by author - report only this author's revisions.",
)
filter_group.add_option(
"-m",
"--message",
dest="message_filter",
metavar="MESSAGE",
help="Filter by message - report on only revisions with messages matching this case-insensitive regex.",
)
filter_group.add_option(
"-p",
"--path",
dest="path_filter",
metavar="PATH",
help="Filter by path - report on only revisions with updates to directories or fully-qualified file names matching this case-insensitive regex.",
)
filter_group.add_option(
"-r",
"--range",
dest="range_filter",
metavar="RANGE",
help="Range - range of revisions to report - e.g. -r400:456. Can be either by revision number, date in {YYYY-MM-DD} format, or by special value such as HEAD, BASE, PREV, TODAY or YESTERDAY. TODAY can also be used by itself to show only today's checkins.",
)
filter_group.add_option(
"-l",
"--limit",
type="int",
default=10,
metavar="LIMIT",
help="Limit - Look at only this many revisions. (Note that the limit is applied *before* some other filtering options, so you may end up seeing fewer revisions than you specify here.) Defaults to %default. Set to 0 for no limit.",
)
filter_group.add_option(
"-o",
"--operation",
dest="operation_filter",
type="str",
metavar="OPERATION",
action="callback",
callback=check_multiple_valid_option_values,
callback_kwargs={"valid_values": "AMD"},
help="Filter by file operation - report on only revisions where the specified file operations (A, M or D) have been performed on the specified paths.",
)
filter_group.add_option(
"-s",
"--stop-on-copy",
action="store_true",
help="Stop on copy - don't show revisions from before current branch was taken.",
)
parser.add_option_group(filter_group)
output_group = OptionGroup(parser, "Output Options", "Options to specify the content and format of the output.")
output_group.add_option(
"-f",
"--format",
choices=(list("HCX")),
default="H",
help="Output format. One of H (humane), C (comma separated) or X (XML). Defaults to %default.",
)
output_group.add_option(
"-d",
"--detail",
action="store_true",
help="Detail - display details of individual files and directories modified.",
)
output_group.add_option(
"-i",
"--invert",
action="store_true",
help="Invert - show revisions in reverse order, i.e. oldest first. Useful if you aren't piping the output into less.",
)
output_group.add_option(
"-t",
"--timezone",
choices=(list("lsLS")),
default="L",
metavar="ZONE",
help="Timezone - Timezone to use for display. One of l (local), s (server), L (local, omit), or S (server, omit). Defaults to %default.",
)
parser.add_option_group(output_group)
options, args = parser.parse_args(argv)
script, args = args[0], args[1:]
return options, script, args
def main(argv):
loadConfigFile()
#Command Line arguments and options
#
usage = "usage: prog command [options]\n\n"
parser = optparse.OptionParser(usage=usage, version="%prog 1.2")
group1 = optparse.OptionGroup(parser, "Options for Accounts and Media")
group1.add_option("--query",
action="store",
dest="query",
default=_CFG['query'],
help="Provide a query to limit the search (ex id:2444)")
group1.add_option(
"--accountid",
action="store",
dest="acctId",
default=_CFG['accountId'],
help="Process a single account if accountID is provided.")
group1.add_option("--mediaid",
action="store",
dest="mediaId",
default="",
help="Display all info on a specific media item")
#######################
group2 = optparse.OptionGroup(parser,
"Update Commands - Require permissions",
"These options update the media items")
group2.add_option("--delete-masters",
action="store_true",
dest="deletemaster",
default=False,
help="Delete the digital masters")
group2.add_option("--high-as-master",
action="store_true",
dest="highasmaster",
default=False,
help="Set the highest MP4 rendition as the new master")
group2.add_option("--reingest-from-highest",
action="store_true",
dest="reingesthighest",
default=False,
help="Re-ingest content using highest rendition.")
#######################
group3 = optparse.OptionGroup(
parser, "Level of Information to return",
"These options define what information is returned")
group3.add_option("--countonly",
action="store_true",
dest="countonly",
default=False,
help="Just return the number of items in each account")
group3.add_option("--itemsonly",
action="store_true",
dest="itemsonly",
default=False,
help="Process items only (no masters or renditions")
#######################
group4 = optparse.OptionGroup(parser, "Default Overrides",
"Override defaults in configuration file")
group4.add_option("--outputdir",
action="store",
dest="outputdir",
default=_CFG['outputdir'],
help="Output Directory (Default=./)")
group4.add_option("--keyname",
action="store",
dest="keyname",
default=_CFG['keyname'],
help="Key Name in config file to use")
group4.add_option("--ingest-profile",
action="store",
dest="ingestprofile",
default=_CFG['ingestprofile'],
help="Ingest profile to use for ingestion/re-ingestion")
group4.add_option("--keyfile",
action="store",
dest="keyfile",
default=_CFG['keyfile'],
help="Special case. Ask Dave")
group4.add_option(
"--itemcsv",
action="store",
dest="itemcsv",
default="",
help=
"Added for Josef. Path to a file with MediaIDs to process. (just skips items that arent in file)"
)
#######################
group5 = optparse.OptionGroup(parser, "Debugging and screen output")
group5.add_option("--verbose",
action="store_true",
dest="verbose",
default=_CFG['verbose'],
help="Display more info on screen")
group5.add_option("--debug",
action="store",
dest="debug",
type="int",
default=_CFG['debug'],
help="Enable debugging (1=basic or 2=json 3=print keys)")
group5.add_option(
"--passitems",
action="store",
dest="passitems",
default=_CFG['passitems'],
type="int",
help=
"Number of items to process per query call (max is 100, default is 50)"
)
group5.add_option("--limit",
action="store",
dest="limit",
default=_CFG['limit'],
type="int",
help="Limit the number of media items to this count")
group5.add_option("--skipto",
action="store",
dest="skiptoid",
default=_CFG['skiptoid'],
help="Skip until this ID then start processing)")
group5.add_option("--testme",
action="store_true",
dest="testme",
default="",
help="Test Code.")
#######################
group6 = optparse.OptionGroup(parser, "Ingest and Content Item Related ")
group6.add_option("--ingest-from-json",
action="store",
dest="ingestjsonfile",
default="",
help="Ingest from a JSON Feed File (supports --limit)")
group6.add_option("--ingest-from-MRSS",
action="store",
dest="ingestmrssfile",
default="",
help="Ingest from a MRSS Feed File (supports --limit)")
group6.add_option("--get-ingest-profiles",
action="store_true",
dest="getingestprofiles",
default=False,
help="Get ingestion profile info for accountid")
group6.add_option(
"--copy-item-fields",
action="store_true",
dest="copyitemfields",
default=False,
help=
"Copy the field to an alternate field (specified in the config file)")
group6.add_option(
"--delete-media-items",
action="store_true",
dest="deletemediaitems",
default=False,
help="Delete Content Items for account (BE CAREFUL). uses --query")
parser.add_option_group(group1)
parser.add_option_group(group2)
parser.add_option_group(group3)
parser.add_option_group(group4)
parser.add_option_group(group5)
parser.add_option_group(group6)
(options, args) = parser.parse_args()
# Override config file with command line options
#
_CFG['keyname'] = options.keyname
_CFG['accountId'] = options.acctId
_CFG['debug'] = options.debug
_CFG['limit'] = options.limit
_CFG['skiptoid'] = options.skiptoid
_CFG['mediaId'] = options.mediaId
_CFG['query'] = options.query
_CFG['outputDir'] = options.outputdir
_CFG['itemsonly'] = options.itemsonly
_CFG['passitems'] = options.passitems
_CFG['keyfile'] = options.keyfile
_CFG['ingestprofile'] = options.ingestprofile
_CFG['verbose'] = options.verbose
_CFG['countonly'] = options.countonly
_CFG['deletemaster'] = options.deletemaster
_CFG['highasmaster'] = options.highasmaster
_CFG['reingesthighest'] = options.reingesthighest
_CFG['itemcsv'] = options.itemcsv
_CFG['handles'] = {'real': 0, 'shared': 0, 'error': 0, 'results': 0}
# _CFG['fieldStr'] = "{id},{name},{reference_id},{masterId},{masterRate},{masterSize},{duration},{created_at},{updated_at},{state},{rendSize},{rendCount},{rendBitrates},{HLSRendSize},{HLSRendCount},{HLSResolutions},{HLSBitrates},{rendLgEncRate},{rendLgRes},{rendSize},{rendLgSize},{rendLgUrl}"
# _CFG['resultStr'] = "{accountId},{totalCount},{itemcount},{shared},{skipped},{duration},{masterSize},{rendCount},{rendSize},{HLSRendCount},{HLSRendSize},{rendLgSize}"
_CFG['totalFields'] = {
'masterSize': 0,
'masterCount': 0,
'duration': 0,
'rendCount': 0,
'rendSize': 0,
'rendLgSize': 0,
'HLSRendSize': 0,
'HLSRendCount': 0,
'skipped': 0,
'shared': 0,
'itemcount': 0,
'totalCount': 0
}
_CFG['itemFields'] = {
'masterSize': 0,
'duration': 0,
'masterId': 0,
'masterRate': 0,
'id': 0,
'name': 0,
'created_at': 0,
'updated_at': 0,
'state': 0
}
_CFG['rendFields'] = {
'rendCount': 0,
'rendSize': 0,
'rendLgSize': 0,
'rendBitrates': 0,
'rendCodecs': 0,
'rendLgId': 0,
'rendLgEncRate': 0,
'rendLgRes': 0,
'rendLgUrl': 0,
'HLSRendCount': 0,
'HLSRendSize': 0,
'HLSResolutions': 0,
'HLSBitrates': 0
}
# AuthToken (gets renewed every 4 minutes)
_CFG['token'] = ""
_CFG['tokenLastUpdated'] = 0
#if len(argv) == 1: parser.print_help(); sys.exit()
#if not _CFG['mediaId']: printCfg(_CFG)
printCfg(_CFG)
sys.stdout.flush()
# Change the config options based on passed in flags where necessary
#
if _CFG['mediaId']:
_CFG['level'] = {
'items': 1,
'masters': 1,
'renditions': 1,
'hlsrenditions': 1
}
if options.countonly:
_CFG['level'] = {
'items': 0,
'masters': 0,
'renditions': 0,
'hlsrenditions': 0
}
if options.itemsonly:
_CFG['level'] = {
'items': 1,
'masters': 0,
'renditions': 0,
'hlsrenditions': 0
}
# need to fetch renditions to set high master.
if _CFG.get('highasmaster', 0) or _CFG.get('reingest', 0):
_CFG['level']['renditions'] = 1
if options.testme:
testme()
sys.exit()
elif options.getingestprofiles:
getIngestionProfiles()
sys.exit()
elif options.ingestjsonfile:
ingestFromJsonFile(options.ingestjsonfile, _CFG['accountId'])
sys.exit()
elif options.ingestmrssfile:
ingestFromMRSSFile(options.ingestmrssfile, _CFG['accountId'])
sys.exit()
elif options.copyitemfields:
copyItemFields()
sys.exit()
elif options.deletemediaitems:
deleteMediaItems()
sys.exit()
else:
processAll()
sys.exit()
action="store_true",
default=False,
help='Search everything at scihub')
querygroup.add_option('-d',
'--download',
action="store_true",
default=False,
help='Download new scenes')
querygroup.add_option('-i',
'--intersectsWith',
action="store",
dest="intersectsWith",
metavar='<ARG>',
default='',
help='WKT format POINT,LINE, or POLYGON')
parser.add_option_group(querygroup)
opts, remainder = parser.parse_args(sys.argv[1:])
opt_dict = vars(opts)
#open a connection to the scihub
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, BASE_URL, USERNAME, PASSWORD)
authhandler = urllib2.HTTPBasicAuthHandler(passman)
opener = urllib2.build_opener(authhandler)
conn = database_connection()
if opt_dict['opensearch']:
scihub_opensearch(conn, opener, opt_dict['days'])
if opt_dict['download']:
download_new(opener, conn)
Other Example:
scihub_archive.py --download
scihub_archive.py --opensearch
scihub_archive.py --opensearch --download
scihub_archive.py --download --intersectsWith='POINT(-155.3 19.4)'
"""
parser = MyParser(description=desc, epilog=epi, version='1.0')
querygroup = optparse.OptionGroup(parser, "Parameters", "These options are used to control what is done")
querygroup.add_option('-o','--opensearch', action="store_true", default=False, help='Search scihub with opensearch and create archive.db')
querygroup.add_option('--days', action="store", dest="days", metavar='<ARG>', default='4',help='Number of days in past to search (default=4)')
querygroup.add_option('-a','--all', action="store_true", default=False, help='Search everything at scihub')
querygroup.add_option('-d','--download', action="store_true", default=False, help='Download new scenes')
querygroup.add_option('-i','--intersectsWith', action="store", dest="intersectsWith", metavar='<ARG>', default='',help='WKT format POINT,LINE, or POLYGON')
parser.add_option_group(querygroup)
opts, remainder = parser.parse_args(sys.argv[1:])
opt_dict= vars(opts)
#open a connection to the scihub
passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
passman.add_password(None, BASE_URL, USERNAME,PASSWORD)
authhandler = urllib2.HTTPBasicAuthHandler(passman)
opener = urllib2.build_opener(authhandler)
conn = database_connection()
if opt_dict['opensearch']:
scihub_opensearch(conn,opener,opt_dict['days'])
if opt_dict['download']:
download_new(opener,conn)
def main():
# Silence upload.py.
rietveld.upload.verbosity = 0
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option('-u',
'--user',
metavar='<email>',
default=os.environ.get('USER'),
help='Filter on user, default=%default')
parser.add_option('-b',
'--begin',
metavar='<date>',
help='Filter issues created after the date (mm/dd/yy)')
parser.add_option('-e',
'--end',
metavar='<date>',
help='Filter issues created before the date (mm/dd/yy)')
quarter_begin, quarter_end = get_quarter_of(datetime.today() -
relativedelta(months=2))
parser.add_option(
'-Q',
'--last_quarter',
action='store_true',
help='Use last quarter\'s dates, i.e. %s to %s' %
(quarter_begin.strftime('%Y-%m-%d'), quarter_end.strftime('%Y-%m-%d')))
parser.add_option('-Y',
'--this_year',
action='store_true',
help='Use this year\'s dates')
parser.add_option('-w',
'--week_of',
metavar='<date>',
help='Show issues for week of the date (mm/dd/yy)')
parser.add_option(
'-W',
'--last_week',
action='count',
help='Show last week\'s issues. Use more times for more weeks.')
parser.add_option(
'-a',
'--auth',
action='store_true',
help='Ask to authenticate for instances with no auth cookie')
parser.add_option('-d',
'--deltas',
action='store_true',
help='Fetch deltas for changes (slow).')
activity_types_group = optparse.OptionGroup(
parser, 'Activity Types',
'By default, all activity will be looked up and '
'printed. If any of these are specified, only '
'those specified will be searched.')
activity_types_group.add_option('-c',
'--changes',
action='store_true',
help='Show changes.')
activity_types_group.add_option('-i',
'--issues',
action='store_true',
help='Show issues.')
activity_types_group.add_option('-r',
'--reviews',
action='store_true',
help='Show reviews.')
parser.add_option_group(activity_types_group)
output_format_group = optparse.OptionGroup(
parser, 'Output Format',
'By default, all activity will be printed in the '
'following format: {url} {title}. This can be '
'changed for either all activity types or '
'individually for each activity type. The format '
'is defined as documented for '
'string.format(...). The variables available for '
'all activity types are url, title and author. '
'Format options for specific activity types will '
'override the generic format.')
output_format_group.add_option(
'-f',
'--output-format',
metavar='<format>',
default=u'{url} {title}',
help='Specifies the format to use when printing all your activity.')
output_format_group.add_option(
'--output-format-changes',
metavar='<format>',
default=None,
help='Specifies the format to use when printing changes. Supports the '
'additional variable {reviewers}')
output_format_group.add_option(
'--output-format-issues',
metavar='<format>',
default=None,
help='Specifies the format to use when printing issues. Supports the '
'additional variable {owner}.')
output_format_group.add_option(
'--output-format-reviews',
metavar='<format>',
default=None,
help='Specifies the format to use when printing reviews.')
output_format_group.add_option(
'--output-format-heading',
metavar='<format>',
default=u'{heading}:',
help='Specifies the format to use when printing headings.')
output_format_group.add_option(
'-m',
'--markdown',
action='store_true',
help='Use markdown-friendly output (overrides --output-format '
'and --output-format-heading)')
output_format_group.add_option(
'-j',
'--json',
action='store_true',
help='Output json data (overrides other format options)')
parser.add_option_group(output_format_group)
auth.add_auth_options(parser)
parser.add_option('-v',
'--verbose',
action='store_const',
dest='verbosity',
default=logging.WARN,
const=logging.INFO,
help='Output extra informational messages.')
parser.add_option('-q',
'--quiet',
action='store_const',
dest='verbosity',
const=logging.ERROR,
help='Suppress non-error messages.')
parser.add_option(
'-o',
'--output',
metavar='<file>',
help='Where to output the results. By default prints to stdout.')
# Remove description formatting
parser.format_description = (lambda _: parser.description) # pylint: disable=no-member
options, args = parser.parse_args()
options.local_user = os.environ.get('USER')
if args:
parser.error('Args unsupported')
if not options.user:
parser.error('USER is not set, please use -u')
options.user = username(options.user)
logging.basicConfig(level=options.verbosity)
# python-keyring provides easy access to the system keyring.
try:
import keyring # pylint: disable=unused-import,unused-variable,F0401
except ImportError:
logging.warning('Consider installing python-keyring')
if not options.begin:
if options.last_quarter:
begin, end = quarter_begin, quarter_end
elif options.this_year:
begin, end = get_year_of(datetime.today())
elif options.week_of:
begin, end = (get_week_of(
datetime.strptime(options.week_of, '%m/%d/%y')))
elif options.last_week:
begin, end = (
get_week_of(datetime.today() -
timedelta(days=1 + 7 * options.last_week)))
else:
begin, end = (get_week_of(datetime.today() - timedelta(days=1)))
else:
begin = datetime.strptime(options.begin, '%m/%d/%y')
if options.end:
end = datetime.strptime(options.end, '%m/%d/%y')
else:
end = datetime.today()
options.begin, options.end = begin, end
if options.markdown:
options.output_format = ' * [{title}]({url})'
options.output_format_heading = '### {heading} ###'
logging.info('Searching for activity by %s', options.user)
logging.info('Using range %s to %s', options.begin, options.end)
my_activity = MyActivity(options)
if not (options.changes or options.reviews or options.issues):
options.changes = True
options.issues = True
options.reviews = True
# First do any required authentication so none of the user interaction has to
# wait for actual work.
if options.changes:
my_activity.auth_for_changes()
if options.reviews:
my_activity.auth_for_reviews()
logging.info('Looking up activity.....')
try:
if options.changes:
my_activity.get_changes()
if options.reviews:
my_activity.get_reviews()
if options.issues:
my_activity.get_issues()
except auth.AuthenticationError as e:
logging.error('auth.AuthenticationError: %s', e)
output_file = None
try:
if options.output:
output_file = open(options.output, 'w')
logging.info('Printing output to "%s"', options.output)
sys.stdout = output_file
except (IOError, OSError) as e:
logging.error('Unable to write output: %s', e)
else:
if options.json:
my_activity.dump_json()
else:
my_activity.print_changes()
my_activity.print_reviews()
my_activity.print_issues()
finally:
if output_file:
logging.info('Done printing to file.')
sys.stdout = sys.__stdout__
output_file.close()
return 0
def main(argv):
loadConfigFile()
#Command Line arguments and options
#
usage = "usage: prog command [options]\n\n"
parser = optparse.OptionParser(usage=usage, version="%prog 1.2")
group1 = optparse.OptionGroup(parser, "Options for Accounts and Media")
group1.add_option("--query", action="store", dest="query", default=_CFG['query'],
help="Provide a query to limit the search (ex id:2444)")
group1.add_option("--accountid", action="store", dest="acctId",default=_CFG['accountId'],
help="Process a single account if accountID is provided.")
group1.add_option("--mediaid", action="store", dest="mediaId", default="",
help="Display all info on a specific media item")
#######################
group2 = optparse.OptionGroup(parser, "Update Commands - Require permissions",
"These options update the media items")
group2.add_option("--delete-masters", action="store_true", dest="deletemaster", default=False,
help="Delete the digital masters")
group2.add_option("--high-as-master", action="store_true", dest="highasmaster", default=False,
help="Set the highest MP4 rendition as the new master")
group2.add_option("--reingest-from-highest", action="store_true", dest="reingesthighest", default=False,
help="Re-ingest content using highest rendition.")
#######################
group3 = optparse.OptionGroup(parser, "Level of Information to return",
"These options define what information is returned")
group3.add_option("--countonly", action="store_true", dest="countonly", default=False,
help="Just return the number of items in each account")
group3.add_option("--itemsonly", action="store_true", dest="itemsonly", default=False,
help="Process items only (no masters or renditions")
#######################
group4 = optparse.OptionGroup(parser, "Default Overrides",
"Override defaults in configuration file")
group4.add_option("--outputdir", action="store", dest="outputdir", default=_CFG['outputdir'],
help="Output Directory (Default=./)")
group4.add_option("--keyname", action="store", dest="keyname", default=_CFG['keyname'],
help="Key Name in config file to use")
group4.add_option("--ingest-profile", action="store", dest="ingestprofile", default=_CFG['ingestprofile'],
help="Ingest profile to use for ingestion/re-ingestion")
group4.add_option("--keyfile", action="store", dest="keyfile", default=_CFG['keyfile'],
help="Special case. Ask Dave")
group4.add_option("--itemcsv", action="store", dest="itemcsv", default="",
help="Added for Josef. Path to a file with MediaIDs to process. (just skips items that arent in file)")
#######################
group5 = optparse.OptionGroup(parser, "Debugging and screen output")
group5.add_option("--verbose", action="store_true", dest="verbose", default=_CFG['verbose'],
help="Display more info on screen")
group5.add_option("--debug", action="store", dest="debug", type="int",default=_CFG['debug'],
help="Enable debugging (1=basic or 2=json 3=print keys)")
group5.add_option("--passitems", action="store", dest="passitems", default=_CFG['passitems'], type="int",
help="Number of items to process per query call (max is 100, default is 50)")
group5.add_option("--limit", action="store", dest="limit",default=_CFG['limit'],type="int",
help="Limit the number of media items to this count")
group5.add_option("--skipto", action="store", dest="skiptoid",default=_CFG['skiptoid'],
help="Skip until this ID then start processing)")
group5.add_option("--testme", action="store_true", dest="testme", default="",
help="Test Code.")
#######################
group6 = optparse.OptionGroup(parser, "Ingest and Content Item Related ")
group6.add_option("--ingest-from-json", action="store", dest="ingestjsonfile", default="",
help="Ingest from a JSON Feed File (supports --limit)")
group6.add_option("--ingest-from-MRSS", action="store", dest="ingestmrssfile", default="",
help="Ingest from a MRSS Feed File (supports --limit)")
group6.add_option("--get-ingest-profiles", action="store_true", dest="getingestprofiles", default=False,
help="Get ingestion profile info for accountid")
group6.add_option("--copy-item-fields", action="store_true", dest="copyitemfields", default=False,
help="Copy the field to an alternate field (specified in the config file)")
group6.add_option("--delete-media-items", action="store_true", dest="deletemediaitems", default=False,
help="Delete Content Items for account (BE CAREFUL). uses --query")
parser.add_option_group(group1)
parser.add_option_group(group2)
parser.add_option_group(group3)
parser.add_option_group(group4)
parser.add_option_group(group5)
parser.add_option_group(group6)
(options, args) = parser.parse_args()
# Override config file with command line options
#
_CFG['keyname'] = options.keyname
_CFG['accountId'] = options.acctId
_CFG['debug'] = options.debug
_CFG['limit'] = options.limit
_CFG['skiptoid'] = options.skiptoid
_CFG['mediaId'] = options.mediaId
_CFG['query'] = options.query
_CFG['outputDir'] = options.outputdir
_CFG['itemsonly'] = options.itemsonly
_CFG['passitems'] = options.passitems
_CFG['keyfile'] = options.keyfile
_CFG['ingestprofile'] = options.ingestprofile
_CFG['verbose'] = options.verbose
_CFG['countonly'] = options.countonly
_CFG['deletemaster'] = options.deletemaster
_CFG['highasmaster'] = options.highasmaster
_CFG['reingesthighest'] = options.reingesthighest
_CFG['itemcsv'] = options.itemcsv
_CFG['handles'] = { 'real':0, 'shared':0, 'error':0, 'results':0 }
# _CFG['fieldStr'] = "{id},{name},{reference_id},{masterId},{masterRate},{masterSize},{duration},{created_at},{updated_at},{state},{rendSize},{rendCount},{rendBitrates},{HLSRendSize},{HLSRendCount},{HLSResolutions},{HLSBitrates},{rendLgEncRate},{rendLgRes},{rendSize},{rendLgSize},{rendLgUrl}"
# _CFG['resultStr'] = "{accountId},{totalCount},{itemcount},{shared},{skipped},{duration},{masterSize},{rendCount},{rendSize},{HLSRendCount},{HLSRendSize},{rendLgSize}"
_CFG['totalFields'] = { 'masterSize':0, 'masterCount':0, 'duration':0,'rendCount':0, 'rendSize':0, 'rendLgSize':0, 'HLSRendSize':0, 'HLSRendCount':0,
'skipped':0, 'shared':0, 'itemcount':0 , 'totalCount':0}
_CFG['itemFields'] = { 'masterSize':0,'duration':0,'masterId':0, 'masterRate':0, 'id':0, 'name':0,'created_at':0,'updated_at':0, 'state':0 }
_CFG['rendFields'] = { 'rendCount':0, 'rendSize':0, 'rendLgSize':0, 'rendBitrates':0, 'rendCodecs':0, 'rendLgId':0, 'rendLgEncRate':0, 'rendLgRes':0,'rendLgUrl':0,
'HLSRendCount':0, 'HLSRendSize':0, 'HLSResolutions':0,'HLSBitrates':0 }
# AuthToken (gets renewed every 4 minutes)
_CFG['token'] = ""
_CFG['tokenLastUpdated'] = 0
#if len(argv) == 1: parser.print_help(); sys.exit()
#if not _CFG['mediaId']: printCfg(_CFG)
printCfg(_CFG)
sys.stdout.flush()
# Change the config options based on passed in flags where necessary
#
if _CFG['mediaId']:
_CFG['level'] = { 'items': 1, 'masters': 1, 'renditions': 1, 'hlsrenditions': 1}
if options.countonly:
_CFG['level'] = { 'items': 0, 'masters': 0, 'renditions': 0, 'hlsrenditions': 0}
if options.itemsonly:
_CFG['level'] = { 'items': 1, 'masters': 0, 'renditions': 0, 'hlsrenditions': 0}
# need to fetch renditions to set high master.
if _CFG.get('highasmaster',0) or _CFG.get('reingest',0):
_CFG['level']['renditions'] = 1;
if options.testme:
testme()
sys.exit()
elif options.getingestprofiles:
getIngestionProfiles()
sys.exit()
elif options.ingestjsonfile:
ingestFromJsonFile(options.ingestjsonfile, _CFG['accountId'])
sys.exit()
elif options.ingestmrssfile:
ingestFromMRSSFile(options.ingestmrssfile, _CFG['accountId'])
sys.exit()
elif options.copyitemfields:
copyItemFields()
sys.exit()
elif options.deletemediaitems:
deleteMediaItems()
sys.exit()
else:
processAll()
sys.exit()
def main():
# Silence upload.py.
rietveld.upload.verbosity = 0
parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
parser.add_option(
'-u', '--user', metavar='<email>',
default=os.environ.get('USER'),
help='Filter on user, default=%default')
parser.add_option(
'-b', '--begin', metavar='<date>',
help='Filter issues created after the date (mm/dd/yy)')
parser.add_option(
'-e', '--end', metavar='<date>',
help='Filter issues created before the date (mm/dd/yy)')
quarter_begin, quarter_end = get_quarter_of(datetime.today() -
relativedelta(months=2))
parser.add_option(
'-Q', '--last_quarter', action='store_true',
help='Use last quarter\'s dates, i.e. %s to %s' % (
quarter_begin.strftime('%Y-%m-%d'), quarter_end.strftime('%Y-%m-%d')))
parser.add_option(
'-Y', '--this_year', action='store_true',
help='Use this year\'s dates')
parser.add_option(
'-w', '--week_of', metavar='<date>',
help='Show issues for week of the date (mm/dd/yy)')
parser.add_option(
'-W', '--last_week', action='count',
help='Show last week\'s issues. Use more times for more weeks.')
parser.add_option(
'-a', '--auth',
action='store_true',
help='Ask to authenticate for instances with no auth cookie')
parser.add_option(
'-d', '--deltas',
action='store_true',
help='Fetch deltas for changes.')
parser.add_option(
'--no-referenced-issues',
action='store_true',
help='Do not fetch issues referenced by owned changes. Useful in '
'combination with --changes-by-issue when you only want to list '
'issues that have also been modified in the same time period.')
parser.add_option(
'--skip-own-issues-without-changes',
action='store_true',
help='Skips listing own issues without changes when showing changes '
'grouped by referenced issue(s). See --changes-by-issue for more '
'details.')
activity_types_group = optparse.OptionGroup(parser, 'Activity Types',
'By default, all activity will be looked up and '
'printed. If any of these are specified, only '
'those specified will be searched.')
activity_types_group.add_option(
'-c', '--changes',
action='store_true',
help='Show changes.')
activity_types_group.add_option(
'-i', '--issues',
action='store_true',
help='Show issues.')
activity_types_group.add_option(
'-r', '--reviews',
action='store_true',
help='Show reviews.')
activity_types_group.add_option(
'--changes-by-issue', action='store_true',
help='Show changes grouped by referenced issue(s).')
parser.add_option_group(activity_types_group)
output_format_group = optparse.OptionGroup(parser, 'Output Format',
'By default, all activity will be printed in the '
'following format: {url} {title}. This can be '
'changed for either all activity types or '
'individually for each activity type. The format '
'is defined as documented for '
'string.format(...). The variables available for '
'all activity types are url, title and author. '
'Format options for specific activity types will '
'override the generic format.')
output_format_group.add_option(
'-f', '--output-format', metavar='<format>',
default=u'{url} {title}',
help='Specifies the format to use when printing all your activity.')
output_format_group.add_option(
'--output-format-changes', metavar='<format>',
default=None,
help='Specifies the format to use when printing changes. Supports the '
'additional variable {reviewers}')
output_format_group.add_option(
'--output-format-issues', metavar='<format>',
default=None,
help='Specifies the format to use when printing issues. Supports the '
'additional variable {owner}.')
output_format_group.add_option(
'--output-format-reviews', metavar='<format>',
default=None,
help='Specifies the format to use when printing reviews.')
output_format_group.add_option(
'--output-format-heading', metavar='<format>',
default=u'{heading}:',
help='Specifies the format to use when printing headings.')
output_format_group.add_option(
'--output-format-no-url', default='{title}',
help='Specifies the format to use when printing activity without url.')
output_format_group.add_option(
'-m', '--markdown', action='store_true',
help='Use markdown-friendly output (overrides --output-format '
'and --output-format-heading)')
output_format_group.add_option(
'-j', '--json', action='store_true',
help='Output json data (overrides other format options)')
parser.add_option_group(output_format_group)
auth.add_auth_options(parser)
parser.add_option(
'-v', '--verbose',
action='store_const',
dest='verbosity',
default=logging.WARN,
const=logging.INFO,
help='Output extra informational messages.'
)
parser.add_option(
'-q', '--quiet',
action='store_const',
dest='verbosity',
const=logging.ERROR,
help='Suppress non-error messages.'
)
parser.add_option(
'-M', '--merged-only',
action='store_true',
dest='merged_only',
default=False,
help='Shows only changes that have been merged.')
parser.add_option(
'-C', '--completed-issues',
action='store_true',
dest='completed_issues',
default=False,
help='Shows only monorail issues that have completed (Fixed|Verified) '
'by the user.')
parser.add_option(
'-o', '--output', metavar='<file>',
help='Where to output the results. By default prints to stdout.')
# Remove description formatting
parser.format_description = (
lambda _: parser.description) # pylint: disable=no-member
options, args = parser.parse_args()
options.local_user = os.environ.get('USER')
if args:
parser.error('Args unsupported')
if not options.user:
parser.error('USER is not set, please use -u')
options.user = username(options.user)
logging.basicConfig(level=options.verbosity)
# python-keyring provides easy access to the system keyring.
try:
import keyring # pylint: disable=unused-import,unused-variable,F0401
except ImportError:
logging.warning('Consider installing python-keyring')
if not options.begin:
if options.last_quarter:
begin, end = quarter_begin, quarter_end
elif options.this_year:
begin, end = get_year_of(datetime.today())
elif options.week_of:
begin, end = (get_week_of(datetime.strptime(options.week_of, '%m/%d/%y')))
elif options.last_week:
begin, end = (get_week_of(datetime.today() -
timedelta(days=1 + 7 * options.last_week)))
else:
begin, end = (get_week_of(datetime.today() - timedelta(days=1)))
else:
begin = dateutil.parser.parse(options.begin)
if options.end:
end = dateutil.parser.parse(options.end)
else:
end = datetime.today()
options.begin, options.end = begin, end
if options.markdown:
options.output_format_heading = '### {heading}\n'
options.output_format = ' * [{title}]({url})'
options.output_format_no_url = ' * {title}'
logging.info('Searching for activity by %s', options.user)
logging.info('Using range %s to %s', options.begin, options.end)
my_activity = MyActivity(options)
my_activity.show_progress('Loading data')
if not (options.changes or options.reviews or options.issues or
options.changes_by_issue):
options.changes = True
options.issues = True
options.reviews = True
# First do any required authentication so none of the user interaction has to
# wait for actual work.
if options.changes or options.changes_by_issue:
my_activity.auth_for_changes()
if options.reviews:
my_activity.auth_for_reviews()
logging.info('Looking up activity.....')
try:
if options.changes or options.changes_by_issue:
my_activity.get_changes()
if options.reviews:
my_activity.get_reviews()
if options.issues or options.changes_by_issue:
my_activity.get_issues()
if not options.no_referenced_issues:
my_activity.get_referenced_issues()
except auth.AuthenticationError as e:
logging.error('auth.AuthenticationError: %s', e)
my_activity.show_progress('\n')
my_activity.print_access_errors()
output_file = None
try:
if options.output:
output_file = open(options.output, 'w')
logging.info('Printing output to "%s"', options.output)
sys.stdout = output_file
except (IOError, OSError) as e:
logging.error('Unable to write output: %s', e)
else:
if options.json:
my_activity.dump_json()
else:
if options.changes:
my_activity.print_changes()
if options.reviews:
my_activity.print_reviews()
if options.issues:
my_activity.print_issues()
if options.changes_by_issue:
my_activity.print_changes_by_issue(
options.skip_own_issues_without_changes)
finally:
if output_file:
logging.info('Done printing to file.')
sys.stdout = sys.__stdout__
output_file.close()
return 0
def main():
parser = optparse.OptionParser(option_class=SGEOption,
usage="usage: %prog [options]", version='%%prog %s' % pkg_resources.require("qutiepy")[0].version)
parser.add_option('--first-match', action='store_const', dest='action', const=action_firstMatch,
help='Only display the first record which matches')
parser.add_option('--last-match', action='store_const', dest='action', const=action_lastMatch,
help='Only display the last record which matches')
parser.add_option('--dry-run', action='store_true', default=False, dest='dry_run',
help='Only helpful for debugging, just prepairs to walk the file but never actually does anything')
group = optparse.OptionGroup(parser, "Grouping Predicates",
"Filter rows by field values. Filters are grouped using prefix notation, "
"by default a row must match all filters to be printed.")
group.add_option(SGEAndOption())
group.add_option(SGEOrOption())
group.add_option(SGENotOption())
group.add_option(SGEGroupEndOption())
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Time Filters',
"Filter on time based fields by giving a date")
group.add_option(SGEBeforeOption('--submitted-before', 'submission_time'))
group.add_option(SGEAfterOption('--submitted-after', 'submission_time'))
group.add_option(SGEBeforeOption('--started-before', 'start_time'))
group.add_option(SGEAfterOption('--started-after', 'start_time'))
group.add_option(SGEBeforeOption('--ended-before', 'end_time'))
group.add_option(SGEAfterOption('--ended-after', 'end_time'))
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Glob Filters',
"Match fields using unix style globing")
group.add_option(SGEGlobOption('--queue', 'qname'))
group.add_option(SGEGlobOption('--host', 'hostname'))
group.add_option(SGEGlobOption('--group', 'group'))
group.add_option(SGEGlobOption('--owner', 'owner'))
group.add_option(SGEGlobOption('--job', 'job_name'))
group.add_option(SGEGlobOption('--granted-pe', 'granted_pe'))
parser.add_option_group(group)
group = optparse.OptionGroup(parser, 'Range Filters',
"Filter field by specifiying ranges of numeric values. "
"...NUM is less than or equal to NUM, NUM is equal to a number, "
"NUM..NUM is every value between the two numbers including the endpoints, "
"NUM... is every value greater than or equal to NUM")
group.add_option(SGERangeOption('--job-number', 'job_number'))
group.add_option(SGERangeOption('--exit-status', 'exit_status'))
group.add_option(SGERangeOption('--slots', 'slots'))
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "format", ""
"available field names: " +
', '.join(qutiepy.sge_accounting.SGEAccountingRow.fields()))
group.add_option('--header', action='store_true', dest='print_header', default=False,
help="Print the format string out before the first record as a header")
group.add_option('--format', action='store', default=DEFAULT_TEMPLATE, dest='output_template',
help="Format the output by using template strings. $$ will be replaced by a $. "
"$var_name or ${var_name} will be replaced by the value from matched records.")
group.add_option('--filter', action='store', default=None, dest='str_filter')
parser.add_option_group(group)
(options, args) = parser.parse_args()
if options.dry_run:
print('This is a test it was only a test.')
sys.exit(0)
account = qutiepy.sge_accounting.SGEAccountingFile()
str_filter = getattr(options, 'str_filter', None)
if str_filter:
p = qutiepy.filter.Parser.Parser()
filter = p.parse(str_filter)
else:
filter = getattr(options, 'filter', None)
if filter:
records = itertools.ifilter(filter, account)
else:
records = iter(account)
try:
templ = string.Template(options.output_template)
if options.print_header:
print(options.output_template)
action = action_formatAll
if options.action:
action = options.action
if action(templ, records):
sys.exit(0)
else:
print('No matching records', file=sys.stderr)
sys.exit(1)
except IndexError as ex:
print('Malformed format option', ex[0], file=sys.stderr)
sys.exit(2)
except KeyboardInterrupt:
sys.exit(0)
def main():
# make sure we run in UTC timezone
import os
os.environ['TZ'] = 'UTC'
# Check the invocation arguments
parser = OptionParser(
'%prog [options]',
description='run the resource assignment editor web service')
parser.add_option(
'--webserver_port',
dest='webserver_port',
type='int',
default=7412,
help='port number on which to host the webservice, default: %default')
parser.add_option('-q',
'--broker',
dest='broker',
type='string',
default=DEFAULT_BROKER,
help='Address of the qpid broker, default: %default')
parser.add_option(
'--exchange',
dest='exchange',
type='string',
default=DEFAULT_BUSNAME,
help='Name of the bus exchange on the qpid broker, default: %default')
parser.add_option('-V',
'--verbose',
dest='verbose',
action='store_true',
help='verbose logging')
parser.add_option_group(dbcredentials.options_group(parser))
parser.set_defaults(dbcredentials="RADB")
(options, args) = parser.parse_args()
logging.basicConfig(
format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG if options.verbose else logging.INFO)
global _radb_dbcreds
_radb_dbcreds = dbcredentials.parse_options(options)
if _radb_dbcreds.database:
logger.info("Using dbcreds for direct RADB access: %s" %
_radb_dbcreds.stringWithHiddenPassword())
else:
_radb_dbcreds = None
global rarpc
rarpc = RADBRPC.create(exchange=options.exchange, broker=options.broker)
global otdbrpc
otdbrpc = OTDBRPC.create(exchange=options.exchange, broker=options.broker)
global curpc
curpc = CleanupRPC.create(exchange=options.exchange, broker=options.broker)
global sqrpc
sqrpc = StorageQueryRPC.create(exchange=options.exchange,
timeout=10,
broker=options.broker)
global momqueryrpc
momqueryrpc = MoMQueryRPC.create(exchange=options.exchange,
timeout=10,
broker=options.broker)
global changeshandler
changeshandler = ChangesHandler(exchange=options.exchange,
broker=options.broker,
momqueryrpc=momqueryrpc,
radbrpc=rarpc,
sqrpc=sqrpc)
with changeshandler, rarpc, otdbrpc, curpc, sqrpc, momqueryrpc:
'''Start the webserver'''
app.run(debug=options.verbose,
threaded=True,
host='0.0.0.0',
port=options.webserver_port)
