def main():
"""
Utility to perform actions on HTML
Required arguments:
-a | --action <action_name> : Action to perform. 2 valid options exist so far:
'retrieve-content' or 'strip-tags'
-u | --url <url> : URL for the HTML content
"""
exit_code = 0
logs.init(logfile='html')
ap = args_util.get_parser()
ap.add_argument('-a',
'--action',
type=str,
help='Action to perform. 2 valid options exist so far: '
'\'retrieve-content\' or \'strip-tags\'')
ap.add_argument('-u', '--url', type=str, help='URL for the HTML content')
ap.add_argument('-t',
'--tag',
type=str,
default=None,
help='Optional HTML tag to strip and return the content')
ap.add_argument('--text',
type=str,
default=True,
help='Optional text within HTML tags to strip')
args = ap.parse_args()
assert args.action, ('--action is required. The valid options are: '
'\'retrieve-content\' or \'strip-tags\'')
assert args.url, '--url is required'
try:
html = retrieve_html(args.url)
if html is None:
print 'HTML was not retrieved'
exit_code = 1
exit(exit_code)
if args.action == 'retrieve-content':
contents = extract_content_and_language(html, url=args.url)
if contents is None:
print 'Content could not be retrieved. Check log.'
else:
for content in contents:
print content, ' = ', contents[content]
elif args.action == 'strip-tags':
print strip_tags(html, node_text=args.text, tag=args.tag)
else:
log.error('Invalid action provided: \'%s\'' % args.action)
print 'Invalid action provided: \'%s\'' % args.action
print 'Must be either \'retrieve-content\' or \'strip-tags\''
exit_code = 2
except Exception as e:
log.exception('Exception encountered: %s' % e)
exit_code = 1
exit(exit_code)
def main():
"""start the game"""
# initiate logging system
logs.init()
# set caption and title screen
pg.display.set_caption("Stickman's New World")
pg.display.set_icon(PICS['game_icon'])
# place starting image on screen
SURFACE.blit(PICS['title_screen'], (0, 0))
continue_ = True
logging.debug('starting game')
while continue_:
if MAIN_GAME_STATE.get('TERMINAL') is not None:
MAIN_GAME_STATE['TERMINAL'].threaded_update()
SURFACE.blit(PICS['title_screen'], (0, 0))
SURFACE.blit(MAIN_GAME_STATE['CURSOR'], pg.mouse.get_pos())
for event in pg.event.get():
check_quit(event)
if event.type == MOUSEBUTTONDOWN:
# on to the next loop
continue_ = not continue_
pg.display.update()
music.check(MAIN_GAME_STATE)
choose_function()
def init():
conductor.init()
logs.init()
# TODO: if host mysql server do not start, db.init() will raise
# mysql.connector.errors.InterfaceError.
lines = db.init()
compute.init(lines=lines)
def initial_setup():
""" This makes folder required to keep all data files
"""
try:
os.mkdir('data')
except OSError as e:
# folder data already exists this means maybe files init already
# exists so we create the log file using below function
logs.init()
logs.info('Error:' + str(e))
def main():
"""
Utility to perform actions on HTML
Required arguments:
-a | --action <action_name> : Action to perform. 2 valid options exist so far:
'retrieve-content' or 'strip-tags'
-u | --url <url> : URL for the HTML content
"""
exit_code = 0
logs.init(logfile='html')
ap = args_util.get_parser()
ap.add_argument('-a', '--action', type=str,
help='Action to perform. 2 valid options exist so far: '
'\'retrieve-content\' or \'strip-tags\'')
ap.add_argument('-u', '--url', type=str, help='URL for the HTML content')
ap.add_argument('-t', '--tag', type=str, default=None,
help='Optional HTML tag to strip and return the content')
ap.add_argument('--text', type=str, default=True,
help='Optional text within HTML tags to strip')
args = ap.parse_args()
assert args.action, ('--action is required. The valid options are: '
'\'retrieve-content\' or \'strip-tags\'')
assert args.url, '--url is required'
try:
html = retrieve_html(args.url)
if html is None:
print 'HTML was not retrieved'
exit_code = 1
exit(exit_code)
if args.action == 'retrieve-content':
contents = extract_content_and_language(html, url=args.url)
if contents is None:
print 'Content could not be retrieved. Check log.'
else:
for content in contents:
print content, ' = ', contents[content]
elif args.action == 'strip-tags':
print strip_tags(html, node_text=args.text, tag=args.tag)
else:
log.error('Invalid action provided: \'%s\'' % args.action)
print 'Invalid action provided: \'%s\'' % args.action
print 'Must be either \'retrieve-content\' or \'strip-tags\''
exit_code = 2
except Exception as e:
log.exception('Exception encountered: %s' % e)
exit_code = 1
exit(exit_code)
def check_logs_file():
""" Checks if the file size has exceeded 512kB. If so upload the file,
delete it and then create new one.
"""
file_name = logs.FILE_NAME
size = file_operations.get_file_size(file_name)
if size > FILE_SIZE:
if upload_file.upload(file_name):
file_operations.delete_file(file_name)
# this will create logs file afresh file once again
logs.init()
# limitations under the License. """A module to handle running a fuzz target for a specified amount of time.""" import collections import logging import os import shutil import stat import tempfile import clusterfuzz.environment import clusterfuzz.fuzz import config_utils import logs logs.init() # Use len_control=0 since we don't have enough time fuzzing for len_control to # make sense (probably). LIBFUZZER_OPTIONS_BATCH = ['-len_control=0'] # Use a fixed seed for determinism for code change fuzzing. LIBFUZZER_OPTIONS_CODE_CHANGE = LIBFUZZER_OPTIONS_BATCH + ['-seed=1337'] LIBFUZZER_OPTIONS_NO_REPORT_OOM = ['-rss_limit_mb=0'] # The number of reproduce attempts for a crash. REPRODUCE_ATTEMPTS = 10 REPRODUCE_TIME_SECONDS = 30 # Seconds on top of duration until a timeout error is raised. BUFFER_TIME = 10
def main():
"""
A little utility to handle reading and writing streams
to and from a queue.
--pub <queue> : publish what's read from stdin to <queue>
--sub <queue> : read from <queue> and write the messages to stdout
--cat : when used with --pub, write all published messages to stdout
--clean : check in incoming and outgoing messages.
Verify the message is correct JSON and add
an embersId if needed.
--log_file : Path to write the log file to
--log_level : Logging level
Other standard EMBERS options (e.g. --verbose).
"""
import args
import message
global log
ap = args.get_parser()
ap.add_argument('--clean', action="store_true",
help='Verify message format and add standard fields such as embersId.')
ap.add_argument('--addfeed', action="store_true", help='Add feed and feedPath fields to published message.')
ap.add_argument('--cat', action="store_true", help='Write all published messages to stdout.')
ap.add_argument('--rm', nargs="+", help="delete queue")
arg = ap.parse_args()
log = logs.getLogger(log_name=arg.log_file)
logs.init(arg, l=arg.log_level, logfile=arg.log_file)
init(arg)
if arg.rm and not arg.sub:
for queue in arg.rm:
print "Deleting", queue,
queue = ikqueue.Queue(queue)
queue.maybe_bind(connect())
queue.delete()
print "."
return
try:
# need to use the raw/utf handler unless we are doing clean
marshal = UnicodeMarshal()
if arg.clean or arg.addfeed:
marshal = JsonMarshal()
if arg.sub is None and os.environ.get('UPSTART_JOB') is None:
arg.sub = '-' # stdin
subq = open(arg.sub, 'r') #, marshal=marshal, ssh_key=arg.ssh_key, ssh_conn=arg.tunnel)
if arg.pub is None and os.environ.get('UPSTART_JOB') is None:
arg.pub = '-' # stdout
pubq = open(arg.pub, 'w', capture=arg.cat, marshal=marshal)
except Exception as e:
log.exception("Exception opening queues: %s" % e)
# "Human-readable" queue name can be retrieved as
#
# sname = subq.get_name()
# pname = pubq.get_name()
rc = 0
try:
it = subq.__iter__()
while True:
m = ''
try:
m = it.next()
if arg.clean:
m = message.clean(m)
if m:
if arg.addfeed:
m = message.add_embers_ids(m, feed=pubq.get_name(), feedPath=pubq.get_name())
pubq.write(m)
except StopIteration:
break
except KeyboardInterrupt:
break
except Exception as e:
rc += 1
if m:
log.exception('Could not process message %s: %s' % (m, e))
else:
log.exception('Unknown processing error %s' % e)
except KeyboardInterrupt:
pass
except Exception as e:
rc = 1
log.exception('Top level exception %s' % e)
return rc
def main():
"""
A little utility to handle reading and writing streams
to and from a queue.
--pub <queue> : publish what's read from stdin to <queue>
--sub <queue> : read from <queue> and write the messages to stdout
--cat : when used with --pub, write all published messages to stdout
--clean : check in incoming and outgoing messages.
Verify the message is correct JSON and add
an embersId if needed.
--log_file : Path to write the log file to
--log_level : Logging level
Other standard EMBERS options (e.g. --verbose).
"""
import args
import message
global log
ap = args.get_parser()
ap.add_argument(
'--clean',
action="store_true",
help='Verify message format and add standard fields such as embersId.')
ap.add_argument('--addfeed',
action="store_true",
help='Add feed and feedPath fields to published message.')
ap.add_argument('--cat',
action="store_true",
help='Write all published messages to stdout.')
ap.add_argument('--rm', nargs="+", help="delete queue")
arg = ap.parse_args()
log = logs.getLogger(log_name=arg.log_file)
logs.init(arg, l=arg.log_level, logfile=arg.log_file)
init(arg)
if arg.rm and not arg.sub:
for queue in arg.rm:
print "Deleting", queue,
queue = ikqueue.Queue(queue)
queue.maybe_bind(connect())
queue.delete()
print "."
return
try:
# need to use the raw/utf handler unless we are doing clean
marshal = UnicodeMarshal()
if arg.clean or arg.addfeed:
marshal = JsonMarshal()
if arg.sub is None and os.environ.get('UPSTART_JOB') is None:
arg.sub = '-' # stdin
subq = open(
arg.sub,
'r') #, marshal=marshal, ssh_key=arg.ssh_key, ssh_conn=arg.tunnel)
if arg.pub is None and os.environ.get('UPSTART_JOB') is None:
arg.pub = '-' # stdout
pubq = open(arg.pub, 'w', capture=arg.cat, marshal=marshal)
except Exception as e:
log.exception("Exception opening queues: %s" % e)
# "Human-readable" queue name can be retrieved as
#
# sname = subq.get_name()
# pname = pubq.get_name()
rc = 0
try:
it = subq.__iter__()
while True:
m = ''
try:
m = it.next()
if arg.clean:
m = message.clean(m)
if m:
if arg.addfeed:
m = message.add_embers_ids(m,
feed=pubq.get_name(),
feedPath=pubq.get_name())
pubq.write(m)
except StopIteration:
break
except KeyboardInterrupt:
break
except Exception as e:
rc += 1
if m:
log.exception('Could not process message %s: %s' % (m, e))
else:
log.exception('Unknown processing error %s' % e)
except KeyboardInterrupt:
pass
except Exception as e:
rc = 1
log.exception('Top level exception %s' % e)
return rc
def main():
import args
import logs
# preliminary stab at some useful command line functionality (only cluster is currently needed)
ap = args.get_parser()
ap.add_argument('-c',
'--cluster',
action="store_true",
help='Switch to output names of cluster nodes.')
ap.add_argument(
'--get',
metavar='REQUEST',
nargs='?',
type=str,
help=
'Information to get from embers.conf [service|data|cluster|host|input|output]'
)
ap.add_argument('--s3only',
action="store_true",
help="Write debug messages.")
ap.add_argument('--s3onlyprod',
action="store_true",
help="Write debug messages.")
ap.add_argument('--prefixes',
action="store_true",
help="Write debug messages.")
ap.add_argument('--prefixpairs',
action="store_true",
help="Write debug messages.")
group = ap.add_mutually_exclusive_group()
group.add_argument('--host',
metavar='HOST',
nargs='?',
type=str,
help='Name of the host to get information about')
group.add_argument(
'--data',
metavar='DATA',
nargs='?',
type=str,
help='Name of the data (queue, etc.) to get information about')
arg = ap.parse_args()
logs.init(arg)
init(arg)
if arg.get:
assert arg.get in ('service', 'data', 'cluster', 'host', 'inputs',
'outputs', 'services'), 'Improper get request'
try:
if arg.s3only or arg.s3onlyprod:
if arg.s3only:
prodOnly = False
else:
prodOnly = True
plst = get_all_s3only(prodOnly)
if plst:
print string.join(plst)
return
if arg.prefixpairs:
plst, qlst = get_all_prefixpairs()
for i in range(len(plst)):
print "%s %s" % (qlst[i], plst[i])
return
if arg.prefixes:
plst = get_all_prefixes()
for p in plst:
print p
return
if arg.host:
assert arg.host in conf.get(
'cluster'), 'Host is not listed in embers.conf'
if arg.get == 'services':
print conf.get('cluster')[arg.host]['services']
if arg.get == 'data':
print[
data for service in conf['cluster'][arg.host]['services']
for data in conf['services'][service]['outputs']
]
if arg.data:
assert arg.data in conf.get(
'data'), 'Data is not listed in embers.conf'
(host, service) = get_host_and_service(arg.data)
if arg.get == 'host':
print host
if arg.get == 'service':
print service
if arg.service:
assert arg.service in conf.get(
'services'), 'Service is not listed in embers.conf'
if arg.get == 'host':
print[
host for host in conf['cluster']
if arg.service in conf['cluster'][host]['services']
]
if arg.get == 'inputs':
print conf['services'][arg.service]['inputs']
if arg.get == 'outputs':
print conf['services'][arg.service]['outputs']
if arg.cluster or arg.get == 'cluster':
print ' '.join(conf.get('cluster', ''))
if arg.get == 'data' and not (arg.service or arg.data or arg.host):
print ' '.join(conf.get('data', ''))
if arg.get == 'host' and not (arg.service or arg.data or arg.host):
print ' '.join(conf.get('cluster', ''))
if arg.get == 'service' and not (arg.service or arg.data or arg.host):
print ' '.join(conf.get('services', ''))
except Exception, e:
log.error("Requested information not in embers.conf, error %s", e)
def main():
import args
import logs
# preliminary stab at some useful command line functionality (only cluster is currently needed)
ap = args.get_parser()
ap.add_argument('-c', '--cluster', action="store_true",
help='Switch to output names of cluster nodes.')
ap.add_argument('--get', metavar='REQUEST', nargs='?', type=str,
help='Information to get from embers.conf [service|data|cluster|host|input|output]')
ap.add_argument('--s3only', action="store_true", help="Write debug messages.")
ap.add_argument('--s3onlyprod', action="store_true", help="Write debug messages.")
ap.add_argument('--prefixes', action="store_true", help="Write debug messages.")
ap.add_argument('--prefixpairs', action="store_true", help="Write debug messages.")
group = ap.add_mutually_exclusive_group()
group.add_argument('--host', metavar='HOST', nargs='?', type=str,
help='Name of the host to get information about')
group.add_argument('--data', metavar='DATA', nargs='?', type=str,
help='Name of the data (queue, etc.) to get information about')
arg = ap.parse_args()
logs.init(arg)
init(arg)
if arg.get:
assert arg.get in ('service','data','cluster','host','inputs','outputs','services'), 'Improper get request'
try:
if arg.s3only or arg.s3onlyprod:
if arg.s3only:
prodOnly = False
else:
prodOnly = True
plst = get_all_s3only(prodOnly)
if plst:
print string.join(plst)
return
if arg.prefixpairs:
plst, qlst = get_all_prefixpairs()
for i in range(len(plst)):
print "%s %s" % (qlst[i], plst[i])
return
if arg.prefixes:
plst = get_all_prefixes()
for p in plst:
print p
return
if arg.host:
assert arg.host in conf.get('cluster'), 'Host is not listed in embers.conf'
if arg.get == 'services':
print conf.get('cluster')[arg.host]['services']
if arg.get == 'data':
print [data for service in conf['cluster'][arg.host]['services'] for data in conf['services'][service]['outputs']]
if arg.data:
assert arg.data in conf.get('data'), 'Data is not listed in embers.conf'
(host, service) = get_host_and_service(arg.data)
if arg.get == 'host':
print host
if arg.get == 'service':
print service
if arg.service:
assert arg.service in conf.get('services'), 'Service is not listed in embers.conf'
if arg.get == 'host':
print [host for host in conf['cluster'] if arg.service in conf['cluster'][host]['services']]
if arg.get == 'inputs':
print conf['services'][arg.service]['inputs']
if arg.get == 'outputs':
print conf['services'][arg.service]['outputs']
if arg.cluster or arg.get == 'cluster':
print ' '.join(conf.get('cluster',''))
if arg.get == 'data' and not (arg.service or arg.data or arg.host):
print ' '.join(conf.get('data',''))
if arg.get == 'host' and not (arg.service or arg.data or arg.host):
print ' '.join(conf.get('cluster',''))
if arg.get == 'service' and not (arg.service or arg.data or arg.host):
print ' '.join(conf.get('services',''))
except Exception, e:
log.error("Requested information not in embers.conf, error %s", e)
def main():
"""A little utility to handle reading and writing streams
to and from a queue.
--pub <queue> : publish what's read from stdin to <queue>
--sub <queue> : read from <queue> and write the messages to stdout
--cat : when used with --pub, write all published messages to stdout
--clean : check in incoming and outgoing messages.
Verify the message is correct JSON and add
an embersId if needed.
Other standard EMBERS options (e.g. --verbose).
"""
import args
import logs
import message
ap = args.get_parser()
ap.add_argument('--clean', action="store_true",
help='Verify message format and add standard fields such as embersId.')
ap.add_argument('--cat', action="store_true",
help='Write all published messages to stdout.')
arg = ap.parse_args()
logs.init(arg)
init(arg)
assert arg.sub or arg.pub, "Need to subscribe or publish to something."
# need to use the raw/utf handler unless we are doing clean
marshal = UnicodeMarshal()
if arg.clean:
marshal = JsonMarshal()
subq = None
if arg.sub: # read a queue
subq = open(arg.sub, 'sub', marshal=marshal, ssh_key=arg.ssh_key, ssh_conn=arg.tunnel)
else: # read stdin
subq = StreamQueue(sys.stdin, marshal=marshal)
pubq = None
if arg.pub: # send to queue
pubq = open(arg.pub, 'pub', capture=arg.cat, marshal=marshal)
else: # send to stdout
pubq = StreamQueue(sys.stdout, mode='w', marshal=marshal)
rc = 0
try:
it = subq.__iter__()
while True:
m = ''
try:
m = it.next()
if arg.clean:
m = message.clean(m)
if m:
pubq.write(m)
except StopIteration:
break
except KeyboardInterrupt:
break
except:
rc += 1
if m:
log.exception('Could not process message %s' % (m,))
else:
log.exception('Unknown processing error')
except KeyboardInterrupt:
pass
except:
rc = 1
log.exception('Top level exception')
return rc
def main():
"""A little utility to handle reading and writing streams
to and from a queue.
--pub <queue> : publish what's read from stdin to <queue>
--sub <queue> : read from <queue> and write the messages to stdout
--cat : when used with --pub, write all published messages to stdout
--clean : check in incoming and outgoing messages.
Verify the message is correct JSON and add
an embersId if needed.
Other standard EMBERS options (e.g. --verbose).
"""
import args
import logs
import message
ap = args.get_parser()
ap.add_argument(
'--clean',
action="store_true",
help='Verify message format and add standard fields such as embersId.')
ap.add_argument('--cat',
action="store_true",
help='Write all published messages to stdout.')
arg = ap.parse_args()
logs.init(arg)
init(arg)
assert arg.sub or arg.pub, "Need to subscribe or publish to something."
# need to use the raw/utf handler unless we are doing clean
marshal = UnicodeMarshal()
if arg.clean:
marshal = JsonMarshal()
subq = None
if arg.sub: # read a queue
subq = open(arg.sub,
'sub',
marshal=marshal,
ssh_key=arg.ssh_key,
ssh_conn=arg.tunnel)
else: # read stdin
subq = StreamQueue(sys.stdin, marshal=marshal)
pubq = None
if arg.pub: # send to queue
pubq = open(arg.pub, 'pub', capture=arg.cat, marshal=marshal)
else: # send to stdout
pubq = StreamQueue(sys.stdout, mode='w', marshal=marshal)
rc = 0
try:
it = subq.__iter__()
while True:
m = ''
try:
m = it.next()
if arg.clean:
m = message.clean(m)
if m:
pubq.write(m)
except StopIteration:
break
except KeyboardInterrupt:
break
except:
rc += 1
if m:
log.exception('Could not process message %s' % (m, ))
else:
log.exception('Unknown processing error')
except KeyboardInterrupt:
pass
except:
rc = 1
log.exception('Top level exception')
return rc