def cmdLineParse():
parser = argparse.ArgumentParser(description='Download GPS data over SAR coverage.',\
formatter_class=argparse.RawTextHelpFormatter,\
epilog=INTRODUCTION+'\n'+EXAMPLE)
parser.add_argument('-p', dest='station', help='GPS stations name')
parser.add_argument('-s', dest='start', help='start date.')
parser.add_argument('-e', dest='end', help='end date.')
parser.add_argument('-t',
dest='utc_time',
help='troposphere of UTC time, e.g., 14:00')
parser.add_argument('-o', dest='out', help='output file name.')
parser.add_argument(
'--inside',
action="store_true",
default=False,
help=
'Constraining stations inside the SAR coverage, otherwise, in the corner rectangle region.'
)
parser.add_argument(
'--extend_search',
dest='extend_search',
help='extend the search region based on the box corner.')
inps = parser.parse_args()
if not inps.station:
parser.print_usage()
sys.exit(
os.path.basename(sys.argv[0]) +
': error: GPS station should be provided.')
return inps
def cmdLineParse():
parser = argparse.ArgumentParser(description='Download GPS data over SAR coverage.',\
formatter_class=argparse.RawTextHelpFormatter,\
epilog=INTRODUCTION+'\n'+EXAMPLE)
parser.add_argument('-f', dest='search_gps', help='GPS stations file')
parser.add_argument('-s', dest='start', help='start date.')
parser.add_argument('-e', dest='end', help='end date.')
parser.add_argument('-t',
dest='run_time',
help='downloading time in pegasus.')
parser.add_argument('-u',
dest='utc_time',
help='UTC time for troposphere.')
parser.add_argument('-m',
dest='memory',
help='downloading memory in pegasus.')
inps = parser.parse_args()
if not inps.search_gps:
parser.print_usage()
sys.exit(
os.path.basename(sys.argv[0]) +
': error: GPS station file should be provided.')
return inps
def run_command_line(parser, args):
if not args.config:
parser.print_usage()
return
logging.basicConfig(
format='%(message)s',
level=(logging.DEBUG if args.very_verbose else
logging.INFO if args.verbose else logging.WARNING))
# Configuration and settings (command line arguments override config file options)
logger.debug("Parsing options from %s", args.config)
with open(args.config) as config:
options = config_to_options(config.read())
# Allow command line arguments to override options
override_options(options['connection'], args)
override_options(options['settings'], args)
# Set charting context, which determines how charts are rendered
set_chart_context("paper")
# Set output directory if required
if args.output_directory:
logger.info("Changing working directory to %s" % args.output_directory)
os.chdir(args.output_directory)
# Query JIRA and run calculators
jira = get_jira_client(options['connection'])
logger.info("Running calculators")
query_manager = QueryManager(jira, options['settings'])
run_calculators(CALCULATORS, query_manager, options['settings'])
def main():
from optparse import OptionParser
parser = OptionParser()
parser.add_option('-c', '--connnection', dest='connection_string', type='string', help='SQL connection URI such as sqlite:///full/path/to/database.db')
parser.add_option('-s', '--search', dest='search', type='string', help='Archive search results such as #foo')
parser.add_option('-u', '--user', dest='user', type='string', help='Archive user timeline')
parser.add_option('--sql', action='store_true', help='Print backend-specific SQL commands to create database tables')
(options, args) = parser.parse_args()
if not options.connection_string:
parser.print_usage()
print 'Try the --help option'
sys.exit(1)
connect_sql(options.connection_string)
if options.sql:
print_sql()
sys.exit(0)
twitter_search = connect_twitter()
# process command line
if options.search:
print '*** Archiving search: %s' % options.search
sa = SearchArchiver(options.search, twitter_search)
archive_loop(sa)
if options.user:
for user in options.user.split(','):
print '*** Archiving user timelime: %s' % user
ta = TimelineArchiver(user, twitter_search)
archive_loop(ta)
print ''
def cmdLineParse():
parser = argparse.ArgumentParser(description='Download GPS data over SAR coverage.',\
formatter_class=argparse.RawTextHelpFormatter,\
epilog=INTRODUCTION+'\n'+EXAMPLE)
parser.add_argument('RefDate', help='Reference date of differential InSAR')
parser.add_argument('-d', dest='date', help='date for estimation.')
parser.add_argument('--datetxt', dest='datetxt', help='text file of date.')
parser.add_argument('--Atm',
action="store_true",
default=False,
help='Geting SAR LOS tropospheric delay.')
parser.add_argument('--Def',
action="store_true",
default=False,
help='Getting SAR LOS deformation.')
inps = parser.parse_args()
if not inps.date and not inps.datetxt:
parser.print_usage()
sys.exit(
os.path.basename(sys.argv[0]) +
': error: date and date_txt File, at least one is needed.')
return inps
def cmdLineParse():
parser = argparse.ArgumentParser(description='Compare InSAR results and GPS results both for APS and deformation.',\
formatter_class=argparse.RawTextHelpFormatter,\
epilog=INTRODUCTION+'\n'+EXAMPLE)
parser.add_argument('UNW', help='unwrapped interferograms.')
parser.add_argument('width', help='width of the unwrap image.')
parser.add_argument('-m',
dest='master',
help='Master date of interferograms.')
parser.add_argument('-s',
dest='slave',
help='Slave date of interferograms.')
parser.add_argument('-r',
'--RefGPS',
dest='ref_gps',
help='Referenced GPS station.')
parser.add_argument('--max_dist',
dest='max_dist',
help='maximum search radius of InSAR results.')
parser.add_argument('--diff_atm',
dest='diff_atm',
help='differential file of GPS ASP results.')
parser.add_argument('--diff_def',
dest='diff_def',
help='differential file of GPS Def results.')
parser.add_argument('-t',
'--trans',
dest='gps_coord',
help='GPS coordinates transfomration file.')
parser.add_argument('--byteorder',
dest='byteorder',
help='byte order of the unwrap image.')
parser.add_argument(
'--AtmDir',
dest='AtmDir',
help='GPS-based atm datasets directory. [default: current directory.]')
parser.add_argument(
'--DefDir',
dest='DefDir',
help='GPS-based def datasets directory.[default: current directory.]')
parser.add_argument('--Atm',
action="store_true",
default=False,
help='Comparing APS results of InSAR and GPS.')
parser.add_argument('--Def',
action="store_true",
default=False,
help='Comparing DEF results of InSAR and GPS.')
inps = parser.parse_args()
if not inps.Atm and not inps.Def:
parser.print_usage()
sys.exit(
os.path.basename(sys.argv[0]) +
': error: Comparison of Atm and Def, at least one is needed.')
return inps
def contribute_to_subparser(self, parser):
parser.set_defaults(function=lambda args: parser.print_usage())
subparsers = parser.add_subparsers(title='commands')
setup_parser = subparsers.add_parser(
'setup', help='setup Google Drive API access through OAuth')
setup_parser.set_defaults(function=self.setup)
def cmdLineParse():
parser = argparse.ArgumentParser(description='Download GPS data over SAR coverage.',\
formatter_class=argparse.RawTextHelpFormatter,\
epilog=INTRODUCTION+'\n'+EXAMPLE)
parser.add_argument('date',help='SAR date.')
parser.add_argument('--station', dest='station_name', help='GPS station name.')
parser.add_argument('--station_txt', dest='station_txt', help='GPS station txet file.')
inps = parser.parse_args()
if not inps.station_name and not inps.station_txt:
parser.print_usage()
sys.exit(os.path.basename(sys.argv[0])+': error: station name or station txt should provide at least one.')
return inps
def main():
parser, arguments = parse_arguments()
if arguments.file:
cookies_init()
try:
with open(arguments.file) as urls:
download_executor(list(urls), arguments.concurrent)
except Exception as e:
print(e)
elif arguments.url:
cookies_init()
try:
urls = [arguments.url]
download_executor(urls, arguments.concurrent)
except Exception as e:
print(e)
else:
parser.print_usage()
sys.exit(1)
def main():
param0 = ["-y", "station", "-q", "format=text", "-q", "level=network"]
param1 = ["-y", "station", "-q", "format=xml", "-q", "level=response"]
param2 = ["-y", "dataselect", "-z"]
nets = set()
def add_param0(option, opt_str, value, parser):
param0.append(opt_str)
param0.append(value)
def add_param1(option, opt_str, value, parser):
param1.append(opt_str)
param1.append(value)
def add_param2(option, opt_str, value, parser):
param2.append(opt_str)
param2.append(value)
def add_param12(option, opt_str, value, parser):
add_param1(option, opt_str, value, parser)
add_param2(option, opt_str, value, parser)
def add_param(option, opt_str, value, parser):
add_param0(option, opt_str, value, parser)
add_param1(option, opt_str, value, parser)
add_param2(option, opt_str, value, parser)
parser = optparse.OptionParser(
usage="Usage: %prog [-h|--help] [OPTIONS] -o file",
version="%prog " + VERSION)
parser.set_defaults(url="http://geofon.gfz-potsdam.de/eidaws/routing/1/",
timeout=600,
retries=10,
retry_wait=60,
threads=5)
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="verbose mode")
parser.add_option("-u",
"--url",
type="string",
action="callback",
callback=add_param,
help="URL of routing service (default %default)")
parser.add_option("-N",
"--network",
type="string",
action="callback",
callback=add_param12,
help="network code or pattern")
parser.add_option("-S",
"--station",
type="string",
action="callback",
callback=add_param12,
help="station code or pattern")
parser.add_option("-L",
"--location",
type="string",
action="callback",
callback=add_param12,
help="location code or pattern")
parser.add_option("-C",
"--channel",
type="string",
action="callback",
callback=add_param12,
help="channel code or pattern")
parser.add_option("-s",
"--starttime",
type="string",
action="callback",
callback=add_param12,
help="start time")
parser.add_option("-e",
"--endtime",
type="string",
action="callback",
callback=add_param12,
help="end time")
parser.add_option("-t",
"--timeout",
type="int",
action="callback",
callback=add_param,
help="request timeout in seconds (default %default)")
parser.add_option("-r",
"--retries",
type="int",
action="callback",
callback=add_param,
help="number of retries (default %default)")
parser.add_option(
"-w",
"--retry-wait",
type="int",
action="callback",
callback=add_param,
help="seconds to wait before each retry (default %default)")
parser.add_option(
"-n",
"--threads",
type="int",
action="callback",
callback=add_param,
help="maximum number of download threads (default %default)")
parser.add_option("-c",
"--credentials-file",
type="string",
action="callback",
callback=add_param2,
help="URL,user,password file (CSV format) for queryauth")
parser.add_option("-a",
"--auth-file",
type="string",
action="callback",
callback=add_param2,
help="file that contains the auth token")
parser.add_option("-p",
"--post-file",
type="string",
action="callback",
callback=add_param12,
help="request file in FDSNWS POST format")
parser.add_option("-f",
"--arclink-file",
type="string",
action="callback",
callback=add_param12,
help="request file in ArcLink format")
parser.add_option("-b",
"--breqfast-file",
type="string",
action="callback",
callback=add_param12,
help="request file in breq_fast format")
parser.add_option("-d",
"--dataless",
action="store_true",
default=False,
help="create dataless SEED volume")
parser.add_option("-l",
"--label",
type="string",
help="label of SEED volume")
parser.add_option("-o",
"--output-file",
type="string",
help="file where SEED data is written")
parser.add_option("-z",
"--no-citation",
action="store_true",
default=False,
help="suppress network citation info")
parser.add_option("-Z",
"--no-check",
action="store_true",
default=False,
help="suppress checking received routes and data")
(options, args) = parser.parse_args()
if args or not options.output_file:
parser.print_usage(sys.stderr)
return 1
def log_alert(s):
if sys.stderr.isatty():
s = "\033[31m" + s + "\033[m"
sys.stderr.write(s + '\n')
sys.stderr.flush()
def log_notice(s):
if sys.stderr.isatty():
s = "\033[32m" + s + "\033[m"
sys.stderr.write(s + '\n')
sys.stderr.flush()
def log_verbose(s):
sys.stderr.write(s + '\n')
sys.stderr.flush()
def log_silent(s):
pass
logs.error = log_alert
logs.warning = log_alert
logs.notice = log_notice
logs.info = (log_silent, log_verbose)[options.verbose]
logs.debug = log_silent
try:
proc = exec_fetch(param1, None, options.verbose, options.no_check)
except OSError as e:
logs.error(str(e))
logs.error("error running fdsnws_fetch")
return 1
inv = fdsnxml.Inventory()
with tempfile.TemporaryFile() as fd:
shutil.copyfileobj(proc.stdout, fd)
proc.stdout.close()
proc.wait()
if proc.returncode != 0:
logs.error("error running fdsnws_fetch")
return 1
if fd.tell():
fd.seek(0)
try:
inv.load_fdsnxml(fd)
except fdsnxml.Error as e:
logs.error(str(e))
return 1
seed_volume = fseed.SEEDVolume(inv, ORGANIZATION, options.label, False)
if options.dataless:
for net in iterinv(inv.network):
for sta in iterinv(net.station):
for loc in iterinv(sta.sensorLocation):
for cha in iterinv(loc.stream):
try:
seed_volume.add_chan(net.code, sta.code, loc.code,
cha.code, cha.start, cha.end)
except fseed.SEEDError as e:
logs.warning("%s.%s.%s.%s.%s: %s" %
(net.code, sta.code, loc.code,
cha.code, cha.start.isoformat(), e))
else:
try:
proc = exec_fetch(param2, None, options.verbose, options.no_check)
except OSError as e:
logs.error(str(e))
logs.error("error running fdsnws_fetch")
return 1
try:
for rec in mseedlite.Input(proc.stdout):
try:
seed_volume.add_data(rec)
except fseed.SEEDError as e:
logs.warning("%s.%s.%s.%s.%s: %s" %
(rec.net.code, rec.sta.code, rec.loc.code,
rec.cha.code, rec.cha.start.isoformat(), e))
nets.add((rec.net, rec.begin_time.year))
except mseedlite.MSeedError as e:
logs.error(str(e))
proc.stdout.close()
proc.wait()
if proc.returncode != 0:
logs.error("error running fdsnws_fetch")
return 1
with open(options.output_file, "wb") as fd:
try:
seed_volume.output(fd)
except fseed.SEEDError as e:
logs.error(str(e))
return 1
if nets and not options.no_citation:
logs.info("retrieving network citation info")
get_citation(nets, param0, options.verbose)
return 0
def main():
qp = {}
def add_qp(option, opt_str, value, parser):
if option.dest == 'query':
try:
(p, v) = value.split('=', 1)
qp[p] = v
except ValueError:
raise optparse.OptionValueError("%s expects parameter=value" %
opt_str)
else:
qp[option.dest] = value
parser = optparse.OptionParser(
usage="Usage: %prog [-h|--help] [OPTIONS] -o file",
version="%prog " + VERSION,
add_help_option=False)
parser.set_defaults(url="http://geofon.gfz-potsdam.de/eidaws/routing/1/",
timeout=600,
retries=10,
retry_wait=60,
threads=5)
parser.add_option("-h",
"--help",
action="store_true",
default=False,
help="show help message and exit")
parser.add_option("-l",
"--longhelp",
action="store_true",
default=False,
help="show extended help message and exit")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="verbose mode")
parser.add_option("-u",
"--url",
type="string",
help="URL of routing service (default %default)")
parser.add_option("-y",
"--service",
type="string",
action="callback",
callback=add_qp,
help="target service (default dataselect)")
parser.add_option("-N",
"--network",
type="string",
action="callback",
callback=add_qp,
help="network code or pattern")
parser.add_option("-S",
"--station",
type="string",
action="callback",
callback=add_qp,
help="station code or pattern")
parser.add_option("-L",
"--location",
type="string",
action="callback",
callback=add_qp,
help="location code or pattern")
parser.add_option("-C",
"--channel",
type="string",
action="callback",
callback=add_qp,
help="channel code or pattern")
parser.add_option("-s",
"--starttime",
type="string",
action="callback",
callback=add_qp,
help="start time")
parser.add_option("-e",
"--endtime",
type="string",
action="callback",
callback=add_qp,
help="end time")
parser.add_option("-q",
"--query",
type="string",
action="callback",
callback=add_qp,
metavar="PARAMETER=VALUE",
help="additional query parameter")
parser.add_option("-t",
"--timeout",
type="int",
help="request timeout in seconds (default %default)")
parser.add_option("-r",
"--retries",
type="int",
help="number of retries (default %default)")
parser.add_option("-w",
"--retry-wait",
type="int",
help="seconds to wait before each retry "
"(default %default)")
parser.add_option("-n",
"--threads",
type="int",
help="maximum number of download threads "
"(default %default)")
parser.add_option("-c",
"--credentials-file",
type="string",
help="URL,user,password file (CSV format) for queryauth")
parser.add_option("-a",
"--auth-file",
type="string",
help="file that contains the auth token")
parser.add_option("-p",
"--post-file",
type="string",
help="request file in FDSNWS POST format")
parser.add_option("-f",
"--arclink-file",
type="string",
help="request file in ArcLink format")
parser.add_option("-b",
"--breqfast-file",
type="string",
help="request file in breq_fast format")
parser.add_option("-o",
"--output-file",
type="string",
help="file where downloaded data is written")
parser.add_option("-z",
"--no-citation",
action="store_true",
default=False,
help="suppress network citation info")
parser.add_option("-Z",
"--no-check",
action="store_true",
default=False,
help="suppress checking received routes and data")
(options, args) = parser.parse_args()
if options.help:
print(__doc__.split("Usage Examples", 1)[0], end="")
parser.print_help()
return 0
if options.longhelp:
print(__doc__)
parser.print_help()
return 0
if args or not options.output_file:
parser.print_usage(sys.stderr)
return 1
if bool(options.post_file) + bool(options.arclink_file) + \
bool(options.breqfast_file) > 1:
msg("only one of (--post-file, --arclink-file, --breqfast-file) "
"can be used")
return 1
try:
cred = {}
authdata = None
postdata = None
chans_to_check = set()
if options.credentials_file:
with open(options.credentials_file) as fd:
try:
for (url, user, passwd) in csv.reader(fd):
cred[url] = (user, passwd)
except (ValueError, csv.Error):
raise Error("error parsing %s" % options.credentials_file)
except UnicodeDecodeError:
raise Error("invalid unicode character found in %s" %
options.credentials_file)
if options.auth_file:
with open(options.auth_file, 'rb') as fd:
authdata = fd.read()
else:
try:
with open(DEFAULT_TOKEN_LOCATION, 'rb') as fd:
authdata = fd.read()
options.auth_file = DEFAULT_TOKEN_LOCATION
except IOError:
pass
if authdata:
msg("using token in %s:" % options.auth_file, options.verbose)
try:
proc = subprocess.Popen(['gpg', '--decrypt'],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate(authdata)
if not out:
if isinstance(err, bytes):
err = err.decode('utf-8')
msg(err)
return 1
if isinstance(out, bytes):
out = out.decode('utf-8')
msg(out, options.verbose)
except OSError as e:
msg(str(e))
if options.post_file:
try:
with open(options.post_file) as fd:
postdata = fd.read()
except UnicodeDecodeError:
raise Error("invalid unicode character found in %s" %
options.post_file)
else:
parser = None
if options.arclink_file:
parser = ArclinkParser()
try:
parser.parse(options.arclink_file)
except UnicodeDecodeError:
raise Error("invalid unicode character found in %s" %
options.arclink_file)
elif options.breqfast_file:
parser = BreqParser()
try:
parser.parse(options.breqfast_file)
except UnicodeDecodeError:
raise Error("invalid unicode character found in %s" %
options.breqfast_file)
if parser is not None:
if parser.failstr:
msg(parser.failstr)
return 1
postdata = parser.postdata
if not options.no_check:
if postdata:
for line in postdata.splitlines():
nslc = line.split()[:4]
if nslc[2] == '--': nslc[2] = ''
chans_to_check.add('.'.join(nslc))
else:
net = qp.get('network', '*')
sta = qp.get('station', '*')
loc = qp.get('location', '*')
cha = qp.get('channel', '*')
for n in net.split(','):
for s in sta.split(','):
for l in loc.split(','):
for c in cha.split(','):
if l == '--': l = ''
chans_to_check.add('.'.join((n, s, l, c)))
url = RoutingURL(urlparse.urlparse(options.url), qp)
dest = open(options.output_file, 'wb')
nets = route(url, cred, authdata, postdata, dest, chans_to_check,
options.timeout, options.retries, options.retry_wait,
options.threads, options.verbose)
if nets and not options.no_citation:
msg("retrieving network citation info", options.verbose)
get_citation(nets, options)
else:
msg("", options.verbose)
msg(
"In case of problems with your request, plese use the contact "
"form at\n\n"
" http://www.orfeus-eu.org/organization/contact/form/"
"?recipient=EIDA\n", options.verbose)
except (IOError, Error) as e:
msg(str(e))
return 1
return 0
try:
_update_db_tools_labs(cfg)
except Exception, e:
traceback.print_exc(file = sys.stderr)
email('Failed to build database for %s' % cfg.lang_code, logfiles)
sys.exit(1)
utils.mkdir_p(cfg.log_dir)
for logfile in logfiles:
os.rename(logfile, os.path.join(cfg.log_dir, logfile))
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Update the CitationHunt databases.')
parser.add_argument('lang_code',
help='One of the language codes in ../config.py')
args = parser.parse_args()
if not (utils.running_in_tools_labs() and utils.running_in_virtualenv()):
print >>sys.stderr, 'Not running in a virtualenv in Tools Labs!'
sys.exit(1)
if args.lang_code not in config.LANG_CODES_TO_LANG_NAMES:
print >>sys.stderr, 'Invalid lang code! Use one of: ',
print >>sys.stderr, config.LANG_CODES_TO_LANG_NAMES.keys()
parser.print_usage()
sys.exit(1)
cfg = config.get_localized_config(args.lang_code)
update_db_tools_labs(cfg)
def main():
param0 = ["-y", "station", "-q", "format=text", "-q", "level=network"]
param1 = ["-y", "station", "-q", "format=text", "-q", "level=channel"]
param2 = ["-y", "dataselect", "-z"]
times = {"starttime": datetime.datetime(1900, 1, 1), "endtime": datetime.datetime(2100, 1, 1)}
nets = set()
def add_param0(option, opt_str, value, parser):
param0.append(opt_str)
param0.append(value)
def add_param1(option, opt_str, value, parser):
param1.append(opt_str)
param1.append(value)
def add_param2(option, opt_str, value, parser):
param2.append(opt_str)
param2.append(value)
def add_param(option, opt_str, value, parser):
add_param0(option, opt_str, value, parser)
add_param1(option, opt_str, value, parser)
add_param2(option, opt_str, value, parser)
def add_time(option, opt_str, value, parser):
add_param1(option, opt_str, value, parser)
try:
t = dateutil.parser.parse(value)
except ValueError as e:
raise optparse.OptionValueError("option '%s': invalid time value: '%s'" % (opt_str, value))
if t.tzinfo is not None:
t = t.astimezone(dateutil.tz.tzutc()).replace(tzinfo=None)
times[option.dest] = t
parser = optparse.OptionParser(
usage="Usage: %prog [-h|--help] [OPTIONS] -o directory",
version="%prog " + VERSION)
parser.set_defaults(
url="http://geofon.gfz-potsdam.de/eidaws/routing/1/",
timeout=600,
retries=10,
retry_wait=60,
threads=5,
max_lines=1000,
max_timespan=1440)
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="verbose mode")
parser.add_option("-u", "--url", type="string", action="callback",
callback=add_param,
help="URL of routing service (default %default)")
parser.add_option("-N", "--network", type="string", action="callback",
callback=add_param1,
help="network code or pattern")
parser.add_option("-S", "--station", type="string", action="callback",
callback=add_param1,
help="station code or pattern")
parser.add_option("-L", "--location", type="string", action="callback",
callback=add_param1,
help="location code or pattern")
parser.add_option("-C", "--channel", type="string", action="callback",
callback=add_param1,
help="channel code or pattern")
parser.add_option("-s", "--starttime", type="string", action="callback",
callback=add_time,
help="start time")
parser.add_option("-e", "--endtime", type="string", action="callback",
callback=add_time,
help="end time")
parser.add_option("-t", "--timeout", type="int", action="callback",
callback=add_param,
help="request timeout in seconds (default %default)")
parser.add_option("-r", "--retries", type="int", action="callback",
callback=add_param,
help="number of retries (default %default)")
parser.add_option("-w", "--retry-wait", type="int", action="callback",
callback=add_param,
help="seconds to wait before each retry (default %default)")
parser.add_option("-n", "--threads", type="int", action="callback",
callback=add_param,
help="maximum number of download threads (default %default)")
parser.add_option("-c", "--credentials-file", type="string", action="callback",
callback=add_param2,
help="URL,user,password file (CSV format) for queryauth")
parser.add_option("-a", "--auth-file", type="string", action="callback",
callback=add_param2,
help="file that contains the auth token")
parser.add_option("-o", "--output-dir", type="string",
help="SDS directory where downloaded data is written")
parser.add_option("-l", "--max-lines", type="int",
help="max lines per request (default %default)")
parser.add_option("-m", "--max-timespan", type="int",
help="max timespan per request in minutes (default %default)")
parser.add_option("-z", "--no-citation", action="store_true", default=False,
help="suppress network citation info")
parser.add_option("-Z", "--no-check", action="store_true", default=False,
help="suppress checking received routes and data")
(options, args) = parser.parse_args()
if args or not options.output_dir:
parser.print_usage(sys.stderr)
return 1
def log_alert(s):
if sys.stderr.isatty():
s = "\033[31m" + s + "\033[m"
sys.stderr.write(s + '\n')
sys.stderr.flush()
def log_notice(s):
if sys.stderr.isatty():
s = "\033[32m" + s + "\033[m"
sys.stderr.write(s + '\n')
sys.stderr.flush()
def log_verbose(s):
sys.stderr.write(s + '\n')
sys.stderr.flush()
def log_silent(s):
pass
logs.error = log_alert
logs.warning = log_alert
logs.notice = log_notice
logs.info = (log_silent, log_verbose)[options.verbose]
logs.debug = log_silent
try:
try:
proc = exec_fetch(param1, None, options.verbose, options.no_check)
except OSError as e:
logs.error(str(e))
logs.error("error running fdsnws_fetch")
return 1
timespan = {}
for line in proc.stdout:
if isinstance(line, bytes):
line = line.decode('utf-8')
if not line or line.startswith('#'):
continue
starttime = max(dateutil.parser.parse(line.split('|')[15]), times['starttime'])
endtime = min(dateutil.parser.parse(line.split('|')[16]), times['endtime'])
if starttime.tzinfo is not None:
starttime = starttime.astimezone(dateutil.tz.tzutc()).replace(tzinfo=None)
if endtime.tzinfo is not None:
endtime = endtime.astimezone(dateutil.tz.tzutc()).replace(tzinfo=None)
try:
ts = timespan[tuple(line.split('|')[:4])]
if ts.start > starttime:
ts.start = starttime
ts.current = starttime
if ts.end < endtime:
ts.end = endtime
except KeyError:
timespan[tuple(line.split('|')[:4])] = Timespan(starttime, endtime)
proc.stdout.close()
proc.wait()
if proc.returncode != 0:
logs.error("error running fdsnws_fetch")
return 1
if os.path.exists(options.output_dir):
scan_sds(options.output_dir, timespan, nets)
while len(timespan) > 0:
postdata = ""
ts_used = random.sample(timespan.items(), min(len(timespan), options.max_lines))
for ((net, sta, loc, cha), ts) in ts_used:
te = min(ts.end, ts.start + datetime.timedelta(minutes=options.max_timespan))
if loc == '':
loc = '--'
postdata += "%s %s %s %s %sZ %sZ\n" \
% (net, sta, loc, cha, ts.start.isoformat(), te.isoformat())
if not isinstance(postdata, bytes):
postdata = postdata.encode('utf-8')
try:
proc = exec_fetch(param2, postdata, options.verbose, options.no_check)
except OSError as e:
logs.error(str(e))
logs.error("error running fdsnws_fetch")
return 1
got_data = False
try:
for rec in mseedlite.Input(proc.stdout):
try:
ts = timespan[(rec.net, rec.sta, rec.loc, rec.cha)]
except KeyError:
logs.warning("unexpected data: %s.%s.%s.%s" % (rec.net, rec.sta, rec.loc, rec.cha))
continue
if rec.end_time <= ts.current:
continue
sds_dir = "%s/%d/%s/%s/%s.D" \
% (options.output_dir, rec.begin_time.year, rec.net, rec.sta, rec.cha)
sds_file = "%s.%s.%s.%s.D.%s" \
% (rec.net, rec.sta, rec.loc, rec.cha, rec.begin_time.strftime('%Y.%j'))
if not os.path.exists(sds_dir):
os.makedirs(sds_dir)
with open(sds_dir + '/' + sds_file, 'ab') as fd:
fd.write(rec.header + rec.data)
ts.current = rec.end_time
nets.add((rec.net, rec.begin_time.year))
got_data = True
except mseedlite.MSeedError as e:
logs.error(str(e))
proc.stdout.close()
proc.wait()
if proc.returncode != 0:
logs.error("error running fdsnws_fetch")
return 1
for ((net, sta, loc, cha), ts) in ts_used:
if not got_data:
# no progress, skip to next segment
ts.start += datetime.timedelta(minutes=options.max_timespan)
else:
# continue from current position
ts.start = ts.current
if ts.start >= ts.end:
# timespan completed
del timespan[(net, sta, loc, cha)]
if nets and not options.no_citation:
logs.info("retrieving network citation info")
get_citation(nets, param0, options.verbose)
except (IOError, Error) as e:
logs.error(str(e))
return 1
return 0
def main():
qp = {}
def add_qp(option, opt_str, value, parser):
if option.dest == 'query':
try:
(p, v) = value.split('=', 1)
qp[p] = v
except ValueError:
raise optparse.OptionValueError("%s expects parameter=value"
% opt_str)
else:
qp[option.dest] = value
parser = optparse.OptionParser(
usage="Usage: %prog [-h|--help] [OPTIONS] -o file",
version="%prog " + VERSION,
add_help_option=False)
parser.set_defaults(
url="http://geofon.gfz-potsdam.de/eidaws/routing/1/",
timeout=600,
retries=10,
retry_wait=60,
threads=5)
parser.add_option("-h", "--help", action="store_true", default=False,
help="show help message and exit")
parser.add_option("-l", "--longhelp", action="store_true", default=False,
help="show extended help message and exit")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="verbose mode")
parser.add_option("-u", "--url", type="string",
help="URL of routing service (default %default)")
parser.add_option("-y", "--service", type="string", action="callback",
callback=add_qp,
help="target service (default dataselect)")
parser.add_option("-N", "--network", type="string", action="callback",
callback=add_qp,
help="network code or pattern")
parser.add_option("-S", "--station", type="string", action="callback",
callback=add_qp,
help="station code or pattern")
parser.add_option("-L", "--location", type="string", action="callback",
callback=add_qp,
help="location code or pattern")
parser.add_option("-C", "--channel", type="string", action="callback",
callback=add_qp,
help="channel code or pattern")
parser.add_option("-s", "--starttime", type="string", action="callback",
callback=add_qp,
help="start time")
parser.add_option("-e", "--endtime", type="string", action="callback",
callback=add_qp,
help="end time")
parser.add_option("-q", "--query", type="string", action="callback",
callback=add_qp, metavar="PARAMETER=VALUE",
help="additional query parameter")
parser.add_option("-t", "--timeout", type="int",
help="request timeout in seconds (default %default)")
parser.add_option("-r", "--retries", type="int",
help="number of retries (default %default)")
parser.add_option("-w", "--retry-wait", type="int",
help="seconds to wait before each retry "
"(default %default)")
parser.add_option("-n", "--threads", type="int",
help="maximum number of download threads "
"(default %default)")
parser.add_option("-c", "--credentials-file", type="string",
help="URL,user,password file (CSV format) for queryauth")
parser.add_option("-a", "--auth-file", type="string",
help="file that contains the auth token")
parser.add_option("-p", "--post-file", type="string",
help="request file in FDSNWS POST format")
parser.add_option("-f", "--arclink-file", type="string",
help="request file in ArcLink format")
parser.add_option("-b", "--breqfast-file", type="string",
help="request file in breq_fast format")
parser.add_option("-o", "--output-file", type="string",
help="file where downloaded data is written")
parser.add_option("-z", "--no-citation", action="store_true", default=False,
help="suppress network citation info")
(options, args) = parser.parse_args()
if options.help:
print(__doc__.split("Usage Examples", 1)[0], end="")
parser.print_help()
return 0
if options.longhelp:
print(__doc__)
parser.print_help()
return 0
if args or not options.output_file:
parser.print_usage()
return 1
if bool(options.post_file) + bool(options.arclink_file) + \
bool(options.breqfast_file) > 1:
msg("only one of (--post-file, --arclink-file, --breqfast-file) "
"can be used")
return 1
try:
cred = {}
authdata = None
postdata = None
if options.credentials_file:
with open(options.credentials_file) as fd:
try:
for (url, user, passwd) in csv.reader(fd):
cred[url] = (user, passwd)
except (ValueError, csv.Error):
raise Error("error parsing %s" % options.credentials_file)
if options.auth_file:
with open(options.auth_file) as fd:
authdata = fd.read()
if options.post_file:
with open(options.post_file) as fd:
postdata = fd.read()
else:
parser = None
if options.arclink_file:
parser = ArclinkParser()
parser.parse(options.arclink_file)
elif options.breqfast_file:
parser = BreqParser()
parser.parse(options.breqfast_file)
if parser is not None:
if parser.failstr:
msg(parser.failstr)
return 1
postdata = parser.postdata
url = RoutingURL(urlparse.urlparse(options.url), qp)
dest = open(options.output_file, 'wb')
nets = route(url, cred, authdata, postdata, dest, options.timeout,
options.retries, options.retry_wait, options.threads,
options.verbose)
if nets and not options.no_citation:
msg("retrieving network citation info", options.verbose)
get_citation(nets, options)
else:
msg("", options.verbose)
msg("In case of problems with your request, plese use the contact "
"form at\n\n"
" http://www.orfeus-eu.org/organization/contact/form/\n",
options.verbose)
except (IOError, Error) as e:
msg(str(e))
return 1
return 0
def main():
qp = {}
def add_qp(option, opt_str, value, parser):
if option.dest == 'query':
try:
(p, v) = value.split('=', 1)
qp[p] = v
except ValueError:
raise optparse.OptionValueError("%s expects parameter=value" %
opt_str)
else:
qp[option.dest] = value
parser = optparse.OptionParser(
usage="Usage: %prog [-h|--help] [OPTIONS] -o file",
version="%prog " + VERSION,
add_help_option=False)
parser.set_defaults(url="http://geofon.gfz-potsdam.de/eidaws/routing/1/",
timeout=600,
retries=10,
retry_wait=60,
threads=5)
parser.add_option("-h",
"--help",
action="store_true",
default=False,
help="show help message and exit")
parser.add_option("-l",
"--longhelp",
action="store_true",
default=False,
help="show extended help message and exit")
parser.add_option("-v",
"--verbose",
action="store_true",
default=False,
help="verbose mode")
parser.add_option("-u",
"--url",
type="string",
help="URL of routing service (default %default)")
parser.add_option("-y",
"--service",
type="string",
action="callback",
callback=add_qp,
help="target service (default dataselect)")
parser.add_option("-N",
"--network",
type="string",
action="callback",
callback=add_qp,
help="network code or pattern")
parser.add_option("-S",
"--station",
type="string",
action="callback",
callback=add_qp,
help="station code or pattern")
parser.add_option("-L",
"--location",
type="string",
action="callback",
callback=add_qp,
help="location code or pattern")
parser.add_option("-C",
"--channel",
type="string",
action="callback",
callback=add_qp,
help="channel code or pattern")
parser.add_option("-s",
"--starttime",
type="string",
action="callback",
callback=add_qp,
help="start time")
parser.add_option("-e",
"--endtime",
type="string",
action="callback",
callback=add_qp,
help="end time")
parser.add_option("-q",
"--query",
type="string",
action="callback",
callback=add_qp,
metavar="PARAMETER=VALUE",
help="additional query parameter")
parser.add_option("-t",
"--timeout",
type="int",
help="request timeout in seconds (default %default)")
parser.add_option("-r",
"--retries",
type="int",
help="number of retries (default %default)")
parser.add_option("-w",
"--retry-wait",
type="int",
help="seconds to wait before each retry "
"(default %default)")
parser.add_option("-n",
"--threads",
type="int",
help="maximum number of download threads "
"(default %default)")
parser.add_option("-c",
"--credentials-file",
type="string",
help="URL,user,password file (CSV format) for queryauth")
parser.add_option("-a",
"--auth-file",
type="string",
help="file that contains the auth token")
parser.add_option("-p",
"--post-file",
type="string",
help="request file in FDSNWS POST format")
parser.add_option("-f",
"--arclink-file",
type="string",
help="request file in ArcLink format")
parser.add_option("-b",
"--breqfast-file",
type="string",
help="request file in breq_fast format")
parser.add_option("-o",
"--output-file",
type="string",
help="file where downloaded data is written")
parser.add_option("-z",
"--no-citation",
action="store_true",
default=False,
help="suppress network citation info")
(options, args) = parser.parse_args()
if options.help:
print(__doc__.split("Usage Examples", 1)[0], end="")
parser.print_help()
return 0
if options.longhelp:
print(__doc__)
parser.print_help()
return 0
if args or not options.output_file:
parser.print_usage()
return 1
if bool(options.post_file) + bool(options.arclink_file) + \
bool(options.breqfast_file) > 1:
msg("only one of (--post-file, --arclink-file, --breqfast-file) "
"can be used")
return 1
try:
cred = {}
authdata = None
postdata = None
if options.credentials_file:
with open(options.credentials_file) as fd:
try:
for (url, user, passwd) in csv.reader(fd):
cred[url] = (user, passwd)
except (ValueError, csv.Error):
raise Error("error parsing %s" % options.credentials_file)
if options.auth_file:
with open(options.auth_file) as fd:
authdata = fd.read()
if options.post_file:
with open(options.post_file) as fd:
postdata = fd.read()
else:
parser = None
if options.arclink_file:
parser = ArclinkParser()
parser.parse(options.arclink_file)
elif options.breqfast_file:
parser = BreqParser()
parser.parse(options.breqfast_file)
if parser is not None:
if parser.failstr:
msg(parser.failstr)
return 1
postdata = parser.postdata
url = RoutingURL(urlparse.urlparse(options.url), qp)
dest = open(options.output_file, 'wb')
nets = route(url, cred, authdata, postdata, dest, options.timeout,
options.retries, options.retry_wait, options.threads,
options.verbose)
if nets and not options.no_citation:
msg("retrieving network citation info", options.verbose)
get_citation(nets, options)
else:
msg("", options.verbose)
msg(
"In case of problems with your request, plese use the contact "
"form at\n\n"
" http://www.orfeus-eu.org/organization/contact/form/\n",
options.verbose)
except (IOError, Error) as e:
msg(str(e))
return 1
return 0
def main():
param0 = ["-y", "station", "-q", "format=text", "-q", "level=network"]
param1 = ["-y", "station", "-q", "format=text", "-q", "level=channel"]
param2 = ["-y", "dataselect", "-z"]
times = {"starttime": datetime.datetime(1900, 1, 1), "endtime": datetime.datetime(2100, 1, 1)}
nets = set()
def add_param0(option, opt_str, value, parser):
param0.append(opt_str)
param0.append(value)
def add_param1(option, opt_str, value, parser):
param1.append(opt_str)
param1.append(value)
def add_param2(option, opt_str, value, parser):
param2.append(opt_str)
param2.append(value)
def add_param(option, opt_str, value, parser):
add_param0(option, opt_str, value, parser)
add_param1(option, opt_str, value, parser)
add_param2(option, opt_str, value, parser)
def add_time(option, opt_str, value, parser):
add_param1(option, opt_str, value, parser)
try:
t = dateutil.parser.parse(value)
except ValueError as e:
raise optparse.OptionValueError("option '%s': invalid time value: '%s'" % (opt_str, value))
if t.tzinfo is not None:
t = t.astimezone(dateutil.tz.tzutc()).replace(tzinfo=None)
times[option.dest] = t
parser = optparse.OptionParser(
usage="Usage: %prog [-h|--help] [OPTIONS] -o directory",
version="%prog " + VERSION)
parser.set_defaults(
url="http://geofon.gfz-potsdam.de/eidaws/routing/1/",
timeout=600,
retries=10,
retry_wait=60,
threads=5,
max_lines=1000,
max_timespan=1440)
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="verbose mode")
parser.add_option("-u", "--url", type="string", action="callback",
callback=add_param,
help="URL of routing service (default %default)")
parser.add_option("-N", "--network", type="string", action="callback",
callback=add_param1,
help="network code or pattern")
parser.add_option("-S", "--station", type="string", action="callback",
callback=add_param1,
help="station code or pattern")
parser.add_option("-L", "--location", type="string", action="callback",
callback=add_param1,
help="location code or pattern")
parser.add_option("-C", "--channel", type="string", action="callback",
callback=add_param1,
help="channel code or pattern")
parser.add_option("-s", "--starttime", type="string", action="callback",
callback=add_time,
help="start time")
parser.add_option("-e", "--endtime", type="string", action="callback",
callback=add_time,
help="end time")
parser.add_option("-t", "--timeout", type="int", action="callback",
callback=add_param,
help="request timeout in seconds (default %default)")
parser.add_option("-r", "--retries", type="int", action="callback",
callback=add_param,
help="number of retries (default %default)")
parser.add_option("-w", "--retry-wait", type="int", action="callback",
callback=add_param,
help="seconds to wait before each retry (default %default)")
parser.add_option("-n", "--threads", type="int", action="callback",
callback=add_param,
help="maximum number of download threads (default %default)")
parser.add_option("-c", "--credentials-file", type="string", action="callback",
callback=add_param2,
help="URL,user,password file (CSV format) for queryauth")
parser.add_option("-a", "--auth-file", type="string", action="callback",
callback=add_param2,
help="file that contains the auth token")
parser.add_option("-o", "--output-dir", type="string",
help="SDS directory where downloaded data is written")
parser.add_option("-l", "--max-lines", type="int",
help="max lines per request (default %default)")
parser.add_option("-m", "--max-timespan", type="int",
help="max timespan per request in minutes (default %default)")
parser.add_option("-z", "--no-citation", action="store_true", default=False,
help="suppress network citation info")
parser.add_option("-Z", "--no-check", action="store_true", default=False,
help="suppress checking received routes and data")
(options, args) = parser.parse_args()
if args or not options.output_dir:
parser.print_usage(sys.stderr)
return 1
def log_alert(s):
if sys.stderr.isatty():
s = "\033[31m" + s + "\033[m"
sys.stderr.write(s + '\n')
sys.stderr.flush()
def log_notice(s):
if sys.stderr.isatty():
s = "\033[32m" + s + "\033[m"
sys.stderr.write(s + '\n')
sys.stderr.flush()
def log_verbose(s):
sys.stderr.write(s + '\n')
sys.stderr.flush()
def log_silent(s):
pass
logs.error = log_alert
logs.warning = log_alert
logs.notice = log_notice
logs.info = (log_silent, log_verbose)[options.verbose]
logs.debug = log_silent
try:
try:
proc = exec_fetch(param1, None, options.verbose, options.no_check)
except OSError as e:
logs.error(str(e))
logs.error("error running fdsnws_fetch")
return 1
timespan = {}
for line in proc.stdout:
if isinstance(line, bytes):
line = line.decode('utf-8')
if not line or line.startswith('#'):
continue
starttime = max(dateutil.parser.parse(line.split('|')[15]), times['starttime'])
try:
endtime = min(dateutil.parser.parse(line.split('|')[16]), times['endtime'])
except ValueError:
# dateutil.parser.parse('') now causes ValueError instead of current time
endtime = min(datetime.datetime.now(), times['endtime'])
if starttime.tzinfo is not None:
starttime = starttime.astimezone(dateutil.tz.tzutc()).replace(tzinfo=None)
if endtime.tzinfo is not None:
endtime = endtime.astimezone(dateutil.tz.tzutc()).replace(tzinfo=None)
try:
ts = timespan[tuple(line.split('|')[:4])]
if ts.start > starttime:
ts.start = starttime
ts.current = starttime
if ts.end < endtime:
ts.end = endtime
except KeyError:
timespan[tuple(line.split('|')[:4])] = Timespan(starttime, endtime)
proc.stdout.close()
proc.wait()
if proc.returncode != 0:
logs.error("error running fdsnws_fetch")
return 1
if os.path.exists(options.output_dir):
scan_sds(options.output_dir, timespan, nets)
while len(timespan) > 0:
postdata = ""
ts_used = random.sample(timespan.items(), min(len(timespan), options.max_lines))
for ((net, sta, loc, cha), ts) in ts_used:
te = min(ts.end, ts.start + datetime.timedelta(minutes=options.max_timespan))
if loc == '':
loc = '--'
postdata += "%s %s %s %s %sZ %sZ\n" \
% (net, sta, loc, cha, ts.start.isoformat(), te.isoformat())
if not isinstance(postdata, bytes):
postdata = postdata.encode('utf-8')
try:
proc = exec_fetch(param2, postdata, options.verbose, options.no_check)
except OSError as e:
logs.error(str(e))
logs.error("error running fdsnws_fetch")
return 1
got_data = False
try:
for rec in mseedlite.Input(proc.stdout):
try:
ts = timespan[(rec.net, rec.sta, rec.loc, rec.cha)]
except KeyError:
logs.warning("unexpected data: %s.%s.%s.%s" % (rec.net, rec.sta, rec.loc, rec.cha))
continue
if rec.end_time <= ts.current:
continue
sds_dir = "%s/%d/%s/%s/%s.D" \
% (options.output_dir, rec.begin_time.year, rec.net, rec.sta, rec.cha)
sds_file = "%s.%s.%s.%s.D.%s" \
% (rec.net, rec.sta, rec.loc, rec.cha, rec.begin_time.strftime('%Y.%j'))
if not os.path.exists(sds_dir):
os.makedirs(sds_dir)
with open(sds_dir + '/' + sds_file, 'ab') as fd:
fd.write(rec.header + rec.data)
ts.current = rec.end_time
nets.add((rec.net, rec.begin_time.year))
got_data = True
except mseedlite.MSeedError as e:
logs.error(str(e))
proc.stdout.close()
proc.wait()
if proc.returncode != 0:
logs.error("error running fdsnws_fetch")
return 1
for ((net, sta, loc, cha), ts) in ts_used:
if not got_data:
# no progress, skip to next segment
ts.start += datetime.timedelta(minutes=options.max_timespan)
else:
# continue from current position
ts.start = ts.current
if ts.start >= ts.end:
# timespan completed
del timespan[(net, sta, loc, cha)]
if nets and not options.no_citation:
logs.info("retrieving network citation info")
get_citation(nets, param0, options.verbose)
except (IOError, Error) as e:
logs.error(str(e))
return 1
return 0
def contribute_to_subparser(self, parser):
parser.set_defaults(function=lambda args: parser.print_usage())
subparsers = parser.add_subparsers(title='commands')
setup_parser = subparsers.add_parser('setup', help='setup Google Drive API access through OAuth')
setup_parser.set_defaults(function=self.setup)
default=False,
help='Import allCountries.txt')
parser.add_option(
'--import-alt',
dest='importAlt',
action='store_true',
default=False,
help='Import alternateNames.txt (must happen before --import-gn)')
(options, args) = parser.parse_args()
if len(args) == 0:
filepath = os.getcwd()
elif len(args) == 1:
filepath = args[1]
else:
parser.print_usage()
exit()
# === Do some initial setup of useful globals ===
# --- Like the configuration file :) ---
localdir = os.path.dirname(os.path.abspath(__file__))
_config = SafeConfigParser()
fileList = ["%s/geonames.cfg" % localdir]
if options.configFile is not None:
fileList.append(options.configFile)
_config.read(fileList)
# --- And the MySQL connection ---
_geonamesDB = MySQL.connect(_config.get('MySQL', 'host'),
_config.get('MySQL', 'user'),
_config.get('MySQL', 'password'),
def main():
# === Extract options ===
parser = OptionParser(usage="usage: %prog [options] <# of seconds to audit>")
parser.add_option("-c", "--config", dest='configFile', default=None, help='Path to configuration file')
parser.add_option("-g", "--gracePeriod", dest='gracePeriod', default=0, help='Number of seconds from now backwards to ignore')
parser.add_option("-i", "--historyFile", dest='historyFile', default=None, help='Stores any pending transactions and the last run time')
parser.add_option('-l', "--logFile", dest='logFile', default=None, help='Saves a log of all Amazon transactions')
(options, args) = parser.parse_args()
if len(args) != 1:
parser.print_usage()
exit()
startTime = datetime.fromtimestamp(int(time.time()) - int(args[0]), pytz.utc)
endTime = datetime.fromtimestamp(int(time.time()) - int(options.gracePeriod), pytz.utc)
print("AWS refund audit requested from %s to %s" % (startTime.isoformat(), endTime.isoformat()))
# === Get the configuration options ===
config = SafeConfigParser()
fileList = ['./amazon-config.cfg']
if options.configFile is not None:
fileList.append(options.configFile)
config.read(fileList)
# === Open up ze STOMP ===
sc = DistStomp(config.get('Stomp', 'server'), config.getint('Stomp', 'port'))
sc.connect()
# === Connection to Amazon ===
aws = Amazon(
awsEndpoint = config.get('AwsConfig', 'endpoint'),
awsAccessKey = config.get('AwsConfig', 'accessKey'),
awsSecret = config.get('AwsConfig', 'secretKey')
)
# === Connection to MySQL ===
dbcon = MySQL.connect(
config.get('MySQL', 'host'),
config.get('MySQL', 'user'),
config.get('MySQL', 'password'),
config.get('MySQL', 'schema')
)
# === Open up the history and log files ===
# If the history file exists, it will modify the start time of this script to be the end time of the
# history file.
hfile = None
historyStart = startTime
historyEnd = endTime
if options.historyFile and os.path.exists(options.historyFile):
hfile = open(options.historyFile, 'r')
if hfile.readline().strip() == AWS_HISTORY_FILE_VERSTR:
historyStart = dateutil.parser.parse(hfile.readline().strip())
historyEnd = dateutil.parser.parse(hfile.readline().strip())
startTime = historyEnd
print("History file modified search period, now %s to %s" % (startTime.isoformat(), endTime.isoformat()))
else:
print('Not starting with a valid history file.')
sfile = None
if options.logFile:
sfile = open(options.logFile, 'a')
sfile.write("!!! Starting run for dates %s -> %s\n" % (startTime.isoformat(), endTime.isoformat()))
# === Sanity checks ===
if endTime < startTime:
startTime = endTime
# === Main Application ===
# --- Process all previously pending transactions from the history file. If the transaction is still in some form
# of pending, add it back to the history list.
historyCount = 0
historyList = []
historyStats = {
'Success': 0,
'Pending': 0,
'Failed': 0,
'Ignored': 0
}
if hfile:
print("Processing history file")
for txn in hfile:
historyCount += 1
txn = json.loads(txn)
result = processTransaction(txn, dbcon, aws, sc, sfile, config)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
hfile.close()
# --- Obtain AWS history ---
print("Obtaining AWS transactions for the period %s -> %s" % (startTime.isoformat(), endTime.isoformat()))
awsTransactions = aws.getAccountActivity(startTime, endDate=endTime, fpsOperation='Pay')
print("Obtained %d transactions" % len(awsTransactions))
# --- Main loop: checks each aws transaction against the Civi database; adding it if it doesn't exist ---
txncount = 0
for txn in awsTransactions:
txncount += 1
result = processTransaction(txn, dbcon, aws, sc, sfile, config)
historyStats[result] += 1
if result == 'Pending':
historyList.append(txn)
print("\n--- Finished processing of messages. ---\n")
# --- Prepare the history file for write ---
if options.historyFile:
print("Rewriting history file with %d transactions" % len(historyList))
hfile = open(options.historyFile, 'w')
hfile.write("%s\n%s\n%s\n" % (AWS_HISTORY_FILE_VERSTR, historyStart.isoformat(), endTime.isoformat()))
for txn in historyList:
hfile.write("%s\n" % json.dumps(txn))
print("Flushing history file in preparation for main loop")
hfile.flush()
# --- Final statistics ---
print("%d new AWS messages" % txncount)
print(" Additionally %d messages were processed from history" % historyCount)
print("This resulted in the following:")
for entry in historyStats.items():
print(" %s Messages: %d" % entry)
# === Final Application Cleanup ===
print("\nCleaning up.")
dbcon.close()
sc.disconnect()
if hfile:
hfile.close()
if sfile:
sfile.close()
time.sleep(1) # Let the STOMP library catch up
