def process(geoclient,r,i,capture=False):
rawaddr = makeaddr(r)
log.info(f'{i}: {rawaddr} ..')
log.info(f'capture = {capture}')
d = OrderedDict(r)
if capture and detect_object(i):
log.info(f'{i}: status = SKIP')
return None
try:
status, keytup, response = do_single(geoclient,rawaddr)
log.info(f'{i}: status = {status}')
if capture:
capture_object(i,response)
d['code'] = status.get('code')
d['bbl'] = keytup.get('bbl') if keytup else None
d['bin'] = keytup.get('bin') if keytup else None
d['error'] = status.get('error')
d['message'] = keytup.get('message') if keytup else None
d['message2'] = keytup.get('message2') if keytup else None
except Exception as e:
errmsg = str(e)
log.info(f'{i}: ERROR {errmsg}')
log.error(e)
d['error'] = errmsg
return d
def procmulti(geoclient,records,capture=False,loud=False):
for i,r in enumerate(records):
log.info(f'proc {i} ..')
d = process(geoclient,r,i,capture)
if loud:
print(f'd[{i}] = {d}')
if d is not None:
yield d
def __call__(self) -> bool:
""" Attempt the transfer task """
# The context manager sets the attempt start and finish timestamps
with self:
task = self.task
log.info(f"Attempting transfer of "
f"{task.source.address} on {task.source.filesystem} to "
f"{task.target.address} on {task.target.filesystem}")
with ThreadPoolExecutor(max_workers=1) as executor:
# TODO Different/multiple checksum algorithms
properties = executor.submit(self._get_source_properties,
"md5")
# Run task in main tread and join on properties thread
success = self.task()
source_size, source_checksums = properties.result()
if not success:
log.warning(
f"Attempt failed with exit code {success.exit_code}")
else:
log.info(f"Data copied; verifying...")
try:
target_size = self.size(_TARGET)
if source_size != target_size:
log.warning(f"Attempt failed: "
f"Source is {source_size} bytes; "
f"target is {target_size} bytes")
success = _MISMATCHED_SIZE
raise _VerificationFailure()
# TODO Different/multiple checksum algorithms
source_checksum = source_checksums["md5"]
target_checksum = self.checksum(_TARGET, "md5")
if source_checksum != target_checksum:
log.warning(f"Attempt failed: "
f"Source has checksum {source_checksum}; "
f"target has checksum {target_checksum}")
success = _MISMATCHED_CHECKSUM
raise _VerificationFailure()
# TODO Data metadata: There is no need to set this
# for each intermediary stage; just set the final
# target metadata to that of the original source.
# Think about how to implement this...
except _VerificationFailure:
pass
self.exit_code = success
return bool(success)
def api_fetch(prefix):
if prefix != 'address.json':
return errmsg('invalid service base')
try:
log.debug("query_string = %s" % request.query_string)
return resolve_query(request.query_string)
except Exception as e:
log.info("exception = %s" % e)
log.exception(e)
return errmsg('internal error')
def resolve(callf,query):
try:
param = split_query(query)
except ValueError as e:
return errmsg('invalid query string')
try:
r = callf(param)
except Exception as e:
log.info("exception = %s" % e)
return errmsg('internal error')
return jsonify(r,sort_keys=True)
def main(*args: str) -> None:
# Expected environment variables (commented out entries are
# optional, listed for documentation's sake)
envvars = {
"PG_HOST": "PostgreSQL hostname",
# "PG_PORT": "PostgreSQL port [5432]",
"PG_DATABASE": "PostgreSQL database name",
"PG_USERNAME": "******",
"PG_PASSWORD": "******",
"LSF_CONFIG": "Path to LSF cluster configuration directory",
"LSF_GROUP": "LSF Fairshare group to run under",
"PREP_QUEUE": "LSF queue to use for the preparation phase",
"TRANSFER_QUEUE": "LSF queue to use for the transfer phase",
"IRODS_BASE": "Base iRODS collection into which to transfer"
# "MAX_ATTEMPTS": "Maximum attempts per transfer task [3]"
# "SHEPHERD_LOG": "Logging directory [pwd]"
# "DAISYCHAIN": "Automatically daisychain transfer workers [Yes]"
}
# Mode delegation routines
delegate = {
"submit": submit,
"resume": resume,
"status": status,
"__prepare": prepare,
"__transfer": transfer
}
# Check the appropriate environment variables are set to connect to
# the PostgreSQL database, otherwise bail out
if any(env not in os.environ for env in envvars):
log.critical("Incomplete environment variables")
column_width = max(map(len, envvars))
for env, desc in envvars.items():
log.info(f"* {env:{column_width}} {desc}")
sys.exit(1)
# Check the binary is running in a known mode, otherwise bail out
mode, *mode_args = args
if mode not in delegate:
log.critical(f"No such mode \"{mode}\"")
user_modes = ", ".join(mode for mode in delegate
if not mode.startswith("__"))
log.info(f"Valid user modes: {user_modes}")
sys.exit(1)
# Delegate to appropriate mode
delegate[mode](*mode_args)
def submit(fofn: str, subcollection: str, metadata: str) -> None:
""" Submit a FoFN job to the executioner """
# Set logging directory, if not already
if "SHEPHERD_LOG" not in os.environ:
os.environ["SHEPHERD_LOG"] = str(T.Path(".").resolve())
log_dir = T.Path(os.environ["SHEPHERD_LOG"]).resolve()
log.to_file(log_dir / "submit.log")
fofn_path = T.Path(fofn).resolve()
irods_base = os.environ["IRODS_BASE"]
metadata_path = T.Path(metadata).resolve()
_LOG_HEADER()
log.info(f"Logging to {log_dir}")
log.info(
f"Will transfer contents of {fofn_path} to {irods_base}/{subcollection}"
)
log.info(f"Will apply metadata from {metadata_path} to each file")
state = _GET_STATE()
job = State.Job(state, client_id=_CLIENT)
job.max_attempts = max_attempts = int(os.getenv("MAX_ATTEMPTS", "3"))
job.set_metadata(fofn=str(fofn_path),
irods_base=irods_base,
subcollection=subcollection,
logs=str(log_dir),
shitty_metadata=str(metadata_path),
DAISYCHAIN=_DAISYCHAIN) # NOTE For debugging
log.info(
f"Created new job with ID {job.job_id}, with up to {max_attempts} attempts per task"
)
executor = _GET_EXECUTOR()
prep_options = LSFSubmissionOptions(cores=1,
memory=1000,
group=os.environ["LSF_GROUP"],
queue=os.environ["PREP_QUEUE"])
prep_worker = Exec.Job(f"\"{_BINARY}\" __prepare {job.job_id}")
prep_worker.stdout = prep_worker.stderr = log_dir / "prep.log"
prep_runner, *_ = executor.submit(prep_worker, prep_options)
log.info(f"Preparation phase submitted with LSF ID {prep_runner.job}")
_submit_transfer(job, executor)
def load_hybrid_conf(confdir,args):
"""Loads the requisite config files for the hybrid service (subject to
arg switches), and returns them as a tuple of (dataconf,geoconf)."""
metaconf = load_file(confdir,'hybrid-settings.json')
dataconf = load_file(confdir,'postgres.json')
if args.mock:
usemock = True
elif args.nomock:
usemock = False
else:
usemock = metaconf['mock']
log.info("mock = %s, port = %d" % (usemock,args.port))
suffix = 'mock' if usemock else 'live';
geofile = "nycgeo-%s.json" % suffix
geoconf = load_file(confdir,geofile)
log.info("siteurl = '%s'" % geoconf.get('siteurl'))
return dataconf,geoconf
def get(self,suburl):
log.info("siteurl = %s" % self.siteurl)
log.info("suburl = %s" % suburl)
t0 = time.time()
r = requests.get((self.siteurl+suburl).encode('utf-8'),verify=self.verify)
t1 = time.time()
dt = 1000*(t1-t0)
log.info("status = %d in %.2f ms" % (r.status_code,dt))
for k,v in r.headers.items():
log.info("header - '%s': '%s'" % (k,v))
return r,dt
def main():
global THROW,LOUD
args = parse_args()
LOUD = args.loud
THROW = args.throw
log.info('hi')
log.debug('yow')
nycgeopath = "config/nycgeo-live.json"
geoconf = json.loads(open(nycgeopath,"r").read())
geoclient = SimpleGeoClient(**geoconf)
print(f'agent = {geoclient}')
if args.rawaddr:
status, keytup, response = do_single(geoclient,args.rawaddr)
print(f'status = {status}, keytup = {keytup}')
else:
infile = args.infile
print(f'slurp from {infile} ..')
inrecs = ioany.read_recs(infile)
inrecs = islice(inrecs,args.limit)
do_multi(geoclient,inrecs,capture=args.capture)
print('done')
def resume(job_id: str, force: T.Optional[str] = None) -> None:
""" Resume job """
_LOG_HEADER()
state = _GET_STATE()
job = State.Job(state, client_id=_CLIENT, job_id=int(job_id))
log_dir = T.Path(job.metadata.logs)
log.to_file(log_dir / "resume.log")
if job.status.phase(_PREPARE).start is None:
log.error(
f"Preparation phase for job {job_id} has yet to start; cannot resume."
)
sys.exit(1)
if job.status.complete:
log.info(f"Job {job_id} has already completed.")
sys.exit(0)
if not job.status.phase(_PREPARE).complete:
log.warning(
f"Preparation phase for job {job_id} is still in progress.")
if force != "--force":
log.error(
f"Cannot resume a job in preparation without the --force option."
)
sys.exit(1)
log.info(f"Resuming job {job_id}...")
resumed = State.Job(state,
client_id=_CLIENT,
job_id=int(job_id),
force_restart=True)
executor = _GET_EXECUTOR()
_submit_transfer(resumed, executor)
def _submit_transfer(job: State.Job, executor: Exec.BaseExecutor) -> None:
# Submit the transfer workers
log_dir = T.Path(job.metadata.logs)
# NOTE We're only dealing with the Lustre-iRODS tuple, so this is
# simplified considerably. In a multi-route context, the maximum
# concurrency should be a function of the pairwise minimum of
# filesystems' maximum concurrencies for each stage of the route.
# That function could be, e.g.: max for maximum speed, but also
# maximum waste (in terms of redundant workers); min (or some lower
# constant) for zero wastage, but longer flight times. The
# arithmetic mean would probably be a good thing to go for, without
# implementing complicated dynamic load handling...
max_concurrency = min(fs.max_concurrency for fs in _FILESYSTEMS)
transfer_worker, transfer_options = _transfer_worker(job.job_id, log_dir)
transfer_worker.workers = max_concurrency
transfer_runners = transfer_runner, *_ = list(
executor.submit(transfer_worker, transfer_options))
log.info(
f"Transfer phase submitted with LSF ID {transfer_runner.job} and {len(transfer_runners)} workers"
)
def get(self,suburl):
log.info("siteurl = %s" % self.siteurl)
log.info("suburl = %s" % suburl)
t0 = time.time()
r = requests.get((self.siteurl+suburl).encode('utf-8'))
t1 = time.time()
dt = 1000*(t1-t0)
log.info("status = %d in %.2f ms" % (r.status_code,dt))
return r,dt
def get(self, suburl):
log.info("siteurl = %s" % self.siteurl)
log.info("suburl = %s" % suburl)
t0 = time.time()
r = requests.get((self.siteurl + suburl).encode('utf-8'))
t1 = time.time()
dt = 1000 * (t1 - t0)
log.info("status = %d in %.2f ms" % (r.status_code, dt))
return r, dt
def prepare(job_id: str) -> None:
""" Prepare the Lustre to iRODS task from FoFN """
_LOG_HEADER()
state = _GET_STATE()
job = State.Job(state, client_id=_CLIENT, job_id=int(job_id))
# Get the FoFN path and prefix from the client metadata
fofn = T.Path(job.metadata.fofn)
irods_base = T.Path(job.metadata.irods_base)
subcollection = job.metadata.subcollection
if job.status.phase(_PREPARE).start is not None:
raise DataException(
f"Preparation phase has already started for job {job.job_id}")
with job.status.phase(_PREPARE):
log.info("Preparation phase started")
# Setup the transfer route
route = posix_to_irods_factory(*_FILESYSTEMS)
route += strip_common_prefix
route += prefix(irods_base / subcollection)
route += debugging
route += telemetry
tasks = 0
lustre, *_ = _FILESYSTEMS
files = lustre._identify_by_fofn(fofn)
for task in route.plan(files):
log.info(f"{task.source.address} on {task.source.filesystem} to "
f"{task.target.address} on {task.target.filesystem}")
# NOTE With just one step in our route, we have no
# inter-task dependencies; the source size is persisted
# automatically, for subsequent transfer rate calculations.
job += DependentTask(task)
tasks += 1
log.info(f"Added {tasks} tasks to the job")
log.info("Preparation phase complete")
def do_multi(geoclient,records,capture=False,loud=False):
log.info("let's do this ...")
log.info(f'capture = {capture}')
stream = procmulti(geoclient,records,capture,loud)
ioany.save_recs("this.csv",stream)
#!/usr/bin/env python
import sys, argparse
import simplejson as json
from nycgeo.client import SimpleGeoClient
from common.logging import log
parser = argparse.ArgumentParser()
parser.add_argument("--addr", help="address to parse")
parser.add_argument("--tiny", help="fetch a tiny rec", type=int)
parser.add_argument("--mock", help="use the mock service", type=int)
args = parser.parse_args()
print(args)
log.info("info!")
log.debug("debug!")
if args.mock:
configpath = "config/mockgeo-client.json"
else:
configpath = "config/nycgeo.json"
config = json.loads(open(configpath,"r").read())
if args.addr:
rawaddr = args.addr
else:
rawaddr = "529 West 29th St, Manhattan"
print("rawaddr = [%s]" % rawaddr)
agent = SimpleGeoClient(**config)
message = data[message_offset:message_offset + 8]
self.assertEqual(message, expected)
def test_part2_example1(self):
self._test_part2_phase_rounds_output(
"03036732577212944063491565474664" * 10000, 100, "84462026")
if __name__ == "__main__":
# unittest.main(defaultTest="TestPhasing", exit=False, verbosity=0)
# part 1
try:
with Timer() as t:
answer = parse_output(phase_count(parse_input(puzzle_input), 100))
log.info(f"Part One = {answer}")
finally:
log.info(f"It took {t.interval} seconds.")
# part 1
try:
with Timer() as t:
data = puzzle_input * 10000
message_offset = int(data[:7])
data = parse_input(data)
data = phase_count(data, 100)
data = parse_output(data)
message = data[message_offset:message_offset + 8]
log.info(f"Part Two = {message}")
finally:
log.info(f"It took {t.interval} seconds.")
class TestFuelCalculations(unittest.TestCase):
def test_determine_fuel(self):
def test(a, b):
self.assertEqual(determine_fuel(a), b)
test(12, 2)
test(14, 2)
test(1969, 654)
test(100756, 33583)
def test_determine_fuel_recursive(self):
def test(a, b):
self.assertEqual(determine_fuel_recursive(a), b)
test(14, 2)
test(1969, 966)
test(100756, 50346)
def parse_module_masses(s):
return [int(line.strip()) for line in s.split('\n')]
if __name__ == '__main__':
unittest.main(exit=False)
for f in (determine_fuel, determine_fuel_recursive):
module_masses = parse_module_masses(puzzle_input)
fuel = sum(map(f, module_masses))
log.info(f"{f.__name__} = {fuel}")
unittest.main(exit=False, verbosity=0)
computer = intcode_computer.IntcodeComputer()
initial_memory = intcode_computer.util.parse_memory_string(puzzle_input)
# initialize computer
computer.memory.reset()
computer.memory.load(initial_memory)
computer.memory[1] = 12
computer.memory[2] = 2
# run
computer.run()
answer = computer.memory[0]
log.info(f"Part One = {answer}")
for noun in range(99):
for verb in range(99):
computer.memory.reset()
computer.memory.load(initial_memory)
computer.memory[1] = noun
computer.memory[2] = verb
computer.run()
if computer.memory[0] == 19690720:
answer = 100 * noun + verb
log.info(f"Part Two = {answer}")
break
if computer.memory[0] == 19690720:
break
else:
def status(job_id: str) -> None:
""" Report job status to user """
_LOG_HEADER()
state = _GET_STATE()
job = State.Job(state, client_id=_CLIENT, job_id=int(job_id))
current = job.status
prep_phase = current.phase(_PREPARE)
transfer_phase = current.phase(_TRANSFER)
log.info(f"Job ID: {job_id}")
log.info(f"Preparation phase: {_phase_status(prep_phase)}")
if not prep_phase.complete:
log.warning("The following output may be incomplete")
log.info(f"Transfer phase: {_phase_status(transfer_phase)}")
if not transfer_phase.complete:
log.info(f"Pending: {current.pending}")
log.info(f"Running: {current.running}")
log.info(f"Failed: {current.failed}")
log.info(f"Succeeded: {current.succeeded}")
try:
# NOTE This is specific to Lustre-to-iRODS tasks
throughput = current.throughput(*_FILESYSTEMS)
log.info(
f"Transfer rate: {_human_size(throughput.transfer_rate)}B/s per worker"
)
log.info(f"Failure rate: {throughput.failure_rate:.1%}")
except NoThroughputData:
log.info("Transfer rate: No data")
log.info("Failure rate: No data")
def transfer(job_id: str) -> None:
""" Transfer prepared tasks from Lustre to iRODS """
_LOG_HEADER()
state = _GET_STATE()
state.register_filesystems(*_FILESYSTEMS)
job = State.Job(state, client_id=_CLIENT, job_id=int(job_id))
executor = _GET_EXECUTOR()
worker = executor.worker
log.info(f"Transfer phase: Worker {worker.id.worker}")
# This is when we should wrap-up
deadline = _START_TIME + worker.limit(LSFWorkerLimit.Runtime) - _FUDGE_TIME
# HACK: Load metadata
with T.Path(job.metadata.shitty_metadata).open() as metadata_handle:
metadata = json.load(metadata_handle)
# Don't start the transfer phase until preparation has started
while job.status.phase(_PREPARE).start is None:
# Check we're not going to overrun the limit (which shouldn't
# happen when just waiting for the preparation phase to start)
if time.now() > deadline:
log.info("Approaching runtime limit; terminating")
sys.exit(0)
log.info("Waiting for preparation phase to start...")
sleep(_FUDGE_TIME.total_seconds())
# Initialise the transfer phase (idempotent)
job.status.phase(_TRANSFER).init()
if job.status.complete:
log.info("Nothing left do to for this worker")
sys.exit(0)
# Launch follow-on worker, in case we run out of time
# NOTE DAISYCHAIN can be set to abort accidental LSF proliferation
following = job.metadata.DAISYCHAIN == _DAISYCHAIN_TRUE
if following:
follow_on, follow_options = _transfer_worker(job_id,
T.Path(job.metadata.logs))
follow_on.specific_worker = worker.id.worker
follow_on += worker.id
follow_runner, *_ = executor.submit(follow_on, follow_options)
log.info(
f"Follow-on worker submitted with LSF ID {follow_runner.job}; "
"will cancel on completion")
log.info("Starting transfers")
while not job.status.complete:
remaining_time = deadline - time.now()
try:
attempt = job.attempt(remaining_time)
except NoTasksAvailable:
# Check if we're done
current = job.status
if current.phase(_PREPARE) or current.pending > 0:
# Preparation phase is still in progress, or there are
# still pending tasks
log.warning(
"Cannot complete any more tasks in the given run limit; terminating"
)
else:
# All tasks have been prepared and none are pending, so
# cancel the follow-on
log.info("Nothing left do to for this worker")
if following:
log.info(
f"Cancelling follow-on worker with LSF ID {follow_runner.job}"
)
executor.signal(follow_runner, SIGTERM)
# If no tasks are in-flight, then we're finished
if current.running == 0:
log.info(f"All tasks complete")
job.status.phase(_TRANSFER).stop()
sys.exit(0)
# TODO Py3.8 walrus operator would be good here
success = attempt()
if success:
log.info(
f"Successfully transferred and verified {_human_size(attempt.size(DataOrigin.Source))}B"
)
# HACK: Set metadata
target = attempt.task.target
log.info(
f"Applying metadata to {target.address} on {target.filesystem}"
)
target.filesystem.set_metadata(
target.address, **{
**metadata, "source": str(attempt.task.source.address)
})
args = parser.parse_args()
port = args.port if args.port else 5002
app = Flask(__name__)
CORS(app)
dataconf = slurp_json("config/postgres.json")
if args.mock:
geoconf = slurp_json("config/mockgeo-client.json")
else:
geoconf = slurp_json("config/nycgeo.json")
log.info("mock = %s, port = %d" % (bool(args.mock),port))
log.info("siteurl = '%s'" % geoconf.get('siteurl'))
agent = lookuptool.hybrid.instance(dataconf,geoconf)
def errmsg(message):
return json.dumps({'error':message})
def jsonify(r):
return json.dumps(r,sort_keys=True)
def wrapsafe(callf,rawarg):
try:
return callf(rawarg)
except Exception as e:
log.debug("exception = %s" % e)
# print_tb(e.__traceback__)
self._test_phase_rounds_output(
"03036732577212944063491565474664" * 10000, 100, "84462026")
def test_part2_example2(self):
self._test_phase_rounds_output(
"02935109699940807407585447034323" * 10000, 100, "78725270")
def test_part2_example3(self):
self._test_phase_rounds_output(
"03081770884921959731165446850517" * 10000, 100, "53553731")
if __name__ == "__main__":
unittest.main(defaultTest="TestPhasing", exit=False, verbosity=0)
# part 1
try:
with Timer() as t:
answer = parse_output(phase_count(parse_input(puzzle_input), 100))
log.info(f"Part One, 100 Phases = {answer}")
finally:
log.info(f"It took {t} seconds.")
# part 2
# real_signal = parse_input(puzzle_input) * 10000
#
# part 2
# answer = determine_shortest_length_to_intersection(paths)
# log.info(f"Part Two, Shortest Wire Length to Intersection = {answer}")
self.assertEqual(length, b)
def test_shortest_length_to_intersection1(self):
self._test_shortest_length_to_intersection(
"R75,D30,R83,U83,L12,D49,R71,U7,L72\n"
"U62,R66,U55,R34,D71,R55,D58,R83",
610
)
def test_shortest_length_to_intersection2(self):
self._test_shortest_length_to_intersection(
"R98,U47,R26,D63,R33,U87,L62,D20,R33,U53,R51\n"
"U98,R91,D20,R16,D67,R40,U7,R15,U6,R7",
410
)
if __name__ == "__main__":
unittest.main(exit=False, verbosity=0)
vectors = parse_vectors(puzzle_input)
paths = list(map(trace_vectors, vectors))
intersections = determine_intersections(paths)
# part 1
answer = determine_distance_to_closest_intersection(paths)
log.info(f"Part One, Distance to Closest Intersection to Origin = {answer}")
# part 2
answer = determine_shortest_length_to_intersection(paths)
log.info(f"Part Two, Shortest Wire Length to Intersection = {answer}")gps
_START_TIME = time.now()
_FUDGE_TIME = time.delta(minutes=3)
_FILESYSTEMS = (POSIXFilesystem(name="Lustre", max_concurrency=50),
iRODSFilesystem(name="iRODS", max_concurrency=10))
# These are lambdas because we haven't, at this point, checked the
# necessary environment variables are set
_GET_EXECUTOR = lambda: LSF(T.Path(os.environ["LSF_CONFIG"]))
_GET_STATE = lambda: State.PostgreSQL(database=os.environ["PG_DATABASE"],
user=os.environ["PG_USERNAME"],
password=os.environ["PG_PASSWORD"],
host=os.environ["PG_HOST"],
port=int(os.getenv("PG_PORT", "5432")))
_LOG_HEADER = lambda: log.info(
f"Shepherd: {_CLIENT} {cli_version} / lib {lib_version}")
_DAISYCHAIN_TRUE = "Yes"
_DAISYCHAIN = os.getenv("DAISYCHAIN", _DAISYCHAIN_TRUE)
# Convenience aliases
_PREPARE = JobPhase.Preparation
_TRANSFER = JobPhase.Transfer
def main(*args: str) -> None:
# Expected environment variables (commented out entries are
# optional, listed for documentation's sake)
envvars = {
"PG_HOST": "PostgreSQL hostname",
# "PG_PORT": "PostgreSQL port [5432]",
