def process(geoclient,r,i,capture=False): rawaddr = makeaddr(r) log.info(f'{i}: {rawaddr} ..') log.info(f'capture = {capture}') d = OrderedDict(r) if capture and detect_object(i): log.info(f'{i}: status = SKIP') return None try: status, keytup, response = do_single(geoclient,rawaddr) log.info(f'{i}: status = {status}') if capture: capture_object(i,response) d['code'] = status.get('code') d['bbl'] = keytup.get('bbl') if keytup else None d['bin'] = keytup.get('bin') if keytup else None d['error'] = status.get('error') d['message'] = keytup.get('message') if keytup else None d['message2'] = keytup.get('message2') if keytup else None except Exception as e: errmsg = str(e) log.info(f'{i}: ERROR {errmsg}') log.error(e) d['error'] = errmsg return d
def procmulti(geoclient,records,capture=False,loud=False): for i,r in enumerate(records): log.info(f'proc {i} ..') d = process(geoclient,r,i,capture) if loud: print(f'd[{i}] = {d}') if d is not None: yield d
def __call__(self) -> bool: """ Attempt the transfer task """ # The context manager sets the attempt start and finish timestamps with self: task = self.task log.info(f"Attempting transfer of " f"{task.source.address} on {task.source.filesystem} to " f"{task.target.address} on {task.target.filesystem}") with ThreadPoolExecutor(max_workers=1) as executor: # TODO Different/multiple checksum algorithms properties = executor.submit(self._get_source_properties, "md5") # Run task in main tread and join on properties thread success = self.task() source_size, source_checksums = properties.result() if not success: log.warning( f"Attempt failed with exit code {success.exit_code}") else: log.info(f"Data copied; verifying...") try: target_size = self.size(_TARGET) if source_size != target_size: log.warning(f"Attempt failed: " f"Source is {source_size} bytes; " f"target is {target_size} bytes") success = _MISMATCHED_SIZE raise _VerificationFailure() # TODO Different/multiple checksum algorithms source_checksum = source_checksums["md5"] target_checksum = self.checksum(_TARGET, "md5") if source_checksum != target_checksum: log.warning(f"Attempt failed: " f"Source has checksum {source_checksum}; " f"target has checksum {target_checksum}") success = _MISMATCHED_CHECKSUM raise _VerificationFailure() # TODO Data metadata: There is no need to set this # for each intermediary stage; just set the final # target metadata to that of the original source. # Think about how to implement this... except _VerificationFailure: pass self.exit_code = success return bool(success)
def api_fetch(prefix): if prefix != 'address.json': return errmsg('invalid service base') try: log.debug("query_string = %s" % request.query_string) return resolve_query(request.query_string) except Exception as e: log.info("exception = %s" % e) log.exception(e) return errmsg('internal error')
def resolve(callf,query): try: param = split_query(query) except ValueError as e: return errmsg('invalid query string') try: r = callf(param) except Exception as e: log.info("exception = %s" % e) return errmsg('internal error') return jsonify(r,sort_keys=True)
def main(*args: str) -> None: # Expected environment variables (commented out entries are # optional, listed for documentation's sake) envvars = { "PG_HOST": "PostgreSQL hostname", # "PG_PORT": "PostgreSQL port [5432]", "PG_DATABASE": "PostgreSQL database name", "PG_USERNAME": "******", "PG_PASSWORD": "******", "LSF_CONFIG": "Path to LSF cluster configuration directory", "LSF_GROUP": "LSF Fairshare group to run under", "PREP_QUEUE": "LSF queue to use for the preparation phase", "TRANSFER_QUEUE": "LSF queue to use for the transfer phase", "IRODS_BASE": "Base iRODS collection into which to transfer" # "MAX_ATTEMPTS": "Maximum attempts per transfer task [3]" # "SHEPHERD_LOG": "Logging directory [pwd]" # "DAISYCHAIN": "Automatically daisychain transfer workers [Yes]" } # Mode delegation routines delegate = { "submit": submit, "resume": resume, "status": status, "__prepare": prepare, "__transfer": transfer } # Check the appropriate environment variables are set to connect to # the PostgreSQL database, otherwise bail out if any(env not in os.environ for env in envvars): log.critical("Incomplete environment variables") column_width = max(map(len, envvars)) for env, desc in envvars.items(): log.info(f"* {env:{column_width}} {desc}") sys.exit(1) # Check the binary is running in a known mode, otherwise bail out mode, *mode_args = args if mode not in delegate: log.critical(f"No such mode \"{mode}\"") user_modes = ", ".join(mode for mode in delegate if not mode.startswith("__")) log.info(f"Valid user modes: {user_modes}") sys.exit(1) # Delegate to appropriate mode delegate[mode](*mode_args)
def submit(fofn: str, subcollection: str, metadata: str) -> None: """ Submit a FoFN job to the executioner """ # Set logging directory, if not already if "SHEPHERD_LOG" not in os.environ: os.environ["SHEPHERD_LOG"] = str(T.Path(".").resolve()) log_dir = T.Path(os.environ["SHEPHERD_LOG"]).resolve() log.to_file(log_dir / "submit.log") fofn_path = T.Path(fofn).resolve() irods_base = os.environ["IRODS_BASE"] metadata_path = T.Path(metadata).resolve() _LOG_HEADER() log.info(f"Logging to {log_dir}") log.info( f"Will transfer contents of {fofn_path} to {irods_base}/{subcollection}" ) log.info(f"Will apply metadata from {metadata_path} to each file") state = _GET_STATE() job = State.Job(state, client_id=_CLIENT) job.max_attempts = max_attempts = int(os.getenv("MAX_ATTEMPTS", "3")) job.set_metadata(fofn=str(fofn_path), irods_base=irods_base, subcollection=subcollection, logs=str(log_dir), shitty_metadata=str(metadata_path), DAISYCHAIN=_DAISYCHAIN) # NOTE For debugging log.info( f"Created new job with ID {job.job_id}, with up to {max_attempts} attempts per task" ) executor = _GET_EXECUTOR() prep_options = LSFSubmissionOptions(cores=1, memory=1000, group=os.environ["LSF_GROUP"], queue=os.environ["PREP_QUEUE"]) prep_worker = Exec.Job(f"\"{_BINARY}\" __prepare {job.job_id}") prep_worker.stdout = prep_worker.stderr = log_dir / "prep.log" prep_runner, *_ = executor.submit(prep_worker, prep_options) log.info(f"Preparation phase submitted with LSF ID {prep_runner.job}") _submit_transfer(job, executor)
def load_hybrid_conf(confdir,args): """Loads the requisite config files for the hybrid service (subject to arg switches), and returns them as a tuple of (dataconf,geoconf).""" metaconf = load_file(confdir,'hybrid-settings.json') dataconf = load_file(confdir,'postgres.json') if args.mock: usemock = True elif args.nomock: usemock = False else: usemock = metaconf['mock'] log.info("mock = %s, port = %d" % (usemock,args.port)) suffix = 'mock' if usemock else 'live'; geofile = "nycgeo-%s.json" % suffix geoconf = load_file(confdir,geofile) log.info("siteurl = '%s'" % geoconf.get('siteurl')) return dataconf,geoconf
def get(self,suburl): log.info("siteurl = %s" % self.siteurl) log.info("suburl = %s" % suburl) t0 = time.time() r = requests.get((self.siteurl+suburl).encode('utf-8'),verify=self.verify) t1 = time.time() dt = 1000*(t1-t0) log.info("status = %d in %.2f ms" % (r.status_code,dt)) for k,v in r.headers.items(): log.info("header - '%s': '%s'" % (k,v)) return r,dt
def main(): global THROW,LOUD args = parse_args() LOUD = args.loud THROW = args.throw log.info('hi') log.debug('yow') nycgeopath = "config/nycgeo-live.json" geoconf = json.loads(open(nycgeopath,"r").read()) geoclient = SimpleGeoClient(**geoconf) print(f'agent = {geoclient}') if args.rawaddr: status, keytup, response = do_single(geoclient,args.rawaddr) print(f'status = {status}, keytup = {keytup}') else: infile = args.infile print(f'slurp from {infile} ..') inrecs = ioany.read_recs(infile) inrecs = islice(inrecs,args.limit) do_multi(geoclient,inrecs,capture=args.capture) print('done')
def resume(job_id: str, force: T.Optional[str] = None) -> None: """ Resume job """ _LOG_HEADER() state = _GET_STATE() job = State.Job(state, client_id=_CLIENT, job_id=int(job_id)) log_dir = T.Path(job.metadata.logs) log.to_file(log_dir / "resume.log") if job.status.phase(_PREPARE).start is None: log.error( f"Preparation phase for job {job_id} has yet to start; cannot resume." ) sys.exit(1) if job.status.complete: log.info(f"Job {job_id} has already completed.") sys.exit(0) if not job.status.phase(_PREPARE).complete: log.warning( f"Preparation phase for job {job_id} is still in progress.") if force != "--force": log.error( f"Cannot resume a job in preparation without the --force option." ) sys.exit(1) log.info(f"Resuming job {job_id}...") resumed = State.Job(state, client_id=_CLIENT, job_id=int(job_id), force_restart=True) executor = _GET_EXECUTOR() _submit_transfer(resumed, executor)
def _submit_transfer(job: State.Job, executor: Exec.BaseExecutor) -> None: # Submit the transfer workers log_dir = T.Path(job.metadata.logs) # NOTE We're only dealing with the Lustre-iRODS tuple, so this is # simplified considerably. In a multi-route context, the maximum # concurrency should be a function of the pairwise minimum of # filesystems' maximum concurrencies for each stage of the route. # That function could be, e.g.: max for maximum speed, but also # maximum waste (in terms of redundant workers); min (or some lower # constant) for zero wastage, but longer flight times. The # arithmetic mean would probably be a good thing to go for, without # implementing complicated dynamic load handling... max_concurrency = min(fs.max_concurrency for fs in _FILESYSTEMS) transfer_worker, transfer_options = _transfer_worker(job.job_id, log_dir) transfer_worker.workers = max_concurrency transfer_runners = transfer_runner, *_ = list( executor.submit(transfer_worker, transfer_options)) log.info( f"Transfer phase submitted with LSF ID {transfer_runner.job} and {len(transfer_runners)} workers" )
def get(self,suburl): log.info("siteurl = %s" % self.siteurl) log.info("suburl = %s" % suburl) t0 = time.time() r = requests.get((self.siteurl+suburl).encode('utf-8')) t1 = time.time() dt = 1000*(t1-t0) log.info("status = %d in %.2f ms" % (r.status_code,dt)) return r,dt
def get(self, suburl): log.info("siteurl = %s" % self.siteurl) log.info("suburl = %s" % suburl) t0 = time.time() r = requests.get((self.siteurl + suburl).encode('utf-8')) t1 = time.time() dt = 1000 * (t1 - t0) log.info("status = %d in %.2f ms" % (r.status_code, dt)) return r, dt
def prepare(job_id: str) -> None: """ Prepare the Lustre to iRODS task from FoFN """ _LOG_HEADER() state = _GET_STATE() job = State.Job(state, client_id=_CLIENT, job_id=int(job_id)) # Get the FoFN path and prefix from the client metadata fofn = T.Path(job.metadata.fofn) irods_base = T.Path(job.metadata.irods_base) subcollection = job.metadata.subcollection if job.status.phase(_PREPARE).start is not None: raise DataException( f"Preparation phase has already started for job {job.job_id}") with job.status.phase(_PREPARE): log.info("Preparation phase started") # Setup the transfer route route = posix_to_irods_factory(*_FILESYSTEMS) route += strip_common_prefix route += prefix(irods_base / subcollection) route += debugging route += telemetry tasks = 0 lustre, *_ = _FILESYSTEMS files = lustre._identify_by_fofn(fofn) for task in route.plan(files): log.info(f"{task.source.address} on {task.source.filesystem} to " f"{task.target.address} on {task.target.filesystem}") # NOTE With just one step in our route, we have no # inter-task dependencies; the source size is persisted # automatically, for subsequent transfer rate calculations. job += DependentTask(task) tasks += 1 log.info(f"Added {tasks} tasks to the job") log.info("Preparation phase complete")
def do_multi(geoclient,records,capture=False,loud=False): log.info("let's do this ...") log.info(f'capture = {capture}') stream = procmulti(geoclient,records,capture,loud) ioany.save_recs("this.csv",stream)
#!/usr/bin/env python import sys, argparse import simplejson as json from nycgeo.client import SimpleGeoClient from common.logging import log parser = argparse.ArgumentParser() parser.add_argument("--addr", help="address to parse") parser.add_argument("--tiny", help="fetch a tiny rec", type=int) parser.add_argument("--mock", help="use the mock service", type=int) args = parser.parse_args() print(args) log.info("info!") log.debug("debug!") if args.mock: configpath = "config/mockgeo-client.json" else: configpath = "config/nycgeo.json" config = json.loads(open(configpath,"r").read()) if args.addr: rawaddr = args.addr else: rawaddr = "529 West 29th St, Manhattan" print("rawaddr = [%s]" % rawaddr) agent = SimpleGeoClient(**config)
message = data[message_offset:message_offset + 8] self.assertEqual(message, expected) def test_part2_example1(self): self._test_part2_phase_rounds_output( "03036732577212944063491565474664" * 10000, 100, "84462026") if __name__ == "__main__": # unittest.main(defaultTest="TestPhasing", exit=False, verbosity=0) # part 1 try: with Timer() as t: answer = parse_output(phase_count(parse_input(puzzle_input), 100)) log.info(f"Part One = {answer}") finally: log.info(f"It took {t.interval} seconds.") # part 1 try: with Timer() as t: data = puzzle_input * 10000 message_offset = int(data[:7]) data = parse_input(data) data = phase_count(data, 100) data = parse_output(data) message = data[message_offset:message_offset + 8] log.info(f"Part Two = {message}") finally: log.info(f"It took {t.interval} seconds.")
class TestFuelCalculations(unittest.TestCase): def test_determine_fuel(self): def test(a, b): self.assertEqual(determine_fuel(a), b) test(12, 2) test(14, 2) test(1969, 654) test(100756, 33583) def test_determine_fuel_recursive(self): def test(a, b): self.assertEqual(determine_fuel_recursive(a), b) test(14, 2) test(1969, 966) test(100756, 50346) def parse_module_masses(s): return [int(line.strip()) for line in s.split('\n')] if __name__ == '__main__': unittest.main(exit=False) for f in (determine_fuel, determine_fuel_recursive): module_masses = parse_module_masses(puzzle_input) fuel = sum(map(f, module_masses)) log.info(f"{f.__name__} = {fuel}")
unittest.main(exit=False, verbosity=0) computer = intcode_computer.IntcodeComputer() initial_memory = intcode_computer.util.parse_memory_string(puzzle_input) # initialize computer computer.memory.reset() computer.memory.load(initial_memory) computer.memory[1] = 12 computer.memory[2] = 2 # run computer.run() answer = computer.memory[0] log.info(f"Part One = {answer}") for noun in range(99): for verb in range(99): computer.memory.reset() computer.memory.load(initial_memory) computer.memory[1] = noun computer.memory[2] = verb computer.run() if computer.memory[0] == 19690720: answer = 100 * noun + verb log.info(f"Part Two = {answer}") break if computer.memory[0] == 19690720: break else:
def status(job_id: str) -> None: """ Report job status to user """ _LOG_HEADER() state = _GET_STATE() job = State.Job(state, client_id=_CLIENT, job_id=int(job_id)) current = job.status prep_phase = current.phase(_PREPARE) transfer_phase = current.phase(_TRANSFER) log.info(f"Job ID: {job_id}") log.info(f"Preparation phase: {_phase_status(prep_phase)}") if not prep_phase.complete: log.warning("The following output may be incomplete") log.info(f"Transfer phase: {_phase_status(transfer_phase)}") if not transfer_phase.complete: log.info(f"Pending: {current.pending}") log.info(f"Running: {current.running}") log.info(f"Failed: {current.failed}") log.info(f"Succeeded: {current.succeeded}") try: # NOTE This is specific to Lustre-to-iRODS tasks throughput = current.throughput(*_FILESYSTEMS) log.info( f"Transfer rate: {_human_size(throughput.transfer_rate)}B/s per worker" ) log.info(f"Failure rate: {throughput.failure_rate:.1%}") except NoThroughputData: log.info("Transfer rate: No data") log.info("Failure rate: No data")
def transfer(job_id: str) -> None: """ Transfer prepared tasks from Lustre to iRODS """ _LOG_HEADER() state = _GET_STATE() state.register_filesystems(*_FILESYSTEMS) job = State.Job(state, client_id=_CLIENT, job_id=int(job_id)) executor = _GET_EXECUTOR() worker = executor.worker log.info(f"Transfer phase: Worker {worker.id.worker}") # This is when we should wrap-up deadline = _START_TIME + worker.limit(LSFWorkerLimit.Runtime) - _FUDGE_TIME # HACK: Load metadata with T.Path(job.metadata.shitty_metadata).open() as metadata_handle: metadata = json.load(metadata_handle) # Don't start the transfer phase until preparation has started while job.status.phase(_PREPARE).start is None: # Check we're not going to overrun the limit (which shouldn't # happen when just waiting for the preparation phase to start) if time.now() > deadline: log.info("Approaching runtime limit; terminating") sys.exit(0) log.info("Waiting for preparation phase to start...") sleep(_FUDGE_TIME.total_seconds()) # Initialise the transfer phase (idempotent) job.status.phase(_TRANSFER).init() if job.status.complete: log.info("Nothing left do to for this worker") sys.exit(0) # Launch follow-on worker, in case we run out of time # NOTE DAISYCHAIN can be set to abort accidental LSF proliferation following = job.metadata.DAISYCHAIN == _DAISYCHAIN_TRUE if following: follow_on, follow_options = _transfer_worker(job_id, T.Path(job.metadata.logs)) follow_on.specific_worker = worker.id.worker follow_on += worker.id follow_runner, *_ = executor.submit(follow_on, follow_options) log.info( f"Follow-on worker submitted with LSF ID {follow_runner.job}; " "will cancel on completion") log.info("Starting transfers") while not job.status.complete: remaining_time = deadline - time.now() try: attempt = job.attempt(remaining_time) except NoTasksAvailable: # Check if we're done current = job.status if current.phase(_PREPARE) or current.pending > 0: # Preparation phase is still in progress, or there are # still pending tasks log.warning( "Cannot complete any more tasks in the given run limit; terminating" ) else: # All tasks have been prepared and none are pending, so # cancel the follow-on log.info("Nothing left do to for this worker") if following: log.info( f"Cancelling follow-on worker with LSF ID {follow_runner.job}" ) executor.signal(follow_runner, SIGTERM) # If no tasks are in-flight, then we're finished if current.running == 0: log.info(f"All tasks complete") job.status.phase(_TRANSFER).stop() sys.exit(0) # TODO Py3.8 walrus operator would be good here success = attempt() if success: log.info( f"Successfully transferred and verified {_human_size(attempt.size(DataOrigin.Source))}B" ) # HACK: Set metadata target = attempt.task.target log.info( f"Applying metadata to {target.address} on {target.filesystem}" ) target.filesystem.set_metadata( target.address, **{ **metadata, "source": str(attempt.task.source.address) })
args = parser.parse_args() port = args.port if args.port else 5002 app = Flask(__name__) CORS(app) dataconf = slurp_json("config/postgres.json") if args.mock: geoconf = slurp_json("config/mockgeo-client.json") else: geoconf = slurp_json("config/nycgeo.json") log.info("mock = %s, port = %d" % (bool(args.mock),port)) log.info("siteurl = '%s'" % geoconf.get('siteurl')) agent = lookuptool.hybrid.instance(dataconf,geoconf) def errmsg(message): return json.dumps({'error':message}) def jsonify(r): return json.dumps(r,sort_keys=True) def wrapsafe(callf,rawarg): try: return callf(rawarg) except Exception as e: log.debug("exception = %s" % e) # print_tb(e.__traceback__)
self._test_phase_rounds_output( "03036732577212944063491565474664" * 10000, 100, "84462026") def test_part2_example2(self): self._test_phase_rounds_output( "02935109699940807407585447034323" * 10000, 100, "78725270") def test_part2_example3(self): self._test_phase_rounds_output( "03081770884921959731165446850517" * 10000, 100, "53553731") if __name__ == "__main__": unittest.main(defaultTest="TestPhasing", exit=False, verbosity=0) # part 1 try: with Timer() as t: answer = parse_output(phase_count(parse_input(puzzle_input), 100)) log.info(f"Part One, 100 Phases = {answer}") finally: log.info(f"It took {t} seconds.") # part 2 # real_signal = parse_input(puzzle_input) * 10000 # # part 2 # answer = determine_shortest_length_to_intersection(paths) # log.info(f"Part Two, Shortest Wire Length to Intersection = {answer}")
self.assertEqual(length, b) def test_shortest_length_to_intersection1(self): self._test_shortest_length_to_intersection( "R75,D30,R83,U83,L12,D49,R71,U7,L72\n" "U62,R66,U55,R34,D71,R55,D58,R83", 610 ) def test_shortest_length_to_intersection2(self): self._test_shortest_length_to_intersection( "R98,U47,R26,D63,R33,U87,L62,D20,R33,U53,R51\n" "U98,R91,D20,R16,D67,R40,U7,R15,U6,R7", 410 ) if __name__ == "__main__": unittest.main(exit=False, verbosity=0) vectors = parse_vectors(puzzle_input) paths = list(map(trace_vectors, vectors)) intersections = determine_intersections(paths) # part 1 answer = determine_distance_to_closest_intersection(paths) log.info(f"Part One, Distance to Closest Intersection to Origin = {answer}") # part 2 answer = determine_shortest_length_to_intersection(paths) log.info(f"Part Two, Shortest Wire Length to Intersection = {answer}")gps
_START_TIME = time.now() _FUDGE_TIME = time.delta(minutes=3) _FILESYSTEMS = (POSIXFilesystem(name="Lustre", max_concurrency=50), iRODSFilesystem(name="iRODS", max_concurrency=10)) # These are lambdas because we haven't, at this point, checked the # necessary environment variables are set _GET_EXECUTOR = lambda: LSF(T.Path(os.environ["LSF_CONFIG"])) _GET_STATE = lambda: State.PostgreSQL(database=os.environ["PG_DATABASE"], user=os.environ["PG_USERNAME"], password=os.environ["PG_PASSWORD"], host=os.environ["PG_HOST"], port=int(os.getenv("PG_PORT", "5432"))) _LOG_HEADER = lambda: log.info( f"Shepherd: {_CLIENT} {cli_version} / lib {lib_version}") _DAISYCHAIN_TRUE = "Yes" _DAISYCHAIN = os.getenv("DAISYCHAIN", _DAISYCHAIN_TRUE) # Convenience aliases _PREPARE = JobPhase.Preparation _TRANSFER = JobPhase.Transfer def main(*args: str) -> None: # Expected environment variables (commented out entries are # optional, listed for documentation's sake) envvars = { "PG_HOST": "PostgreSQL hostname", # "PG_PORT": "PostgreSQL port [5432]",