def login_to_qrz(session: requests.Session, call_sign: str,
password: str) -> requests.Session:
return chain(
format_qrz_login_data,
to(post(session=session, url='https://www.qrz.com/login'), kw='data'),
to(return_session_if_login_verified(session=session),
kw='response'))((call_sign, password))
def main(arrl_session: requests.Session,
qrz_session: requests.Session) -> dict:
return chain(
collect_arrl_data,
split_to(handle_arrl_data(qrz_session=qrz_session),
kws=['missing_countries',
'contacted_call_signs']))(session=arrl_session)
def setup(own_call_sign: str, arrl_pass: str,
qrz_pass: str) -> (requests.Session, requests.Session):
return chain(
start_session,
to(concurrently(
login_to_lotw_arrl(password=arrl_pass, call_sign=own_call_sign),
login_to_qrz(password=qrz_pass, call_sign=own_call_sign)),
kw='session'))()
def handle_arrl_data(contacted_call_signs: tuple, missing_countries: dict,
qrz_session: requests.Session) -> dict:
return chain(
map(func=handle_country(filter_func=filterer(
iterable=contacted_call_signs, kw='string'),
qrz_session=qrz_session),
kw='country'), filter_empty_values,
dict)(iterable=missing_countries.items())
def handle_country(country: str, filter_func: callable,
qrz_session: requests.Session) -> (str, list):
return country[1], chain(
filter_func,
to(map(func=get_call_sign_data(qrz_session=qrz_session),
kw='call_sign'),
kw='iterable'),
)(func=startswith(substring=country[0]))
def login_to_lotw_arrl(session: requests.Session, call_sign: str,
password: str) -> requests.Session:
return chain(
format_arrl_login_data,
to(post(session=session, url='https://lotw.arrl.org/lotwuser/login'),
kw='data'),
to(return_session_if_login_verified(session=session),
kw='response'))(call_sign, password)
__license__ = "GNU GPL3"
__version__ = "0.2.0"
__maintainer__ = "Nicholas Logan"
__email__ = "*****@*****.**"
__status__ = "Prototype"
import requests
from lxml import etree
import re
import pprint
from functional import chain, concurrently, to, split_to, map, filterer, zipper, curry
start_session = requests.session
filter_empty_values = to(filterer(func=lambda item: item[1], kw='item'),
kw='iterable')
format_arrl_login_data = 'login={}&password={}&acct_sel=&thisForm=login'.format
format_qrz_login_data = to(chain(zipper(iter1=('username', 'password')), dict),
kw='iter2')
@curry
def post(session: requests.Session, url: str, data: str
or dict) -> requests.Response:
return session.post(url, data=data)
@curry
def startswith(string: str, substring: str) -> bool:
return string.startswith(substring)
def get_missing_credits(session: requests.Session) -> dict:
def __call__(self):
pandda_start_time = time.time()
working_phil = parse_phil_args(master_phil=pandda_phil,
args=self.args,
blank_arg_prepend=None,
home_scope=None)
# # TODO: remove
# print("printing welcome")
# sys.stdout.flush()
# # welcome()
#
# # TODO: remove
# print("getting phil")
# sys.stdout.flush()
# p = working_phil.extract()
#
# # TODO: remove
# print("making directories")
# sys.stdout.flush()
# out_dir = easy_directory(os.path.abspath(p.pandda.output.out_dir))
# _log_dir = easy_directory(os.path.join(out_dir, 'logs'))
# # TODO: remove
# print("made directories")
# sys.stdout.flush()
#
# _log_filename = 'pandda-{}.log'.format(time.strftime("%Y-%m-%d-%H%M", time.gmtime()))
#
# # TODO: remove
# print("got log fileename")
# sys.stdout.flush()
# _def_filename = _log_filename.replace('.log', '.def')
# _eff_filename = _log_filename.replace('.log', '.eff')
#
# # TODO: remove
# print("makeing log")
# sys.stdout.flush()
# log = Log(log_file=os.path.join(_log_dir, _log_filename), verbose=p.settings.verbose)
#
# # TODO: remove
# print("args processor")
# sys.stdout.flush()
# pandda_arg_processor = PanddaArgsProcessor(log, p.pandda)
# args = pandda_arg_processor()
# params = args.params
# settings = p.settings
# # TODO: remove
# print("got settings")
# sys.stdout.flush()
#
# pickled_dataset_meta = Meta({'number_of_datasets': 0, 'dataset_labels': [], 'dataset_pickle_list': []})
################################################################################################################
# Maps options to code abstractions
pandda_config = PanDDAConfig(working_phil)
# Get Dataset
pandda_dataset = mdc.dataset.dataset.MultiCrystalDataset(
dataloader=pandda_config.dataloader,
sample_loader=pandda_config.sample_loader)
reference = pandda_config.get_reference(pandda_dataset.datasets)
pandda_dataset.sample_loader.reference = reference
# transform dataset
dataset = chain(pandda_dataset, [
pandda_config.check_data, pandda_config.scale_diffraction,
pandda_config.filter_structure, pandda_config.filter_wilson,
pandda_config.align
])
# Get grid
grid = pandda_config.get_grid(reference)
# Get file tree
tree = pandda_config.pandda_output(dataset)
# Define event Model
pandda_event_model = PanDDAEventModel(
pandda_config.statistical_model,
pandda_config.clusterer,
pandda_config.event_finder,
bdc_calculator=pandda_config.bdc_calculator,
statistics=[],
map_maker=pandda_config.map_maker,
event_table_maker=pandda_config.event_table_maker,
cpus=pandda_config["args"]["cpus"],
tree=tree)
# Get partitions
partitions = pandda_config.partitioner(dataset)
# instatiate a dataloader for the datasets
# dataloader = DefaultPanDDADataloader(min_train_datasets=60,
# max_test_datasets=60)
# Get the datasets to iterate over
ds = [(idx, d) for idx, d in pandda_config.dataloader(dataset)]
# Iterate over resolution shells
for shell_num, shell_dataset in ds:
client = get_client()
# ###############################################
# Get resolution
# ###############################################
resolutions_test = max([
dts.data.summary.high_res for dtag, dts in
shell_dataset.partition_datasets("test").items()
])
resolutions_train = max([
dts.data.summary.high_res for dtag, dts in
shell_dataset.partition_datasets("train").items()
])
max_res = max(resolutions_test, resolutions_train)
# ###############################################
# Instantiate sheel variable names
# ###############################################
# Dataset names
dtags = set(
shell_dataset.partition_datasets("test").keys() +
shell_dataset.partition_datasets("train").keys())
dask_dtags = {
"{}".format(dtag.replace("-", "_")): dtag
for dtag in dtags
}
train_dtags = [
dtag for dtag in dask_dtags
if (dask_dtags[dtag] in shell_dataset.partition_datasets(
"train").keys())
]
test_dtags = [
dtag for dtag in dask_dtags
if (dask_dtags[dtag] in shell_dataset.partition_datasets(
"test").keys())
]
# ###############################################
# Truncate datasets
# ###############################################
# TODO: move to imports section
truncated_reference, truncated_datasets = PanddaDiffractionDataTruncater(
)(shell_dataset.datasets, reference)
# ###############################################
# Load computed variables into dask
# ###############################################
# Rename trucnated datasets
for ddtag, dtag in dask_dtags.items():
truncated_datasets[ddtag] = truncated_datasets[dtag]
# record max res of shell datasets
shell_max_res = max_res
# ###############################################
# Generate maps
# ###############################################
# Generate reference map for shell
shell_ref_map = delayed(get_reference_map)(
pandda_config.reference_map_getter, reference, shell_max_res,
grid)
# Load maps
xmaps = {}
for dtag in dask_dtags:
xmaps[dtag] = delayed(load_sample)(pandda_config.map_loader,
truncated_datasets[dtag],
grid, shell_ref_map,
shell_max_res)
# ###############################################
# Fit statistical model to trianing sets
# ###############################################
xmaps_persisted_futures = client.persist(
[xmaps[dtag] for dtag in dask_dtags])
xmaps_computed = {
dtag: client.compute(xmaps_persisted_futures[i]).result()
for i, dtag in enumerate(dask_dtags)
}
shell_fit_model = fit(
pandda_config.statistical_model,
[xmaps_computed[dtag] for dtag in train_dtags],
[xmaps_computed[dtag] for dtag in test_dtags])
shell_fit_model_scattered = client.scatter(shell_fit_model)
xmaps_scattered = client.scatter(
[xmaps_computed[dtag] for dtag in dask_dtags])
xmaps_scattered_dict = {
dtag: xmaps_scattered[i]
for i, dtag in enumerate(dask_dtags)
}
grid_scattered = client.scatter(grid)
# ###############################################
# Find events
# ###############################################
zmaps = {}
clusters = {}
events = {}
bdcs = {}
for dtag in dask_dtags:
# Get z maps by evaluating model on maps
zmaps[dtag] = delayed(evaluate_model)(
shell_fit_model_scattered, xmaps_scattered_dict[dtag])
# Cluster outlying points in z maps
clusters[dtag] = delayed(cluster_outliers)(
pandda_config.clusterer, truncated_datasets[dtag],
zmaps[dtag], grid_scattered)
# Find events by filtering the clusters
events[dtag] = delayed(filter_clusters)(
pandda_config.event_finder, truncated_datasets[dtag],
clusters[dtag], grid_scattered)
events_persisted_futures = client.persist(
[events[dtag] for dtag in dask_dtags])
events_computed = {
dtag: client.compute(events_persisted_futures[i]).result()
for i, dtag in enumerate(dask_dtags)
}
events_scattered = client.scatter(
[events_computed[dtag] for dtag in dask_dtags])
events_scattered_dict = {
dtag: xmaps_scattered[i]
for i, dtag in enumerate(dask_dtags)
}
# Calculate background correction factors
for dtag in dask_dtags:
bdcs[dtag] = delayed(estimate_bdcs)(
pandda_config.bdc_calculator, truncated_datasets[dtag],
xmaps_scattered_dict[dtag], shell_ref_map, events[dtag],
grid_scattered)
# Criticise each indiidual dataset (generate statistics, event map and event table)
event_maps = {}
for dtag in dask_dtags:
event_maps[dtag] = delayed(make_event_map)(
pandda_config.map_maker, tree, pandda_config.map_loader,
truncated_datasets[dtag], shell_ref_map, events[dtag],
bdcs[dtag])
event_maps_persisted_futures = client.persist(
[event_maps[dtag] for dtag in dask_dtags])
event_maps_computed = {
dtag: client.compute(event_maps_persisted_futures[i]).result()
for i, dtag in enumerate(dask_dtags)
}
shell_maps = delayed(make_shell_maps)(pandda_config.map_maker,
tree, shell_num, reference,
shell_ref_map)
shell_maps_persisted_futures = client.persist(shell_maps)
shell_maps_computed = shell_maps_persisted_futures.result()
event_table = delayed(make_event_table)(
pandda_config.event_table_maker, tree, shell_num,
shell_dataset, events_computed)
event_table_persisted_future = client.persist(event_table)
event_table_computed = event_table_persisted_future.result()
client.close()