def is_bam(infile: str) -> bool:
u"""
check if input file is bam or sam file
:param infile: path to input file
:return: Boolean
"""
try:
create = False
if not os.path.exists(infile + ".bai"):
create = True
elif os.path.getctime(infile + ".bai") < os.path.getctime(infile):
try:
os.remove(infile + ".bai")
create = True
except PermissionError as err:
logger.warn(err)
create = False
else:
try:
with pysam.AlignmentFile(infile) as r:
r.check_index()
except ValueError:
create = True
if create:
logger.info(f"Creating index for {infile}")
pysam.index(infile)
return True
except pysam.utils.SamtoolsError:
return False
def prepare_bigwig_list(
bigwig: str,
region,
process: int = 1,
clustering: bool = False,
clustering_method: str = "ward",
distance_metric: str = "euclidean",
do_scale: bool = False,
) -> List[Bigwig]:
u"""
Prepare bam list
:return: [list of bam files, dict recorded the share y details]
"""
if not bigwig or not os.path.exists(bigwig):
return []
# load bam list
bigwig_list = OrderedDict()
assign_colors = set()
with open(bigwig) as r:
for line in r:
lines = re.split(r"\t| {2,}", line.strip())
if not os.path.exists(lines[0]) and not os.path.isfile(lines[0]):
logger.warn(f"Wrong input path {lines[0]}")
continue
path = lines[0]
if not is_bigwig(path):
continue
alias = lines[1] if len(lines) > 1 else os.path.basename(lines[0])
if len(lines) > 2 and lines[-1] in COLOR_MAP:
col = lines[-1]
else:
assign_colors.add(alias)
col = COLOR_MAP[len(assign_colors) - 1 % len(COLOR_MAP)]
if alias not in bigwig_list.keys():
bigwig_list[alias] = Bigwig(
[path],
clustering=clustering,
clustering_method=clustering_method,
distance_metric=distance_metric,
do_scale=do_scale,
alias=alias,
color_map=col)
bigwig_list[alias].raster = region.raster
else:
bigwig_list[alias].files.append(path)
with Pool(process) as p:
res = p.map(prepare_bigwig,
[[x, region] for x in bigwig_list.values()])
return res
def create_task(repository_id: str) -> Optional[str]:
# TODO: Allow overrides with environment variables?
project_id = dynamodb_client.get_asana_id_from_github_node_id(
repository_id)
if project_id is None:
logger.warn(f"No project id found for repository id {repository_id}")
return None
else:
due_date_str = asana_helpers.default_due_date_str()
return asana_client.create_task(project_id, due_date_str=due_date_str)
def saveWorkbook(self):
logger.critical("saving workbook")
# remove empty default sheet
try:
self.book.remove_sheet(self.book.get_sheet_by_name("Sheet"))
except Exception as e:
logger.warn(f'{e}')
self.book.save(self.fileName)
def _handle_status_webhook(payload: dict) -> HttpResponse:
commit_id = payload["commit"]["node_id"]
pull_request = graphql_client.get_pull_request_for_commit(commit_id)
if pull_request is None:
# This could happen for commits that get pushed outside of the normal
# pull request flow. These should just be silently ignored.
logger.warn(f"No pull request found for commit id {commit_id}")
return HttpResponse("200")
with dynamodb_lock(pull_request.id()):
github_logic.maybe_automerge_pull_request(pull_request)
github_controller.upsert_pull_request(pull_request)
return HttpResponse("200")
def create_attachments(body_text: str, task_id: str) -> None:
attachments = _extract_attachments(body_text)
for attachment in attachments:
try:
with urllib.request.urlopen(attachment.file_url) as f:
attachment_contents = f.read()
asana_client.create_attachment_on_task(
task_id,
attachment_contents,
attachment.file_name,
attachment.image_type,
)
except Exception as error:
logger.warn(
"Attachment creation failed. Creating task comment anyway.")
def rebalance_with_available_cash(self):
# if not self._is_all_markets_open():
# raise SystemError(
# "Not all of the markets are open currently. "
# "Try again in better time when both London and NYE are trading")
cash_balance = self._broker_interface.request_cash_balance()
if cash_balance <= 0:
raise SystemError(
"Insufficient funds to starts a rebalancing and ordering process."
)
logger.info(f"Cash balance: {cash_balance}")
cash_balance = 5000.0 # TESTING
logger.warn(f"Using constant: {cash_balance} for testing")
self._current_positions = self._broker_interface.request_all_holdings()
logger.info(f"Current positions: {self._current_positions}")
if not self._current_positions:
raise SystemError(
"No tradeable holdings was received from the broker")
assets_before_balance = self._create_rebalance_entites_and_enrich_data(
self._current_positions)
logger.info(f"Assets before rebalance: {assets_before_balance}")
self._fetch_latest_market_data()
rebalanced_assets = self._rebalancer.rebalance_by_contribution(
assets_before_balance, cash_balance - CASH_MARGIN)
logger.info(f"Assets after rebalance: {rebalanced_assets}")
self._perform_assets_gap_ordering(rebalanced_assets)
def _perform_data_request(self, conids_str, persist_if_not_valid=True):
jres = requests.get(
f"{API_URL}/iserver/marketdata/snapshot?conids={conids_str}",
verify=False,
timeout=REQUESTS_TIMEOUT_SEC)
res_content = json.loads(jres.content)
if not self._validate_result(res_content):
if self._retry_attempts > RETRIEVAL_RETRY_ATTEMPTS_THRESHOLD or not persist_if_not_valid:
raise SystemError(
"Retrieval for market data failed, the data received is not valid. Stopping."
)
self._retry_attempts += 1
sleep(1)
logger.warn(
f"Retrieval for market data failed. Retry #{self._retry_attempts}"
)
return self._perform_data_request(conids_str)
return res_content
def prepare_bam_list(bam,
color_factor,
colors,
share_y_by=-1,
plot_by=None,
barcodes=None,
kind: str = "bam",
show_mean: bool = False):
u"""
Prepare bam list
:return: [list of bam files, dict recorded the share y details]
"""
if is_bam(bam):
return [
BamInfo(path=bam,
alias=clean_star_filename(bam),
title=None,
label=None,
color=colors[0],
kind=kind)
], {}
# load barcode groups
barcodes_group = load_barcode(barcodes) if barcodes else {}
colors = load_colors(bam, barcodes, color_factor, colors)
# load bam list
shared_y = {} # {sample group: [BamInfo...]}
bam_list = OrderedDict()
with open(bam) as r:
for line in r:
lines = re.split(r"\t| {2,}", line.strip())
if not os.path.exists(lines[0]) and not os.path.isfile(lines[0]):
logger.warn(f"wrong input path {lines[0]}")
continue
if kind == "bam" and not is_bam(lines[0]):
raise ValueError(f"{lines[0]} seem not ba a valid BAM file")
temp_barcodes = barcodes_group.get(
clean_star_filename(lines[0]),
barcodes_group.get(lines[1], {}) if len(lines) > 1 else {})
if not barcodes_group:
key = lines[1] if len(lines) > 1 else clean_star_filename(
lines[0])
temp_barcodes[key] = None
for alias, barcode in temp_barcodes.items():
tmp = BamInfo(path=lines[0],
alias=alias,
title="",
label=alias,
color=colors[alias],
barcodes=set(barcode) if barcode else None,
kind=kind)
tmp.show_mean = show_mean
if alias not in bam_list.keys():
bam_list[alias] = tmp
else:
bam_list[alias] += tmp
if plot_by is not None:
if plot_by < 0:
tmp = shared_y.get("", [])
tmp.append(alias)
shared_y[""] = tmp
else:
try:
tmp = shared_y.get(lines[plot_by - 1], [])
tmp.append(alias)
shared_y[lines[plot_by - 1]] = tmp
except IndexError as err:
logger.error(err)
logger.error("Wrong --plot-by index")
logger.error(f"Your --plot-by is {plot_by}")
logger.error(f"Your error line in {lines}")
exit(err)
elif share_y_by > 0:
try:
tmp = shared_y.get(lines[share_y_by - 1], [])
tmp.append(alias)
shared_y[lines[share_y_by - 1]] = tmp
except IndexError as err:
logger.error(err)
logger.error("Wrong --share-y-by index")
logger.error(f"Your --share-y-by is {share_y_by}")
logger.error(f"Your error line in {lines}")
exit(err)
if len(bam_list) == 0:
logger.error(
"Cannot find any input bam file, please check the bam path or the input list"
)
exit(1)
if not barcodes:
return list(bam_list.values()), {
i: [bam_list[k] for k in j]
for i, j in shared_y.items()
}
return [bam_list[x] for x in colors.keys() if x in bam_list.keys()
], {i: [bam_list[k] for k in j]
for i, j in shared_y.items()}