def list(self, request):
action = request.GET.get('action')
hadoop_settings = settings.get_hadoop_settings(["paths", "actions"])
if action == 'check':
running, stopped = check_hadoop_services(True)
num, output = check_hadoop_applications()
return Response(
{
"running_services": running,
"stopped_services": stopped,
"number_of_running_applications": num,
"running_applications": output
},
status=status.HTTP_200_OK)
if action == 'restart':
restart = hadoop_settings["paths"]["hadoop_home"] + hadoop_settings[
"paths"]["sbin"] + "/" + hadoop_settings["actions"]['stop']
output = run_command(restart, True)
action = 'start'
action = hadoop_settings["paths"]["hadoop_home"] + hadoop_settings[
"paths"]["sbin"] + "/" + hadoop_settings["actions"][action]
output = run_command(action, True)
return Response({"output": output}, status=status.HTTP_200_OK)
def create(self, request):
action = request.GET.get('action')
hadoop_settings = settings.get_hadoop_settings(["paths", "actions"])
if action == 'upload':
path = request.data.get("path")
dest = request.data.get("dest")
files = request.FILES
print files
# Try to upload
try:
action = hadoop_settings["paths"][
"hadoop_home"] + hadoop_settings["paths"][
"bin"] + "/hadoop " + hadoop_settings["actions"][
action] + " " + path + " " + dest
run_command(action)
except:
raise CustomRequestFailed(
"Something went wrong while trying to upload.")
elif action == 'run':
path = request.data.get("path")
args = request.data.get("args")
# Try to run
try:
action = hadoop_settings["paths"][
"hadoop_home"] + hadoop_settings["paths"][
"bin"] + "/hadoop " + hadoop_settings["actions"][
action] + " " + path + " " + args
run_command(action)
except:
raise CustomRequestFailed(
"Something went wrong while trying to run the job.")
return Response(status=status.HTTP_200_OK)
def build_project(project_dir, workspace, scheme):
log(f"Installing CocoaPods for '{os.path.join(project_dir, workspace)}'")
rv, out, err = run_command(["pod", "install", "--verbose"],
working_dir=project_dir)
if rv != 0:
log(f"Failed to install CocoaPod for '{project_dir}; output={out}, error={err}'"
)
return rv
log(f"Building project '{os.path.join(project_dir, workspace)}'")
build_command = [
"xcodebuild",
"build",
"-workspace",
workspace,
"-scheme",
scheme,
"-sdk",
"iphonesimulator",
]
rv, out, err = run_command(build_command, working_dir=project_dir)
if rv != 0:
log(f"Failed to build '{project_dir}; output={out}, error={err}'")
return rv
def create_archive(framework, project_file, build_for_device):
if build_for_device:
archive_path = f"{IOS_DEVICE_ARCHIVE_PATH}{framework}"
destination = "generic/platform=iOS"
else:
archive_path = f"{IOS_SIMULATOR_ARCHIVE_PATH}{framework}"
destination = "generic/platform=iOS Simulator"
cmd = [
"xcodebuild",
"archive",
"-project",
project_file,
"-scheme",
framework,
"-destination",
destination,
"-archivePath",
archive_path,
"SKIP_INSTALL=NO",
"BUILD_LIBRARY_FOR_DISTRIBUTION=YES"
]
(exit_code, out, err) = run_command(cmd, keepalive_interval=300, timeout=7200)
if exit_code == 0:
log(f"Created iOS archive {framework} {destination}")
else:
log(f"Could not create xcodebuild archive: {framework} output: {out}; error: {err}")
sys.exit(exit_code)
def get_php_configs(ssh_path, user, host, debug=False):
cmd = '"{0}" -ssh {1}@{2} "/usr/bin/php -i | grep \'upload_max_filesize\|post_max_size\'"'.format(ssh_path, user, host)
if debug:
print(cmd)
output, error = functions.run_command(cmd, printError=debug)
if error:
return {'error': error}
d = {}
for var in output.strip().split('\n'):
vals = var.split(' => ')
d[vals[0]] = vals[1]
return d
def map_framework_to_project(framework_list):
framework_map = {}
cmd = [
"xcodebuild",
"-project",
"AWSiOSSDKv2.xcodeproj",
"-list",
]
(exit_code, out, err) = run_command(cmd, keepalive_interval=300, timeout=7200)
if exit_code == 0:
log(f"List of schema found")
else:
log(f"Xcodebuild list failed: output: {out}; error: {err}")
sys.exit(exit_code)
for framework in framework_list:
if framework not in str(out):
framework_map[framework] = "./AWSAuthSDK/AWSAuthSDK.xcodeproj"
else:
framework_map[framework] = "AWSiOSSDKv2.xcodeproj"
return framework_map
def create_checksum(archive_path, spm_manifest_repo, version):
framework_to_checksum = {}
for framework in xcframeworks:
final_archive_name_with_ext = f"{framework}-{version}.zip"
zipfile_path = os.path.join(archive_path, final_archive_name_with_ext)
cmd = [
"swift", "package", "--package-path", spm_manifest_repo,
"compute-checksum", zipfile_path
]
(exit_code, out, err) = run_command(cmd,
keepalive_interval=300,
timeout=7200)
if exit_code == 0:
logging.info(f"Created check sum for archive {framework} {out}")
else:
logging.error(
f"Could not create checksum for archive: {framework} output: {out}; error: {err}"
)
sys.exit(exit_code)
framework_to_checksum[framework] = out.decode("utf-8").rstrip()
return framework_to_checksum
if os.path.exists(xcframework):
log(f"skipping {framework}...")
else:
log(f"Creating XCF for {framework}")
cmd = [
"xcodebuild",
"-create-xcframework",
"-framework",
ios_device_framework,
"-debug-symbols",
ios_device_debug_symbols,
"-framework",
ios_simulator_framework,
"-debug-symbols",
ios_simulator_debug_symbols,
"-output",
xcframework
]
(exit_code, out, err) = run_command(cmd, keepalive_interval=300, timeout=7200)
if exit_code == 0:
log(f"Created XCFramework for {framework}")
else:
log(f"Could not create XCFramework: {framework} output: {out}; error: {err}")
sys.exit(exit_code)
if os.path.exists(IOS_DEVICE_ARCHIVE_PATH):
shutil.rmtree(IOS_DEVICE_ARCHIVE_PATH)
if os.path.exists(IOS_SIMULATOR_ARCHIVE_PATH):
shutil.rmtree(IOS_SIMULATOR_ARCHIVE_PATH)
def process(bedgraph, junc, genome, output, strand, minjxncount, min_percent,
min_reads, window_size):
# === calculate window read counts. incorporate introns if --junc provided. ===
gc_strand, outfile_temp_unionbg = ".", None # @2020.10.10 by Zhang yiming - init variable
if junc:
if re.search(r'\.bed$', junc):
logger.info('converting junction .bed file to .bedgraph {}',
str(datetime.now().time()))
# --- get overlapping junction regions ---
temp_jxn_intersect = output + '.temp.junction.intersect.txt'
if strand == 0:
temp_jxn_file = junc
pb.BedTool(junc).genome_coverage(bg=True, g=genome).intersect(
wo=True, b=temp_jxn_file).saveas(temp_jxn_intersect)
else:
if strand == 1:
gc_strand = '+'
elif strand == -1:
gc_strand = '-'
# get this strand's junctions only & filter junctions by read counts
temp_jxn_file = output + '.' + str(strand) + '.jxn.bed'
o = open(temp_jxn_file, 'w')
with open(junc, 'r') as f:
for line in f:
(chrom, start, end, name, cov,
strand) = line.rstrip().split('\t')
if strand == gc_strand and int(cov) >= minjxncount:
o.write(line)
o.close()
# get junction genomecov & intersect to get the # reads per genomecov region
pb.BedTool(junc).genome_coverage(
bg=True, g=genome, strand=gc_strand).intersect(
wo=True, b=temp_jxn_file).saveas(temp_jxn_intersect)
# --- get bedgraph of junction regions ---
jxn2cov = {}
with open(temp_jxn_intersect, 'r') as f:
for line in f:
if strand == 0:
(cchrom, cstart, cend, ccov, jchrom, jstart, jend,
jname, jcov, overlap) = line.rstrip().split('\t')
else:
(cchrom, cstart, cend, ccov, jchrom, jstart, jend,
jname, jcov, jstrand,
overlap) = line.rstrip().split('\t')
jxn = ':'.join([cchrom, cstart, cend])
if jxn not in jxn2cov:
jxn2cov[jxn] = int(jcov)
else:
jxn2cov[jxn] += int(jcov)
outfile_temp_jxn_bg_unsorted = output + '.' + str(
strand) + '.jxn.bedgraph.unsorted'
o = open(outfile_temp_jxn_bg_unsorted, 'w')
for jxn in jxn2cov:
o.write('\t'.join(jxn.split(':')) + '\t' + str(jxn2cov[jxn]) +
'\n')
o.close()
outfile_temp_jxn_bg = output + '.' + str(strand) + '.jxn.bedgraph'
sort_bedfile(outfile_temp_jxn_bg_unsorted, outfile_temp_jxn_bg)
# --- cleanup ---
os.remove(temp_jxn_intersect)
os.remove(outfile_temp_jxn_bg_unsorted)
if strand != 0:
os.remove(temp_jxn_file)
elif re.search(r'\.bedgraph$', junc):
outfile_temp_jxn_bg = junc
else:
logger.error(
'EXIT: File extension not recognized. Please input .bed or .bedgraph --junc file'
)
sys.exit(1)
# combine bedgraphs: using subprocess because pybedtools complains about unionbedg format not being bed
logger.info('merging junctions with bedgraph {}',
str(datetime.now().time()))
outfile_temp_unionbg = output + '.unionbedg'
j = open(outfile_temp_unionbg, 'w')
cmd = ['bedtools', 'unionbedg', '-i', bedgraph, outfile_temp_jxn_bg]
stdout, stderr = run_command(cmd, stdoutfile=j)
j.close()
os.remove(outfile_temp_jxn_bg)
# write output
outfile_temp_merge_bg_jxn = output + '.bg_jxn.bedgraph'
m = open(outfile_temp_merge_bg_jxn, 'w')
with open(outfile_temp_unionbg, 'r') as f:
for line in f:
(chrom, start, end, count1, count2) = line.split('\t')
if '+' in count1 or '+' in count2:
count1 = scientific2int(count1) if '+' in count1 else int(
count1)
count2 = scientific2int(count2) if '+' in count2 else int(
count2)
m.write('\t'.join(
map(str, [chrom, start, end,
int(count1) + int(count2)])) + '\n')
m.close()
logger.info('calculating window read counts: {}',
str(datetime.now().time()))
window2count_length, chr2starts, chr2ends = window_count(
outfile_temp_merge_bg_jxn, window_size)
else:
logger.error(
'No --junc file was input. It is recommended to include this input.'
)
logger.info('calculating window read counts: {}',
str(datetime.now().time()))
window2count_length, chr2starts, chr2ends = window_count(
bedgraph, window_size)
outfile_temp_merge_bg_jxn = bedgraph
logger.info('{} total windows {}', len(window2count_length.keys()),
str(datetime.now().time()))
# === merge consecutive windows ===
logger.info('merging consecutive windows: {}', str(datetime.now().time()))
outfile_temp_windows = output + '.windows'
outfile_temp_bg_merged = output + '.bg.merged'
transcripts_trimmed = merge_windows(window2count_length, min_percent,
min_reads, outfile_temp_merge_bg_jxn,
outfile_temp_bg_merged,
outfile_temp_windows)
if transcripts_trimmed != 0:
logger.info('total transcripts after merging {}',
len(transcripts_trimmed))
# === get strand ===
if strand == 1:
strand = "+"
elif strand == -1:
strand = "-"
elif strand == 0:
strand = 0
else:
logger.error('DIED: did not recognize strand {}', strand)
sys.exit()
# === print output -> unsorted bed format ===
logger.info("printing output (unsorted): {}",
str(datetime.now().time()))
transcripts_trimmed_list = list(sorted(transcripts_trimmed.keys()))
o = open(output, 'w+')
for ind, tx in enumerate(transcripts_trimmed_list):
(chrom, start, end) = tx
# if not chrom.startswith('chr'):
# chrom = 'chr' + chrom
if strand != 0:
o.write('\t'.join([
chrom,
str(start),
str(end), 'tx_' + str(ind), '0', strand
]) + '\n')
else:
o.write('\t'.join(
[chrom, str(start),
str(end), 'tx_' + str(ind)]) + '\n')
o.close()
# === output: sorted bed format & delete the temp unsorted file ===
sort_bedfile(output, output, sort_by_bedtools=True)
# === clean up ===
os.remove(outfile_temp_windows)
if junc:
if outfile_temp_unionbg:
os.remove(outfile_temp_unionbg)
os.remove(outfile_temp_merge_bg_jxn)
user = sys.argv[1]
token = sys.argv[2]
title = sys.argv[3]
body = sys.argv[4]
base = sys.argv[5]
head = sys.argv[6]
github_org = sys.argv[7]
repo = sys.argv[8]
log(f"Creating pull request for {repo}")
auth_user = f"{user}:{token}"
data = f'{{"title":"{title}","base":"{base}","head":"{head}", "body":"{body}"}}'
log(f"Pull request data: {data}")
repo_url = f"https://api.github.com/repos/{github_org}/{repo}/pulls"
exit_code, out, err = run_command([
"curl", "--user", auth_user, "--request", "POST", "--data", data, repo_url
])
if exit_code == 0:
log(f"Created pull request for {repo}")
else:
log(f"Could not create pull request for {repo}: output: {out}; error: {err}"
)
sys.exit(exit_code)
import datetime
import sys
from functions import log, run_command
awsprofile = sys.argv[1]
distribution_id = sys.argv[2]
path = sys.argv[3]
invalidation = '{{"Paths":{{"Quantity":1,"Items":["/{0}"]}},"CallerReference":"{0}{1}"}}'.format(
path, datetime.datetime.now())
exit_code, out, err = run_command(
["aws", "configure", "set", "preview.cloudfront", "true"])
if exit_code != 0:
log(f"Failed to configure preview.cloudfront; output={out}, error={err}'")
sys.exit(exit_code)
exit_code, out, err = run_command([
"aws",
"cloudfront",
"create-invalidation",
"--distribution-id",
distribution_id,
"--invalidation-batch",
invalidation,
"--profile",
awsprofile,
])
if exit_code != 0:
log(f"Failed to create invalidation; output={out}, error={err}'")
from framework_list import frameworks
from functions import log, run_command
log("Publishing CocoaPods")
for framework in frameworks:
log(f"Publishing {framework}")
# Most pods take a few minutes to build, and a few seconds to push to trunk. However, the
# AWSiOSSDK podspec can take a long time to build, since it builds each dependent pod as
# part of its linting process, so set the timeout accordingly.
(exit_code, out, err) = run_command(
[
"bundle", "exec", "pod", "trunk", "push", f"{framework}.podspec",
"--allow-warnings", "--synchronous"
],
keepalive_interval=300,
timeout=3600,
)
if exit_code != 0 and "Unable to accept duplicate entry for" in str(out):
log(f"Already published {framework}")
elif exit_code == 0:
log(f"Published {framework}")
else:
log(f"Could not publish {framework}: output: {out}; error: {err}")
sys.exit(exit_code)
if framework == "AWSCore":
log(f"pod repo update after {framework}")
(exit_code, out, err) = run_command(
import os
import sys
from shutil import copyfile
from functions import log, run_command
root = sys.argv[1]
dest = sys.argv[2]
files = {
"LICENSE": "LICENSE",
"LICENSE.APACHE": "LICENSE.APACHE",
"NOTICE": "NOTICE",
"README.md": "README.md",
"CircleciScripts/src/README.html": "src/source.html",
"CircleciScripts/samples/README.html": "samples/samples.html",
}
for source, target in files.items():
s = os.path.join(root, source)
t = os.path.join(dest, target)
target_dir = os.path.dirname(t)
exit_code, out, err = run_command(["mkdir", "-p", target_dir])
if exit_code != 0:
log(f"Failed to make directory '{target_dir}'; output={out}, error={err}"
)
copyfile(s, t)
