def get(self, function):
""" function for get request via api
Parameters
----------
function : str
name of called function
"""
if request.args:
args = request.args
if function == 'create_project':
print('this is the function to create a project: ' +
args['project_name'])
project.create_project(args['project_name'])
print('this is the function to create a project: ' +
args['project_name'])
if function == 'get_project':
return project.get_project()
if function == 'get_project_files':
return project.get_project_files(args['project_name'])
if function == 'get_file':
project.get_file(args['project_name'], args['filename'])
print('this funciton parameteres: ' + args['filename'])
if function == 'delete_project':
project.delete_project(args['project_name'])
return 201
def main(args):
try:
scheduler = get_environment().get_scheduler()
except AttributeError:
scheduler = None
project = get_project()
with Pool() as pool:
project.print_status(scheduler, pool=pool, **vars(args))
def main(args):
env = environment.get_environment(test=args.test)
if args.ppn is None:
try:
args.ppn = env.cores_per_node
except AttributeError:
raise ValueError(
"Did not find a default value for the processors-per-node (ppn)."
"Please provide `--ppn` argument")
project = get_project()
project.submit(env, **vars(args))
def project():
api = request.args['api']
client = request.args['client']
server = request.args['server']
platform = request.args['platform']
data = p.get_project(api, client, server, platform)
response_body = {
"data":
data,
"name":
f"{api.replace(' ', '_')}_{client.replace(' ', '_')}_{server.replace(' ', '_')}_{platform.replace(' ', '_')}"
}
return make_response(jsonify(response_body), 200)
def to_csv(trello_json_path, project_csv_path):
table = project.get_project(trello_json_path).to_table()
with open(project_csv_path, 'wb') as project_csv:
writer = csv.writer(project_csv)
for row in table:
writer.writerow(row)
#!/usr/bin/env python
"""Print the project's status to screen."""
import argparse
from multiprocessing import Pool
from project import get_project
from environment import get_environment
def main(args):
try:
scheduler = get_environment().get_scheduler()
except AttributeError:
scheduler = None
project = get_project()
with Pool() as pool:
project.print_status(scheduler, pool=pool, **vars(args))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
get_project().add_print_status_args(parser)
main(parser.parse_args())
env = environment.get_environment(test=args.test)
if args.ppn is None:
try:
args.ppn = env.cores_per_node
except AttributeError:
raise ValueError(
"Did not find a default value for the processors-per-node (ppn)."
"Please provide `--ppn` argument")
project = get_project()
project.submit(env, **vars(args))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
get_project().add_submit_args(parser)
group = parser.add_argument_group("Execution configuration:",
"Specify the execution environment.")
group.add_argument(
'--np',
type=int,
default=1,
help="Specify the total # of processors to be used per operation.")
group.add_argument(
'--ppn',
type=int,
help="Specify the number of processors allocated to each node.")
parser.add_argument('-t',
'--test',
action='store_true',
help="Use TestEnvironment for testing.")
def main():
parser = argparse.ArgumentParser(description="""imm client
Example:
python3 imm_client.py -a ListProjects -r hz
python3 imm_client.py -a ListProjects -r hz -c ~/.ossutilconfig.immtest -e http://imm.hele.aliyuncs.com
python3 imm_client.py -a preview -r bj -b hatlonely-test-bucket -f test.docx -p ossdocdefault -c ~/.ossutilconfig.hatlonely.sub.test-account-robot -u 1509106610405543 --role "test-ram-account"
""")
parser.add_argument("-c",
"--config",
help="config path",
default=os.path.join(os.path.expanduser("~"),
".ossutilconfig"))
parser.add_argument("-r", "--region", help="region")
parser.add_argument("-i", "--access-key-id", help="access key id")
parser.add_argument("-s", "--access-key-secret", help="access key secret")
parser.add_argument("-a", "--action", help="action")
parser.add_argument("-p", "--project", help="project")
parser.add_argument("-b", "--bucket", help="bucket")
parser.add_argument("-f", "--filename", help="filename")
parser.add_argument("--src-type", help="src type")
parser.add_argument("-t", "--tgt-type", help="target type")
parser.add_argument("--role", help="role", default="test-ram-account")
parser.add_argument("-u", "--uid", help="user id")
parser.add_argument("-e", "--endpoint", help="endpoint")
args = parser.parse_args()
if args.config:
config = util.load_credential_from_local(args.config)
access_key_id = config["accessKeyID"]
access_key_secret = config["accessKeySecret"]
region_id = config["regionID"]
if args.region == "hz":
region_id = "cn-hangzhou"
elif args.region == "sh":
region_id = "cn-shanghai"
elif args.region == "bj":
region_id = "cn-beijing"
region = "oss-" + region_id
endpoint = "https://imm.{}.aliyuncs.com".format(region_id)
sts_endpoint = "https://sts.aliyuncs.com"
if args.endpoint:
endpoint = args.endpoint
if args.project:
proj = args.project
if args.bucket:
bucket = args.bucket
if args.filename:
filename = args.filename
src_type = args.src_type
if args.tgt_type:
tgt_type = args.tgt_type
if args.access_key_id:
access_key_id = args.access_key_id
if args.access_key_secret:
access_key_secret = args.access_key_secret
if args.uid:
uid = args.uid
if args.role:
role = args.role
if args.action == "AssumeRole":
print(
json.dumps(
sts.assume_role(sts_endpoint, access_key_id, access_key_secret,
uid, role)))
return
security_token = ''
if args.uid and args.role:
res = sts.assume_role(sts_endpoint, access_key_id, access_key_secret,
uid, role)
print(json.dumps(res))
access_key_id = res["Credentials"]["AccessKeyId"]
access_key_secret = res["Credentials"]["AccessKeySecret"]
security_token = res["Credentials"]["SecurityToken"]
if args.action == "ListProjects":
project.list_projects(endpoint, access_key_id, access_key_secret,
security_token, region_id)
elif args.action == "PutProject":
project.put_project(endpoint, access_key_id, access_key_secret,
security_token, proj, region_id)
elif args.action == "GetProject":
project.get_project(endpoint, access_key_id, access_key_secret,
security_token, proj, region_id)
elif args.action == "DelProject":
project.del_project(endpoint, access_key_id, access_key_secret,
security_token, proj, region_id)
elif args.action == "ConvertOfficeFormat":
doc.convert_office_format(endpoint, access_key_id, access_key_secret,
proj, region_id, bucket, filename, src_type,
tgt_type)
elif args.action == "preview":
doc.preview(access_key_id, access_key_secret, proj, region, bucket,
filename, security_token)
elif args.action == "GetOfficePreviewURL":
doc.get_office_preview_url(endpoint, access_key_id, access_key_secret,
proj, region_id, bucket, filename)
def extract(generator):
'''
Scrapes Kickstarter pages and parses features into
MongoDB database. This function calls the featurize
function from the featurizer module to insert data
into the MongoDB database.
'''
go = True
progress = 0
skipped = 0
failed = 0
while go:
block_size = random.randint(5, 10)
wait_time = random.randint(2, 4)
wait = False
print '\n'
print 'Block size: {0}'.format(block_size)
for i in xrange(0, block_size):
# founder_id, project_id = (int(x) for x in generator.next())
project_id, founder_id, project_url, founder_url, rewards_url = (
x for x in generator.next())
collection_check = set(db.ksdata.distinct('project_id', {}))
if project_id in collection_check:
print "Already scraped"
skipped += 1
wait = False
else:
try:
project_soup, project_url, status1 = get_project(
project_url)
founder_soup, founder_url, status2 = get_profile(
founder_url)
rewards_soup, rewards_url, status3 = get_rewards(
rewards_url)
if (status1 & status2 & status3) == 200:
featurize(project_id, founder_id, project_url,
founder_url, rewards_url, project_soup,
founder_soup, rewards_soup, collection)
progress += 1
wait = True
except requests.ConnectionError:
failed += 1
print '\n'
print 'Scraped: {}'.format(progress)
print 'Skipped: {}'.format(skipped)
print 'Failed: {}'.format(failed)
print 'Next block: {}s'.format(wait_time)
if wait:
time.sleep(wait_time)
else:
pass