def getMockRewinder(logger, repoID, credentials):
logger.debug("getMock, repoID: " + str(repoID) +
" maxing out calls to github.")
for i in range(0, 1000):
repoJSON = util.get_json(
logger, "https://api.github.com/repositories/" + str(repoID),
credentials)
repo_name = repoJSON['name']
default_branch = repoJSON['default_branch']
logger.debug("getMock - default branch name: " + default_branch)
clone_url = repoJSON['clone_url']
masterURL = re.sub(r'({.*)$', '/' + default_branch,
repoJSON['branches_url'])
masterJSON = util.get_json(logger, masterURL, credentials)
masterSHA = masterJSON['commit']['sha']
masterDate = masterJSON['commit']['commit']['committer']['date']
masterDateS = calendar.timegm(
dateutil.parser.parse(masterDate).utctimetuple())
sourceJSON = json.dumps({
"type": "Mock_Github",
"meta": {
"repoID": str(repoID),
"default_branch": str(default_branch),
"clone_url": clone_url,
"name": repo_name
},
"data": {
"sha": masterSHA,
"commitS": masterDateS
}
})
return rewinder.MockRewinder(repoID, sourceJSON)
def refresh_repos(self, lastRepo):
self.log("refresh_repos, lastRepo: " + str(lastRepo))
sinceLastJSON = util.get_json(
self.logger,
'https://api.github.com/repositories?since=' + str(lastRepo),
self.credentials, True)
if sinceLastJSON is None or len(str(sinceLastJSON)) < 3:
self.exhausted = True
return
repoID = "errorRepoID"
projectCounter = 0
for projectJSON in sinceLastJSON:
repoID = projectJSON['id']
self.lastRepoID = repoID
if repoID > self.stop:
self.logger.info(
"GiPyS - refresh_repos, stopped refreshing with stop:" +
str(self.stop) + " and current repoID: " + str(repoID))
return
try:
languagesJSON = util.get_json(self.logger,
projectJSON['languages_url'],
self.credentials, True)
pythonProjectFound = "Python" in languagesJSON
self.logger.info("GiPyS - refresh_repos, repoID:" +
str(repoID) + " hasPython: " +
str(pythonProjectFound) + " name: " +
projectJSON['name'])
self.logger.debug("GiPyS - refresh_repos, since:" +
str(self.nextID) + " len(repos): " +
str(len(self.repos)) + " languagesJSON: " +
str(languagesJSON))
if pythonProjectFound:
self.repos.append(projectJSON['id'])
except KeyboardInterrupt:
raise
except:
self.logger.warning(
"GiPyS - refresh_repos, failed to finish entire page of repos starting at lastRepo: "
+ str(lastRepo) + " but succeeded up to but excluding: " +
str(repoID) + " which is the first " +
str(projectCounter) +
" projects. this troublesome url will be skipped.")
self.nRRFailures += 1
projectCounter += 1
self.nProjects += 1
# repos can be empty if there are no python projects on this page
if len(self.repos) == 0:
self.logger.info(
"GiPyS - refresh_repos, no repos found on page starting at:" +
str(lastRepo))
self.refresh_repos(repoID)
return
self.logger.info("GiPyS - refresh_repos, finished refreshing, found " +
str(len(self.repos)) + " Python repos in " +
str(projectCounter) +
" Github projects with last scanned repoID: " +
str(repoID))
def get_articles(
article_id: str = None,
article_title: str = None,
tag: str = None,
page_num: int = 1,
page_size: int = 30,
current_user: schemas.UserVerify = Depends(deps.get_current_user)
) -> JSONResponse:
""" Return All Articles"""
if article_id is not None:
data = crud_articles.get_article(article_id=article_id)
if data is None:
return JSONResponse(status_code=500,
content={"message": "No Records Found"})
json_compatible_item_data = jsonable_encoder(data)
return JSONResponse(status_code=200, content=json_compatible_item_data)
else:
data = crud_articles.get_all_articles(tag=tag,
article_title=article_title,
page_num=page_num,
page_size=page_size)
if data is None:
return JSONResponse(status_code=500,
content={"message": "No Articles Found"})
return JSONResponse(status_code=200,
content={
"total_pages": data.pages,
"total_items": data.total_items,
"page_data": {
"page_num": page_num,
"item_count": data.page_size,
"items": get_json(data.items)
}
})
def __init__(self):
self.current_data_json = get_json(config.data_json_filename)
# self.patients_file = get_xlsx(config.patients_xlsx, "patients.xlsx")
self.patients_and_inspections_file = get_xlsx(
config.patients_and_inspections_xlsx, "patients_and_inspections.xlsx"
)
# self.patients_sheet = self.patients_file["Sheet1"]
# self.inspections_file = get_xlsx(config.inspections_xlsx, "inspections.xlsx")
# self.inspections_sheet = self.inspections_file["モトデータ"]
# self.main_summary_sheet = self.inspections_file["総括表"]
self.patients_sheet = self.patients_and_inspections_file["【公開OK】府HP用(陽性者ベース)"]
self.inspections_sheet = self.patients_and_inspections_file["【公開OK】コロナサイト用（日付ベース）"]
self.main_summary_sheet = self.patients_and_inspections_file["【公開OK】ピポット集計"]
self.contacts1_file = get_xlsx(config.contacts1_xlsx, "contacts1.xlsx")
self.contacts1_sheet = self.contacts1_file["Sheet1"]
self.contacts2_file = get_xlsx(config.contacts2_xlsx, "contacts2.xlsx")
self.contacts2_sheet = self.contacts2_file["Sheet1"]
self.patients_count = 3
self.inspections_count = 3
self.contacts1_count = 3
self.contacts2_count = 4
self._data_json = {}
self._patients_json = {}
self._patients_summary_json = {}
self._inspections_summary_json = {}
self._contacts1_summary_json = {}
self._contacts2_summary_json = {}
self._treated_summary_json = {}
self._main_summary_json = {}
self.last_update = datetime.today().astimezone(jst).strftime("%Y/%m/%d %H:%M")
self.get_patients()
self.get_inspections()
self.get_contacts1()
self.get_contacts2()
def get_users(
user_id: str = None,
page_num: int = 1,
current_user: schemas.UserVerify = Depends(deps.get_current_user)
) -> JSONResponse:
""" Return All Users"""
if user_id is not None:
db_user = crud_users.get_user_id(id=user_id)
if db_user is None:
return JSONResponse(status_code=500,
content={"message": "No User Found"})
return JSONResponse(status_code=200, content=db_user.to_json())
else:
db_user = crud_users.get_all_user(page_num=page_num)
if db_user is None:
return JSONResponse(status_code=500,
content={"message": "No Users Found"})
return JSONResponse(status_code=200,
content={
"total_pages": db_user.pages,
"total_items": db_user.total_items,
"page_data": {
"page_num": page_num,
"item_count": db_user.page_size,
"items": get_json(db_user.items)
}
})
def login_user(user: schemas.UserLogIn) -> JSONResponse:
""" Login user and Return Access Token"""
db_user = get_active_user(email=user.email)
print(db_user)
if db_user is None:
return JSONResponse(status_code=400,
content={"message": "Invalid Credentials"})
else:
is_password_correct = crud_login.check_username_password(
email=user.email,
password=user.password)
if is_password_correct is False:
return JSONResponse(status_code=400,
content={"message": "Invalid Credentials"})
else:
uid = str(uuid.uuid4().hex)
crud_login.login_user(user=user, session_id=uid)
access_token_expires = timedelta(
minutes=ProjectSettings.ACCESS_TOKEN_EXPIRE_MINUTES)
token = access_token.create_access_token(
data={"sub": user.email},
expires_delta=access_token_expires)
return JSONResponse(status_code=200,
content={"access_token": token,
"token_type": "Bearer",
"session_id": uid,
"user": get_json(get_user(
email=user.email))})
def get_price(horizon_host, pair):
"""return last trade price as DatedPrice"""
print "fetching latest price for:" + pair["name"]
params = make_trade_params(pair)
json_result = get_json(horizon_host + "/trades", params)
try:
trade_record = json_result["_embedded"]["records"][0]
except IndexError:
return DatedPrice(date=datetime.utcfromtimestamp(0), price=0)
price = float(trade_record["price"]["n"]) / float(trade_record["price"]["d"])
timestamp = parser.parse(trade_record["ledger_close_time"], ignoretz=True)
return DatedPrice(date=timestamp, price=price)
def get_prediction(input_path):
input_img = load_img(input_path, single_input_shape, grayscale)
with get_evaluation_context():
return jsonify(
json.loads(
get_json(
decode_predictions(
model.predict(input_img)
)
)
)
)
def aggregate_pair(horizon_host, pair, start, end, resolution):
"""
fetch all trades from given time period and aggregate
:return a tuple of (base_volume, counter_volume, trade_count)
"""
print "aggregating pair:", pair["name"]
values = (0, 0, 0)
params = make_aggregation_params(pair, start, end, resolution)
url = horizon_host + "/trade_aggregations?" + urlencode(params)
consumed = False
while not consumed:
print "fetching url:", url
json_result = get_json(url)
records = json_result['_embedded']['records']
for record in records:
values = sum_tuples(values, record_to_tuple(record))
consumed = len(records) < PAGE_LIMIT
url = json_result["_links"]["next"]["href"]
return values
base_asset_code = "EURT"
base_asset_issuer = "GAP5LETOV6YIE62YAM56STDANPRDO7ZFDBGSNHJQIYGGKSMOZAHOOS2S"
counter_asset_code = "PHP"
counter_asset_issuer = "GBUQWP3BOUZX34TOND2QV7QQ7K7VJTG6VSE7WMLBTMDJLLAW7YKGU6EP"
[[pair]]
name = "BTC_XEL"
base_asset_code = "BTC"
base_asset_issuer = "GATEMHCCKCY67ZUCKTROYN24ZYT5GK4EQZ65JJLDHKHRUZI3EUEKMTCH"
counter_asset_code = "XEL"
counter_asset_issuer = "GAXELY4AOIRVONF7V25BUPDNKZYIVT6CWURG7R2I6NQU26IQSQODBVCS"
''')
next_url = "https://horizon.stellar.org/assets?limit=200"
while True:
o = get_json(next_url)
next_url = o['_links']['next']['href']
records = o['_embedded']['records']
for r in records:
key = r['asset_code'] + ':' + r['asset_issuer']
if key not in whitelist:
if r['asset_code'] in blacklist_code:
continue
if r['flags']['auth_required']:
account_limit = min_account_holders_auth_required
else:
account_limit = min_account_holders
if r['num_accounts'] < account_limit:
continue
def nCommitsGithubRewinder(logger, repoID, repo_path, report_path,
uniqueSourceID, n, rewinder_type, credentials):
# this is the path of the new directory to clone the project into
repoIDPath = repo_path + str(repoID) + "/"
logger.debug("Factory, repoIDPath: " + repoIDPath)
# get metadata for sourceJSON
repoJSON = util.get_json(
logger, "https://api.github.com/repositories/" + str(repoID),
credentials, True)
repo_name = repoJSON['name']
default_branch = repoJSON['default_branch']
logger.debug("nCoGiRe - default branch name: " + default_branch)
clone_url = repoJSON['clone_url']
# make sure that repo_path exists, or create it.
if not os.path.exists(repo_path):
logger.critical(
"Factory - nCoGiRe, this path does not exist!! repo_path: " +
repo_path)
os.mkdir(repo_path)
# try to empty this folder and clone into a new folder in repo_path
for try_i in range(1, 4):
if (try_i > 1):
logger.info("nCoGiRe, attempt " + str(try_i) +
" to remove contents of repo_path: " + str(repo_path) +
" and then clone using clone_url: " + clone_url)
try:
# try to erase everything in repo_path
repoPathContents = [
os.path.join(repo_path, f) for f in os.listdir(repo_path)
]
logger.info("nCoGiRe, repoPathContents: " + str(repoPathContents))
if repoPathContents:
# shuffle: don't always get stuck behind a problem directory
random.shuffle(repoPathContents)
for x in repoPathContents:
try:
util.erasePath(x, logger)
except Exception as ex:
logger.warning(
"exception when trying to erasePath on: " + x +
" ex: " + str(ex))
# I've had enough problems erasing the repo_path. forget it.
# all that matters is that we can clone into an empty directory,
# and in the long run of course we want to avoid running out of
# disk space, so trying to erase all these directories is good
# make a new directory just for this new project - this should always succeed and allow git to clone, since all repoIDs are unique
os.mkdir(repoIDPath)
git.repo.base.Repo.clone_from(clone_url,
repoIDPath,
None,
branch=default_branch)
break
except Exception as e:
logger.info("Factory - nCoGiRe, unexpected error: " + str(e))
# just to be safe, always do these sanity checks.
# the repoIDPath should exist and contain the new ".git" every time.
if not os.path.exists(repoIDPath):
logger.info("Factory - nCoGiRe, path missing!")
raise RuntimeWarning(
"do not proceed with repoIDPath directory missing - you will rewind the tour_de_source project!"
)
repoIDPathContents = [
os.path.join(repoIDPath, f) for f in os.listdir(repoIDPath)
]
logger.info("contents of repoIDPath: " + str(repoIDPathContents))
if not repoIDPathContents:
logger.info("Factory - nCoGiRe, repoIDPathContents is empty!")
raise RuntimeWarning(
"do not proceed with repoIDPath directory empty - you will rewind the tour_de_source project!"
)
if repoIDPath + ".git" not in repoIDPathContents:
logger.info(
"Factory - nCoGiRe, repoIDPathContents has no git! cloning did not succeed."
)
raise RuntimeWarning(
"do not proceed with repoIDPath directory having no .git folder - you will rewind the tour_de_source project!"
)
# now get the time and sha for commits
commitList = []
repoGit = git.Git(repoIDPath)
hexshas = repoGit.log("--format=format:\"%ct %H\"").split('\n')
for line in hexshas:
cleanedLine = line[line.find('"') + 1:line.rfind('"')]
pair = cleanedLine.split(' ')
# this json is what couples each rewinded data source to a particular github project version. For bitbucket or another source, many of the inner fields should be different, but the three root fields: type, meta and data should be the same for all future sourceJson objects like this.
sourceJson = json.dumps({
"type": "Github",
"meta": {
"repoID": str(repoID),
"default_branch": str(default_branch),
"clone_url": clone_url,
"name": repo_name
},
"data": {
"sha": str(pair[1]),
"commitS": str(pair[0])
}
})
# logger.info("sourceJSON created: " + sourceJson)
if len(pair) == 2:
commitList.append((pair[0], pair[1], sourceJson))
# here we are sorting by the first field of the tuple: the commit time
commitList.sort(reverse=True)
nCommits = len(commitList)
if nCommits <= n:
# return a rewinder with all commits
stack = commitList
else:
# return a rewinder with n commits, spaced out about evenly
unitFloat = nCommits / float(n - 1)
stack = []
# always put the most recent first
stack.append(commitList[0])
# put the rest
for multiplier in range(1, n):
commitIndex = int(multiplier * unitFloat) - 1
stack.append(commitList[commitIndex])
# remember python lists pop at the tail...
stack.reverse()
return rewinder.GitRewinder(repoIDPath, rewinder_type, uniqueSourceID,
stack)
def get_prediction(input_path):
input_img = load_img(input_path, single_input_shape)
with graph.as_default():
return jsonify(
json.loads(
get_json(decode_predictions(model.predict(input_img)))))
