def getdata(self):
save_dir = os.path.join(os.getcwd(), "corona_report/data")
print('save_dir:', save_dir)
if not os.path.exists(save_dir):
os.system('mkdir -p ' + save_dir)
csv_file_name = 'agg_data.csv'
print('...', csv_file_name)
if os.path.exists(os.path.join(save_dir, csv_file_name)):
current_time_epoch = calendar.timegm(time.gmtime())
last_update = os.path.getmtime(
os.path.join(save_dir, csv_file_name))
print("t:", current_time_epoch, last_update)
diff = current_time_epoch - last_update
print("diff:", diff)
if (diff < 20000):
return
print('Updating data...')
df = github.get()
print("back from github.saving")
#print("back from github",df.info())
# sheets need to be sorted by date value
#print('Sorting by datetime...')
df = df.sort_values('datetime')
df.astype(str).to_csv(os.path.join(save_dir, csv_file_name))
def get_data_async():
cached_resources = cache.get("resources_to_cache")
if not cached_resources:
return
for resource in cached_resources:
(data, status, header) = get(resource)
cache.set("data+" + resource, data)
cache.set("headers+" + resource, header)
def calc_labels(self, pr):
label_requirements = []
parsed = urlparse(pr['url'])
orgs_url = '{0}://{1}/user/orgs'.format(parsed.scheme, parsed.netloc)
for org in github.get(orgs_url):
for team_slug, label in self._config:
membership_url = '{0}/teams/{1}/memberships/{2}'.format(org['url'], team_slug, pr['user']['login'])
try:
membership = github.get(membership_url)
# The other possible state is "pending", which means the user has not
# accepted the invitation to join the team yet.
requirement = membership['state'] == 'active'
except exceptions.HTTPError as ex:
# This API returns 404 if the user is not in the team.
if ex.response.status_code != 404:
raise
requirement = False
label_requirements.append((label, requirement))
return label_requirements
def release(config):
log("Releasing Defold %s to GitHub" % (config.version))
if config.github_token is None:
log("No GitHub authorization token")
return
repo = config.github_target_repo
if repo is None:
repo = "defold/defold"
repo = "/repos/%s" % repo
# get the sha for the version we are releasing by
# searching all tags for the version
ref_tag = None
ref_tags = github.get("%s/git/refs/tags" % repo, config.github_token)
if not ref_tags:
log("Unable to get ref tags")
exit(1)
for tag in ref_tags:
if config.version in tag.get("ref"):
ref_tag = tag
break
if not ref_tag:
log("Unable to find ref tag for version %s" % (config.version))
exit(1)
release_tag = github.get(
"%s/git/tags/%s" % (repo, ref_tag.get("object").get("sha")),
config.github_token)
if not release_tag:
log("Unable to get release tag")
exit(1)
release_sha = release_tag.get("object").get("sha")
# get the release on S3
log("Getting S3 release for version %s with sha %s" %
(config.version, release_sha))
s3_release = s3.get_single_release(config.archive_path, config.version,
release_sha)
if not s3_release.get("files"):
log("No files found on S3")
exit(1)
# try to find existing release for the current version
release = None
response = github.get("%s/releases" % repo, config.github_token)
if not response:
log("Unable to get releases")
exit(1)
for r in response:
if r.get("tag_name") == config.version:
release = r
break
# no existing relase, create one!
if not release:
data = {
"tag_name": config.version,
"name": "v" + config.version,
"body": "Defold version " + config.version,
"draft": False,
"prerelease": False
}
release = github.post("%s/releases" % repo,
config.github_token,
json=data)
if not release:
log("Unable to create new release")
exit(1)
# check that what we found/created a release
if not release:
log("Unable to create GitHub release for %s" % (config.version))
exit(1)
# remove existing uploaded assets
log("Deleting existing artifacts from the release")
for asset in release.get("assets", []):
log("Deleting %s" % (asset.get("id")))
github.delete("%s/releases/assets/%s" % (repo, asset.get("id")),
config.github_token)
# upload_url is a Hypermedia link (https://developer.github.com/v3/#hypermedia)
# Example: https://uploads.github.com/repos/defold/defold/releases/25677114/assets{?name,label}
# Can be parsed and expanded using: https://pypi.org/project/uritemplate/
# for now we ignore this and fix it ourselves (note this may break if GitHub
# changes the way uploads are done)
log("Uploading artifacts to GitHub from S3")
upload_url = release.get("upload_url").replace("{?name,label}", "?name=%s")
base_url = "https://" + urlparse.urlparse(config.archive_path).hostname
for file in s3_release.get("files", None):
# download file
download_url = base_url + file.get("path")
filepath = config._download(download_url)
filename = re.sub(r'https://d.defold.com/archive/(.*?)/', '',
download_url)
# file stream upload to GitHub
log("Uploading to GitHub " + filename)
with open(filepath, 'rb') as f:
content_type, _ = mimetypes.guess_type(filename)
headers = {
"Content-Type": content_type or "application/octet-stream"
}
github.post(upload_url % (filename),
config.github_token,
data=f,
headers=headers)
log("Released Defold %s to GitHub" % (config.version))
def release_markdown(config):
"""
Produce a markdown with each release item
Pushes changes to a file releases.md
Currently only grabs bob and editor
"""
log("Releasing Defold %s to GitHub" % (config.version))
if config.github_token is None:
log("No GitHub authorization token")
return
repo = config.github_target_repo
if repo is None:
repo = get_default_repo()
repo = "/repos/%s" % repo
target_url = "%s/contents/RELEASES.md" % repo
release_sha = config.github_sha1
if not release_sha:
release_sha = get_git_sha1()
# get the release on S3
log("Getting S3 release for version %s with sha %s" %
(config.version, release_sha))
s3_release = s3.get_single_release(config.archive_path, config.version,
release_sha)
if not s3_release.get("files"):
log("No files found on S3")
exit(1)
def test_path(path):
if path.endswith('bob.jar'):
return True
if 'Defold-x86_64-' in path:
return True
return False
release_info = {}
base_url = "https://" + urlparse.urlparse(config.archive_path).hostname
for file in s3_release.get("files", None):
# download file
download_url = base_url + file.get("path")
if not test_path(download_url):
continue
if download_url.endswith('bob.jar'):
release_info['url_bob'] = download_url
if 'Defold-x86_64-' in download_url:
editor_platform = 'osx'
if 'win32' in download_url:
editor_platform = 'win'
if 'linux' in download_url:
editor_platform = 'lin'
release_info['url_editor_%s' % editor_platform] = download_url
print "url", download_url
release_info['version'] = config.version
release_info['sha1'] = release_sha
release_info['branch'] = get_git_branch()
release_info['date'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
response = github.get(target_url, config.github_token)
if not response:
log("Unable to get target_url: " + target_url)
exit(1)
release_data = ''
target_sha = None
for r in response:
if r == 'sha':
target_sha = response.get(r)
if r == 'content':
data = response.get(r)
release_data = base64.b64decode(data).decode('ascii')
new_data = MARKDOWN_TEMPLATE % release_info
new_data += '\n\n' + release_data
log("Uploading to GitHub " + target_url)
post_data = {}
post_data['message'] = "Release %s - %s" % (config.version, release_sha)
post_data['content'] = base64.b64encode(
new_data.encode('ascii')).decode('ascii')
post_data[
'sha'] = target_sha # we need the current sha of the target file in order to update it
headers = {"Content-Type": "application/octet-stream"}
github.put(target_url,
config.github_token,
json=post_data,
headers=headers)
log("Released Defold %s - %s to %s" %
(config.version, release_sha, target_url))
def repo():
repo_dict = github.get('repos/cenkalti/github-flask')
return str(repo_dict)
def release(config, tag_name, release_sha, s3_release):
log("Releasing Defold %s to GitHub" % tag_name)
if config.github_token is None:
log("No GitHub authorization token")
return
channel = config.channel
if channel is None:
log("No release channel specified")
return
log("tag name: %s" % tag_name)
log("release sha1: %s" % release_sha)
log("channel: %s" % channel)
source_repo = os.environ.get('GITHUB_REPOSITORY', "defold/defold")
source_repo = "/repos/%s" % source_repo
log("source repo: %s" % source_repo)
target_repo = config.github_target_repo
if target_repo is None:
target_repo = os.environ.get('GITHUB_REPOSITORY', None)
if target_repo is None:
target_repo = "defold/defold"
target_repo = "/repos/%s" % target_repo
log("target repo: %s" % target_repo)
release_name = 'v%s - %s' % (config.version, channel)
draft = False # If true, it won't create a tag
pre_release = channel not in (
'stable', 'beta'
) # If true, it will be marked as "Pre-release" in the UI
if not s3_release.get("files"):
log("No files found on S3 with sha %s" % release_sha)
exit(1)
release = None
response = github.get("%s/releases/tags/%s" % (target_repo, tag_name),
config.github_token)
if response:
release = response
data = {
"tag_name":
tag_name,
"name":
release_name,
"body":
"Defold version %s channel=%s sha1=%s date='%s'" %
(config.version, channel, release_sha,
datetime.now().strftime("%Y-%m-%d %H:%M:%S")),
"draft":
draft,
"prerelease":
pre_release
}
if not release:
log("No release found with tag %s, creating a new one" % tag_name)
release = github.post("%s/releases" % target_repo,
config.github_token,
json=data)
else:
release_id = release['id']
log("Updating existing release: %s - %s - %s" %
(tag_name, release_id, release.get('url')))
release = github.patch("%s/releases/%s" % (target_repo, release_id),
config.github_token,
json=data)
# check that what we created/updated a release
if not release:
log("Unable to update GitHub release for %s" % (config.version))
exit(1)
# remove existing uploaded assets (It's not currently possible to update a release asset
log("Deleting existing artifacts from the release")
for asset in release.get("assets", []):
asset_url = asset.get("url")
log("Deleting %s - %s" % (asset.get("id"), asset.get("name")))
github.delete(asset_url, config.github_token)
# upload_url is a Hypermedia link (https://developer.github.com/v3/#hypermedia)
# Example: https://uploads.github.com/repos/defold/defold/releases/25677114/assets{?name,label}
# Can be parsed and expanded using: https://pypi.org/project/uritemplate/
# for now we ignore this and fix it ourselves (note this may break if GitHub
# changes the way uploads are done)
log("Uploading artifacts to GitHub from S3")
base_url = "https://" + urlparse.urlparse(config.archive_path).hostname
def is_main_file(path):
return os.path.basename(path) in ('bob.jar',
'Defold-x86_64-darwin.dmg',
'Defold-x86_64-linux.zip',
'Defold-x86_64-win32.zip')
urls = set(
) # not sure why some files are reported twice, but we don't want to download/upload them twice
for file in s3_release.get("files", None):
# download file
path = file.get("path")
keep = False
if is_main_file(path):
keep = True
if not keep:
continue
download_url = base_url + path
urls.add(download_url)
upload_url = release.get("upload_url").replace("{?name,label}", "?name=%s")
for download_url in urls:
filepath = config._download(download_url)
filename = re.sub(r'https://%s/archive/(.*?)/' % config.archive_path,
'', download_url)
basename = os.path.basename(filename)
# file stream upload to GitHub
log("Uploading to GitHub " + filename)
with open(filepath, 'rb') as f:
content_type, _ = mimetypes.guess_type(basename)
headers = {
"Content-Type": content_type or "application/octet-stream"
}
name = filename
if is_main_file(path):
name = basename
github.post(upload_url % (name),
config.github_token,
data=f,
headers=headers)
log("Released Defold %s to GitHub" % tag_name)
def _cache(path):
(data, status, headers) = get_from_cache(path)
if data is None and status != 404:
return get(path)
return (data, status, headers)
formatter_class=RawTextHelpFormatter)
p.add_argument('-f','--force', required= False, help='Force', action='store_true')
p.add_argument('-out','--output_folder', required= False, help='output folder')
ns = p.parse_args()
if ns.output_folder:
out = ns.output_folder
else:
out=home = str(Path.home())+"/Desktop/Corona/"
from config import REPO, TMP_FOLDER, TMP_GIT, DATA, KEEP_COLS, NUMERIC_COLS
############ DATA SELECTION ############
df = github.get()
#print (df)
############ COUNTRY SELECTION ############
def get_similar_countries(c, country_list):
pos_countries = get_close_matches(c, country_list)
if len(pos_countries) > 0:
print('\033[1;31m'+c, 'was not listed. did you mean', pos_countries[0].capitalize() + '?\033[0;0m')
#Only delete if its a covidify generated folder
if 'Desktop/covidify-output-' in out:
