def getdata(self): save_dir = os.path.join(os.getcwd(), "corona_report/data") print('save_dir:', save_dir) if not os.path.exists(save_dir): os.system('mkdir -p ' + save_dir) csv_file_name = 'agg_data.csv' print('...', csv_file_name) if os.path.exists(os.path.join(save_dir, csv_file_name)): current_time_epoch = calendar.timegm(time.gmtime()) last_update = os.path.getmtime( os.path.join(save_dir, csv_file_name)) print("t:", current_time_epoch, last_update) diff = current_time_epoch - last_update print("diff:", diff) if (diff < 20000): return print('Updating data...') df = github.get() print("back from github.saving") #print("back from github",df.info()) # sheets need to be sorted by date value #print('Sorting by datetime...') df = df.sort_values('datetime') df.astype(str).to_csv(os.path.join(save_dir, csv_file_name))
def get_data_async(): cached_resources = cache.get("resources_to_cache") if not cached_resources: return for resource in cached_resources: (data, status, header) = get(resource) cache.set("data+" + resource, data) cache.set("headers+" + resource, header)
def calc_labels(self, pr): label_requirements = [] parsed = urlparse(pr['url']) orgs_url = '{0}://{1}/user/orgs'.format(parsed.scheme, parsed.netloc) for org in github.get(orgs_url): for team_slug, label in self._config: membership_url = '{0}/teams/{1}/memberships/{2}'.format(org['url'], team_slug, pr['user']['login']) try: membership = github.get(membership_url) # The other possible state is "pending", which means the user has not # accepted the invitation to join the team yet. requirement = membership['state'] == 'active' except exceptions.HTTPError as ex: # This API returns 404 if the user is not in the team. if ex.response.status_code != 404: raise requirement = False label_requirements.append((label, requirement)) return label_requirements
def release(config): log("Releasing Defold %s to GitHub" % (config.version)) if config.github_token is None: log("No GitHub authorization token") return repo = config.github_target_repo if repo is None: repo = "defold/defold" repo = "/repos/%s" % repo # get the sha for the version we are releasing by # searching all tags for the version ref_tag = None ref_tags = github.get("%s/git/refs/tags" % repo, config.github_token) if not ref_tags: log("Unable to get ref tags") exit(1) for tag in ref_tags: if config.version in tag.get("ref"): ref_tag = tag break if not ref_tag: log("Unable to find ref tag for version %s" % (config.version)) exit(1) release_tag = github.get( "%s/git/tags/%s" % (repo, ref_tag.get("object").get("sha")), config.github_token) if not release_tag: log("Unable to get release tag") exit(1) release_sha = release_tag.get("object").get("sha") # get the release on S3 log("Getting S3 release for version %s with sha %s" % (config.version, release_sha)) s3_release = s3.get_single_release(config.archive_path, config.version, release_sha) if not s3_release.get("files"): log("No files found on S3") exit(1) # try to find existing release for the current version release = None response = github.get("%s/releases" % repo, config.github_token) if not response: log("Unable to get releases") exit(1) for r in response: if r.get("tag_name") == config.version: release = r break # no existing relase, create one! if not release: data = { "tag_name": config.version, "name": "v" + config.version, "body": "Defold version " + config.version, "draft": False, "prerelease": False } release = github.post("%s/releases" % repo, config.github_token, json=data) if not release: log("Unable to create new release") exit(1) # check that what we found/created a release if not release: log("Unable to create GitHub release for %s" % (config.version)) exit(1) # remove existing uploaded assets log("Deleting existing artifacts from the release") for asset in release.get("assets", []): log("Deleting %s" % (asset.get("id"))) github.delete("%s/releases/assets/%s" % (repo, asset.get("id")), config.github_token) # upload_url is a Hypermedia link (https://developer.github.com/v3/#hypermedia) # Example: https://uploads.github.com/repos/defold/defold/releases/25677114/assets{?name,label} # Can be parsed and expanded using: https://pypi.org/project/uritemplate/ # for now we ignore this and fix it ourselves (note this may break if GitHub # changes the way uploads are done) log("Uploading artifacts to GitHub from S3") upload_url = release.get("upload_url").replace("{?name,label}", "?name=%s") base_url = "https://" + urlparse.urlparse(config.archive_path).hostname for file in s3_release.get("files", None): # download file download_url = base_url + file.get("path") filepath = config._download(download_url) filename = re.sub(r'https://d.defold.com/archive/(.*?)/', '', download_url) # file stream upload to GitHub log("Uploading to GitHub " + filename) with open(filepath, 'rb') as f: content_type, _ = mimetypes.guess_type(filename) headers = { "Content-Type": content_type or "application/octet-stream" } github.post(upload_url % (filename), config.github_token, data=f, headers=headers) log("Released Defold %s to GitHub" % (config.version))
def release_markdown(config): """ Produce a markdown with each release item Pushes changes to a file releases.md Currently only grabs bob and editor """ log("Releasing Defold %s to GitHub" % (config.version)) if config.github_token is None: log("No GitHub authorization token") return repo = config.github_target_repo if repo is None: repo = get_default_repo() repo = "/repos/%s" % repo target_url = "%s/contents/RELEASES.md" % repo release_sha = config.github_sha1 if not release_sha: release_sha = get_git_sha1() # get the release on S3 log("Getting S3 release for version %s with sha %s" % (config.version, release_sha)) s3_release = s3.get_single_release(config.archive_path, config.version, release_sha) if not s3_release.get("files"): log("No files found on S3") exit(1) def test_path(path): if path.endswith('bob.jar'): return True if 'Defold-x86_64-' in path: return True return False release_info = {} base_url = "https://" + urlparse.urlparse(config.archive_path).hostname for file in s3_release.get("files", None): # download file download_url = base_url + file.get("path") if not test_path(download_url): continue if download_url.endswith('bob.jar'): release_info['url_bob'] = download_url if 'Defold-x86_64-' in download_url: editor_platform = 'osx' if 'win32' in download_url: editor_platform = 'win' if 'linux' in download_url: editor_platform = 'lin' release_info['url_editor_%s' % editor_platform] = download_url print "url", download_url release_info['version'] = config.version release_info['sha1'] = release_sha release_info['branch'] = get_git_branch() release_info['date'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S") response = github.get(target_url, config.github_token) if not response: log("Unable to get target_url: " + target_url) exit(1) release_data = '' target_sha = None for r in response: if r == 'sha': target_sha = response.get(r) if r == 'content': data = response.get(r) release_data = base64.b64decode(data).decode('ascii') new_data = MARKDOWN_TEMPLATE % release_info new_data += '\n\n' + release_data log("Uploading to GitHub " + target_url) post_data = {} post_data['message'] = "Release %s - %s" % (config.version, release_sha) post_data['content'] = base64.b64encode( new_data.encode('ascii')).decode('ascii') post_data[ 'sha'] = target_sha # we need the current sha of the target file in order to update it headers = {"Content-Type": "application/octet-stream"} github.put(target_url, config.github_token, json=post_data, headers=headers) log("Released Defold %s - %s to %s" % (config.version, release_sha, target_url))
def repo(): repo_dict = github.get('repos/cenkalti/github-flask') return str(repo_dict)
def release(config, tag_name, release_sha, s3_release): log("Releasing Defold %s to GitHub" % tag_name) if config.github_token is None: log("No GitHub authorization token") return channel = config.channel if channel is None: log("No release channel specified") return log("tag name: %s" % tag_name) log("release sha1: %s" % release_sha) log("channel: %s" % channel) source_repo = os.environ.get('GITHUB_REPOSITORY', "defold/defold") source_repo = "/repos/%s" % source_repo log("source repo: %s" % source_repo) target_repo = config.github_target_repo if target_repo is None: target_repo = os.environ.get('GITHUB_REPOSITORY', None) if target_repo is None: target_repo = "defold/defold" target_repo = "/repos/%s" % target_repo log("target repo: %s" % target_repo) release_name = 'v%s - %s' % (config.version, channel) draft = False # If true, it won't create a tag pre_release = channel not in ( 'stable', 'beta' ) # If true, it will be marked as "Pre-release" in the UI if not s3_release.get("files"): log("No files found on S3 with sha %s" % release_sha) exit(1) release = None response = github.get("%s/releases/tags/%s" % (target_repo, tag_name), config.github_token) if response: release = response data = { "tag_name": tag_name, "name": release_name, "body": "Defold version %s channel=%s sha1=%s date='%s'" % (config.version, channel, release_sha, datetime.now().strftime("%Y-%m-%d %H:%M:%S")), "draft": draft, "prerelease": pre_release } if not release: log("No release found with tag %s, creating a new one" % tag_name) release = github.post("%s/releases" % target_repo, config.github_token, json=data) else: release_id = release['id'] log("Updating existing release: %s - %s - %s" % (tag_name, release_id, release.get('url'))) release = github.patch("%s/releases/%s" % (target_repo, release_id), config.github_token, json=data) # check that what we created/updated a release if not release: log("Unable to update GitHub release for %s" % (config.version)) exit(1) # remove existing uploaded assets (It's not currently possible to update a release asset log("Deleting existing artifacts from the release") for asset in release.get("assets", []): asset_url = asset.get("url") log("Deleting %s - %s" % (asset.get("id"), asset.get("name"))) github.delete(asset_url, config.github_token) # upload_url is a Hypermedia link (https://developer.github.com/v3/#hypermedia) # Example: https://uploads.github.com/repos/defold/defold/releases/25677114/assets{?name,label} # Can be parsed and expanded using: https://pypi.org/project/uritemplate/ # for now we ignore this and fix it ourselves (note this may break if GitHub # changes the way uploads are done) log("Uploading artifacts to GitHub from S3") base_url = "https://" + urlparse.urlparse(config.archive_path).hostname def is_main_file(path): return os.path.basename(path) in ('bob.jar', 'Defold-x86_64-darwin.dmg', 'Defold-x86_64-linux.zip', 'Defold-x86_64-win32.zip') urls = set( ) # not sure why some files are reported twice, but we don't want to download/upload them twice for file in s3_release.get("files", None): # download file path = file.get("path") keep = False if is_main_file(path): keep = True if not keep: continue download_url = base_url + path urls.add(download_url) upload_url = release.get("upload_url").replace("{?name,label}", "?name=%s") for download_url in urls: filepath = config._download(download_url) filename = re.sub(r'https://%s/archive/(.*?)/' % config.archive_path, '', download_url) basename = os.path.basename(filename) # file stream upload to GitHub log("Uploading to GitHub " + filename) with open(filepath, 'rb') as f: content_type, _ = mimetypes.guess_type(basename) headers = { "Content-Type": content_type or "application/octet-stream" } name = filename if is_main_file(path): name = basename github.post(upload_url % (name), config.github_token, data=f, headers=headers) log("Released Defold %s to GitHub" % tag_name)
def _cache(path): (data, status, headers) = get_from_cache(path) if data is None and status != 404: return get(path) return (data, status, headers)
formatter_class=RawTextHelpFormatter) p.add_argument('-f','--force', required= False, help='Force', action='store_true') p.add_argument('-out','--output_folder', required= False, help='output folder') ns = p.parse_args() if ns.output_folder: out = ns.output_folder else: out=home = str(Path.home())+"/Desktop/Corona/" from config import REPO, TMP_FOLDER, TMP_GIT, DATA, KEEP_COLS, NUMERIC_COLS ############ DATA SELECTION ############ df = github.get() #print (df) ############ COUNTRY SELECTION ############ def get_similar_countries(c, country_list): pos_countries = get_close_matches(c, country_list) if len(pos_countries) > 0: print('\033[1;31m'+c, 'was not listed. did you mean', pos_countries[0].capitalize() + '?\033[0;0m') #Only delete if its a covidify generated folder if 'Desktop/covidify-output-' in out: